From da576cf4738bcaa32da795bf1a4021a0c34db450 Mon Sep 17 00:00:00 2001 From: Pablo Marquez Tello Date: Thu, 30 Apr 2026 15:34:39 +0100 Subject: [PATCH] perf: Enable signed requantized max pooling with differing qinfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable QASYMM8_SIGNED NHWC MAX pooling with differing input/output quantization info in the asm wrapper. Fix the A64/SVE2/SME signed requantized max pooling kernels to apply input and output offsets correctly, and add NEON pooling validation and execution coverage for the supported signed padded MAX cases. Keep AVG pooling unchanged and leave SME on the existing guarded path. Measured uplift vs generic fallback: - Neon™: 6.28x (112x112x64), 6.64x (56x56x256) - SVE: 6.48x, 7.22x - SME: 4.14x, 9.15x Signed-off-by: Pablo Marquez Tello Change-Id: I6105404638ce11e32ec3a1275eea26e0b795b98a --- .../generic.cpp | 463 ++++++++++-------- .../generic.cpp | 342 +++++++------ .../generic.cpp | 344 +++++++------ .../CpuPool2dAssemblyWrapperKernel.cpp | 4 - tests/validation/NEON/PoolingLayer.cpp | 51 ++ 5 files changed, 675 insertions(+), 529 deletions(-) diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp index 797a8f9235..94bf38c29b 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/a64_s8q_nhwc_max_generic_depthfirst/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2024 Arm Limited. + * Copyright (c) 2021-2024, 2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,20 +43,20 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( { __asm__ __volatile__( "cmp %x[n_channels], #0x40\n" - "mov x9, #0x0\n" + "mov x9, #0\n" "mov x28, #0x10\n" // cntb _, ALL, #1 "mov x27, #0x20\n" // cntb _, ALL, #2 "mov x26, #0x30\n" // cntb _, ALL, #3 "blt 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" + "movi v6.16b, #0x80\n" "movi v9.16b, #0x80\n" - "movi v8.16b, #0x80\n" "mov x24, %x[inptrs]\n" + "movi v8.16b, #0x80\n" "movi v7.16b, #0x80\n" - "movi v6.16b, #0x80\n" "cbz x25, 4f\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" "add x24, x24, #0x20\n" @@ -80,7 +80,7 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "2:" // 4-vectors of channels: 4 inputs loop "smax v23.16b, v5.16b, v4.16b\n" "smax v19.16b, v3.16b, v2.16b\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "smax v22.16b, v1.16b, v0.16b\n" "smax v18.16b, v31.16b, v30.16b\n" @@ -106,14 +106,14 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "smax v16.16b, v20.16b, v16.16b\n" "ldr q28, [x21, x27]\n" "ldr q27, [x20, x27]\n" - "smax v9.16b, v9.16b, v19.16b\n" - "smax v8.16b, v8.16b, v18.16b\n" + "smax v6.16b, v6.16b, v19.16b\n" + "smax v9.16b, v9.16b, v18.16b\n" "ldr q26, [x23, x26]\n" "ldr q20, [x22, x26]\n" - "smax v7.16b, v7.16b, v17.16b\n" + "smax v8.16b, v8.16b, v17.16b\n" "ldr q25, [x21, x26]\n" "ldr q24, [x20, x26]\n" - "smax v6.16b, v6.16b, v16.16b\n" + "smax v7.16b, v7.16b, v16.16b\n" "bgt 2b\n" "3:" // 4-vectors of channels: 4 inputs tail "smax v23.16b, v5.16b, v4.16b\n" @@ -128,10 +128,10 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "smax v18.16b, v22.16b, v18.16b\n" "smax v17.16b, v21.16b, v17.16b\n" "smax v16.16b, v20.16b, v16.16b\n" - "smax v9.16b, v9.16b, v19.16b\n" - "smax v8.16b, v8.16b, v18.16b\n" - "smax v7.16b, v7.16b, v17.16b\n" - "smax v6.16b, v6.16b, v16.16b\n" + "smax v6.16b, v6.16b, v19.16b\n" + "smax v9.16b, v9.16b, v18.16b\n" + "smax v8.16b, v8.16b, v17.16b\n" + "smax v7.16b, v7.16b, v16.16b\n" "4:" // 4-vectors of channels: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" @@ -142,134 +142,155 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "ldr q18, [x20, x28]\n" "ldr q17, [x20, x27]\n" "ldr q16, [x20, x26]\n" - "smax v9.16b, v9.16b, v19.16b\n" - "smax v8.16b, v8.16b, v18.16b\n" - "smax v7.16b, v7.16b, v17.16b\n" - "smax v6.16b, v6.16b, v16.16b\n" + "smax v6.16b, v6.16b, v19.16b\n" + "smax v9.16b, v9.16b, v18.16b\n" + "smax v8.16b, v8.16b, v17.16b\n" + "smax v7.16b, v7.16b, v16.16b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End - "sxtl v23.8h, v9.8b\n" - "sxtl2 v19.8h, v9.16b\n" - "add x21, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x21, %x[quant_params], %[offsetof_qp_input_offset]\n" + "sxtl v23.8h, v6.8b\n" + "sxtl2 v19.8h, v6.16b\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" + "ld1r { v6.4s }, [x21]\n" + "ld1r { v5.4s }, [x20]\n" + "sxtl v22.8h, v9.8b\n" + "sxtl2 v18.8h, v9.16b\n" + "sxtl v21.8h, v8.8b\n" + "sxtl2 v17.8h, v8.16b\n" + "add x21, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" "ld1r { v4.4s }, [x21]\n" "ld1r { v3.4s }, [x20]\n" - "sxtl v22.8h, v8.8b\n" - "sxtl2 v18.8h, v8.16b\n" - "sxtl v21.8h, v7.8b\n" - "sxtl2 v20.8h, v7.16b\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" + "sxtl v20.8h, v7.8b\n" + "sxtl2 v16.8h, v7.16b\n" + "neg v6.4s, v6.4s\n" + "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" + "movi v2.4s, #0x7f\n" "sub %x[n_channels], %x[n_channels], #0x40\n" - "ld1r { v2.4s }, [x20]\n" - "sxtl v17.8h, v6.8b\n" - "sxtl2 v16.8h, v6.16b\n" + "ld1r { v1.4s }, [x20]\n" "cmp %x[n_channels], #0x40\n" - "sxtl v1.4s, v23.4h\n" - "sxtl2 v23.4s, v23.8h\n" - "sxtl v0.4s, v19.4h\n" - "sxtl2 v19.4s, v19.8h\n" - "sxtl v31.4s, v22.4h\n" - "sxtl2 v22.4s, v22.8h\n" - "sxtl v30.4s, v18.4h\n" - "sxtl2 v18.4s, v18.8h\n" - "sxtl v29.4s, v21.4h\n" - "sxtl2 v21.4s, v21.8h\n" - "sxtl v28.4s, v20.4h\n" - "sxtl2 v27.4s, v20.8h\n" - "sxtl v26.4s, v17.4h\n" - "sxtl2 v20.4s, v17.8h\n" - "sxtl v25.4s, v16.4h\n" - "sxtl2 v24.4s, v16.8h\n" - "srshl v1.4s, v1.4s, v4.4s\n" - "srshl v23.4s, v23.4s, v4.4s\n" - "srshl v0.4s, v0.4s, v4.4s\n" - "srshl v19.4s, v19.4s, v4.4s\n" - "srshl v31.4s, v31.4s, v4.4s\n" - "srshl v22.4s, v22.4s, v4.4s\n" - "srshl v30.4s, v30.4s, v4.4s\n" - "srshl v18.4s, v18.4s, v4.4s\n" - "srshl v29.4s, v29.4s, v4.4s\n" - "srshl v21.4s, v21.4s, v4.4s\n" - "srshl v28.4s, v28.4s, v4.4s\n" - "srshl v27.4s, v27.4s, v4.4s\n" - "srshl v26.4s, v26.4s, v4.4s\n" - "srshl v20.4s, v20.4s, v4.4s\n" - "srshl v25.4s, v25.4s, v4.4s\n" - "srshl v24.4s, v24.4s, v4.4s\n" - "sqrdmulh v1.4s, v1.4s, v3.4s\n" - "sqrdmulh v23.4s, v23.4s, v3.4s\n" - "sqrdmulh v0.4s, v0.4s, v3.4s\n" - "sqrdmulh v19.4s, v19.4s, v3.4s\n" - "sqrdmulh v31.4s, v31.4s, v3.4s\n" - "sqrdmulh v22.4s, v22.4s, v3.4s\n" - "sqrdmulh v30.4s, v30.4s, v3.4s\n" - "sqrdmulh v18.4s, v18.4s, v3.4s\n" - "sqrdmulh v29.4s, v29.4s, v3.4s\n" - "sqrdmulh v21.4s, v21.4s, v3.4s\n" - "sqrdmulh v28.4s, v28.4s, v3.4s\n" - "sqrdmulh v27.4s, v27.4s, v3.4s\n" - "sqrdmulh v26.4s, v26.4s, v3.4s\n" - "sqrdmulh v20.4s, v20.4s, v3.4s\n" - "sqrdmulh v25.4s, v25.4s, v3.4s\n" - "sqrdmulh v24.4s, v24.4s, v3.4s\n" - "movi v17.4s, #0x7f\n" - "srshl v1.4s, v1.4s, v2.4s\n" - "srshl v23.4s, v23.4s, v2.4s\n" - "srshl v0.4s, v0.4s, v2.4s\n" - "srshl v19.4s, v19.4s, v2.4s\n" - "srshl v31.4s, v31.4s, v2.4s\n" - "srshl v22.4s, v22.4s, v2.4s\n" - "srshl v30.4s, v30.4s, v2.4s\n" - "srshl v18.4s, v18.4s, v2.4s\n" - "srshl v29.4s, v29.4s, v2.4s\n" - "srshl v21.4s, v21.4s, v2.4s\n" - "srshl v28.4s, v28.4s, v2.4s\n" - "srshl v27.4s, v27.4s, v2.4s\n" - "srshl v26.4s, v26.4s, v2.4s\n" - "srshl v20.4s, v20.4s, v2.4s\n" - "srshl v25.4s, v25.4s, v2.4s\n" - "srshl v24.4s, v24.4s, v2.4s\n" - "not v16.16b, v17.16b\n" - "smax v1.4s, v1.4s, v16.4s\n" - "smax v23.4s, v23.4s, v16.4s\n" - "smax v0.4s, v0.4s, v16.4s\n" - "smax v19.4s, v19.4s, v16.4s\n" - "smax v31.4s, v31.4s, v16.4s\n" - "smax v22.4s, v22.4s, v16.4s\n" - "smax v30.4s, v30.4s, v16.4s\n" - "smax v18.4s, v18.4s, v16.4s\n" - "smax v29.4s, v29.4s, v16.4s\n" - "smax v21.4s, v21.4s, v16.4s\n" - "smax v28.4s, v28.4s, v16.4s\n" - "smax v27.4s, v27.4s, v16.4s\n" - "smax v26.4s, v26.4s, v16.4s\n" - "smax v20.4s, v20.4s, v16.4s\n" - "smax v25.4s, v25.4s, v16.4s\n" - "smax v24.4s, v24.4s, v16.4s\n" - "smin v1.4s, v1.4s, v17.4s\n" - "smin v23.4s, v23.4s, v17.4s\n" - "smin v0.4s, v0.4s, v17.4s\n" - "smin v19.4s, v19.4s, v17.4s\n" - "smin v31.4s, v31.4s, v17.4s\n" - "smin v22.4s, v22.4s, v17.4s\n" - "smin v30.4s, v30.4s, v17.4s\n" - "smin v18.4s, v18.4s, v17.4s\n" - "smin v29.4s, v29.4s, v17.4s\n" - "smin v21.4s, v21.4s, v17.4s\n" - "smin v28.4s, v28.4s, v17.4s\n" - "smin v27.4s, v27.4s, v17.4s\n" - "smin v26.4s, v26.4s, v17.4s\n" - "smin v20.4s, v20.4s, v17.4s\n" - "smin v25.4s, v25.4s, v17.4s\n" - "smin v24.4s, v24.4s, v17.4s\n" - "uzp1 v23.16b, v1.16b, v23.16b\n" - "uzp1 v19.16b, v0.16b, v19.16b\n" - "uzp1 v22.16b, v31.16b, v22.16b\n" - "uzp1 v18.16b, v30.16b, v18.16b\n" - "uzp1 v21.16b, v29.16b, v21.16b\n" - "uzp1 v17.16b, v28.16b, v27.16b\n" - "uzp1 v20.16b, v26.16b, v20.16b\n" - "uzp1 v16.16b, v25.16b, v24.16b\n" + "not v0.16b, v2.16b\n" + "saddw v31.4s, v6.4s, v23.4h\n" + "saddw2 v23.4s, v6.4s, v23.8h\n" + "saddw v30.4s, v6.4s, v19.4h\n" + "saddw2 v19.4s, v6.4s, v19.8h\n" + "saddw v29.4s, v6.4s, v22.4h\n" + "saddw2 v22.4s, v6.4s, v22.8h\n" + "saddw v28.4s, v6.4s, v18.4h\n" + "saddw2 v18.4s, v6.4s, v18.8h\n" + "saddw v27.4s, v6.4s, v21.4h\n" + "saddw2 v21.4s, v6.4s, v21.8h\n" + "saddw v26.4s, v6.4s, v17.4h\n" + "saddw2 v17.4s, v6.4s, v17.8h\n" + "saddw v25.4s, v6.4s, v20.4h\n" + "saddw2 v20.4s, v6.4s, v20.8h\n" + "saddw v24.4s, v6.4s, v16.4h\n" + "saddw2 v16.4s, v6.4s, v16.8h\n" + "srshl v31.4s, v31.4s, v5.4s\n" + "srshl v23.4s, v23.4s, v5.4s\n" + "srshl v30.4s, v30.4s, v5.4s\n" + "srshl v19.4s, v19.4s, v5.4s\n" + "srshl v29.4s, v29.4s, v5.4s\n" + "srshl v22.4s, v22.4s, v5.4s\n" + "srshl v28.4s, v28.4s, v5.4s\n" + "srshl v18.4s, v18.4s, v5.4s\n" + "srshl v27.4s, v27.4s, v5.4s\n" + "srshl v21.4s, v21.4s, v5.4s\n" + "srshl v26.4s, v26.4s, v5.4s\n" + "srshl v17.4s, v17.4s, v5.4s\n" + "srshl v25.4s, v25.4s, v5.4s\n" + "srshl v20.4s, v20.4s, v5.4s\n" + "srshl v24.4s, v24.4s, v5.4s\n" + "srshl v16.4s, v16.4s, v5.4s\n" + "sqrdmulh v31.4s, v31.4s, v4.4s\n" + "sqrdmulh v23.4s, v23.4s, v4.4s\n" + "sqrdmulh v30.4s, v30.4s, v4.4s\n" + "sqrdmulh v19.4s, v19.4s, v4.4s\n" + "sqrdmulh v29.4s, v29.4s, v4.4s\n" + "sqrdmulh v22.4s, v22.4s, v4.4s\n" + "sqrdmulh v28.4s, v28.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v27.4s, v27.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v26.4s, v26.4s, v4.4s\n" + "sqrdmulh v17.4s, v17.4s, v4.4s\n" + "sqrdmulh v25.4s, v25.4s, v4.4s\n" + "sqrdmulh v20.4s, v20.4s, v4.4s\n" + "sqrdmulh v24.4s, v24.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" + "srshl v31.4s, v31.4s, v3.4s\n" + "srshl v23.4s, v23.4s, v3.4s\n" + "srshl v30.4s, v30.4s, v3.4s\n" + "srshl v19.4s, v19.4s, v3.4s\n" + "srshl v29.4s, v29.4s, v3.4s\n" + "srshl v22.4s, v22.4s, v3.4s\n" + "srshl v28.4s, v28.4s, v3.4s\n" + "srshl v18.4s, v18.4s, v3.4s\n" + "srshl v27.4s, v27.4s, v3.4s\n" + "srshl v21.4s, v21.4s, v3.4s\n" + "srshl v26.4s, v26.4s, v3.4s\n" + "srshl v17.4s, v17.4s, v3.4s\n" + "srshl v25.4s, v25.4s, v3.4s\n" + "srshl v20.4s, v20.4s, v3.4s\n" + "srshl v24.4s, v24.4s, v3.4s\n" + "srshl v16.4s, v16.4s, v3.4s\n" + "add v31.4s, v31.4s, v1.4s\n" + "add v23.4s, v23.4s, v1.4s\n" + "add v30.4s, v30.4s, v1.4s\n" + "add v19.4s, v19.4s, v1.4s\n" + "add v29.4s, v29.4s, v1.4s\n" + "add v22.4s, v22.4s, v1.4s\n" + "add v28.4s, v28.4s, v1.4s\n" + "add v18.4s, v18.4s, v1.4s\n" + "add v27.4s, v27.4s, v1.4s\n" + "add v21.4s, v21.4s, v1.4s\n" + "add v26.4s, v26.4s, v1.4s\n" + "add v17.4s, v17.4s, v1.4s\n" + "add v25.4s, v25.4s, v1.4s\n" + "add v20.4s, v20.4s, v1.4s\n" + "add v24.4s, v24.4s, v1.4s\n" + "add v16.4s, v16.4s, v1.4s\n" + "smax v31.4s, v31.4s, v0.4s\n" + "smax v23.4s, v23.4s, v0.4s\n" + "smax v30.4s, v30.4s, v0.4s\n" + "smax v19.4s, v19.4s, v0.4s\n" + "smax v29.4s, v29.4s, v0.4s\n" + "smax v22.4s, v22.4s, v0.4s\n" + "smax v28.4s, v28.4s, v0.4s\n" + "smax v18.4s, v18.4s, v0.4s\n" + "smax v27.4s, v27.4s, v0.4s\n" + "smax v21.4s, v21.4s, v0.4s\n" + "smax v26.4s, v26.4s, v0.4s\n" + "smax v17.4s, v17.4s, v0.4s\n" + "smax v25.4s, v25.4s, v0.4s\n" + "smax v20.4s, v20.4s, v0.4s\n" + "smax v24.4s, v24.4s, v0.4s\n" + "smax v16.4s, v16.4s, v0.4s\n" + "smin v31.4s, v31.4s, v2.4s\n" + "smin v23.4s, v23.4s, v2.4s\n" + "smin v30.4s, v30.4s, v2.4s\n" + "smin v19.4s, v19.4s, v2.4s\n" + "smin v29.4s, v29.4s, v2.4s\n" + "smin v22.4s, v22.4s, v2.4s\n" + "smin v28.4s, v28.4s, v2.4s\n" + "smin v18.4s, v18.4s, v2.4s\n" + "smin v27.4s, v27.4s, v2.4s\n" + "smin v21.4s, v21.4s, v2.4s\n" + "smin v26.4s, v26.4s, v2.4s\n" + "smin v17.4s, v17.4s, v2.4s\n" + "smin v25.4s, v25.4s, v2.4s\n" + "smin v20.4s, v20.4s, v2.4s\n" + "smin v24.4s, v24.4s, v2.4s\n" + "smin v16.4s, v16.4s, v2.4s\n" + "uzp1 v23.16b, v31.16b, v23.16b\n" + "uzp1 v19.16b, v30.16b, v19.16b\n" + "uzp1 v22.16b, v29.16b, v22.16b\n" + "uzp1 v18.16b, v28.16b, v18.16b\n" + "uzp1 v21.16b, v27.16b, v21.16b\n" + "uzp1 v17.16b, v26.16b, v17.16b\n" + "uzp1 v20.16b, v25.16b, v20.16b\n" + "uzp1 v16.16b, v24.16b, v16.16b\n" "uzp1 v19.16b, v23.16b, v19.16b\n" "uzp1 v18.16b, v22.16b, v18.16b\n" "uzp1 v17.16b, v21.16b, v17.16b\n" @@ -289,10 +310,10 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "blt 14f\n" "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" - "movi v9.16b, #0x80\n" + "movi v6.16b, #0x80\n" "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" "add x24, x24, #0x20\n" @@ -304,7 +325,7 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "9:" // Single vector of channels: Loop: 4 inputs loop "smax v17.16b, v5.16b, v4.16b\n" "smax v16.16b, v3.16b, v2.16b\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" "add x24, x24, #0x20\n" @@ -313,13 +334,13 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "ldr q3, [x21, x9]\n" "ldr q2, [x20, x9]\n" "smax v16.16b, v17.16b, v16.16b\n" - "smax v9.16b, v9.16b, v16.16b\n" + "smax v6.16b, v6.16b, v16.16b\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 4 inputs tail "smax v17.16b, v5.16b, v4.16b\n" "smax v16.16b, v3.16b, v2.16b\n" "smax v16.16b, v17.16b, v16.16b\n" - "smax v9.16b, v9.16b, v16.16b\n" + "smax v6.16b, v6.16b, v16.16b\n" "11:" // Single vector of channels: Loop: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 13f\n" @@ -327,47 +348,56 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "ldr x20, [x24], #0x8\n" "subs x21, x21, #0x1\n" "ldr q16, [x20, x9]\n" - "smax v9.16b, v9.16b, v16.16b\n" + "smax v6.16b, v6.16b, v16.16b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "sxtl v17.8h, v9.8b\n" - "sxtl2 v16.8h, v9.16b\n" - "add x21, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x21, %x[quant_params], %[offsetof_qp_input_offset]\n" + "sxtl v17.8h, v6.8b\n" + "sxtl2 v26.8h, v6.16b\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" + "ld1r { v16.4s }, [x21]\n" + "ld1r { v25.4s }, [x20]\n" + "add x21, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" "ld1r { v24.4s }, [x21]\n" "ld1r { v23.4s }, [x20]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" + "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" "movi v22.4s, #0x7f\n" "ld1r { v21.4s }, [x20]\n" "sub %x[n_channels], %x[n_channels], #0x10\n" - "sxtl v20.4s, v17.4h\n" - "sxtl2 v17.4s, v17.8h\n" - "sxtl v19.4s, v16.4h\n" - "sxtl2 v18.4s, v16.8h\n" + "neg v16.4s, v16.4s\n" + "not v20.16b, v22.16b\n" "cmp %x[n_channels], #0x10\n" - "not v16.16b, v22.16b\n" - "srshl v20.4s, v20.4s, v24.4s\n" - "srshl v17.4s, v17.4s, v24.4s\n" - "srshl v19.4s, v19.4s, v24.4s\n" - "srshl v18.4s, v18.4s, v24.4s\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "sqrdmulh v17.4s, v17.4s, v23.4s\n" - "sqrdmulh v19.4s, v19.4s, v23.4s\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "srshl v20.4s, v20.4s, v21.4s\n" - "srshl v17.4s, v17.4s, v21.4s\n" - "srshl v19.4s, v19.4s, v21.4s\n" - "srshl v18.4s, v18.4s, v21.4s\n" - "smax v20.4s, v20.4s, v16.4s\n" - "smax v17.4s, v17.4s, v16.4s\n" - "smax v19.4s, v19.4s, v16.4s\n" - "smax v18.4s, v18.4s, v16.4s\n" - "smin v20.4s, v20.4s, v22.4s\n" - "smin v17.4s, v17.4s, v22.4s\n" + "saddw v19.4s, v16.4s, v17.4h\n" + "saddw2 v17.4s, v16.4s, v17.8h\n" + "saddw v18.4s, v16.4s, v26.4h\n" + "saddw2 v16.4s, v16.4s, v26.8h\n" + "srshl v19.4s, v19.4s, v25.4s\n" + "srshl v17.4s, v17.4s, v25.4s\n" + "srshl v18.4s, v18.4s, v25.4s\n" + "srshl v16.4s, v16.4s, v25.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "sqrdmulh v17.4s, v17.4s, v24.4s\n" + "sqrdmulh v18.4s, v18.4s, v24.4s\n" + "sqrdmulh v16.4s, v16.4s, v24.4s\n" + "srshl v19.4s, v19.4s, v23.4s\n" + "srshl v17.4s, v17.4s, v23.4s\n" + "srshl v18.4s, v18.4s, v23.4s\n" + "srshl v16.4s, v16.4s, v23.4s\n" + "add v19.4s, v19.4s, v21.4s\n" + "add v17.4s, v17.4s, v21.4s\n" + "add v18.4s, v18.4s, v21.4s\n" + "add v16.4s, v16.4s, v21.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v16.4s, v16.4s, v20.4s\n" "smin v19.4s, v19.4s, v22.4s\n" + "smin v17.4s, v17.4s, v22.4s\n" "smin v18.4s, v18.4s, v22.4s\n" - "uzp1 v17.16b, v20.16b, v17.16b\n" - "uzp1 v16.16b, v19.16b, v18.16b\n" + "smin v16.4s, v16.4s, v22.4s\n" + "uzp1 v17.16b, v19.16b, v17.16b\n" + "uzp1 v16.16b, v18.16b, v16.16b\n" "uzp1 v16.16b, v17.16b, v16.16b\n" "str q16, [%x[outptr], x9]\n" "add x9, x9, #0x10\n" @@ -376,17 +406,17 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "14:" // Oddments "lsr x25, %x[n_valid_cells], #0x2\n" "add %x[outptr], %x[outptr], x9\n" - "movi v9.16b, #0x80\n" + "movi v6.16b, #0x80\n" "mov x24, %x[inptrs]\n" "cbz x25, 24f\n" "15:" // Oddments: 4 inputs loop - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "add x24, x24, #0x20\n" - "movi v5.16b, #0x0\n" - "movi v4.16b, #0x0\n" - "movi v3.16b, #0x0\n" - "movi v2.16b, #0x0\n" + "movi v5.16b, #0\n" + "movi v4.16b, #0\n" + "movi v3.16b, #0\n" + "movi v2.16b, #0\n" "add x23, x23, x9\n" "add x22, x22, x9\n" "add x21, x21, x9\n" @@ -485,14 +515,14 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "smax v16.16b, v3.16b, v2.16b\n" "subs x25, x25, #0x1\n" "smax v16.16b, v17.16b, v16.16b\n" - "smax v9.16b, v9.16b, v16.16b\n" + "smax v6.16b, v6.16b, v16.16b\n" "bgt 15b\n" "24:" // Oddments: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 34f\n" "25:" // Oddments: Single input loop "ldr x23, [x24], #0x8\n" - "movi v5.16b, #0x0\n" + "movi v5.16b, #0\n" "add x23, x23, x9\n" "tbz %x[n_channels], #3, 29f\n" "ldr d5, [x23], #0x8\n" @@ -540,45 +570,54 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "ldr b5, [x23], #0x1\n" "33:" // Oddments: Single input loop: Load: Bit 3: End "subs x21, x21, #0x1\n" - "smax v9.16b, v9.16b, v5.16b\n" + "smax v6.16b, v6.16b, v5.16b\n" "bgt 25b\n" "34:" // Oddments: Single input loop: End - "sxtl v17.8h, v9.8b\n" - "sxtl2 v16.8h, v9.16b\n" - "add x21, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x21, %x[quant_params], %[offsetof_qp_input_offset]\n" + "sxtl v17.8h, v6.8b\n" + "sxtl2 v26.8h, v6.16b\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" + "ld1r { v16.4s }, [x21]\n" + "ld1r { v25.4s }, [x20]\n" + "add x21, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" "ld1r { v24.4s }, [x21]\n" "ld1r { v23.4s }, [x20]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" + "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" "movi v22.4s, #0x7f\n" "ld1r { v21.4s }, [x20]\n" - "sxtl v20.4s, v17.4h\n" - "sxtl2 v17.4s, v17.8h\n" - "sxtl v19.4s, v16.4h\n" - "sxtl2 v18.4s, v16.8h\n" - "not v16.16b, v22.16b\n" - "srshl v20.4s, v20.4s, v24.4s\n" - "srshl v17.4s, v17.4s, v24.4s\n" - "srshl v19.4s, v19.4s, v24.4s\n" - "srshl v18.4s, v18.4s, v24.4s\n" - "sqrdmulh v20.4s, v20.4s, v23.4s\n" - "sqrdmulh v17.4s, v17.4s, v23.4s\n" - "sqrdmulh v19.4s, v19.4s, v23.4s\n" - "sqrdmulh v18.4s, v18.4s, v23.4s\n" - "srshl v20.4s, v20.4s, v21.4s\n" - "srshl v17.4s, v17.4s, v21.4s\n" - "srshl v19.4s, v19.4s, v21.4s\n" - "srshl v18.4s, v18.4s, v21.4s\n" - "smax v20.4s, v20.4s, v16.4s\n" - "smax v17.4s, v17.4s, v16.4s\n" - "smax v19.4s, v19.4s, v16.4s\n" - "smax v18.4s, v18.4s, v16.4s\n" - "smin v20.4s, v20.4s, v22.4s\n" - "smin v17.4s, v17.4s, v22.4s\n" + "neg v16.4s, v16.4s\n" + "not v20.16b, v22.16b\n" + "saddw v19.4s, v16.4s, v17.4h\n" + "saddw2 v17.4s, v16.4s, v17.8h\n" + "saddw v18.4s, v16.4s, v26.4h\n" + "saddw2 v16.4s, v16.4s, v26.8h\n" + "srshl v19.4s, v19.4s, v25.4s\n" + "srshl v17.4s, v17.4s, v25.4s\n" + "srshl v18.4s, v18.4s, v25.4s\n" + "srshl v16.4s, v16.4s, v25.4s\n" + "sqrdmulh v19.4s, v19.4s, v24.4s\n" + "sqrdmulh v17.4s, v17.4s, v24.4s\n" + "sqrdmulh v18.4s, v18.4s, v24.4s\n" + "sqrdmulh v16.4s, v16.4s, v24.4s\n" + "srshl v19.4s, v19.4s, v23.4s\n" + "srshl v17.4s, v17.4s, v23.4s\n" + "srshl v18.4s, v18.4s, v23.4s\n" + "srshl v16.4s, v16.4s, v23.4s\n" + "add v19.4s, v19.4s, v21.4s\n" + "add v17.4s, v17.4s, v21.4s\n" + "add v18.4s, v18.4s, v21.4s\n" + "add v16.4s, v16.4s, v21.4s\n" + "smax v19.4s, v19.4s, v20.4s\n" + "smax v17.4s, v17.4s, v20.4s\n" + "smax v18.4s, v18.4s, v20.4s\n" + "smax v16.4s, v16.4s, v20.4s\n" "smin v19.4s, v19.4s, v22.4s\n" + "smin v17.4s, v17.4s, v22.4s\n" "smin v18.4s, v18.4s, v22.4s\n" - "uzp1 v17.16b, v20.16b, v17.16b\n" - "uzp1 v16.16b, v19.16b, v18.16b\n" + "smin v16.4s, v16.4s, v22.4s\n" + "uzp1 v17.16b, v19.16b, v17.16b\n" + "uzp1 v16.16b, v18.16b, v16.16b\n" "uzp1 v16.16b, v17.16b, v16.16b\n" "tbz %x[n_channels], #3, 38f\n" "st1 { v16.d }[0], [%x[outptr]], #0x8\n" @@ -627,7 +666,7 @@ void a64_s8q_nhwc_max_generic_depthfirst_impl( "42:" // Oddments: Store: Bit 3: End "43:" // End : [n_channels] "+&r" (n_channels), [outptr] "+&r" (outptr) - : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [quant_params] "r" (&qp) + : [inptrs] "r" (inptrs), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_input_offset] "I" (offsetof(Requantize32, input_offset)), [offsetof_qp_output_offset] "I" (offsetof(Requantize32, output_offset)), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [quant_params] "r" (&qp) : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28" ); } diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_max_generic_depthfirst/generic.cpp index 6895fd2011..1c91baf41b 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_s8q_nhwc_max_generic_depthfirst/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2024 Arm Limited. + * Copyright (c) 2022-2024, 2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -43,7 +43,7 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( { __asm__ __volatile__( ".inst 0xd503477f // SMSTART ZA\n" - "mov x9, #0x0\n" + "mov x9, #0\n" "cntb x28\n" "cntb x27, ALL, MUL #2\n" "cntb x26, ALL, MUL #3\n" @@ -52,16 +52,16 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "whilelt p2.b, x27, %x[n_channels]\n" "whilelt p1.b, x26, %x[n_channels]\n" "ptrue p0.b\n" - "b.none 7f\n" + "b.eq 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" "mov z5.b, #0x80\n" - "mov z3.b, #0x80\n" + "mov z4.b, #0x80\n" "mov x24, %x[inptrs]\n" + "mov z3.b, #0x80\n" "mov z2.b, #0x80\n" - "mov z4.b, #0x80\n" "cbz x25, 4f\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "subs x25, x25, #0x1\n" "ldp x21, x20, [x24, #0x10]\n" "add x24, x24, #0x20\n" @@ -85,7 +85,7 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z1\n smax z19.b, p0/M, z19.b, z0.b\n" "smax z23.b, p0/M, z23.b, z31.b\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "subs x25, x25, #0x1\n" "movprfx z18, z30\n smax z18.b, p0/M, z18.b, z29.b\n" "smax z22.b, p0/M, z22.b, z28.b\n" @@ -106,11 +106,11 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "ld1b { z30.b }, p3/Z, [x23, x28]\n" "smax z5.b, p0/M, z5.b, z19.b\n" "ld1b { z29.b }, p3/Z, [x22, x28]\n" - "smax z3.b, p0/M, z3.b, z18.b\n" + "smax z4.b, p0/M, z4.b, z18.b\n" "ld1b { z22.b }, p3/Z, [x21, x28]\n" - "smax z2.b, p0/M, z2.b, z17.b\n" + "smax z3.b, p0/M, z3.b, z17.b\n" "ld1b { z28.b }, p3/Z, [x20, x28]\n" - "smax z4.b, p0/M, z4.b, z16.b\n" + "smax z2.b, p0/M, z2.b, z16.b\n" "ld1b { z17.b }, p2/Z, [x23, x27]\n" "ld1b { z27.b }, p2/Z, [x22, x27]\n" "ld1b { z21.b }, p2/Z, [x21, x27]\n" @@ -134,9 +134,9 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "smax z17.b, p0/M, z17.b, z21.b\n" "smax z16.b, p0/M, z16.b, z20.b\n" "smax z5.b, p0/M, z5.b, z19.b\n" - "smax z3.b, p0/M, z3.b, z18.b\n" - "smax z2.b, p0/M, z2.b, z17.b\n" - "smax z4.b, p0/M, z4.b, z16.b\n" + "smax z4.b, p0/M, z4.b, z18.b\n" + "smax z3.b, p0/M, z3.b, z17.b\n" + "smax z2.b, p0/M, z2.b, z16.b\n" "4:" // 4-vectors of channels: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" @@ -148,129 +148,150 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "ld1b { z17.b }, p2/Z, [x20, x27]\n" "ld1b { z16.b }, p1/Z, [x20, x26]\n" "smax z5.b, p0/M, z5.b, z19.b\n" - "smax z3.b, p0/M, z3.b, z18.b\n" - "smax z2.b, p0/M, z2.b, z17.b\n" - "smax z4.b, p0/M, z4.b, z16.b\n" + "smax z4.b, p0/M, z4.b, z18.b\n" + "smax z3.b, p0/M, z3.b, z17.b\n" + "smax z2.b, p0/M, z2.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End - ".inst 0x4508a0b3 // sshllb z19.h, z5.b, #0x0\n" - ".inst 0x4508a4b8 // sshllt z24.h, z5.b, #0x0\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "add x21, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - ".inst 0x4508a076 // sshllb z22.h, z3.b, #0x0\n" - ".inst 0x4508a472 // sshllt z18.h, z3.b, #0x0\n" + "add x20, %x[quant_params], %[offsetof_qp_input_offset]\n" + ".inst 0x4508a0b3 // sshllb z19.h, z5.b, #0\n" + ".inst 0x4508a4b6 // sshllt z22.h, z5.b, #0\n" + "add x21, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" + "ld1rw { z5.s }, p0/Z, [x20]\n" + ".inst 0x4508a095 // sshllb z21.h, z4.b, #0\n" + ".inst 0x4508a492 // sshllt z18.h, z4.b, #0\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + ".inst 0x4508a074 // sshllb z20.h, z3.b, #0\n" + ".inst 0x4508a471 // sshllt z17.h, z3.b, #0\n" + "ld1rw { z4.s }, p0/Z, [x21]\n" + "add x21, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" + ".inst 0x4508a058 // sshllb z24.h, z2.b, #0\n" + ".inst 0x4508a457 // sshllt z23.h, z2.b, #0\n" "ld1rw { z3.s }, p0/Z, [x20]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - ".inst 0x4508a055 // sshllb z21.h, z2.b, #0x0\n" - ".inst 0x4508a454 // sshllt z20.h, z2.b, #0x0\n" + "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" + "neg z5.s, p0/M, z5.s\n" "ld1rw { z2.s }, p0/Z, [x21]\n" - ".inst 0x4508a097 // sshllb z23.h, z4.b, #0x0\n" - ".inst 0x4508a491 // sshllt z17.h, z4.b, #0x0\n" + "mov z1.s, #0x7f\n" "ld1rw { z16.s }, p0/Z, [x20]\n" - ".inst 0x4510a261 // sshllb z1.s, z19.h, #0x0\n" - ".inst 0x4510a673 // sshllt z19.s, z19.h, #0x0\n" - ".inst 0x4510a300 // sshllb z0.s, z24.h, #0x0\n" - ".inst 0x4510a71f // sshllt z31.s, z24.h, #0x0\n" - ".inst 0x4510a2de // sshllb z30.s, z22.h, #0x0\n" - ".inst 0x4510a6d6 // sshllt z22.s, z22.h, #0x0\n" - ".inst 0x4510a25d // sshllb z29.s, z18.h, #0x0\n" - ".inst 0x4510a652 // sshllt z18.s, z18.h, #0x0\n" - ".inst 0x4510a2bc // sshllb z28.s, z21.h, #0x0\n" - ".inst 0x4510a6b5 // sshllt z21.s, z21.h, #0x0\n" - ".inst 0x4510a29b // sshllb z27.s, z20.h, #0x0\n" - ".inst 0x4510a694 // sshllt z20.s, z20.h, #0x0\n" - ".inst 0x4510a2fa // sshllb z26.s, z23.h, #0x0\n" - ".inst 0x4510a6f9 // sshllt z25.s, z23.h, #0x0\n" - ".inst 0x4510a238 // sshllb z24.s, z17.h, #0x0\n" - ".inst 0x4510a637 // sshllt z23.s, z17.h, #0x0\n" - ".inst 0x44828061 // srshl z1.s, p0/M, z1.s, z3.s\n" - ".inst 0x44828073 // srshl z19.s, p0/M, z19.s, z3.s\n" - ".inst 0x44828060 // srshl z0.s, p0/M, z0.s, z3.s\n" - ".inst 0x4482807f // srshl z31.s, p0/M, z31.s, z3.s\n" - ".inst 0x4482807e // srshl z30.s, p0/M, z30.s, z3.s\n" - ".inst 0x44828076 // srshl z22.s, p0/M, z22.s, z3.s\n" - ".inst 0x4482807d // srshl z29.s, p0/M, z29.s, z3.s\n" - ".inst 0x44828072 // srshl z18.s, p0/M, z18.s, z3.s\n" - ".inst 0x4482807c // srshl z28.s, p0/M, z28.s, z3.s\n" - ".inst 0x44828075 // srshl z21.s, p0/M, z21.s, z3.s\n" - ".inst 0x4482807b // srshl z27.s, p0/M, z27.s, z3.s\n" - ".inst 0x44828074 // srshl z20.s, p0/M, z20.s, z3.s\n" - ".inst 0x4482807a // srshl z26.s, p0/M, z26.s, z3.s\n" - ".inst 0x44828079 // srshl z25.s, p0/M, z25.s, z3.s\n" - ".inst 0x44828078 // srshl z24.s, p0/M, z24.s, z3.s\n" - ".inst 0x44828077 // srshl z23.s, p0/M, z23.s, z3.s\n" - ".inst 0x04a27421 // sqrdmulh z1.s, z1.s, z2.s\n" - ".inst 0x04a27673 // sqrdmulh z19.s, z19.s, z2.s\n" - ".inst 0x04a27400 // sqrdmulh z0.s, z0.s, z2.s\n" - ".inst 0x04a277ff // sqrdmulh z31.s, z31.s, z2.s\n" - ".inst 0x04a277de // sqrdmulh z30.s, z30.s, z2.s\n" - ".inst 0x04a276d6 // sqrdmulh z22.s, z22.s, z2.s\n" - ".inst 0x04a277bd // sqrdmulh z29.s, z29.s, z2.s\n" - ".inst 0x04a27652 // sqrdmulh z18.s, z18.s, z2.s\n" - ".inst 0x04a2779c // sqrdmulh z28.s, z28.s, z2.s\n" - ".inst 0x04a276b5 // sqrdmulh z21.s, z21.s, z2.s\n" - ".inst 0x04a2777b // sqrdmulh z27.s, z27.s, z2.s\n" - ".inst 0x04a27694 // sqrdmulh z20.s, z20.s, z2.s\n" - ".inst 0x04a2775a // sqrdmulh z26.s, z26.s, z2.s\n" - ".inst 0x04a27739 // sqrdmulh z25.s, z25.s, z2.s\n" - ".inst 0x04a27718 // sqrdmulh z24.s, z24.s, z2.s\n" - ".inst 0x04a276f7 // sqrdmulh z23.s, z23.s, z2.s\n" - "mov z17.s, #0x7f\n" - ".inst 0x44828201 // srshl z1.s, p0/M, z1.s, z16.s\n" - ".inst 0x44828213 // srshl z19.s, p0/M, z19.s, z16.s\n" - ".inst 0x44828200 // srshl z0.s, p0/M, z0.s, z16.s\n" - ".inst 0x4482821f // srshl z31.s, p0/M, z31.s, z16.s\n" - ".inst 0x4482821e // srshl z30.s, p0/M, z30.s, z16.s\n" - ".inst 0x44828216 // srshl z22.s, p0/M, z22.s, z16.s\n" - ".inst 0x4482821d // srshl z29.s, p0/M, z29.s, z16.s\n" - ".inst 0x44828212 // srshl z18.s, p0/M, z18.s, z16.s\n" - ".inst 0x4482821c // srshl z28.s, p0/M, z28.s, z16.s\n" - ".inst 0x44828215 // srshl z21.s, p0/M, z21.s, z16.s\n" - ".inst 0x4482821b // srshl z27.s, p0/M, z27.s, z16.s\n" - ".inst 0x44828214 // srshl z20.s, p0/M, z20.s, z16.s\n" - ".inst 0x4482821a // srshl z26.s, p0/M, z26.s, z16.s\n" - ".inst 0x44828219 // srshl z25.s, p0/M, z25.s, z16.s\n" - ".inst 0x44828218 // srshl z24.s, p0/M, z24.s, z16.s\n" - ".inst 0x44828217 // srshl z23.s, p0/M, z23.s, z16.s\n" - "not z16.s, p0/M, z17.s\n" - "smax z1.s, p0/M, z1.s, z16.s\n" - "smax z19.s, p0/M, z19.s, z16.s\n" + ".inst 0x459340a0 // saddwb z0.s, z5.s, z19.h\n" + ".inst 0x459344b3 // saddwt z19.s, z5.s, z19.h\n" + ".inst 0x459640bf // saddwb z31.s, z5.s, z22.h\n" + ".inst 0x459644be // saddwt z30.s, z5.s, z22.h\n" + ".inst 0x459540bd // saddwb z29.s, z5.s, z21.h\n" + ".inst 0x459544b6 // saddwt z22.s, z5.s, z21.h\n" + ".inst 0x459240bc // saddwb z28.s, z5.s, z18.h\n" + ".inst 0x459244b2 // saddwt z18.s, z5.s, z18.h\n" + ".inst 0x459440bb // saddwb z27.s, z5.s, z20.h\n" + ".inst 0x459444b5 // saddwt z21.s, z5.s, z20.h\n" + ".inst 0x459140b4 // saddwb z20.s, z5.s, z17.h\n" + ".inst 0x459144b1 // saddwt z17.s, z5.s, z17.h\n" + ".inst 0x459840ba // saddwb z26.s, z5.s, z24.h\n" + ".inst 0x459844b9 // saddwt z25.s, z5.s, z24.h\n" + ".inst 0x459740b8 // saddwb z24.s, z5.s, z23.h\n" + ".inst 0x459744b7 // saddwt z23.s, z5.s, z23.h\n" + ".inst 0x44828080 // srshl z0.s, p0/M, z0.s, z4.s\n" + ".inst 0x44828093 // srshl z19.s, p0/M, z19.s, z4.s\n" + ".inst 0x4482809f // srshl z31.s, p0/M, z31.s, z4.s\n" + ".inst 0x4482809e // srshl z30.s, p0/M, z30.s, z4.s\n" + ".inst 0x4482809d // srshl z29.s, p0/M, z29.s, z4.s\n" + ".inst 0x44828096 // srshl z22.s, p0/M, z22.s, z4.s\n" + ".inst 0x4482809c // srshl z28.s, p0/M, z28.s, z4.s\n" + ".inst 0x44828092 // srshl z18.s, p0/M, z18.s, z4.s\n" + ".inst 0x4482809b // srshl z27.s, p0/M, z27.s, z4.s\n" + ".inst 0x44828095 // srshl z21.s, p0/M, z21.s, z4.s\n" + ".inst 0x44828094 // srshl z20.s, p0/M, z20.s, z4.s\n" + ".inst 0x44828091 // srshl z17.s, p0/M, z17.s, z4.s\n" + ".inst 0x4482809a // srshl z26.s, p0/M, z26.s, z4.s\n" + ".inst 0x44828099 // srshl z25.s, p0/M, z25.s, z4.s\n" + ".inst 0x44828098 // srshl z24.s, p0/M, z24.s, z4.s\n" + ".inst 0x44828097 // srshl z23.s, p0/M, z23.s, z4.s\n" + ".inst 0x04a37400 // sqrdmulh z0.s, z0.s, z3.s\n" + ".inst 0x04a37673 // sqrdmulh z19.s, z19.s, z3.s\n" + ".inst 0x04a377ff // sqrdmulh z31.s, z31.s, z3.s\n" + ".inst 0x04a377de // sqrdmulh z30.s, z30.s, z3.s\n" + ".inst 0x04a377bd // sqrdmulh z29.s, z29.s, z3.s\n" + ".inst 0x04a376d6 // sqrdmulh z22.s, z22.s, z3.s\n" + ".inst 0x04a3779c // sqrdmulh z28.s, z28.s, z3.s\n" + ".inst 0x04a37652 // sqrdmulh z18.s, z18.s, z3.s\n" + ".inst 0x04a3777b // sqrdmulh z27.s, z27.s, z3.s\n" + ".inst 0x04a376b5 // sqrdmulh z21.s, z21.s, z3.s\n" + ".inst 0x04a37694 // sqrdmulh z20.s, z20.s, z3.s\n" + ".inst 0x04a37631 // sqrdmulh z17.s, z17.s, z3.s\n" + ".inst 0x04a3775a // sqrdmulh z26.s, z26.s, z3.s\n" + ".inst 0x04a37739 // sqrdmulh z25.s, z25.s, z3.s\n" + ".inst 0x04a37718 // sqrdmulh z24.s, z24.s, z3.s\n" + ".inst 0x04a376f7 // sqrdmulh z23.s, z23.s, z3.s\n" + ".inst 0x44828040 // srshl z0.s, p0/M, z0.s, z2.s\n" + ".inst 0x44828053 // srshl z19.s, p0/M, z19.s, z2.s\n" + ".inst 0x4482805f // srshl z31.s, p0/M, z31.s, z2.s\n" + ".inst 0x4482805e // srshl z30.s, p0/M, z30.s, z2.s\n" + ".inst 0x4482805d // srshl z29.s, p0/M, z29.s, z2.s\n" + ".inst 0x44828056 // srshl z22.s, p0/M, z22.s, z2.s\n" + ".inst 0x4482805c // srshl z28.s, p0/M, z28.s, z2.s\n" + ".inst 0x44828052 // srshl z18.s, p0/M, z18.s, z2.s\n" + ".inst 0x4482805b // srshl z27.s, p0/M, z27.s, z2.s\n" + ".inst 0x44828055 // srshl z21.s, p0/M, z21.s, z2.s\n" + ".inst 0x44828054 // srshl z20.s, p0/M, z20.s, z2.s\n" + ".inst 0x44828051 // srshl z17.s, p0/M, z17.s, z2.s\n" + ".inst 0x4482805a // srshl z26.s, p0/M, z26.s, z2.s\n" + ".inst 0x44828059 // srshl z25.s, p0/M, z25.s, z2.s\n" + ".inst 0x44828058 // srshl z24.s, p0/M, z24.s, z2.s\n" + ".inst 0x44828057 // srshl z23.s, p0/M, z23.s, z2.s\n" + "add z0.s, z0.s, z16.s\n" + "add z19.s, z19.s, z16.s\n" + "add z31.s, z31.s, z16.s\n" + "add z30.s, z30.s, z16.s\n" + "add z29.s, z29.s, z16.s\n" + "add z22.s, z22.s, z16.s\n" + "add z28.s, z28.s, z16.s\n" + "add z18.s, z18.s, z16.s\n" + "add z27.s, z27.s, z16.s\n" + "add z21.s, z21.s, z16.s\n" + "add z20.s, z20.s, z16.s\n" + "add z17.s, z17.s, z16.s\n" + "add z26.s, z26.s, z16.s\n" + "add z25.s, z25.s, z16.s\n" + "add z24.s, z24.s, z16.s\n" + "add z23.s, z23.s, z16.s\n" + "not z16.s, p0/M, z1.s\n" "smax z0.s, p0/M, z0.s, z16.s\n" + "smax z19.s, p0/M, z19.s, z16.s\n" "smax z31.s, p0/M, z31.s, z16.s\n" "smax z30.s, p0/M, z30.s, z16.s\n" - "smax z22.s, p0/M, z22.s, z16.s\n" "smax z29.s, p0/M, z29.s, z16.s\n" - "smax z18.s, p0/M, z18.s, z16.s\n" + "smax z22.s, p0/M, z22.s, z16.s\n" "smax z28.s, p0/M, z28.s, z16.s\n" - "smax z21.s, p0/M, z21.s, z16.s\n" + "smax z18.s, p0/M, z18.s, z16.s\n" "smax z27.s, p0/M, z27.s, z16.s\n" + "smax z21.s, p0/M, z21.s, z16.s\n" "smax z20.s, p0/M, z20.s, z16.s\n" + "smax z17.s, p0/M, z17.s, z16.s\n" "smax z26.s, p0/M, z26.s, z16.s\n" "smax z25.s, p0/M, z25.s, z16.s\n" "smax z24.s, p0/M, z24.s, z16.s\n" "smax z23.s, p0/M, z23.s, z16.s\n" - "smin z1.s, p0/M, z1.s, z17.s\n" - "smin z19.s, p0/M, z19.s, z17.s\n" - "smin z0.s, p0/M, z0.s, z17.s\n" - "smin z31.s, p0/M, z31.s, z17.s\n" - "smin z30.s, p0/M, z30.s, z17.s\n" - "smin z22.s, p0/M, z22.s, z17.s\n" - "smin z29.s, p0/M, z29.s, z17.s\n" - "smin z18.s, p0/M, z18.s, z17.s\n" - "smin z28.s, p0/M, z28.s, z17.s\n" - "smin z21.s, p0/M, z21.s, z17.s\n" - "trn1 z19.h, z1.h, z19.h\n" - "smin z27.s, p0/M, z27.s, z17.s\n" - "smin z20.s, p0/M, z20.s, z17.s\n" - "trn1 z16.h, z0.h, z31.h\n" - "smin z26.s, p0/M, z26.s, z17.s\n" - "smin z25.s, p0/M, z25.s, z17.s\n" - "trn1 z22.h, z30.h, z22.h\n" - "smin z24.s, p0/M, z24.s, z17.s\n" - "smin z23.s, p0/M, z23.s, z17.s\n" - "trn1 z18.h, z29.h, z18.h\n" - "trn1 z21.h, z28.h, z21.h\n" - "trn1 z17.h, z27.h, z20.h\n" + "smin z0.s, p0/M, z0.s, z1.s\n" + "smin z19.s, p0/M, z19.s, z1.s\n" + "smin z31.s, p0/M, z31.s, z1.s\n" + "smin z30.s, p0/M, z30.s, z1.s\n" + "smin z29.s, p0/M, z29.s, z1.s\n" + "smin z22.s, p0/M, z22.s, z1.s\n" + "smin z28.s, p0/M, z28.s, z1.s\n" + "smin z18.s, p0/M, z18.s, z1.s\n" + "smin z27.s, p0/M, z27.s, z1.s\n" + "smin z21.s, p0/M, z21.s, z1.s\n" + "trn1 z19.h, z0.h, z19.h\n" + "smin z20.s, p0/M, z20.s, z1.s\n" + "smin z17.s, p0/M, z17.s, z1.s\n" + "trn1 z16.h, z31.h, z30.h\n" + "smin z26.s, p0/M, z26.s, z1.s\n" + "smin z25.s, p0/M, z25.s, z1.s\n" + "trn1 z22.h, z29.h, z22.h\n" + "smin z24.s, p0/M, z24.s, z1.s\n" + "smin z23.s, p0/M, z23.s, z1.s\n" + "trn1 z18.h, z28.h, z18.h\n" + "trn1 z21.h, z27.h, z21.h\n" + "trn1 z17.h, z20.h, z17.h\n" "trn1 z20.b, z19.b, z16.b\n" "trn1 z19.h, z26.h, z25.h\n" "trn1 z16.h, z24.h, z23.h\n" @@ -286,16 +307,16 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "st1b { z16.b }, p1, [%x[outptr], x26]\n" "incb x26, ALL, MUL #4\n" "whilelt p1.b, x26, %x[n_channels]\n" - "b.any 1b\n" + "b.ne 1b\n" "7:" // Single vector of channels "whilelt p4.b, x9, %x[n_channels]\n" - "b.none 14f\n" + "b.eq 14f\n" "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z5.b, #0x80\n" "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "subs x25, x25, #0x1\n" "ldp x21, x20, [x24, #0x10]\n" "add x24, x24, #0x20\n" @@ -307,7 +328,7 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "9:" // Single vector of channels: Loop: 4 inputs loop "movprfx z16, z1\n smax z16.b, p0/M, z16.b, z0.b\n" "movprfx z17, z23\n smax z17.b, p0/M, z17.b, z31.b\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "subs x25, x25, #0x1\n" "ldp x21, x20, [x24, #0x10]\n" "add x24, x24, #0x20\n" @@ -333,51 +354,60 @@ void sme_s8q_nhwc_max_generic_depthfirst_impl( "smax z5.b, p0/M, z5.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - ".inst 0x4508a0b1 // sshllb z17.h, z5.b, #0x0\n" - ".inst 0x4508a4b0 // sshllt z16.h, z5.b, #0x0\n" + "add x21, %x[quant_params], %[offsetof_qp_input_offset]\n" + ".inst 0x4508a0b1 // sshllb z17.h, z5.b, #0\n" + ".inst 0x4508a4ba // sshllt z26.h, z5.b, #0\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "add x21, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "ld1rw { z16.s }, p0/Z, [x21]\n" + "add x22, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x21, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" + "mov z25.s, #0x7f\n" "ld1rw { z24.s }, p0/Z, [x20]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - "mov z23.s, #0x7f\n" + "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" + "ld1rw { z23.s }, p0/Z, [x22]\n" + "neg z16.s, p0/M, z16.s\n" "ld1rw { z22.s }, p0/Z, [x21]\n" "ld1rw { z21.s }, p0/Z, [x20]\n" - ".inst 0x4510a234 // sshllb z20.s, z17.h, #0x0\n" - ".inst 0x4510a631 // sshllt z17.s, z17.h, #0x0\n" - ".inst 0x4510a213 // sshllb z19.s, z16.h, #0x0\n" - ".inst 0x4510a612 // sshllt z18.s, z16.h, #0x0\n" - "not z16.s, p0/M, z23.s\n" - ".inst 0x44828314 // srshl z20.s, p0/M, z20.s, z24.s\n" - ".inst 0x44828311 // srshl z17.s, p0/M, z17.s, z24.s\n" + "not z20.s, p0/M, z25.s\n" + ".inst 0x45914213 // saddwb z19.s, z16.s, z17.h\n" + ".inst 0x45914611 // saddwt z17.s, z16.s, z17.h\n" + ".inst 0x459a4212 // saddwb z18.s, z16.s, z26.h\n" + ".inst 0x459a4610 // saddwt z16.s, z16.s, z26.h\n" ".inst 0x44828313 // srshl z19.s, p0/M, z19.s, z24.s\n" + ".inst 0x44828311 // srshl z17.s, p0/M, z17.s, z24.s\n" ".inst 0x44828312 // srshl z18.s, p0/M, z18.s, z24.s\n" - ".inst 0x04b67694 // sqrdmulh z20.s, z20.s, z22.s\n" - ".inst 0x04b67631 // sqrdmulh z17.s, z17.s, z22.s\n" - ".inst 0x04b67673 // sqrdmulh z19.s, z19.s, z22.s\n" - ".inst 0x04b67652 // sqrdmulh z18.s, z18.s, z22.s\n" - ".inst 0x448282b4 // srshl z20.s, p0/M, z20.s, z21.s\n" - ".inst 0x448282b1 // srshl z17.s, p0/M, z17.s, z21.s\n" - ".inst 0x448282b3 // srshl z19.s, p0/M, z19.s, z21.s\n" - ".inst 0x448282b2 // srshl z18.s, p0/M, z18.s, z21.s\n" - "smax z20.s, p0/M, z20.s, z16.s\n" - "smax z17.s, p0/M, z17.s, z16.s\n" - "smax z19.s, p0/M, z19.s, z16.s\n" - "smax z18.s, p0/M, z18.s, z16.s\n" - "smin z20.s, p0/M, z20.s, z23.s\n" - "smin z17.s, p0/M, z17.s, z23.s\n" - "smin z19.s, p0/M, z19.s, z23.s\n" - "smin z18.s, p0/M, z18.s, z23.s\n" - "trn1 z17.h, z20.h, z17.h\n" - "trn1 z16.h, z19.h, z18.h\n" + ".inst 0x44828310 // srshl z16.s, p0/M, z16.s, z24.s\n" + ".inst 0x04b77673 // sqrdmulh z19.s, z19.s, z23.s\n" + ".inst 0x04b77631 // sqrdmulh z17.s, z17.s, z23.s\n" + ".inst 0x04b77652 // sqrdmulh z18.s, z18.s, z23.s\n" + ".inst 0x04b77610 // sqrdmulh z16.s, z16.s, z23.s\n" + ".inst 0x448282d3 // srshl z19.s, p0/M, z19.s, z22.s\n" + ".inst 0x448282d1 // srshl z17.s, p0/M, z17.s, z22.s\n" + ".inst 0x448282d2 // srshl z18.s, p0/M, z18.s, z22.s\n" + ".inst 0x448282d0 // srshl z16.s, p0/M, z16.s, z22.s\n" + "add z19.s, z19.s, z21.s\n" + "add z17.s, z17.s, z21.s\n" + "add z18.s, z18.s, z21.s\n" + "add z16.s, z16.s, z21.s\n" + "smax z19.s, p0/M, z19.s, z20.s\n" + "smax z17.s, p0/M, z17.s, z20.s\n" + "smax z18.s, p0/M, z18.s, z20.s\n" + "smax z16.s, p0/M, z16.s, z20.s\n" + "smin z19.s, p0/M, z19.s, z25.s\n" + "smin z17.s, p0/M, z17.s, z25.s\n" + "smin z18.s, p0/M, z18.s, z25.s\n" + "smin z16.s, p0/M, z16.s, z25.s\n" + "trn1 z17.h, z19.h, z17.h\n" + "trn1 z16.h, z18.h, z16.h\n" "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x9]\n" "incb x9\n" "whilelt p4.b, x9, %x[n_channels]\n" - "b.any 8b\n" + "b.ne 8b\n" "14:" // End ".inst 0xd503467f // SMSTOP\n" : - : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [outptr] "r" (outptr), [quant_params] "r" (&qp) + : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_input_offset] "I" (offsetof(Requantize32, input_offset)), [offsetof_qp_output_offset] "I" (offsetof(Requantize32, output_offset)), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [outptr] "r" (outptr), [quant_params] "r" (&qp) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp index 5033aa9d73..0b3f4b88ec 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sve_s8q_nhwc_max_generic_depthfirst/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2024 Arm Limited. + * Copyright (c) 2021-2024, 2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -42,7 +42,7 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( ) { __asm__ __volatile__( - "mov x9, #0x0\n" + "mov x9, #0\n" "cntb x28\n" "cntb x27, ALL, MUL #2\n" "cntb x26, ALL, MUL #3\n" @@ -51,16 +51,16 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "whilelt p2.b, x28, %x[n_channels]\n" "whilelt p1.b, x27, %x[n_channels]\n" "whilelt p0.b, x26, %x[n_channels]\n" - "b.none 7f\n" + "b.eq 7f\n" "1:" // 4-vectors of channels "lsr x25, %x[n_valid_cells], #0x2\n" "mov z6.b, #0x80\n" - "mov z3.b, #0x80\n" - "mov x24, %x[inptrs]\n" "mov z5.b, #0x80\n" + "mov x24, %x[inptrs]\n" "mov z4.b, #0x80\n" + "mov z3.b, #0x80\n" "cbz x25, 4f\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" "add x24, x24, #0x20\n" @@ -84,7 +84,7 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "2:" // 4-vectors of channels: 4 inputs loop "movprfx z19, z2\n smax z19.b, p4/M, z19.b, z1.b\n" "smax z23.b, p4/M, z23.b, z0.b\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "movprfx z18, z31\n smax z18.b, p4/M, z18.b, z30.b\n" "smax z22.b, p4/M, z22.b, z29.b\n" @@ -107,13 +107,13 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "ld1b { z22.b }, p2/Z, [x21, x28]\n" "ld1b { z29.b }, p2/Z, [x20, x28]\n" "smax z6.b, p4/M, z6.b, z19.b\n" - "smax z3.b, p4/M, z3.b, z18.b\n" + "smax z5.b, p4/M, z5.b, z18.b\n" "ld1b { z28.b }, p1/Z, [x23, x27]\n" "ld1b { z27.b }, p1/Z, [x22, x27]\n" - "smax z5.b, p4/M, z5.b, z17.b\n" + "smax z4.b, p4/M, z4.b, z17.b\n" "ld1b { z21.b }, p1/Z, [x21, x27]\n" "ld1b { z26.b }, p1/Z, [x20, x27]\n" - "smax z4.b, p4/M, z4.b, z16.b\n" + "smax z3.b, p4/M, z3.b, z16.b\n" "ld1b { z16.b }, p0/Z, [x23, x26]\n" "ld1b { z25.b }, p0/Z, [x22, x26]\n" "ld1b { z20.b }, p0/Z, [x21, x26]\n" @@ -133,9 +133,9 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "smax z17.b, p4/M, z17.b, z21.b\n" "smax z16.b, p4/M, z16.b, z20.b\n" "smax z6.b, p4/M, z6.b, z19.b\n" - "smax z3.b, p4/M, z3.b, z18.b\n" - "smax z5.b, p4/M, z5.b, z17.b\n" - "smax z4.b, p4/M, z4.b, z16.b\n" + "smax z5.b, p4/M, z5.b, z18.b\n" + "smax z4.b, p4/M, z4.b, z17.b\n" + "smax z3.b, p4/M, z3.b, z16.b\n" "4:" // 4-vectors of channels: After loop "ands x21, %x[n_valid_cells], #0x3\n" "beq 6f\n" @@ -147,128 +147,149 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "ld1b { z17.b }, p1/Z, [x20, x27]\n" "ld1b { z16.b }, p0/Z, [x20, x26]\n" "smax z6.b, p4/M, z6.b, z19.b\n" - "smax z3.b, p4/M, z3.b, z18.b\n" - "smax z5.b, p4/M, z5.b, z17.b\n" - "smax z4.b, p4/M, z4.b, z16.b\n" + "smax z5.b, p4/M, z5.b, z18.b\n" + "smax z4.b, p4/M, z4.b, z17.b\n" + "smax z3.b, p4/M, z3.b, z16.b\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End - ".inst 0x4508a0d3 // sshllb z19.h, z6.b, #0x0\n" - ".inst 0x4508a4d1 // sshllt z17.h, z6.b, #0x0\n" - "add x21, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" - ".inst 0x4508a072 // sshllb z18.h, z3.b, #0x0\n" - ".inst 0x4508a478 // sshllt z24.h, z3.b, #0x0\n" - "ld1rw { z3.s }, p4/Z, [x21]\n" - "ld1rw { z2.s }, p4/Z, [x20]\n" - ".inst 0x4508a0b5 // sshllb z21.h, z5.b, #0x0\n" - ".inst 0x4508a4b7 // sshllt z23.h, z5.b, #0x0\n" + "add x21, %x[quant_params], %[offsetof_qp_input_offset]\n" + ".inst 0x4508a0d3 // sshllb z19.h, z6.b, #0\n" + ".inst 0x4508a4d1 // sshllt z17.h, z6.b, #0\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" + "ld1rw { z6.s }, p4/Z, [x21]\n" + ".inst 0x4508a0b2 // sshllb z18.h, z5.b, #0\n" + ".inst 0x4508a4b0 // sshllt z16.h, z5.b, #0\n" + "ld1rw { z5.s }, p4/Z, [x20]\n" + ".inst 0x4508a094 // sshllb z20.h, z4.b, #0\n" + ".inst 0x4508a498 // sshllt z24.h, z4.b, #0\n" + "add x21, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" - ".inst 0x4508a096 // sshllb z22.h, z4.b, #0x0\n" - ".inst 0x4508a494 // sshllt z20.h, z4.b, #0x0\n" - "ld1rw { z16.s }, p4/Z, [x20]\n" - ".inst 0x4510a261 // sshllb z1.s, z19.h, #0x0\n" - ".inst 0x4510a673 // sshllt z19.s, z19.h, #0x0\n" - ".inst 0x4510a220 // sshllb z0.s, z17.h, #0x0\n" - ".inst 0x4510a631 // sshllt z17.s, z17.h, #0x0\n" - ".inst 0x4510a25f // sshllb z31.s, z18.h, #0x0\n" - ".inst 0x4510a652 // sshllt z18.s, z18.h, #0x0\n" - ".inst 0x4510a31e // sshllb z30.s, z24.h, #0x0\n" - ".inst 0x4510a71d // sshllt z29.s, z24.h, #0x0\n" - ".inst 0x44829061 // srshl z1.s, p4/M, z1.s, z3.s\n" - ".inst 0x44829073 // srshl z19.s, p4/M, z19.s, z3.s\n" - ".inst 0x4510a2bc // sshllb z28.s, z21.h, #0x0\n" - ".inst 0x4510a6b5 // sshllt z21.s, z21.h, #0x0\n" - ".inst 0x44829060 // srshl z0.s, p4/M, z0.s, z3.s\n" - ".inst 0x44829071 // srshl z17.s, p4/M, z17.s, z3.s\n" - ".inst 0x4510a2fb // sshllb z27.s, z23.h, #0x0\n" - ".inst 0x4510a6fa // sshllt z26.s, z23.h, #0x0\n" + ".inst 0x4508a077 // sshllb z23.h, z3.b, #0\n" + ".inst 0x4508a476 // sshllt z22.h, z3.b, #0\n" + "ld1rw { z4.s }, p4/Z, [x21]\n" + "ld1rw { z3.s }, p4/Z, [x20]\n" + "neg z6.s, p4/M, z6.s\n" + "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" + "mov z2.s, #0x7f\n" + "ld1rw { z1.s }, p4/Z, [x20]\n" + "not z0.s, p4/M, z2.s\n" + ".inst 0x459340df // saddwb z31.s, z6.s, z19.h\n" + ".inst 0x459344d3 // saddwt z19.s, z6.s, z19.h\n" + ".inst 0x459140de // saddwb z30.s, z6.s, z17.h\n" + ".inst 0x459144d1 // saddwt z17.s, z6.s, z17.h\n" + ".inst 0x459240dd // saddwb z29.s, z6.s, z18.h\n" + ".inst 0x459244d2 // saddwt z18.s, z6.s, z18.h\n" + ".inst 0x459040dc // saddwb z28.s, z6.s, z16.h\n" + ".inst 0x459044d0 // saddwt z16.s, z6.s, z16.h\n" + ".inst 0x448290bf // srshl z31.s, p4/M, z31.s, z5.s\n" + ".inst 0x448290b3 // srshl z19.s, p4/M, z19.s, z5.s\n" + ".inst 0x459440d5 // saddwb z21.s, z6.s, z20.h\n" + ".inst 0x459444d4 // saddwt z20.s, z6.s, z20.h\n" + ".inst 0x448290be // srshl z30.s, p4/M, z30.s, z5.s\n" + ".inst 0x448290b1 // srshl z17.s, p4/M, z17.s, z5.s\n" + ".inst 0x459840db // saddwb z27.s, z6.s, z24.h\n" + ".inst 0x459844da // saddwt z26.s, z6.s, z24.h\n" + ".inst 0x448290bd // srshl z29.s, p4/M, z29.s, z5.s\n" + ".inst 0x448290b2 // srshl z18.s, p4/M, z18.s, z5.s\n" + ".inst 0x459740d9 // saddwb z25.s, z6.s, z23.h\n" + ".inst 0x459744d8 // saddwt z24.s, z6.s, z23.h\n" + ".inst 0x448290bc // srshl z28.s, p4/M, z28.s, z5.s\n" + ".inst 0x448290b0 // srshl z16.s, p4/M, z16.s, z5.s\n" + ".inst 0x459640d7 // saddwb z23.s, z6.s, z22.h\n" + ".inst 0x459644d6 // saddwt z22.s, z6.s, z22.h\n" + ".inst 0x448290b5 // srshl z21.s, p4/M, z21.s, z5.s\n" + ".inst 0x448290b4 // srshl z20.s, p4/M, z20.s, z5.s\n" + ".inst 0x448290bb // srshl z27.s, p4/M, z27.s, z5.s\n" + ".inst 0x448290ba // srshl z26.s, p4/M, z26.s, z5.s\n" + ".inst 0x04a477ff // sqrdmulh z31.s, z31.s, z4.s\n" + ".inst 0x04a47673 // sqrdmulh z19.s, z19.s, z4.s\n" + ".inst 0x448290b9 // srshl z25.s, p4/M, z25.s, z5.s\n" + ".inst 0x448290b8 // srshl z24.s, p4/M, z24.s, z5.s\n" + ".inst 0x04a477de // sqrdmulh z30.s, z30.s, z4.s\n" + ".inst 0x04a47631 // sqrdmulh z17.s, z17.s, z4.s\n" + ".inst 0x448290b7 // srshl z23.s, p4/M, z23.s, z5.s\n" + ".inst 0x448290b6 // srshl z22.s, p4/M, z22.s, z5.s\n" + ".inst 0x04a477bd // sqrdmulh z29.s, z29.s, z4.s\n" + ".inst 0x04a47652 // sqrdmulh z18.s, z18.s, z4.s\n" + ".inst 0x04a4779c // sqrdmulh z28.s, z28.s, z4.s\n" + ".inst 0x04a47610 // sqrdmulh z16.s, z16.s, z4.s\n" ".inst 0x4482907f // srshl z31.s, p4/M, z31.s, z3.s\n" - ".inst 0x44829072 // srshl z18.s, p4/M, z18.s, z3.s\n" - ".inst 0x4510a2d9 // sshllb z25.s, z22.h, #0x0\n" - ".inst 0x4510a6d8 // sshllt z24.s, z22.h, #0x0\n" + ".inst 0x44829073 // srshl z19.s, p4/M, z19.s, z3.s\n" + ".inst 0x04a476b5 // sqrdmulh z21.s, z21.s, z4.s\n" + ".inst 0x04a47694 // sqrdmulh z20.s, z20.s, z4.s\n" ".inst 0x4482907e // srshl z30.s, p4/M, z30.s, z3.s\n" + ".inst 0x44829071 // srshl z17.s, p4/M, z17.s, z3.s\n" + ".inst 0x04a4777b // sqrdmulh z27.s, z27.s, z4.s\n" + ".inst 0x04a4775a // sqrdmulh z26.s, z26.s, z4.s\n" ".inst 0x4482907d // srshl z29.s, p4/M, z29.s, z3.s\n" - ".inst 0x4510a297 // sshllb z23.s, z20.h, #0x0\n" - ".inst 0x4510a696 // sshllt z22.s, z20.h, #0x0\n" + ".inst 0x44829072 // srshl z18.s, p4/M, z18.s, z3.s\n" + ".inst 0x04a47739 // sqrdmulh z25.s, z25.s, z4.s\n" + ".inst 0x04a47718 // sqrdmulh z24.s, z24.s, z4.s\n" ".inst 0x4482907c // srshl z28.s, p4/M, z28.s, z3.s\n" + ".inst 0x44829070 // srshl z16.s, p4/M, z16.s, z3.s\n" + ".inst 0x04a476f7 // sqrdmulh z23.s, z23.s, z4.s\n" + ".inst 0x04a476d6 // sqrdmulh z22.s, z22.s, z4.s\n" ".inst 0x44829075 // srshl z21.s, p4/M, z21.s, z3.s\n" + ".inst 0x44829074 // srshl z20.s, p4/M, z20.s, z3.s\n" ".inst 0x4482907b // srshl z27.s, p4/M, z27.s, z3.s\n" ".inst 0x4482907a // srshl z26.s, p4/M, z26.s, z3.s\n" - ".inst 0x04a27421 // sqrdmulh z1.s, z1.s, z2.s\n" - ".inst 0x04a27673 // sqrdmulh z19.s, z19.s, z2.s\n" + "add z31.s, z31.s, z1.s\n" + "add z19.s, z19.s, z1.s\n" ".inst 0x44829079 // srshl z25.s, p4/M, z25.s, z3.s\n" ".inst 0x44829078 // srshl z24.s, p4/M, z24.s, z3.s\n" - ".inst 0x04a27400 // sqrdmulh z0.s, z0.s, z2.s\n" - ".inst 0x04a27631 // sqrdmulh z17.s, z17.s, z2.s\n" + "add z30.s, z30.s, z1.s\n" + "add z17.s, z17.s, z1.s\n" ".inst 0x44829077 // srshl z23.s, p4/M, z23.s, z3.s\n" ".inst 0x44829076 // srshl z22.s, p4/M, z22.s, z3.s\n" - ".inst 0x04a277ff // sqrdmulh z31.s, z31.s, z2.s\n" - ".inst 0x04a27652 // sqrdmulh z18.s, z18.s, z2.s\n" - ".inst 0x04a277de // sqrdmulh z30.s, z30.s, z2.s\n" - ".inst 0x04a277bd // sqrdmulh z29.s, z29.s, z2.s\n" - ".inst 0x44829201 // srshl z1.s, p4/M, z1.s, z16.s\n" - ".inst 0x44829213 // srshl z19.s, p4/M, z19.s, z16.s\n" - ".inst 0x04a2779c // sqrdmulh z28.s, z28.s, z2.s\n" - ".inst 0x04a276b5 // sqrdmulh z21.s, z21.s, z2.s\n" - ".inst 0x44829200 // srshl z0.s, p4/M, z0.s, z16.s\n" - ".inst 0x44829211 // srshl z17.s, p4/M, z17.s, z16.s\n" - ".inst 0x04a2777b // sqrdmulh z27.s, z27.s, z2.s\n" - ".inst 0x04a2775a // sqrdmulh z26.s, z26.s, z2.s\n" - ".inst 0x4482921f // srshl z31.s, p4/M, z31.s, z16.s\n" - ".inst 0x44829212 // srshl z18.s, p4/M, z18.s, z16.s\n" - ".inst 0x04a27739 // sqrdmulh z25.s, z25.s, z2.s\n" - ".inst 0x04a27718 // sqrdmulh z24.s, z24.s, z2.s\n" - ".inst 0x4482921e // srshl z30.s, p4/M, z30.s, z16.s\n" - ".inst 0x4482921d // srshl z29.s, p4/M, z29.s, z16.s\n" - ".inst 0x04a276f7 // sqrdmulh z23.s, z23.s, z2.s\n" - ".inst 0x04a276d6 // sqrdmulh z22.s, z22.s, z2.s\n" - ".inst 0x4482921c // srshl z28.s, p4/M, z28.s, z16.s\n" - ".inst 0x44829215 // srshl z21.s, p4/M, z21.s, z16.s\n" - "mov z20.s, #0x7f\n" - ".inst 0x4482921b // srshl z27.s, p4/M, z27.s, z16.s\n" - ".inst 0x4482921a // srshl z26.s, p4/M, z26.s, z16.s\n" - ".inst 0x44829219 // srshl z25.s, p4/M, z25.s, z16.s\n" - ".inst 0x44829218 // srshl z24.s, p4/M, z24.s, z16.s\n" - ".inst 0x44829217 // srshl z23.s, p4/M, z23.s, z16.s\n" - ".inst 0x44829216 // srshl z22.s, p4/M, z22.s, z16.s\n" - "not z16.s, p4/M, z20.s\n" - "smax z1.s, p4/M, z1.s, z16.s\n" - "smax z19.s, p4/M, z19.s, z16.s\n" - "smax z0.s, p4/M, z0.s, z16.s\n" - "smax z17.s, p4/M, z17.s, z16.s\n" - "smax z31.s, p4/M, z31.s, z16.s\n" - "smax z18.s, p4/M, z18.s, z16.s\n" - "smax z30.s, p4/M, z30.s, z16.s\n" - "smax z29.s, p4/M, z29.s, z16.s\n" - "smax z28.s, p4/M, z28.s, z16.s\n" - "smax z21.s, p4/M, z21.s, z16.s\n" - "smax z27.s, p4/M, z27.s, z16.s\n" - "smax z26.s, p4/M, z26.s, z16.s\n" - "smax z25.s, p4/M, z25.s, z16.s\n" - "smax z24.s, p4/M, z24.s, z16.s\n" - "smax z23.s, p4/M, z23.s, z16.s\n" - "smax z22.s, p4/M, z22.s, z16.s\n" - "smin z1.s, p4/M, z1.s, z20.s\n" - "smin z19.s, p4/M, z19.s, z20.s\n" - "smin z0.s, p4/M, z0.s, z20.s\n" - "smin z17.s, p4/M, z17.s, z20.s\n" - "smin z31.s, p4/M, z31.s, z20.s\n" - "smin z18.s, p4/M, z18.s, z20.s\n" - "smin z30.s, p4/M, z30.s, z20.s\n" - "smin z29.s, p4/M, z29.s, z20.s\n" - "smin z28.s, p4/M, z28.s, z20.s\n" - "trn1 z19.h, z1.h, z19.h\n" - "smin z21.s, p4/M, z21.s, z20.s\n" - "smin z27.s, p4/M, z27.s, z20.s\n" - "trn1 z17.h, z0.h, z17.h\n" - "smin z26.s, p4/M, z26.s, z20.s\n" - "smin z25.s, p4/M, z25.s, z20.s\n" - "trn1 z18.h, z31.h, z18.h\n" - "smin z24.s, p4/M, z24.s, z20.s\n" - "smin z23.s, p4/M, z23.s, z20.s\n" - "trn1 z16.h, z30.h, z29.h\n" - "smin z22.s, p4/M, z22.s, z20.s\n" - "trn1 z21.h, z28.h, z21.h\n" + "add z29.s, z29.s, z1.s\n" + "add z18.s, z18.s, z1.s\n" + "add z28.s, z28.s, z1.s\n" + "add z16.s, z16.s, z1.s\n" + "smax z31.s, p4/M, z31.s, z0.s\n" + "smax z19.s, p4/M, z19.s, z0.s\n" + "add z21.s, z21.s, z1.s\n" + "add z20.s, z20.s, z1.s\n" + "smax z30.s, p4/M, z30.s, z0.s\n" + "smax z17.s, p4/M, z17.s, z0.s\n" + "add z27.s, z27.s, z1.s\n" + "add z26.s, z26.s, z1.s\n" + "smax z29.s, p4/M, z29.s, z0.s\n" + "smax z18.s, p4/M, z18.s, z0.s\n" + "add z25.s, z25.s, z1.s\n" + "add z24.s, z24.s, z1.s\n" + "smax z28.s, p4/M, z28.s, z0.s\n" + "smax z16.s, p4/M, z16.s, z0.s\n" + "add z23.s, z23.s, z1.s\n" + "add z22.s, z22.s, z1.s\n" + "smax z21.s, p4/M, z21.s, z0.s\n" + "smax z20.s, p4/M, z20.s, z0.s\n" + "smax z27.s, p4/M, z27.s, z0.s\n" + "smax z26.s, p4/M, z26.s, z0.s\n" + "smax z25.s, p4/M, z25.s, z0.s\n" + "smax z24.s, p4/M, z24.s, z0.s\n" + "smax z23.s, p4/M, z23.s, z0.s\n" + "smax z22.s, p4/M, z22.s, z0.s\n" + "smin z31.s, p4/M, z31.s, z2.s\n" + "smin z19.s, p4/M, z19.s, z2.s\n" + "smin z30.s, p4/M, z30.s, z2.s\n" + "smin z17.s, p4/M, z17.s, z2.s\n" + "smin z29.s, p4/M, z29.s, z2.s\n" + "smin z18.s, p4/M, z18.s, z2.s\n" + "smin z28.s, p4/M, z28.s, z2.s\n" + "smin z16.s, p4/M, z16.s, z2.s\n" + "trn1 z19.h, z31.h, z19.h\n" + "smin z21.s, p4/M, z21.s, z2.s\n" + "smin z20.s, p4/M, z20.s, z2.s\n" + "trn1 z17.h, z30.h, z17.h\n" + "smin z27.s, p4/M, z27.s, z2.s\n" + "smin z26.s, p4/M, z26.s, z2.s\n" + "trn1 z18.h, z29.h, z18.h\n" + "smin z25.s, p4/M, z25.s, z2.s\n" + "smin z24.s, p4/M, z24.s, z2.s\n" + "trn1 z16.h, z28.h, z16.h\n" + "smin z23.s, p4/M, z23.s, z2.s\n" + "smin z22.s, p4/M, z22.s, z2.s\n" + "trn1 z21.h, z21.h, z20.h\n" "trn1 z20.b, z19.b, z17.b\n" "trn1 z17.h, z27.h, z26.h\n" "trn1 z19.h, z25.h, z24.h\n" @@ -285,16 +306,16 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "st1b { z16.b }, p0, [%x[outptr], x26]\n" "incb x26, ALL, MUL #4\n" "whilelt p0.b, x26, %x[n_channels]\n" - "b.any 1b\n" + "b.ne 1b\n" "7:" // Single vector of channels "whilelt p3.b, x9, %x[n_channels]\n" - "b.none 14f\n" + "b.eq 14f\n" "8:" // Single vector of channels: Loop "lsr x25, %x[n_valid_cells], #0x2\n" "mov z6.b, #0x80\n" "mov x24, %x[inptrs]\n" "cbz x25, 11f\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" "add x24, x24, #0x20\n" @@ -306,7 +327,7 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "9:" // Single vector of channels: Loop: 4 inputs loop "movprfx z16, z2\n smax z16.b, p4/M, z16.b, z1.b\n" "movprfx z17, z23\n smax z17.b, p4/M, z17.b, z0.b\n" - "ldp x23, x22, [x24, #0x0]\n" + "ldp x23, x22, [x24, #0]\n" "ldp x21, x20, [x24, #0x10]\n" "subs x25, x25, #0x1\n" "add x24, x24, #0x20\n" @@ -332,50 +353,59 @@ void sve_s8q_nhwc_max_generic_depthfirst_impl( "smax z6.b, p4/M, z6.b, z16.b\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - ".inst 0x4508a0d1 // sshllb z17.h, z6.b, #0x0\n" - ".inst 0x4508a4d0 // sshllt z16.h, z6.b, #0x0\n" - "add x21, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x21, %x[quant_params], %[offsetof_qp_input_offset]\n" + ".inst 0x4508a0d1 // sshllb z17.h, z6.b, #0\n" + ".inst 0x4508a4da // sshllt z26.h, z6.b, #0\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_left_shift]\n" + "ld1rw { z16.s }, p4/Z, [x21]\n" + "ld1rw { z25.s }, p4/Z, [x20]\n" + "add x21, %x[quant_params], %[offsetof_qp_per_layer_mul]\n" + "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" "ld1rw { z24.s }, p4/Z, [x21]\n" "ld1rw { z23.s }, p4/Z, [x20]\n" - "add x20, %x[quant_params], %[offsetof_qp_per_layer_right_shift]\n" + "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" "mov z22.s, #0x7f\n" "ld1rw { z21.s }, p4/Z, [x20]\n" - ".inst 0x4510a234 // sshllb z20.s, z17.h, #0x0\n" - ".inst 0x4510a631 // sshllt z17.s, z17.h, #0x0\n" - ".inst 0x4510a213 // sshllb z19.s, z16.h, #0x0\n" - ".inst 0x4510a612 // sshllt z18.s, z16.h, #0x0\n" - "not z16.s, p4/M, z22.s\n" - ".inst 0x44829314 // srshl z20.s, p4/M, z20.s, z24.s\n" - ".inst 0x44829311 // srshl z17.s, p4/M, z17.s, z24.s\n" - ".inst 0x44829313 // srshl z19.s, p4/M, z19.s, z24.s\n" - ".inst 0x44829312 // srshl z18.s, p4/M, z18.s, z24.s\n" - ".inst 0x04b77694 // sqrdmulh z20.s, z20.s, z23.s\n" - ".inst 0x04b77631 // sqrdmulh z17.s, z17.s, z23.s\n" - ".inst 0x04b77673 // sqrdmulh z19.s, z19.s, z23.s\n" - ".inst 0x04b77652 // sqrdmulh z18.s, z18.s, z23.s\n" - ".inst 0x448292b4 // srshl z20.s, p4/M, z20.s, z21.s\n" - ".inst 0x448292b1 // srshl z17.s, p4/M, z17.s, z21.s\n" - ".inst 0x448292b3 // srshl z19.s, p4/M, z19.s, z21.s\n" - ".inst 0x448292b2 // srshl z18.s, p4/M, z18.s, z21.s\n" - "smax z20.s, p4/M, z20.s, z16.s\n" - "smax z17.s, p4/M, z17.s, z16.s\n" - "smax z19.s, p4/M, z19.s, z16.s\n" - "smax z18.s, p4/M, z18.s, z16.s\n" - "smin z20.s, p4/M, z20.s, z22.s\n" - "smin z17.s, p4/M, z17.s, z22.s\n" + "neg z16.s, p4/M, z16.s\n" + "not z20.s, p4/M, z22.s\n" + ".inst 0x45914213 // saddwb z19.s, z16.s, z17.h\n" + ".inst 0x45914611 // saddwt z17.s, z16.s, z17.h\n" + ".inst 0x459a4212 // saddwb z18.s, z16.s, z26.h\n" + ".inst 0x459a4610 // saddwt z16.s, z16.s, z26.h\n" + ".inst 0x44829333 // srshl z19.s, p4/M, z19.s, z25.s\n" + ".inst 0x44829331 // srshl z17.s, p4/M, z17.s, z25.s\n" + ".inst 0x44829332 // srshl z18.s, p4/M, z18.s, z25.s\n" + ".inst 0x44829330 // srshl z16.s, p4/M, z16.s, z25.s\n" + ".inst 0x04b87673 // sqrdmulh z19.s, z19.s, z24.s\n" + ".inst 0x04b87631 // sqrdmulh z17.s, z17.s, z24.s\n" + ".inst 0x04b87652 // sqrdmulh z18.s, z18.s, z24.s\n" + ".inst 0x04b87610 // sqrdmulh z16.s, z16.s, z24.s\n" + ".inst 0x448292f3 // srshl z19.s, p4/M, z19.s, z23.s\n" + ".inst 0x448292f1 // srshl z17.s, p4/M, z17.s, z23.s\n" + ".inst 0x448292f2 // srshl z18.s, p4/M, z18.s, z23.s\n" + ".inst 0x448292f0 // srshl z16.s, p4/M, z16.s, z23.s\n" + "add z19.s, z19.s, z21.s\n" + "add z17.s, z17.s, z21.s\n" + "add z18.s, z18.s, z21.s\n" + "add z16.s, z16.s, z21.s\n" + "smax z19.s, p4/M, z19.s, z20.s\n" + "smax z17.s, p4/M, z17.s, z20.s\n" + "smax z18.s, p4/M, z18.s, z20.s\n" + "smax z16.s, p4/M, z16.s, z20.s\n" "smin z19.s, p4/M, z19.s, z22.s\n" + "smin z17.s, p4/M, z17.s, z22.s\n" "smin z18.s, p4/M, z18.s, z22.s\n" - "trn1 z17.h, z20.h, z17.h\n" - "trn1 z16.h, z19.h, z18.h\n" + "smin z16.s, p4/M, z16.s, z22.s\n" + "trn1 z17.h, z19.h, z17.h\n" + "trn1 z16.h, z18.h, z16.h\n" "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p3, [%x[outptr], x9]\n" "incb x9\n" "whilelt p3.b, x9, %x[n_channels]\n" - "b.any 8b\n" + "b.ne 8b\n" "14:" // End : - : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [outptr] "r" (outptr), [quant_params] "r" (&qp) + : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [n_valid_cells] "r" (n_valid_cells), [offsetof_qp_input_offset] "I" (offsetof(Requantize32, input_offset)), [offsetof_qp_output_offset] "I" (offsetof(Requantize32, output_offset)), [offsetof_qp_per_layer_left_shift] "I" (offsetof(Requantize32, per_layer_left_shift)), [offsetof_qp_per_layer_mul] "I" (offsetof(Requantize32, per_layer_mul)), [offsetof_qp_per_layer_right_shift] "I" (offsetof(Requantize32, per_layer_right_shift)), [outptr] "r" (outptr), [quant_params] "r" (&qp) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "x9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/cpu/kernels/internal/CpuPool2dAssemblyWrapperKernel.cpp b/src/cpu/kernels/internal/CpuPool2dAssemblyWrapperKernel.cpp index fa1f5935bc..c3da624b6a 100644 --- a/src/cpu/kernels/internal/CpuPool2dAssemblyWrapperKernel.cpp +++ b/src/cpu/kernels/internal/CpuPool2dAssemblyWrapperKernel.cpp @@ -137,10 +137,6 @@ CpuPool2dAssemblyWrapperKernel::validate(const ITensorInfo *src, const ITensorIn ARM_COMPUTE_RETURN_ERROR_ON_MSG(info.pool_type != PoolingType::MAX, "Assembly kernels only support differing src/dst quantization info for " "MAX pooling"); - ARM_COMPUTE_RETURN_ERROR_ON_MSG(src->data_type() == DataType::QASYMM8_SIGNED && - (src_qinfo.offset != 0 || dst_qinfo.offset != 0), - "Assembly kernels only support differing src/dst quantization info for " - "QASYMM8_SIGNED when both offsets are zero"); const float multiplier = src_qinfo.scale / dst_qinfo.scale; int32_t dst_multiplier{}; int32_t dst_shift{}; diff --git a/tests/validation/NEON/PoolingLayer.cpp b/tests/validation/NEON/PoolingLayer.cpp index 820a7e6264..6d2183b5ad 100644 --- a/tests/validation/NEON/PoolingLayer.cpp +++ b/tests/validation/NEON/PoolingLayer.cpp @@ -136,6 +136,17 @@ const auto PoolingDatasetQASYMM8SignedPaddedMaxDifferentQInfoZeroOffset = make("InputQuantInfo", {QuantizationInfo(0.25f, 0)}), make("OutputQuantInfo", {QuantizationInfo(0.5f, 0)})); +const auto PoolingDatasetQASYMM8SignedPaddedMaxDifferentQInfo = + combine(make("Shape", {TensorShape(7U, 5U, 3U)}), + make("PoolingType", {PoolingType::MAX}), + make("PoolingSize", {Size2D(3, 3)}), + make("PadStride", {PadStrideInfo(2, 2, 1, 1)}), + make("ExcludePadding", {false}), + make("DataType", DataType::QASYMM8_SIGNED), + make("DataLayout", {DataLayout::NHWC}), + make("InputQuantInfo", {QuantizationInfo(0.25f, -11)}), + make("OutputQuantInfo", {QuantizationInfo(0.5f, -7)})); + const auto PoolingDatasetQASYMM8SignedPaddedMaxSameQInfo = combine(make("Shape", {TensorShape(7U, 5U, 3U), TensorShape(8U, 7U, 5U)}), make("PoolingType", {PoolingType::MAX}), @@ -235,6 +246,38 @@ TEST_CASE(ValidatePaddedAvgDifferentQuantizationInfo, framework::DatasetMode::AL ARM_COMPUTE_EXPECT(!is_valid, framework::LogLevel::ERRORS); } +TEST_CASE(ValidatePaddedSignedMaxDifferentQuantizationInfo, framework::DatasetMode::ALL) +{ + TensorInfo input_info(TensorShape(3U, 15U, 11U, 1U), 1, DataType::QASYMM8_SIGNED, DataLayout::NHWC); + TensorInfo output_info(TensorShape(3U, 8U, 6U, 1U), 1, DataType::QASYMM8_SIGNED, DataLayout::NHWC); + const auto pool_info = PoolingLayerInfo(PoolingType::MAX, 3, DataLayout::NHWC, PadStrideInfo(2, 2, 1, 1), false); + + input_info.set_quantization_info(QuantizationInfo(0.25f, -11)); + output_info.set_quantization_info(QuantizationInfo(0.5f, -7)); + input_info.set_is_resizable(false); + output_info.set_is_resizable(false); + + const bool is_valid = bool(NEPoolingLayer::validate(&input_info, &output_info, pool_info)); + + ARM_COMPUTE_EXPECT(is_valid, framework::LogLevel::ERRORS); +} + +TEST_CASE(ValidatePaddedSignedAvgDifferentQuantizationInfo, framework::DatasetMode::ALL) +{ + TensorInfo input_info(TensorShape(3U, 15U, 11U, 1U), 1, DataType::QASYMM8_SIGNED, DataLayout::NHWC); + TensorInfo output_info(TensorShape(3U, 8U, 6U, 1U), 1, DataType::QASYMM8_SIGNED, DataLayout::NHWC); + const auto pool_info = PoolingLayerInfo(PoolingType::AVG, 3, DataLayout::NHWC, PadStrideInfo(2, 2, 1, 1), false); + + input_info.set_quantization_info(QuantizationInfo(0.25f, -11)); + output_info.set_quantization_info(QuantizationInfo(0.5f, -7)); + input_info.set_is_resizable(false); + output_info.set_is_resizable(false); + + const bool is_valid = bool(NEPoolingLayer::validate(&input_info, &output_info, pool_info)); + + ARM_COMPUTE_EXPECT(!is_valid, framework::LogLevel::ERRORS); +} + template using NEPoolingLayerIndicesFixture = PoolingLayerIndicesValidationFixture; @@ -538,6 +581,14 @@ FIXTURE_DATA_TEST_CASE(PaddedMaxDifferentQInfoZeroOffset, // Validate output validate(Accessor(_target), _reference, tolerance_qasymm8_s); } +FIXTURE_DATA_TEST_CASE(PaddedMaxDifferentQInfo, + NEPoolingLayerQuantizedFixture, + framework::DatasetMode::PRECOMMIT, + PoolingDatasetQASYMM8SignedPaddedMaxDifferentQInfo) +{ + // Validate output + validate(Accessor(_target), _reference, tolerance_qasymm8_s); +} FIXTURE_DATA_TEST_CASE(RunSmall, NEPoolingLayerQuantizedFixture, framework::DatasetMode::PRECOMMIT,