/* -*-arm64-*- * vim: syntax=arm64asm * * Copyright (c) 2014 Seppo Tomperi * Copyright (c) 2023 J. Dekker * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "libavutil/aarch64/asm.S" #include "neon.S" .macro hevc_loop_filter_chroma_start bitdepth mov x4, x30 ldr w14, [x2] ldr w15, [x2, #4] .if \bitdepth > 8 lsl w14, w14, #(\bitdepth - 8) lsl w15, w15, #(\bitdepth - 8) .endif adds w2, w14, w15 b.eq 1f dup v16.4h, w14 dup v17.4h, w15 trn1 v16.2d, v16.2d, v17.2d .if \bitdepth > 8 mvni v19.8h, #((0xff << (\bitdepth - 8)) & 0xff), lsl #8 movi v18.8h, #0 .endif neg v17.8h, v16.8h .endm .macro hevc_loop_filter_chroma_body bitdepth .if \bitdepth <= 8 uxtl v20.8h, v0.8b // p1 uxtl v1.8h, v1.8b // p0 uxtl v2.8h, v2.8b // q0 uxtl v23.8h, v3.8b // q1 va .req v20 vb .req v23 .else // required to specify both cases as we are unable to do: v0 .req v20 va .req v0 vb .req v3 .endif sub v5.8h, v2.8h, v1.8h // q0 - p0 sub v6.8h, va.8h, vb.8h // p1 - q1 shl v5.8h, v5.8h, #2 add v5.8h, v6.8h, v5.8h srshr v5.8h, v5.8h, #3 clip v17.8h, v16.8h, v5.8h sqadd v1.8h, v1.8h, v5.8h // p0 + delta sqsub v2.8h, v2.8h, v5.8h // q0 - delta .if \bitdepth <= 8 sqxtun v1.8b, v1.8h sqxtun v2.8b, v2.8h .else clip v18.8h, v19.8h, v1.8h, v2.8h .endif .unreq va .unreq vb .endm function hevc_loop_filter_chroma_body_8_neon, export=0 hevc_loop_filter_chroma_body 8 ret endfunc function hevc_loop_filter_chroma_body_10_neon, export=0 hevc_loop_filter_chroma_body_12_neon: hevc_loop_filter_chroma_body 10 ret endfunc // void ff_hevc_h_loop_filter_chroma_8_neon(uint8_t *_pix, ptrdiff_t _stride, int *_tc, uint8_t *_no_p, uint8_t *_no_q); .macro hevc_h_loop_filter_chroma bitdepth function ff_hevc_h_loop_filter_chroma_\bitdepth\()_neon, export=1 hevc_loop_filter_chroma_start \bitdepth sub x0, x0, x1, lsl #1 .if \bitdepth > 8 ld1 {v0.8h}, [x0], x1 ld1 {v1.8h}, [x0], x1 ld1 {v2.8h}, [x0], x1 ld1 {v3.8h}, [x0] .else ld1 {v0.8b}, [x0], x1 ld1 {v1.8b}, [x0], x1 ld1 {v2.8b}, [x0], x1 ld1 {v3.8b}, [x0] .endif sub x0, x0, x1, lsl #1 bl hevc_loop_filter_chroma_body_\bitdepth\()_neon .if \bitdepth > 8 st1 {v1.8h}, [x0], x1 st1 {v2.8h}, [x0] .else st1 {v1.8b}, [x0], x1 st1 {v2.8b}, [x0] .endif 1: ret x4 endfunc .endm .macro hevc_v_loop_filter_chroma bitdepth function ff_hevc_v_loop_filter_chroma_\bitdepth\()_neon, export=1 hevc_loop_filter_chroma_start \bitdepth .if \bitdepth > 8 sub x0, x0, #4 add x3, x0, x1 lsl x1, x1, #1 ld1 {v0.d}[0], [x0], x1 ld1 {v1.d}[0], [x3], x1 ld1 {v2.d}[0], [x0], x1 ld1 {v3.d}[0], [x3], x1 ld1 {v0.d}[1], [x0], x1 ld1 {v1.d}[1], [x3], x1 ld1 {v2.d}[1], [x0], x1 ld1 {v3.d}[1], [x3], x1 transpose_4x8H v0, v1, v2, v3, v28, v29, v30, v31 .else sub x0, x0, #2 add x3, x0, x1 lsl x1, x1, #1 ld1 {v0.s}[0], [x0], x1 ld1 {v1.s}[0], [x3], x1 ld1 {v2.s}[0], [x0], x1 ld1 {v3.s}[0], [x3], x1 ld1 {v0.s}[1], [x0], x1 ld1 {v1.s}[1], [x3], x1 ld1 {v2.s}[1], [x0], x1 ld1 {v3.s}[1], [x3], x1 transpose_4x8B v0, v1, v2, v3, v28, v29, v30, v31 .endif sub x0, x0, x1, lsl #2 sub x3, x3, x1, lsl #2 bl hevc_loop_filter_chroma_body_\bitdepth\()_neon .if \bitdepth > 8 transpose_4x8H v0, v1, v2, v3, v28, v29, v30, v31 st1 {v0.d}[0], [x0], x1 st1 {v1.d}[0], [x3], x1 st1 {v2.d}[0], [x0], x1 st1 {v3.d}[0], [x3], x1 st1 {v0.d}[1], [x0], x1 st1 {v1.d}[1], [x3], x1 st1 {v2.d}[1], [x0], x1 st1 {v3.d}[1], [x3] .else transpose_4x8B v0, v1, v2, v3, v28, v29, v30, v31 st1 {v0.s}[0], [x0], x1 st1 {v1.s}[0], [x3], x1 st1 {v2.s}[0], [x0], x1 st1 {v3.s}[0], [x3], x1 st1 {v0.s}[1], [x0], x1 st1 {v1.s}[1], [x3], x1 st1 {v2.s}[1], [x0], x1 st1 {v3.s}[1], [x3] .endif 1: ret x4 endfunc .endm hevc_h_loop_filter_chroma 8 hevc_h_loop_filter_chroma 10 hevc_h_loop_filter_chroma 12 hevc_v_loop_filter_chroma 8 hevc_v_loop_filter_chroma 10 hevc_v_loop_filter_chroma 12 .macro hevc_loop_filter_luma_body bitdepth function hevc_loop_filter_luma_body_\bitdepth\()_neon, export=0 .if \bitdepth > 8 lsl w2, w2, #(\bitdepth - 8) // beta <<= BIT_DEPTH - 8 .else uxtl v0.8h, v0.8b uxtl v1.8h, v1.8b uxtl v2.8h, v2.8b uxtl v3.8h, v3.8b uxtl v4.8h, v4.8b uxtl v5.8h, v5.8b uxtl v6.8h, v6.8b uxtl v7.8h, v7.8b .endif ldr w7, [x3] // tc[0] ldr w8, [x3, #4] // tc[1] dup v18.4h, w7 dup v19.4h, w8 trn1 v18.2d, v18.2d, v19.2d .if \bitdepth > 8 shl v18.8h, v18.8h, #(\bitdepth - 8) .endif dup v27.8h, w2 // beta // tc25 shl v19.8h, v18.8h, #2 // * 4 add v19.8h, v19.8h, v18.8h // (tc * 5) srshr v19.8h, v19.8h, #1 // (tc * 5 + 1) >> 1 sshr v17.8h, v27.8h, #2 // beta2 ////// beta_2 check // dp0 = abs(P2 - 2 * P1 + P0) add v22.8h, v3.8h, v1.8h shl v23.8h, v2.8h, #1 sabd v30.8h, v22.8h, v23.8h // dq0 = abs(Q2 - 2 * Q1 + Q0) add v21.8h, v6.8h, v4.8h shl v26.8h, v5.8h, #1 sabd v31.8h, v21.8h, v26.8h // d0 = dp0 + dq0 add v20.8h, v30.8h, v31.8h shl v25.8h, v20.8h, #1 // (d0 << 1) < beta_2 cmgt v23.8h, v17.8h, v25.8h ////// beta check // d0 + d3 < beta mov x9, #0xFFFF00000000FFFF dup v24.2d, x9 and v25.16b, v24.16b, v20.16b addp v25.8h, v25.8h, v25.8h // 1+0 0+1 1+0 0+1 addp v25.4h, v25.4h, v25.4h // 1+0+0+1 1+0+0+1 cmgt v25.4h, v27.4h, v25.4h // lower/upper mask in h[0/1] mov w9, v25.s[0] cmp w9, #0 sxtl v26.4s, v25.4h sxtl v16.2d, v26.2s // full skip mask b.eq 3f // skip both blocks // TODO: we can check the full skip mask with the weak/strong mask to // potentially skip weak or strong calculation entirely if we only have one ////// beta_3 check // abs(P3 - P0) + abs(Q3 - Q0) < beta_3 sshr v17.8h, v17.8h, #1 // beta_3 sabd v20.8h, v0.8h, v3.8h saba v20.8h, v7.8h, v4.8h cmgt v21.8h, v17.8h, v20.8h and v23.16b, v23.16b, v21.16b ////// tc25 check // abs(P0 - Q0) < tc25 sabd v20.8h, v3.8h, v4.8h cmgt v21.8h, v19.8h, v20.8h and v23.16b, v23.16b, v21.16b ////// Generate low/high line max from lines 0/3/4/7 // mask out lines 2/3/5/6 not v20.16b, v24.16b // 0x0000FFFFFFFF0000 orr v23.16b, v23.16b, v20.16b // generate weak/strong mask uminp v23.8h, v23.8h, v23.8h // extend to singles sxtl v23.4s, v23.4h uminp v26.4s, v23.4s, v23.4s // check lines // extract to gpr ext v25.16b, v26.16b, v26.16b, #2 zip1 v17.4s, v26.4s, v26.4s mov w12, v25.s[0] mov w11, #0x0000FFFF mov w13, #0xFFFF0000 // FFFF FFFF -> strong strong // FFFF 0000 -> strong weak // 0000 FFFF -> weak strong // 0000 0000 -> weak weak cmp w12, w13 b.hi 0f // only strong/strong, skip weak nd_p/nd_q calc ////// weak nd_p/nd_q // d0+d3 and v30.16b, v30.16b, v24.16b // d0 __ __ d3 d4 __ __ d7 and v31.16b, v31.16b, v24.16b addp v30.8h, v30.8h, v30.8h // [d0+__ __+d3 d4+__ __+d7] [ ... ] addp v31.8h, v31.8h, v31.8h // [d0+d3 d4+d7] addp v30.4h, v30.4h, v30.4h addp v31.4h, v31.4h, v31.4h // ((beta + (beta >> 1)) >> 3) sshr v21.8h, v27.8h, #1 add v21.8h, v21.8h, v27.8h sshr v21.8h, v21.8h, #3 // nd_p = dp0 + dp3 < ((beta + (beta >> 1)) >> 3) cmgt v30.8h, v21.8h, v30.8h // nd_q = dq0 + dq3 < ((beta + (beta >> 1)) >> 3) cmgt v31.8h, v21.8h, v31.8h sxtl v30.4s, v30.4h sxtl v31.4s, v31.4h sxtl v28.2d, v30.2s sxtl v29.2d, v31.2s cmp w12, w11 b.lo 1f // can only be weak weak, skip strong 0: // STRONG FILTER // P0 = p0 + av_clip(((p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3) - p0, -tc3, tc3); add v21.8h, v2.8h, v3.8h // (p1 + p0 add v21.8h, v4.8h, v21.8h // + q0) shl v21.8h, v21.8h, #1 // * 2 add v22.8h, v1.8h, v5.8h // (p2 + q1) add v21.8h, v22.8h, v21.8h // + srshr v21.8h, v21.8h, #3 // >> 3 sub v21.8h, v21.8h, v3.8h // - p0 // P1 = p1 + av_clip(((p2 + p1 + p0 + q0 + 2) >> 2) - p1, -tc2, tc2); add v22.8h, v1.8h, v2.8h add v23.8h, v3.8h, v4.8h add v22.8h, v22.8h, v23.8h srshr v22.8h, v22.8h, #2 sub v22.8h, v22.8h, v2.8h // P2 = p2 + av_clip(((2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3) - p2, -tc, tc); add v23.8h, v0.8h, v1.8h // p3 + p2 add v24.8h, v3.8h, v4.8h // p0 + q0 shl v23.8h, v23.8h, #1 // * 2 add v23.8h, v23.8h, v24.8h add v24.8h, v1.8h, v2.8h // p2 + p1 add v23.8h, v23.8h, v24.8h srshr v23.8h, v23.8h, #3 sub v23.8h, v23.8h, v1.8h // Q0 = q0 + av_clip(((p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3) - q0, -tc3, tc3); add v24.8h, v3.8h, v4.8h // (p0 + q0 add v24.8h, v5.8h, v24.8h // + q1) shl v24.8h, v24.8h, #1 // * 2 add v25.8h, v2.8h, v6.8h // (p1 + q2) add v24.8h, v25.8h, v24.8h // + srshr v24.8h, v24.8h, #3 // >> 3 sub v24.8h, v24.8h, v4.8h // - q0 // Q1 = q1 + av_clip(((p0 + q0 + q1 + q2 + 2) >> 2) - q1, -tc2, tc2); add v25.8h, v6.8h, v5.8h add v26.8h, v3.8h, v4.8h add v25.8h, v25.8h, v26.8h srshr v25.8h, v25.8h, #2 sub v25.8h, v25.8h, v5.8h // Q2 = q2 + av_clip(((2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3) - q2, -tc, tc); add v26.8h, v7.8h, v6.8h add v27.8h, v6.8h, v5.8h shl v26.8h, v26.8h, #1 add v26.8h, v26.8h, v27.8h add v27.8h, v3.8h, v4.8h add v26.8h, v26.8h, v27.8h srshr v26.8h, v26.8h, #3 sub v26.8h, v26.8h, v6.8h // this clip should work properly shl v30.8h, v18.8h, #1 // tc2 neg v31.8h, v30.8h // -tc2 clip v31.8h, v30.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h and v21.16b, v21.16b, v16.16b and v22.16b, v22.16b, v16.16b and v23.16b, v23.16b, v16.16b and v24.16b, v24.16b, v16.16b and v25.16b, v25.16b, v16.16b and v26.16b, v26.16b, v16.16b add v23.8h, v23.8h, v1.8h // careful add v22.8h, v22.8h, v2.8h add v21.8h, v21.8h, v3.8h add v24.8h, v24.8h, v4.8h add v25.8h, v25.8h, v5.8h add v26.8h, v26.8h, v6.8h cmp w12, w13 b.hi 2f // only strong/strong, skip weak 1: // WEAK FILTER // delta0 = (9 * (q0 - p0) - 3 * (q1 - p1) + 8) >> 4 .if \bitdepth < 12 sub v27.8h, v4.8h, v3.8h // q0 - p0 shl v30.8h, v27.8h, #3 // * 8 add v27.8h, v27.8h, v30.8h // 9 * (q0 - p0) sub v30.8h, v5.8h, v2.8h // q1 - p1 shl v31.8h, v30.8h, #1 // * 2 sub v27.8h, v27.8h, v31.8h sub v27.8h, v27.8h, v30.8h // - 3 * (q1 - p1) srshr v27.8h, v27.8h, #4 .else sub v19.8h, v4.8h, v3.8h // q0 - p0 sub v20.8h, v5.8h, v2.8h // q1 - p1 sshll v30.4s, v19.4h, #3 // * 8 sshll2 v31.4s, v19.8h, #3 shl v27.8h, v20.8h, #1 saddw v30.4s, v30.4s, v19.4h // 9 * (q0 - p0) saddw2 v31.4s, v31.4s, v19.8h saddl v19.4s, v27.4h, v20.4h // 3 * (q1 - p1) saddl2 v20.4s, v27.8h, v20.8h sub v19.4s, v30.4s, v19.4s sub v20.4s, v31.4s, v20.4s sqrshrn v27.4h, v19.4s, #4 sqrshrn2 v27.8h, v20.4s, #4 .endif // delta0 10tc check mask shl v30.8h, v18.8h, #1 // * 2 shl v31.8h, v18.8h, #3 // * 8 add v30.8h, v30.8h, v31.8h // 10 * tc abs v31.8h, v27.8h cmgt v20.8h, v30.8h, v31.8h // abs(delta0) < 10 * tc and v20.16b, v20.16b, v16.16b // combine with full mask neg v31.8h, v18.8h // -tc clip v31.8h, v18.8h, v27.8h // delta0 = av_clip(delta0, -tc, tc) // deltap1 = av_clip((((p2 + p0 + 1) >> 1) - p1 + delta0) >> 1, -tc_2, tc_2) add v30.8h, v1.8h, v3.8h srshr v30.8h, v30.8h, #1 sub v30.8h, v30.8h, v2.8h add v30.8h, v30.8h, v27.8h sshr v30.8h, v30.8h, #1 // p3 p2 p1 p0 q0 q1 q2 q3 // v0 v1 v2 v3 v4 v5 v6 v7 // deltaq1 = av_clip((((q2 + q0 + 1) >> 1) - q1 - delta0) >> 1, -tc_2, tc_2); add v31.8h, v6.8h, v4.8h srshr v31.8h, v31.8h, #1 sub v31.8h, v31.8h, v5.8h sub v31.8h, v31.8h, v27.8h sshr v31.8h, v31.8h, #1 // apply nd_p nd_q mask to deltap1/deltaq1 and v30.16b, v30.16b, v28.16b and v31.16b, v31.16b, v29.16b // apply full skip mask to deltap1/deltaq1/delta0 and v30.16b, v30.16b, v20.16b and v27.16b, v27.16b, v20.16b and v31.16b, v31.16b, v20.16b // clip P1/Q1 to -tc_2, tc_2 sshr v18.8h, v18.8h, #1 // tc2 neg v28.8h, v18.8h clip v28.8h, v18.8h, v30.8h, v31.8h // P0 = av_clip_pixel(p0 + delta0) // Q0 = av_clip_pixel(q0 - delta0) add v29.8h, v3.8h, v27.8h // P0 sub v27.8h, v4.8h, v27.8h // Q0 // P1 = av_clip_pixel(p1 + deltap1) // Q1 = av_clip_pixel(q1 + deltaq1) add v30.8h, v2.8h, v30.8h // P1 add v31.8h, v5.8h, v31.8h // Q1 2: // MIX WEAK/STRONG mov v19.16b, v1.16b mov v20.16b, v6.16b // copy selection mask mov v1.16b, v17.16b mov v2.16b, v17.16b mov v3.16b, v17.16b mov v4.16b, v17.16b mov v5.16b, v17.16b mov v6.16b, v17.16b // select bsl v1.16b, v23.16b, v19.16b // P2 strong/orig bsl v2.16b, v22.16b, v30.16b // P1 strong/weak bsl v3.16b, v21.16b, v29.16b // P0 strong/weak bsl v4.16b, v24.16b, v27.16b // Q0 strong/weak bsl v5.16b, v25.16b, v31.16b // Q1 strong/weak bsl v6.16b, v26.16b, v20.16b // Q2 strong/orig // NOTE: Q3/P3 are unchanged .if \bitdepth > 8 movi v19.8h, #0 dup v20.8h, w14 clip v19.8h, v20.8h, v1.8h, v2.8h, v3.8h, v4.8h, v5.8h, v6.8h .else sqxtun v0.8b, v0.8h sqxtun v1.8b, v1.8h sqxtun v2.8b, v2.8h sqxtun v3.8b, v3.8h sqxtun v4.8b, v4.8h sqxtun v5.8b, v5.8h sqxtun v6.8b, v6.8h sqxtun v7.8b, v7.8h .endif ret 3: ret x6 endfunc .endm hevc_loop_filter_luma_body 8 hevc_loop_filter_luma_body 10 hevc_loop_filter_luma_body 12 // hevc_v_loop_filter_luma(uint8_t *pix, ptrdiff_t stride, int beta, const int32_t *tc, const uint8_t *no_p, const uint8_t *no_q) .macro hevc_loop_filter_luma dir, bitdepth function ff_hevc_\dir\()_loop_filter_luma_\bitdepth\()_neon, export=1 mov x6, x30 .ifc \dir, v .if \bitdepth > 8 sub x0, x0, #8 .else sub x0, x0, #4 .endif .else sub x0, x0, x1, lsl #2 // -4 * xstride .endif mov x10, x0 .if \bitdepth > 8 ld1 {v0.8h}, [x0], x1 ld1 {v1.8h}, [x0], x1 ld1 {v2.8h}, [x0], x1 ld1 {v3.8h}, [x0], x1 ld1 {v4.8h}, [x0], x1 ld1 {v5.8h}, [x0], x1 ld1 {v6.8h}, [x0], x1 ld1 {v7.8h}, [x0] mov w14, #((1 << \bitdepth) - 1) .ifc \dir, v transpose_8x8H v0, v1, v2, v3, v4, v5, v6, v7, v16, v17 .endif .else ld1 {v0.8b}, [x0], x1 ld1 {v1.8b}, [x0], x1 ld1 {v2.8b}, [x0], x1 ld1 {v3.8b}, [x0], x1 ld1 {v4.8b}, [x0], x1 ld1 {v5.8b}, [x0], x1 ld1 {v6.8b}, [x0], x1 ld1 {v7.8b}, [x0] .ifc \dir, v transpose_8x8B v0, v1, v2, v3, v4, v5, v6, v7, v16, v17 .endif .endif bl hevc_loop_filter_luma_body_\bitdepth\()_neon .if \bitdepth > 8 .ifc \dir, v transpose_8x8H v0, v1, v2, v3, v4, v5, v6, v7, v16, v17 .endif st1 {v0.8h}, [x10], x1 st1 {v1.8h}, [x10], x1 st1 {v2.8h}, [x10], x1 st1 {v3.8h}, [x10], x1 st1 {v4.8h}, [x10], x1 st1 {v5.8h}, [x10], x1 st1 {v6.8h}, [x10], x1 st1 {v7.8h}, [x10] .else .ifc \dir, v transpose_8x8B v0, v1, v2, v3, v4, v5, v6, v7, v16, v17 .endif st1 {v0.8b}, [x10], x1 st1 {v1.8b}, [x10], x1 st1 {v2.8b}, [x10], x1 st1 {v3.8b}, [x10], x1 st1 {v4.8b}, [x10], x1 st1 {v5.8b}, [x10], x1 st1 {v6.8b}, [x10], x1 st1 {v7.8b}, [x10] .endif ret x6 endfunc .endm hevc_loop_filter_luma h, 8 hevc_loop_filter_luma h, 10 hevc_loop_filter_luma h, 12 hevc_loop_filter_luma v, 8 hevc_loop_filter_luma v, 10 hevc_loop_filter_luma v, 12