ffmpeg/libavcodec/aarch64/hevcdsp_deblock_neon.S

601 lines
22 KiB
ArmAsm

/* -*-arm64-*-
* vim: syntax=arm64asm
*
* Copyright (c) 2014 Seppo Tomperi <seppo.tomperi@vtt.fi>
* Copyright (c) 2023 J. Dekker <jdek@itanimul.li>
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "libavutil/aarch64/asm.S"
#include "neon.S"
.macro hevc_loop_filter_chroma_start bitdepth
mov x4, x30
ldr w14, [x2]
ldr w15, [x2, #4]
.if \bitdepth > 8
lsl w14, w14, #(\bitdepth - 8)
lsl w15, w15, #(\bitdepth - 8)
.endif
adds w2, w14, w15
b.eq 1f
dup v16.4h, w14
dup v17.4h, w15
trn1 v16.2d, v16.2d, v17.2d
.if \bitdepth > 8
mvni v19.8h, #((0xff << (\bitdepth - 8)) & 0xff), lsl #8
movi v18.8h, #0
.endif
neg v17.8h, v16.8h
.endm
.macro hevc_loop_filter_chroma_body bitdepth
.if \bitdepth <= 8
uxtl v20.8h, v0.8b // p1
uxtl v1.8h, v1.8b // p0
uxtl v2.8h, v2.8b // q0
uxtl v23.8h, v3.8b // q1
va .req v20
vb .req v23
.else // required to specify both cases as we are unable to do: v0 .req v20
va .req v0
vb .req v3
.endif
sub v5.8h, v2.8h, v1.8h // q0 - p0
sub v6.8h, va.8h, vb.8h // p1 - q1
shl v5.8h, v5.8h, #2
add v5.8h, v6.8h, v5.8h
srshr v5.8h, v5.8h, #3
clip v17.8h, v16.8h, v5.8h
sqadd v1.8h, v1.8h, v5.8h // p0 + delta
sqsub v2.8h, v2.8h, v5.8h // q0 - delta
.if \bitdepth <= 8
sqxtun v1.8b, v1.8h
sqxtun v2.8b, v2.8h
.else
clip v18.8h, v19.8h, v1.8h, v2.8h
.endif
.unreq va
.unreq vb
.endm
function hevc_loop_filter_chroma_body_8_neon, export=0
hevc_loop_filter_chroma_body 8
ret
endfunc
function hevc_loop_filter_chroma_body_10_neon, export=0
hevc_loop_filter_chroma_body_12_neon:
hevc_loop_filter_chroma_body 10
ret
endfunc
// void ff_hevc_h_loop_filter_chroma_8_neon(uint8_t *_pix, ptrdiff_t _stride, int *_tc, uint8_t *_no_p, uint8_t *_no_q);
.macro hevc_h_loop_filter_chroma bitdepth
function ff_hevc_h_loop_filter_chroma_\bitdepth\()_neon, export=1
hevc_loop_filter_chroma_start \bitdepth
sub x0, x0, x1, lsl #1
.if \bitdepth > 8
ld1 {v0.8h}, [x0], x1
ld1 {v1.8h}, [x0], x1
ld1 {v2.8h}, [x0], x1
ld1 {v3.8h}, [x0]
.else
ld1 {v0.8b}, [x0], x1
ld1 {v1.8b}, [x0], x1
ld1 {v2.8b}, [x0], x1
ld1 {v3.8b}, [x0]
.endif
sub x0, x0, x1, lsl #1
bl hevc_loop_filter_chroma_body_\bitdepth\()_neon
.if \bitdepth > 8
st1 {v1.8h}, [x0], x1
st1 {v2.8h}, [x0]
.else
st1 {v1.8b}, [x0], x1
st1 {v2.8b}, [x0]
.endif
1: ret x4
endfunc
.endm
.macro hevc_v_loop_filter_chroma bitdepth
function ff_hevc_v_loop_filter_chroma_\bitdepth\()_neon, export=1
hevc_loop_filter_chroma_start \bitdepth
.if \bitdepth > 8
sub x0, x0, #4
add x3, x0, x1
lsl x1, x1, #1
ld1 {v0.d}[0], [x0], x1
ld1 {v1.d}[0], [x3], x1
ld1 {v2.d}[0], [x0], x1
ld1 {v3.d}[0], [x3], x1
ld1 {v0.d}[1], [x0], x1
ld1 {v1.d}[1], [x3], x1
ld1 {v2.d}[1], [x0], x1
ld1 {v3.d}[1], [x3], x1
transpose_4x8H v0, v1, v2, v3, v28, v29, v30, v31
.else
sub x0, x0, #2
add x3, x0, x1
lsl x1, x1, #1
ld1 {v0.s}[0], [x0], x1
ld1 {v1.s}[0], [x3], x1
ld1 {v2.s}[0], [x0], x1
ld1 {v3.s}[0], [x3], x1
ld1 {v0.s}[1], [x0], x1
ld1 {v1.s}[1], [x3], x1
ld1 {v2.s}[1], [x0], x1
ld1 {v3.s}[1], [x3], x1
transpose_4x8B v0, v1, v2, v3, v28, v29, v30, v31
.endif
sub x0, x0, x1, lsl #2
sub x3, x3, x1, lsl #2
bl hevc_loop_filter_chroma_body_\bitdepth\()_neon
.if \bitdepth > 8
transpose_4x8H v0, v1, v2, v3, v28, v29, v30, v31
st1 {v0.d}[0], [x0], x1
st1 {v1.d}[0], [x3], x1
st1 {v2.d}[0], [x0], x1
st1 {v3.d}[0], [x3], x1
st1 {v0.d}[1], [x0], x1
st1 {v1.d}[1], [x3], x1
st1 {v2.d}[1], [x0], x1
st1 {v3.d}[1], [x3]
.else
transpose_4x8B v0, v1, v2, v3, v28, v29, v30, v31
st1 {v0.s}[0], [x0], x1
st1 {v1.s}[0], [x3], x1
st1 {v2.s}[0], [x0], x1
st1 {v3.s}[0], [x3], x1
st1 {v0.s}[1], [x0], x1
st1 {v1.s}[1], [x3], x1
st1 {v2.s}[1], [x0], x1
st1 {v3.s}[1], [x3]
.endif
1: ret x4
endfunc
.endm
hevc_h_loop_filter_chroma 8
hevc_h_loop_filter_chroma 10
hevc_h_loop_filter_chroma 12
hevc_v_loop_filter_chroma 8
hevc_v_loop_filter_chroma 10
hevc_v_loop_filter_chroma 12
.macro hevc_loop_filter_luma_body bitdepth
function hevc_loop_filter_luma_body_\bitdepth\()_neon, export=0
.if \bitdepth > 8
lsl w2, w2, #(\bitdepth - 8) // beta <<= BIT_DEPTH - 8
.else
uxtl v0.8h, v0.8b
uxtl v1.8h, v1.8b
uxtl v2.8h, v2.8b
uxtl v3.8h, v3.8b
uxtl v4.8h, v4.8b
uxtl v5.8h, v5.8b
uxtl v6.8h, v6.8b
uxtl v7.8h, v7.8b
.endif
ldr w7, [x3] // tc[0]
ldr w8, [x3, #4] // tc[1]
dup v18.4h, w7
dup v19.4h, w8
trn1 v18.2d, v18.2d, v19.2d
.if \bitdepth > 8
shl v18.8h, v18.8h, #(\bitdepth - 8)
.endif
dup v27.8h, w2 // beta
// tc25
shl v19.8h, v18.8h, #2 // * 4
add v19.8h, v19.8h, v18.8h // (tc * 5)
srshr v19.8h, v19.8h, #1 // (tc * 5 + 1) >> 1
sshr v17.8h, v27.8h, #2 // beta2
////// beta_2 check
// dp0 = abs(P2 - 2 * P1 + P0)
add v22.8h, v3.8h, v1.8h
shl v23.8h, v2.8h, #1
sabd v30.8h, v22.8h, v23.8h
// dq0 = abs(Q2 - 2 * Q1 + Q0)
add v21.8h, v6.8h, v4.8h
shl v26.8h, v5.8h, #1
sabd v31.8h, v21.8h, v26.8h
// d0 = dp0 + dq0
add v20.8h, v30.8h, v31.8h
shl v25.8h, v20.8h, #1
// (d0 << 1) < beta_2
cmgt v23.8h, v17.8h, v25.8h
////// beta check
// d0 + d3 < beta
mov x9, #0xFFFF00000000FFFF
dup v24.2d, x9
and v25.16b, v24.16b, v20.16b
addp v25.8h, v25.8h, v25.8h // 1+0 0+1 1+0 0+1
addp v25.4h, v25.4h, v25.4h // 1+0+0+1 1+0+0+1
cmgt v25.4h, v27.4h, v25.4h // lower/upper mask in h[0/1]
mov w9, v25.s[0]
cmp w9, #0
sxtl v26.4s, v25.4h
sxtl v16.2d, v26.2s // full skip mask
b.eq 3f // skip both blocks
// TODO: we can check the full skip mask with the weak/strong mask to
// potentially skip weak or strong calculation entirely if we only have one
////// beta_3 check
// abs(P3 - P0) + abs(Q3 - Q0) < beta_3
sshr v17.8h, v17.8h, #1 // beta_3
sabd v20.8h, v0.8h, v3.8h
saba v20.8h, v7.8h, v4.8h
cmgt v21.8h, v17.8h, v20.8h
and v23.16b, v23.16b, v21.16b
////// tc25 check
// abs(P0 - Q0) < tc25
sabd v20.8h, v3.8h, v4.8h
cmgt v21.8h, v19.8h, v20.8h
and v23.16b, v23.16b, v21.16b
////// Generate low/high line max from lines 0/3/4/7
// mask out lines 2/3/5/6
not v20.16b, v24.16b // 0x0000FFFFFFFF0000
orr v23.16b, v23.16b, v20.16b
// generate weak/strong mask
uminp v23.8h, v23.8h, v23.8h // extend to singles
sxtl v23.4s, v23.4h
uminp v26.4s, v23.4s, v23.4s // check lines
// extract to gpr
ext v25.16b, v26.16b, v26.16b, #2
zip1 v17.4s, v26.4s, v26.4s
mov w12, v25.s[0]
mov w11, #0x0000FFFF
mov w13, #0xFFFF0000
// FFFF FFFF -> strong strong
// FFFF 0000 -> strong weak
// 0000 FFFF -> weak strong
// 0000 0000 -> weak weak
cmp w12, w13
b.hi 0f // only strong/strong, skip weak nd_p/nd_q calc
////// weak nd_p/nd_q
// d0+d3
and v30.16b, v30.16b, v24.16b // d0 __ __ d3 d4 __ __ d7
and v31.16b, v31.16b, v24.16b
addp v30.8h, v30.8h, v30.8h // [d0+__ __+d3 d4+__ __+d7] [ ... ]
addp v31.8h, v31.8h, v31.8h // [d0+d3 d4+d7]
addp v30.4h, v30.4h, v30.4h
addp v31.4h, v31.4h, v31.4h
// ((beta + (beta >> 1)) >> 3)
sshr v21.8h, v27.8h, #1
add v21.8h, v21.8h, v27.8h
sshr v21.8h, v21.8h, #3
// nd_p = dp0 + dp3 < ((beta + (beta >> 1)) >> 3)
cmgt v30.8h, v21.8h, v30.8h
// nd_q = dq0 + dq3 < ((beta + (beta >> 1)) >> 3)
cmgt v31.8h, v21.8h, v31.8h
sxtl v30.4s, v30.4h
sxtl v31.4s, v31.4h
sxtl v28.2d, v30.2s
sxtl v29.2d, v31.2s
cmp w12, w11
b.lo 1f // can only be weak weak, skip strong
0: // STRONG FILTER
// P0 = p0 + av_clip(((p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3) - p0, -tc3, tc3);
add v21.8h, v2.8h, v3.8h // (p1 + p0
add v21.8h, v4.8h, v21.8h // + q0)
shl v21.8h, v21.8h, #1 // * 2
add v22.8h, v1.8h, v5.8h // (p2 + q1)
add v21.8h, v22.8h, v21.8h // +
srshr v21.8h, v21.8h, #3 // >> 3
sub v21.8h, v21.8h, v3.8h // - p0
// P1 = p1 + av_clip(((p2 + p1 + p0 + q0 + 2) >> 2) - p1, -tc2, tc2);
add v22.8h, v1.8h, v2.8h
add v23.8h, v3.8h, v4.8h
add v22.8h, v22.8h, v23.8h
srshr v22.8h, v22.8h, #2
sub v22.8h, v22.8h, v2.8h
// P2 = p2 + av_clip(((2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3) - p2, -tc, tc);
add v23.8h, v0.8h, v1.8h // p3 + p2
add v24.8h, v3.8h, v4.8h // p0 + q0
shl v23.8h, v23.8h, #1 // * 2
add v23.8h, v23.8h, v24.8h
add v24.8h, v1.8h, v2.8h // p2 + p1
add v23.8h, v23.8h, v24.8h
srshr v23.8h, v23.8h, #3
sub v23.8h, v23.8h, v1.8h
// Q0 = q0 + av_clip(((p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3) - q0, -tc3, tc3);
add v24.8h, v3.8h, v4.8h // (p0 + q0
add v24.8h, v5.8h, v24.8h // + q1)
shl v24.8h, v24.8h, #1 // * 2
add v25.8h, v2.8h, v6.8h // (p1 + q2)
add v24.8h, v25.8h, v24.8h // +
srshr v24.8h, v24.8h, #3 // >> 3
sub v24.8h, v24.8h, v4.8h // - q0
// Q1 = q1 + av_clip(((p0 + q0 + q1 + q2 + 2) >> 2) - q1, -tc2, tc2);
add v25.8h, v6.8h, v5.8h
add v26.8h, v3.8h, v4.8h
add v25.8h, v25.8h, v26.8h
srshr v25.8h, v25.8h, #2
sub v25.8h, v25.8h, v5.8h
// Q2 = q2 + av_clip(((2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3) - q2, -tc, tc);
add v26.8h, v7.8h, v6.8h
add v27.8h, v6.8h, v5.8h
shl v26.8h, v26.8h, #1
add v26.8h, v26.8h, v27.8h
add v27.8h, v3.8h, v4.8h
add v26.8h, v26.8h, v27.8h
srshr v26.8h, v26.8h, #3
sub v26.8h, v26.8h, v6.8h
// this clip should work properly
shl v30.8h, v18.8h, #1 // tc2
neg v31.8h, v30.8h // -tc2
clip v31.8h, v30.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h
and v21.16b, v21.16b, v16.16b
and v22.16b, v22.16b, v16.16b
and v23.16b, v23.16b, v16.16b
and v24.16b, v24.16b, v16.16b
and v25.16b, v25.16b, v16.16b
and v26.16b, v26.16b, v16.16b
add v23.8h, v23.8h, v1.8h // careful
add v22.8h, v22.8h, v2.8h
add v21.8h, v21.8h, v3.8h
add v24.8h, v24.8h, v4.8h
add v25.8h, v25.8h, v5.8h
add v26.8h, v26.8h, v6.8h
cmp w12, w13
b.hi 2f // only strong/strong, skip weak
1: // WEAK FILTER
// delta0 = (9 * (q0 - p0) - 3 * (q1 - p1) + 8) >> 4
.if \bitdepth < 12
sub v27.8h, v4.8h, v3.8h // q0 - p0
shl v30.8h, v27.8h, #3 // * 8
add v27.8h, v27.8h, v30.8h // 9 * (q0 - p0)
sub v30.8h, v5.8h, v2.8h // q1 - p1
shl v31.8h, v30.8h, #1 // * 2
sub v27.8h, v27.8h, v31.8h
sub v27.8h, v27.8h, v30.8h // - 3 * (q1 - p1)
srshr v27.8h, v27.8h, #4
.else
sub v19.8h, v4.8h, v3.8h // q0 - p0
sub v20.8h, v5.8h, v2.8h // q1 - p1
sshll v30.4s, v19.4h, #3 // * 8
sshll2 v31.4s, v19.8h, #3
shl v27.8h, v20.8h, #1
saddw v30.4s, v30.4s, v19.4h // 9 * (q0 - p0)
saddw2 v31.4s, v31.4s, v19.8h
saddl v19.4s, v27.4h, v20.4h // 3 * (q1 - p1)
saddl2 v20.4s, v27.8h, v20.8h
sub v19.4s, v30.4s, v19.4s
sub v20.4s, v31.4s, v20.4s
sqrshrn v27.4h, v19.4s, #4
sqrshrn2 v27.8h, v20.4s, #4
.endif
// delta0 10tc check mask
shl v30.8h, v18.8h, #1 // * 2
shl v31.8h, v18.8h, #3 // * 8
add v30.8h, v30.8h, v31.8h // 10 * tc
abs v31.8h, v27.8h
cmgt v20.8h, v30.8h, v31.8h // abs(delta0) < 10 * tc
and v20.16b, v20.16b, v16.16b // combine with full mask
neg v31.8h, v18.8h // -tc
clip v31.8h, v18.8h, v27.8h // delta0 = av_clip(delta0, -tc, tc)
// deltap1 = av_clip((((p2 + p0 + 1) >> 1) - p1 + delta0) >> 1, -tc_2, tc_2)
add v30.8h, v1.8h, v3.8h
srshr v30.8h, v30.8h, #1
sub v30.8h, v30.8h, v2.8h
add v30.8h, v30.8h, v27.8h
sshr v30.8h, v30.8h, #1
// p3 p2 p1 p0 q0 q1 q2 q3
// v0 v1 v2 v3 v4 v5 v6 v7
// deltaq1 = av_clip((((q2 + q0 + 1) >> 1) - q1 - delta0) >> 1, -tc_2, tc_2);
add v31.8h, v6.8h, v4.8h
srshr v31.8h, v31.8h, #1
sub v31.8h, v31.8h, v5.8h
sub v31.8h, v31.8h, v27.8h
sshr v31.8h, v31.8h, #1
// apply nd_p nd_q mask to deltap1/deltaq1
and v30.16b, v30.16b, v28.16b
and v31.16b, v31.16b, v29.16b
// apply full skip mask to deltap1/deltaq1/delta0
and v30.16b, v30.16b, v20.16b
and v27.16b, v27.16b, v20.16b
and v31.16b, v31.16b, v20.16b
// clip P1/Q1 to -tc_2, tc_2
sshr v18.8h, v18.8h, #1 // tc2
neg v28.8h, v18.8h
clip v28.8h, v18.8h, v30.8h, v31.8h
// P0 = av_clip_pixel(p0 + delta0)
// Q0 = av_clip_pixel(q0 - delta0)
add v29.8h, v3.8h, v27.8h // P0
sub v27.8h, v4.8h, v27.8h // Q0
// P1 = av_clip_pixel(p1 + deltap1)
// Q1 = av_clip_pixel(q1 + deltaq1)
add v30.8h, v2.8h, v30.8h // P1
add v31.8h, v5.8h, v31.8h // Q1
2: // MIX WEAK/STRONG
mov v19.16b, v1.16b
mov v20.16b, v6.16b
// copy selection mask
mov v1.16b, v17.16b
mov v2.16b, v17.16b
mov v3.16b, v17.16b
mov v4.16b, v17.16b
mov v5.16b, v17.16b
mov v6.16b, v17.16b
// select
bsl v1.16b, v23.16b, v19.16b // P2 strong/orig
bsl v2.16b, v22.16b, v30.16b // P1 strong/weak
bsl v3.16b, v21.16b, v29.16b // P0 strong/weak
bsl v4.16b, v24.16b, v27.16b // Q0 strong/weak
bsl v5.16b, v25.16b, v31.16b // Q1 strong/weak
bsl v6.16b, v26.16b, v20.16b // Q2 strong/orig
// NOTE: Q3/P3 are unchanged
.if \bitdepth > 8
movi v19.8h, #0
dup v20.8h, w14
clip v19.8h, v20.8h, v1.8h, v2.8h, v3.8h, v4.8h, v5.8h, v6.8h
.else
sqxtun v0.8b, v0.8h
sqxtun v1.8b, v1.8h
sqxtun v2.8b, v2.8h
sqxtun v3.8b, v3.8h
sqxtun v4.8b, v4.8h
sqxtun v5.8b, v5.8h
sqxtun v6.8b, v6.8h
sqxtun v7.8b, v7.8h
.endif
ret
3: ret x6
endfunc
.endm
hevc_loop_filter_luma_body 8
hevc_loop_filter_luma_body 10
hevc_loop_filter_luma_body 12
// hevc_v_loop_filter_luma(uint8_t *pix, ptrdiff_t stride, int beta, const int32_t *tc, const uint8_t *no_p, const uint8_t *no_q)
.macro hevc_loop_filter_luma dir, bitdepth
function ff_hevc_\dir\()_loop_filter_luma_\bitdepth\()_neon, export=1
mov x6, x30
.ifc \dir, v
.if \bitdepth > 8
sub x0, x0, #8
.else
sub x0, x0, #4
.endif
.else
sub x0, x0, x1, lsl #2 // -4 * xstride
.endif
mov x10, x0
.if \bitdepth > 8
ld1 {v0.8h}, [x0], x1
ld1 {v1.8h}, [x0], x1
ld1 {v2.8h}, [x0], x1
ld1 {v3.8h}, [x0], x1
ld1 {v4.8h}, [x0], x1
ld1 {v5.8h}, [x0], x1
ld1 {v6.8h}, [x0], x1
ld1 {v7.8h}, [x0]
mov w14, #((1 << \bitdepth) - 1)
.ifc \dir, v
transpose_8x8H v0, v1, v2, v3, v4, v5, v6, v7, v16, v17
.endif
.else
ld1 {v0.8b}, [x0], x1
ld1 {v1.8b}, [x0], x1
ld1 {v2.8b}, [x0], x1
ld1 {v3.8b}, [x0], x1
ld1 {v4.8b}, [x0], x1
ld1 {v5.8b}, [x0], x1
ld1 {v6.8b}, [x0], x1
ld1 {v7.8b}, [x0]
.ifc \dir, v
transpose_8x8B v0, v1, v2, v3, v4, v5, v6, v7, v16, v17
.endif
.endif
bl hevc_loop_filter_luma_body_\bitdepth\()_neon
.if \bitdepth > 8
.ifc \dir, v
transpose_8x8H v0, v1, v2, v3, v4, v5, v6, v7, v16, v17
.endif
st1 {v0.8h}, [x10], x1
st1 {v1.8h}, [x10], x1
st1 {v2.8h}, [x10], x1
st1 {v3.8h}, [x10], x1
st1 {v4.8h}, [x10], x1
st1 {v5.8h}, [x10], x1
st1 {v6.8h}, [x10], x1
st1 {v7.8h}, [x10]
.else
.ifc \dir, v
transpose_8x8B v0, v1, v2, v3, v4, v5, v6, v7, v16, v17
.endif
st1 {v0.8b}, [x10], x1
st1 {v1.8b}, [x10], x1
st1 {v2.8b}, [x10], x1
st1 {v3.8b}, [x10], x1
st1 {v4.8b}, [x10], x1
st1 {v5.8b}, [x10], x1
st1 {v6.8b}, [x10], x1
st1 {v7.8b}, [x10]
.endif
ret x6
endfunc
.endm
hevc_loop_filter_luma h, 8
hevc_loop_filter_luma h, 10
hevc_loop_filter_luma h, 12
hevc_loop_filter_luma v, 8
hevc_loop_filter_luma v, 10
hevc_loop_filter_luma v, 12