From 6d384298ece19af357b7fd3f683afff0d3f40c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Fri, 1 Mar 2024 14:04:02 +0200 Subject: [PATCH] aarch64: hevc: Implement a neon version of put_hevc_epel_h*_8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AWS Graviton 3: put_hevc_epel_h4_8_c: 64.7 put_hevc_epel_h4_8_neon: 25.0 put_hevc_epel_h4_8_i8mm: 21.2 put_hevc_epel_h6_8_c: 130.0 put_hevc_epel_h6_8_neon: 40.7 put_hevc_epel_h6_8_i8mm: 36.5 put_hevc_epel_h8_8_c: 209.0 put_hevc_epel_h8_8_neon: 45.2 put_hevc_epel_h8_8_i8mm: 41.2 put_hevc_epel_h12_8_c: 465.5 put_hevc_epel_h12_8_neon: 104.5 put_hevc_epel_h12_8_i8mm: 86.5 put_hevc_epel_h16_8_c: 830.7 put_hevc_epel_h16_8_neon: 134.2 put_hevc_epel_h16_8_i8mm: 114.0 put_hevc_epel_h24_8_c: 1844.7 put_hevc_epel_h24_8_neon: 282.2 put_hevc_epel_h24_8_i8mm: 277.2 put_hevc_epel_h32_8_c: 3227.5 put_hevc_epel_h32_8_neon: 501.5 put_hevc_epel_h32_8_i8mm: 396.0 put_hevc_epel_h48_8_c: 7229.2 put_hevc_epel_h48_8_neon: 1120.2 put_hevc_epel_h48_8_i8mm: 901.2 put_hevc_epel_h64_8_c: 12869.0 put_hevc_epel_h64_8_neon: 1999.2 put_hevc_epel_h64_8_i8mm: 1610.5 Signed-off-by: Martin Storsjö --- libavcodec/aarch64/hevcdsp_epel_neon.S | 194 +++++++++++++++++++++- libavcodec/aarch64/hevcdsp_init_aarch64.c | 17 ++ 2 files changed, 209 insertions(+), 2 deletions(-) diff --git a/libavcodec/aarch64/hevcdsp_epel_neon.S b/libavcodec/aarch64/hevcdsp_epel_neon.S index d3f0a26f79..419e83529a 100644 --- a/libavcodec/aarch64/hevcdsp_epel_neon.S +++ b/libavcodec/aarch64/hevcdsp_epel_neon.S @@ -1321,8 +1321,6 @@ function ff_hevc_put_hevc_epel_uni_v64_8_neon, export=1 ret endfunc -#if HAVE_I8MM -ENABLE_I8MM .macro EPEL_H_HEADER movrel x5, epel_filters @@ -1332,6 +1330,198 @@ ENABLE_I8MM mov x10, #(MAX_PB_SIZE * 2) .endm +function ff_hevc_put_hevc_epel_h4_8_neon, export=1 + EPEL_H_HEADER + sxtl v0.8h, v30.8b +1: ld1 {v4.8b}, [x1], x2 + subs w3, w3, #1 // height + uxtl v4.8h, v4.8b + ext v5.16b, v4.16b, v4.16b, #2 + ext v6.16b, v4.16b, v4.16b, #4 + ext v7.16b, v4.16b, v4.16b, #6 + mul v16.4h, v4.4h, v0.h[0] + mla v16.4h, v5.4h, v0.h[1] + mla v16.4h, v6.4h, v0.h[2] + mla v16.4h, v7.4h, v0.h[3] + st1 {v16.4h}, [x0], x10 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_epel_h6_8_neon, export=1 + EPEL_H_HEADER + sxtl v0.8h, v30.8b + add x6, x0, #8 +1: ld1 {v3.16b}, [x1], x2 + subs w3, w3, #1 // height + uxtl2 v4.8h, v3.16b + uxtl v3.8h, v3.8b + ext v5.16b, v3.16b, v4.16b, #2 + ext v6.16b, v3.16b, v4.16b, #4 + ext v7.16b, v3.16b, v4.16b, #6 + mul v16.8h, v3.8h, v0.h[0] + mla v16.8h, v5.8h, v0.h[1] + mla v16.8h, v6.8h, v0.h[2] + mla v16.8h, v7.8h, v0.h[3] + st1 {v16.4h}, [x0], x10 + st1 {v16.s}[2], [x6], x10 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_epel_h8_8_neon, export=1 + EPEL_H_HEADER + sxtl v0.8h, v30.8b +1: ld1 {v3.16b}, [x1], x2 + subs w3, w3, #1 // height + uxtl2 v4.8h, v3.16b + uxtl v3.8h, v3.8b + ext v5.16b, v3.16b, v4.16b, #2 + ext v6.16b, v3.16b, v4.16b, #4 + ext v7.16b, v3.16b, v4.16b, #6 + mul v16.8h, v3.8h, v0.h[0] + mla v16.8h, v5.8h, v0.h[1] + mla v16.8h, v6.8h, v0.h[2] + mla v16.8h, v7.8h, v0.h[3] + st1 {v16.8h}, [x0], x10 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_epel_h12_8_neon, export=1 + EPEL_H_HEADER + add x6, x0, #16 + sxtl v0.8h, v30.8b +1: ld1 {v3.16b}, [x1], x2 + subs w3, w3, #1 // height + uxtl2 v4.8h, v3.16b + uxtl v3.8h, v3.8b + ext v5.16b, v3.16b, v4.16b, #2 + ext v6.16b, v3.16b, v4.16b, #4 + ext v7.16b, v3.16b, v4.16b, #6 + ext v20.16b, v4.16b, v4.16b, #2 + ext v21.16b, v4.16b, v4.16b, #4 + ext v22.16b, v4.16b, v4.16b, #6 + mul v16.8h, v3.8h, v0.h[0] + mla v16.8h, v5.8h, v0.h[1] + mla v16.8h, v6.8h, v0.h[2] + mla v16.8h, v7.8h, v0.h[3] + mul v17.4h, v4.4h, v0.h[0] + mla v17.4h, v20.4h, v0.h[1] + mla v17.4h, v21.4h, v0.h[2] + mla v17.4h, v22.4h, v0.h[3] + st1 {v16.8h}, [x0], x10 + st1 {v17.4h}, [x6], x10 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_epel_h16_8_neon, export=1 + EPEL_H_HEADER + sxtl v0.8h, v30.8b +1: ld1 {v1.8b, v2.8b, v3.8b}, [x1], x2 + subs w3, w3, #1 // height + uxtl v1.8h, v1.8b + uxtl v2.8h, v2.8b + uxtl v3.8h, v3.8b + ext v5.16b, v1.16b, v2.16b, #2 + ext v6.16b, v1.16b, v2.16b, #4 + ext v7.16b, v1.16b, v2.16b, #6 + ext v20.16b, v2.16b, v3.16b, #2 + ext v21.16b, v2.16b, v3.16b, #4 + ext v22.16b, v2.16b, v3.16b, #6 + mul v16.8h, v1.8h, v0.h[0] + mla v16.8h, v5.8h, v0.h[1] + mla v16.8h, v6.8h, v0.h[2] + mla v16.8h, v7.8h, v0.h[3] + mul v17.8h, v2.8h, v0.h[0] + mla v17.8h, v20.8h, v0.h[1] + mla v17.8h, v21.8h, v0.h[2] + mla v17.8h, v22.8h, v0.h[3] + st1 {v16.8h, v17.8h}, [x0], x10 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_epel_h24_8_neon, export=1 + EPEL_H_HEADER + sxtl v0.8h, v30.8b +1: ld1 {v1.8b, v2.8b, v3.8b, v4.8b}, [x1], x2 + subs w3, w3, #1 // height + uxtl v1.8h, v1.8b + uxtl v2.8h, v2.8b + uxtl v3.8h, v3.8b + uxtl v4.8h, v4.8b + ext v5.16b, v1.16b, v2.16b, #2 + ext v6.16b, v1.16b, v2.16b, #4 + ext v7.16b, v1.16b, v2.16b, #6 + ext v20.16b, v2.16b, v3.16b, #2 + ext v21.16b, v2.16b, v3.16b, #4 + ext v22.16b, v2.16b, v3.16b, #6 + ext v23.16b, v3.16b, v4.16b, #2 + ext v24.16b, v3.16b, v4.16b, #4 + ext v25.16b, v3.16b, v4.16b, #6 + mul v16.8h, v1.8h, v0.h[0] + mla v16.8h, v5.8h, v0.h[1] + mla v16.8h, v6.8h, v0.h[2] + mla v16.8h, v7.8h, v0.h[3] + mul v17.8h, v2.8h, v0.h[0] + mla v17.8h, v20.8h, v0.h[1] + mla v17.8h, v21.8h, v0.h[2] + mla v17.8h, v22.8h, v0.h[3] + mul v18.8h, v3.8h, v0.h[0] + mla v18.8h, v23.8h, v0.h[1] + mla v18.8h, v24.8h, v0.h[2] + mla v18.8h, v25.8h, v0.h[3] + st1 {v16.8h, v17.8h, v18.8h}, [x0], x10 + b.ne 1b + ret +endfunc + +function ff_hevc_put_hevc_epel_h32_8_neon, export=1 + EPEL_H_HEADER + ld1 {v1.8b}, [x1], #8 + sub x2, x2, w6, uxtw // decrement src stride + mov w7, w6 // original width + sub x2, x2, #8 // decrement src stride + sub x10, x10, w6, uxtw #1 // decrement dst stride + sxtl v0.8h, v30.8b + uxtl v1.8h, v1.8b +1: ld1 {v2.8b, v3.8b}, [x1], #16 + subs w6, w6, #16 // width + uxtl v2.8h, v2.8b + uxtl v3.8h, v3.8b + ext v5.16b, v1.16b, v2.16b, #2 + ext v6.16b, v1.16b, v2.16b, #4 + ext v7.16b, v1.16b, v2.16b, #6 + ext v20.16b, v2.16b, v3.16b, #2 + ext v21.16b, v2.16b, v3.16b, #4 + ext v22.16b, v2.16b, v3.16b, #6 + mul v16.8h, v1.8h, v0.h[0] + mla v16.8h, v5.8h, v0.h[1] + mla v16.8h, v6.8h, v0.h[2] + mla v16.8h, v7.8h, v0.h[3] + mul v17.8h, v2.8h, v0.h[0] + mla v17.8h, v20.8h, v0.h[1] + mla v17.8h, v21.8h, v0.h[2] + mla v17.8h, v22.8h, v0.h[3] + st1 {v16.8h, v17.8h}, [x0], #32 + mov v1.16b, v3.16b + b.gt 1b + subs w3, w3, #1 // height + add x1, x1, x2 + b.le 9f + ld1 {v1.8b}, [x1], #8 + mov w6, w7 + add x0, x0, x10 + uxtl v1.8h, v1.8b + b 1b +9: + ret +endfunc + +#if HAVE_I8MM +ENABLE_I8MM function ff_hevc_put_hevc_epel_h4_8_neon_i8mm, export=1 EPEL_H_HEADER 1: ld1 {v4.8b}, [x1], x2 diff --git a/libavcodec/aarch64/hevcdsp_init_aarch64.c b/libavcodec/aarch64/hevcdsp_init_aarch64.c index 1e9f5e32db..ece911b8d4 100644 --- a/libavcodec/aarch64/hevcdsp_init_aarch64.c +++ b/libavcodec/aarch64/hevcdsp_init_aarch64.c @@ -223,6 +223,10 @@ NEON8_FNPROTO_PARTIAL_4(qpel_uni_w_v, (uint8_t *_dst, ptrdiff_t _dststride, int height, int denom, int wx, int ox, intptr_t mx, intptr_t my, int width),); +NEON8_FNPROTO(epel_h, (int16_t *dst, + const uint8_t *_src, ptrdiff_t _srcstride, + int height, intptr_t mx, intptr_t my, int width),); + NEON8_FNPROTO(epel_h, (int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width), _i8mm); @@ -290,6 +294,17 @@ NEON8_FNPROTO(qpel_bi_hv, (uint8_t *dst, ptrdiff_t dststride, member[8][v][h] = ff_hevc_put_hevc_##fn##48_8_neon##ext; \ member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; +#define NEON8_FNASSIGN_SHARED_32(member, v, h, fn, ext) \ + member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \ + member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon##ext; \ + member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \ + member[4][v][h] = ff_hevc_put_hevc_##fn##12_8_neon##ext; \ + member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \ + member[6][v][h] = ff_hevc_put_hevc_##fn##24_8_neon##ext; \ + member[7][v][h] = \ + member[8][v][h] = \ + member[9][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext; + #define NEON8_FNASSIGN_PARTIAL_4(member, v, h, fn, ext) \ member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \ member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \ @@ -384,6 +399,8 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 0, epel_uni_w_v,); NEON8_FNASSIGN_PARTIAL_4(c->put_hevc_qpel_uni_w, 1, 0, qpel_uni_w_v,); + NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel, 0, 1, epel_h,); + if (have_i8mm(cpu_flags)) { NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm); NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv, _i8mm);