From 042c1159fcf8431725c718a504b1fb40364bf434 Mon Sep 17 00:00:00 2001 From: James Almer Date: Wed, 4 Feb 2015 20:21:56 -0300 Subject: [PATCH] x86/hevcdsp: add ff_hevc_sao_edge_filter_8_{ssse3,avx2} Original x86 intrinsics code and initial yasm port by Pierre-Edouard Lepere. Refactoring and optimizations by James Almer. Benchmarks of BQTerrace_1920x1080_60_qp22.bin with an Intel Core i5-4200U Width 32 158583 decicycles in edge, sao_edge_filter_8 runs, 0 skips 5205 decicycles in ff_hevc_sao_edge_filter_32_8_ssse3, 32767 runs, 1 skips 2942 decicycles in ff_hevc_sao_edge_filter_32_8_avx2, 32767 runs, 1 skips Width 64 705639 decicycles in sao_edge_filter_8, 262144 runs, 0 skips 19224 decicycles in ff_hevc_sao_edge_filter_64_8_ssse3, 262111 runs, 33 skips 10433 decicycles in ff_hevc_sao_edge_filter_64_8_avx2, 262115 runs, 29 skips Signed-off-by: James Almer --- libavcodec/hevc_filter.c | 8 +- libavcodec/hevcdsp.c | 6 +- libavcodec/hevcdsp.h | 4 +- libavcodec/x86/hevc_sao.asm | 171 +++++++++++++++++++++++++++++++++- libavcodec/x86/hevcdsp_init.c | 51 +++++++--- 5 files changed, 221 insertions(+), 19 deletions(-) diff --git a/libavcodec/hevc_filter.c b/libavcodec/hevc_filter.c index cea975da79..54dc708e2c 100644 --- a/libavcodec/hevc_filter.c +++ b/libavcodec/hevc_filter.c @@ -246,7 +246,7 @@ static void restore_tqb_pixels(HEVCContext *s, static void sao_filter_CTB(HEVCContext *s, int x, int y) { - static const uint8_t band_tab[8] = { 0, 1, 2, 2, 3, 3, 4, 4 }; + static const uint8_t sao_tab[8] = { 0, 1, 2, 2, 3, 3, 4, 4 }; HEVCLocalContext *lc = s->HEVClc; int c_idx; int edges[4]; // 0 left 1 top 2 right 3 bottom @@ -312,7 +312,7 @@ static void sao_filter_CTB(HEVCContext *s, int x, int y) int ctb_size_v = (1 << (s->sps->log2_ctb_size)) >> s->sps->vshift[c_idx]; int width = FFMIN(ctb_size_h, (s->sps->width >> s->sps->hshift[c_idx]) - x0); int height = FFMIN(ctb_size_v, (s->sps->height >> s->sps->vshift[c_idx]) - y0); - int tab = band_tab[(FFALIGN(width, 8) >> 3) - 1]; + int tab = sao_tab[(FFALIGN(width, 8) >> 3) - 1]; uint8_t *src = &s->frame->data[c_idx][y0 * stride_src + (x0 << s->sps->pixel_shift)]; int stride_dst; uint8_t *dst; @@ -427,8 +427,8 @@ static void sao_filter_CTB(HEVCContext *s, int x, int y) copy_CTB_to_hv(s, src, stride_src, x0, y0, width, height, c_idx, x_ctb, y_ctb); - s->hevcdsp.sao_edge_filter(src, dst, stride_src, sao->offset_val[c_idx], - sao->eo_class[c_idx], width, height); + s->hevcdsp.sao_edge_filter[tab](src, dst, stride_src, sao->offset_val[c_idx], + sao->eo_class[c_idx], width, height); s->hevcdsp.sao_edge_restore[restore](src, dst, stride_src, stride_dst, sao, diff --git a/libavcodec/hevcdsp.c b/libavcodec/hevcdsp.c index b9ae787527..7dcdfff77d 100644 --- a/libavcodec/hevcdsp.c +++ b/libavcodec/hevcdsp.c @@ -217,7 +217,11 @@ void ff_hevc_dsp_init(HEVCDSPContext *hevcdsp, int bit_depth) hevcdsp->sao_band_filter[2] = \ hevcdsp->sao_band_filter[3] = \ hevcdsp->sao_band_filter[4] = FUNC(sao_band_filter_0, depth); \ - hevcdsp->sao_edge_filter = FUNC(sao_edge_filter, depth); \ + hevcdsp->sao_edge_filter[0] = \ + hevcdsp->sao_edge_filter[1] = \ + hevcdsp->sao_edge_filter[2] = \ + hevcdsp->sao_edge_filter[3] = \ + hevcdsp->sao_edge_filter[4] = FUNC(sao_edge_filter, depth); \ hevcdsp->sao_edge_restore[0] = FUNC(sao_edge_restore_0, depth); \ hevcdsp->sao_edge_restore[1] = FUNC(sao_edge_restore_1, depth); \ \ diff --git a/libavcodec/hevcdsp.h b/libavcodec/hevcdsp.h index c0706fa353..bf7777f0d8 100644 --- a/libavcodec/hevcdsp.h +++ b/libavcodec/hevcdsp.h @@ -62,8 +62,8 @@ typedef struct HEVCDSPContext { int16_t *sao_offset_val, int sao_left_class, int width, int height); /* implicit stride_src parameter has value of 2 * MAX_PB_SIZE + FF_INPUT_BUFFER_PADDING_SIZE */ - void (*sao_edge_filter)(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, - int sao_eo_class, int width, int height); + void (*sao_edge_filter[5])(uint8_t *_dst /* align 16 */, uint8_t *_src /* align 32 */, ptrdiff_t stride_dst, + int16_t *sao_offset_val, int sao_eo_class, int width, int height); void (*sao_edge_restore[2])(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, struct SAOParams *sao, int *borders, int _width, int _height, int c_idx, diff --git a/libavcodec/x86/hevc_sao.asm b/libavcodec/x86/hevc_sao.asm index b389a9f68d..a306a84e81 100644 --- a/libavcodec/x86/hevc_sao.asm +++ b/libavcodec/x86/hevc_sao.asm @@ -23,14 +23,25 @@ %include "libavutil/x86/x86util.asm" -%if ARCH_X86_64 SECTION_RODATA 32 pw_mask10: times 16 dw 0x03FF pw_mask12: times 16 dw 0x0FFF +pb_2: times 32 db 2 +pb_edge_shuffle: times 2 db 1, 2, 0, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 +pb_eo: db -1, 0, 1, 0, 0, -1, 0, 1, -1, -1, 1, 1, 1, -1, -1, 1 +cextern pb_1 SECTION_TEXT +%define MAX_PB_SIZE 64 +%define PADDING_SIZE 32 ; FF_INPUT_BUFFER_PADDING_SIZE + +;****************************************************************************** +;SAO Band Filter +;****************************************************************************** + +%if ARCH_X86_64 %macro HEVC_SAO_BAND_FILTER_INIT 1 and leftq, 31 movd xm0, leftd @@ -239,3 +250,161 @@ HEVC_SAO_BAND_FILTER_16 12, 48, 1 HEVC_SAO_BAND_FILTER_16 12, 64, 2 %endif %endif + +;****************************************************************************** +;SAO Edge Filter +;****************************************************************************** + +%define EDGE_SRCSTRIDE 2 * MAX_PB_SIZE + PADDING_SIZE + +%macro HEVC_SAO_EDGE_FILTER_COMPUTE_8 1 + pminub m4, m1, m2 + pminub m5, m1, m3 + pcmpeqb m2, m4 + pcmpeqb m3, m5 + pcmpeqb m4, m1 + pcmpeqb m5, m1 + psubb m4, m2 + psubb m5, m3 + paddb m4, m6 + paddb m4, m5 + + pshufb m2, m0, m4 +%if %1 > 8 + punpckhbw m5, m7, m1 + punpckhbw m4, m2, m7 + punpcklbw m3, m7, m1 + punpcklbw m2, m7 + pmaddubsw m5, m4 + pmaddubsw m3, m2 + packuswb m3, m5 +%else + punpcklbw m3, m7, m1 + punpcklbw m2, m7 + pmaddubsw m3, m2 + packuswb m3, m3 +%endif +%endmacro + +;void ff_hevc_sao_edge_filter__8_(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, +; int eo, int width, int height); +%macro HEVC_SAO_EDGE_FILTER_8 2-3 +%if WIN64 +cglobal hevc_sao_edge_filter_%1_8, 4, 8, 8, dst, src, dststride, offset, a_stride, b_stride, height, tmp +%define eoq heightq + movsxd eoq, dword r4m + movsx a_strideq, byte [pb_eo+eoq*4+1] + movsx b_strideq, byte [pb_eo+eoq*4+3] + imul a_strideq, EDGE_SRCSTRIDE + imul b_strideq, EDGE_SRCSTRIDE + movsx tmpq, byte [pb_eo+eoq*4] + add a_strideq, tmpq + movsx tmpq, byte [pb_eo+eoq*4+2] + add b_strideq, tmpq + mov heightd, r6m + +%elif ARCH_X86_64 +cglobal hevc_sao_edge_filter_%1_8, 5, 9, 8, dst, src, dststride, offset, eo, a_stride, b_stride, height, tmp +%define tmp2q heightq + movsxd eoq, eod + lea tmp2q, [pb_eo] + movsx a_strideq, byte [tmp2q+eoq*4+1] + movsx b_strideq, byte [tmp2q+eoq*4+3] + imul a_strideq, EDGE_SRCSTRIDE + imul b_strideq, EDGE_SRCSTRIDE + movsx tmpq, byte [tmp2q+eoq*4] + add a_strideq, tmpq + movsx tmpq, byte [tmp2q+eoq*4+2] + add b_strideq, tmpq + mov heightd, r6m + +%else ; ARCH_X86_32 +cglobal hevc_sao_edge_filter_%1_8, 1, 6, 8, dst, src, dststride, a_stride, b_stride, height +%define eoq srcq +%define tmpq heightq +%define tmp2q dststrideq +%define offsetq heightq + mov eoq, r4m + lea tmp2q, [pb_eo] + movsx a_strideq, byte [tmp2q+eoq*4+1] + movsx b_strideq, byte [tmp2q+eoq*4+3] + imul a_strideq, EDGE_SRCSTRIDE + imul b_strideq, EDGE_SRCSTRIDE + movsx tmpq, byte [tmp2q+eoq*4] + add a_strideq, tmpq + movsx tmpq, byte [tmp2q+eoq*4+2] + add b_strideq, tmpq + + mov srcq, srcm + mov offsetq, r3m + mov dststrideq, dststridem +%endif ; ARCH + +%if mmsize > 16 + vbroadcasti128 m0, [offsetq] +%else + movu m0, [offsetq] +%endif + mova m1, [pb_edge_shuffle] + packsswb m0, m0 + mova m7, [pb_1] + pshufb m0, m1 + mova m6, [pb_2] +%if ARCH_X86_32 + mov heightd, r6m +%endif + +align 16 +.loop: + +%if %1 == 8 + movq m1, [srcq] + movq m2, [srcq + a_strideq] + movq m3, [srcq + b_strideq] + HEVC_SAO_EDGE_FILTER_COMPUTE_8 %1 + movq [dstq], m3 +%endif + +%assign i 0 +%rep %2 + mova m1, [srcq + i] + movu m2, [srcq + a_strideq + i] + movu m3, [srcq + b_strideq + i] + HEVC_SAO_EDGE_FILTER_COMPUTE_8 %1 + mov%3 [dstq + i], m3 +%assign i i+mmsize +%endrep + +%if %1 == 48 +INIT_XMM cpuname + + mova m1, [srcq + i] + movu m2, [srcq + a_strideq + i] + movu m3, [srcq + b_strideq + i] + HEVC_SAO_EDGE_FILTER_COMPUTE_8 %1 + mova [dstq + i], m3 +%if cpuflag(avx2) +INIT_YMM cpuname +%endif +%endif + + add dstq, dststrideq + add srcq, EDGE_SRCSTRIDE + dec heightd + jg .loop + RET +%endmacro + +INIT_XMM ssse3 +HEVC_SAO_EDGE_FILTER_8 8, 0 +HEVC_SAO_EDGE_FILTER_8 16, 1, a +HEVC_SAO_EDGE_FILTER_8 32, 2, a +HEVC_SAO_EDGE_FILTER_8 48, 2, a +HEVC_SAO_EDGE_FILTER_8 64, 4, a + +%if HAVE_AVX2_EXTERNAL +INIT_YMM avx2 +HEVC_SAO_EDGE_FILTER_8 32, 1, a +HEVC_SAO_EDGE_FILTER_8 48, 1, u +HEVC_SAO_EDGE_FILTER_8 64, 2, a +%endif diff --git a/libavcodec/x86/hevcdsp_init.c b/libavcodec/x86/hevcdsp_init.c index 5a01ff63cd..ddc30cf675 100644 --- a/libavcodec/x86/hevcdsp_init.c +++ b/libavcodec/x86/hevcdsp_init.c @@ -500,6 +500,37 @@ SAO_BAND_FILTER_FUNCS(8, avx2); SAO_BAND_FILTER_FUNCS(10, avx2); SAO_BAND_FILTER_FUNCS(12, avx2); +#define SAO_BAND_INIT(bitd, opt) do { \ + c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_##bitd##_##opt; \ + c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_##bitd##_##opt; \ + c->sao_band_filter[2] = ff_hevc_sao_band_filter_32_##bitd##_##opt; \ + c->sao_band_filter[3] = ff_hevc_sao_band_filter_48_##bitd##_##opt; \ + c->sao_band_filter[4] = ff_hevc_sao_band_filter_64_##bitd##_##opt; \ +} while (0) + +#define SAO_EDGE_FILTER_FUNCS(bitd, opt) \ +void ff_hevc_sao_edge_filter_8_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \ + int eo, int width, int height); \ +void ff_hevc_sao_edge_filter_16_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \ + int eo, int width, int height); \ +void ff_hevc_sao_edge_filter_32_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \ + int eo, int width, int height); \ +void ff_hevc_sao_edge_filter_48_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \ + int eo, int width, int height); \ +void ff_hevc_sao_edge_filter_64_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, int16_t *sao_offset_val, \ + int eo, int width, int height); \ + +SAO_EDGE_FILTER_FUNCS(8, ssse3); +SAO_EDGE_FILTER_FUNCS(8, avx2); + +#define SAO_EDGE_INIT(bitd, opt) do { \ + c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8_##bitd##_##opt; \ + c->sao_edge_filter[1] = ff_hevc_sao_edge_filter_16_##bitd##_##opt; \ + c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_##bitd##_##opt; \ + c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_##bitd##_##opt; \ + c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_##bitd##_##opt; \ +} while (0) + #define EPEL_LINKS(pointer, my, mx, fname, bitd, opt ) \ PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \ PEL_LINK(pointer, 2, my , mx , fname##6 , bitd, opt ); \ @@ -520,14 +551,6 @@ SAO_BAND_FILTER_FUNCS(12, avx2); PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \ PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt ) -#define SAO_BAND_INIT(bitd, opt) do { \ - c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_##bitd##_##opt; \ - c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_##bitd##_##opt; \ - c->sao_band_filter[2] = ff_hevc_sao_band_filter_32_##bitd##_##opt; \ - c->sao_band_filter[3] = ff_hevc_sao_band_filter_48_##bitd##_##opt; \ - c->sao_band_filter[4] = ff_hevc_sao_band_filter_64_##bitd##_##opt; \ -} while (0) - void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) { int cpu_flags = av_get_cpu_flags(); @@ -555,9 +578,12 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) c->transform_add[2] = ff_hevc_transform_add16_8_sse2; c->transform_add[3] = ff_hevc_transform_add32_8_sse2; } - if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) { - c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3; - c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3; + if (EXTERNAL_SSSE3(cpu_flags)) { + if(ARCH_X86_64) { + c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3; + c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3; + } + SAO_EDGE_INIT(8, ssse3); } if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) { @@ -590,6 +616,9 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) if (ARCH_X86_64) { SAO_BAND_INIT(8, avx2); } + c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_8_avx2; + c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_8_avx2; + c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_8_avx2; c->transform_add[3] = ff_hevc_transform_add32_8_avx2; }