From 1695fc39200d33736779a32393086c3c3d197f0f Mon Sep 17 00:00:00 2001 From: Dai Zhiwei Date: Mon, 24 Jan 2022 21:08:11 +0800 Subject: [PATCH] compressor/zlib: isa-l optimization for zlib algorithm on aarch64 Signed-off-by: Dai Zhiwei --- src/compressor/zlib/CMakeLists.txt | 134 +++++++++++++------- src/compressor/zlib/CompressionPluginZlib.h | 5 + src/compressor/zlib/ZlibCompressor.cc | 4 +- src/test/compressor/test_compression.cc | 6 +- 4 files changed, 97 insertions(+), 52 deletions(-) diff --git a/src/compressor/zlib/CMakeLists.txt b/src/compressor/zlib/CMakeLists.txt index ade57cb08ee..050ff03fa28 100644 --- a/src/compressor/zlib/CMakeLists.txt +++ b/src/compressor/zlib/CMakeLists.txt @@ -1,53 +1,93 @@ # zlib if(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE)) - set(CMAKE_ASM_FLAGS "-i ${PROJECT_SOURCE_DIR}/src/isa-l/igzip/ -i ${PROJECT_SOURCE_DIR}/src/isa-l/include/ ${CMAKE_ASM_FLAGS}") - set(zlib_sources - CompressionPluginZlib.cc - ZlibCompressor.cc - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c - ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c - ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c - ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c - ) - list(APPEND zlib_sources - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_body.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_finish.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_finish.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/rfc1951_lookup.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_sse.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_avx2_4.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_multibinary.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_01.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_04.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_01.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_04.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate_multibinary.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_04.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_06.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_deflate_hash.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_06.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_04.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_04.asm - ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_06.asm - ) -else(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE)) - set(zlib_sources - CompressionPluginZlib.cc - ZlibCompressor.cc - ) -endif(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE)) + set(CMAKE_ASM_FLAGS "-i ${PROJECT_SOURCE_DIR}/src/isa-l/igzip/ -i ${PROJECT_SOURCE_DIR}/src/isa-l/include/ ${CMAKE_ASM_FLAGS}") + set(zlib_sources + CompressionPluginZlib.cc + ZlibCompressor.cc + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c + ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c + ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c + ) + list(APPEND zlib_sources + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_body.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_finish.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_finish.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/rfc1951_lookup.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_sse.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_avx2_4.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_multibinary.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_01.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_04.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_01.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_04.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate_multibinary.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_04.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_06.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_deflate_hash.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_06.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_04.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_04.asm + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_06.asm + ) +elseif(HAVE_ARMV8_SIMD) + set(zlib_asm_sources + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_inflate_multibinary_arm64.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_multibinary_arm64.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_body_aarch64.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_finish_aarch64.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_deflate_icf_body_hash_hist.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_deflate_icf_finish_hash_hist.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_set_long_icf_fg.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_update_histogram.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_hash_aarch64.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_decode_huffman_code_block_aarch64.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_isal_adler32_neon.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/encode_df.S + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/gen_icf_map.S + ) + set(zlib_sources + CompressionPluginZlib.cc + ZlibCompressor.cc + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_multibinary_aarch64_dispatcher.c + ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c + ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c + ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c + ${zlib_asm_sources} + ) + set_source_files_properties(${zlib_asm_sources} PROPERTIES + COMPILE_DEFINITIONS "__ASSEMBLY__" + INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/src/isa-l/igzip;${PROJECT_SOURCE_DIR}/src/isa-l/igzip/aarch64" + ) +else() + set(zlib_sources + CompressionPluginZlib.cc + ZlibCompressor.cc + ) +endif() add_library(ceph_zlib SHARED ${zlib_sources}) target_link_libraries(ceph_zlib ZLIB::ZLIB compressor $<$:ceph-common>) diff --git a/src/compressor/zlib/CompressionPluginZlib.h b/src/compressor/zlib/CompressionPluginZlib.h index fbdb635c6a4..597bc02a5d9 100644 --- a/src/compressor/zlib/CompressionPluginZlib.h +++ b/src/compressor/zlib/CompressionPluginZlib.h @@ -42,6 +42,11 @@ public: ceph_arch_probe(); isal = (ceph_arch_intel_pclmul && ceph_arch_intel_sse41); } +#elif defined(__aarch64__) + if (cct->_conf->compressor_zlib_isal) { + ceph_arch_probe(); + isal = (ceph_arch_aarch64_pmull && ceph_arch_neon); + } #endif if (compressor == 0 || has_isal != isal) { compressor = std::make_shared(cct, isal); diff --git a/src/compressor/zlib/ZlibCompressor.cc b/src/compressor/zlib/ZlibCompressor.cc index 3bedbd8a1eb..a38eb2e7303 100644 --- a/src/compressor/zlib/ZlibCompressor.cc +++ b/src/compressor/zlib/ZlibCompressor.cc @@ -113,7 +113,7 @@ int ZlibCompressor::zlib_compress(const bufferlist &in, bufferlist &out, std::op return 0; } -#if __x86_64__ && defined(HAVE_NASM_X64_AVX2) +#if (__x86_64__ && defined(HAVE_NASM_X64_AVX2)) || defined(__aarch64__) int ZlibCompressor::isal_compress(const bufferlist &in, bufferlist &out, std::optional &compressor_message) { int ret; @@ -174,7 +174,7 @@ int ZlibCompressor::compress(const bufferlist &in, bufferlist &out, std::optiona if (qat_enabled) return qat_accel.compress(in, out, compressor_message); #endif -#if __x86_64__ && defined(HAVE_NASM_X64_AVX2) +#if (__x86_64__ && defined(HAVE_NASM_X64_AVX2)) || defined(__aarch64__) if (isal_enabled) return isal_compress(in, out, compressor_message); else diff --git a/src/test/compressor/test_compression.cc b/src/test/compressor/test_compression.cc index cee9142742b..6ae49daf544 100644 --- a/src/test/compressor/test_compression.cc +++ b/src/test/compressor/test_compression.cc @@ -378,7 +378,7 @@ INSTANTIATE_TEST_SUITE_P( #ifdef HAVE_LZ4 "lz4", #endif -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(__aarch64__) "zlib/isal", #endif "zlib/noisal", @@ -388,7 +388,7 @@ INSTANTIATE_TEST_SUITE_P( #endif "zstd")); -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(__aarch64__) TEST(ZlibCompressor, zlib_isal_compatibility) { @@ -453,7 +453,7 @@ TEST(CompressionPlugin, all) } } -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(__aarch64__) TEST(ZlibCompressor, isal_compress_zlib_decompress_random) {