Merge pull request #44762 from Zhiwei-Dai/wip-zlib-compressor-isal-for-aarch64

compressor/zlib: isa-l optimization for zlib algorithm on aarch64 Reviewed-by: Kefu Chai <tchaikov@gmail.com>
2025-02-21 18:17:42 +00:00 · 2022-07-28 07:34:04 +08:00 · 2022-07-28 07:34:04 +08:00 · 9b76560557
commit 9b76560557
parent e439ada4c2 1695fc3920
4 changed files with 97 additions and 52 deletions
--- a/src/compressor/zlib/CMakeLists.txt
+++ b/src/compressor/zlib/CMakeLists.txt
@ -1,53 +1,93 @@
 # zlib

 if(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE))
-    set(CMAKE_ASM_FLAGS "-i ${PROJECT_SOURCE_DIR}/src/isa-l/igzip/ -i ${PROJECT_SOURCE_DIR}/src/isa-l/include/ ${CMAKE_ASM_FLAGS}")
-	set(zlib_sources
-	  CompressionPluginZlib.cc
-	  ZlibCompressor.cc
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c
-	)
-	list(APPEND zlib_sources
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_body.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_finish.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_finish.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/rfc1951_lookup.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_sse.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_avx2_4.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_multibinary.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_01.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_04.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_01.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_04.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate_multibinary.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_04.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_06.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_deflate_hash.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_06.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_04.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_04.asm
-	  ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_06.asm
-	)
-else(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE))
-	set(zlib_sources
-	  CompressionPluginZlib.cc
-	  ZlibCompressor.cc
-	)
-endif(HAVE_INTEL_SSE4_1 AND HAVE_NASM_X64_AVX2 AND (NOT APPLE))
+  set(CMAKE_ASM_FLAGS "-i ${PROJECT_SOURCE_DIR}/src/isa-l/igzip/ -i ${PROJECT_SOURCE_DIR}/src/isa-l/include/ ${CMAKE_ASM_FLAGS}")
+  set(zlib_sources
+    CompressionPluginZlib.cc
+    ZlibCompressor.cc
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c
+  )
+  list(APPEND zlib_sources
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_body.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_finish.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_finish.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/rfc1951_lookup.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_sse.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_avx2_4.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_multibinary.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_01.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_update_histogram_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_01.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_decode_block_stateless_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate_multibinary.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df_06.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_deflate_hash.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_06.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_gen_icf_map_lh1_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_04.asm
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_set_long_icf_fg_06.asm
+  )
+elseif(HAVE_ARMV8_SIMD)
+  set(zlib_asm_sources
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_inflate_multibinary_arm64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_multibinary_arm64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_body_aarch64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_finish_aarch64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_deflate_icf_body_hash_hist.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_deflate_icf_finish_hash_hist.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_set_long_icf_fg.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/isal_update_histogram.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_deflate_hash_aarch64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_decode_huffman_code_block_aarch64.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_isal_adler32_neon.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/encode_df.S
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/gen_icf_map.S
+  )
+  set(zlib_sources
+    CompressionPluginZlib.cc
+    ZlibCompressor.cc
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/hufftables_c.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/adler32_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/flatten_ll.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/encode_df.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_icf_body.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/igzip_inflate.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/huff_codes.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/proc_heap_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/igzip/aarch64/igzip_multibinary_aarch64_dispatcher.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base_aliases.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc_base.c
+    ${CMAKE_SOURCE_DIR}/src/isa-l/crc/crc64_base.c
+    ${zlib_asm_sources}
+  )
+  set_source_files_properties(${zlib_asm_sources} PROPERTIES
+    COMPILE_DEFINITIONS "__ASSEMBLY__"
+    INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/src/isa-l/igzip;${PROJECT_SOURCE_DIR}/src/isa-l/igzip/aarch64"
+  )
+else()
+  set(zlib_sources
+    CompressionPluginZlib.cc
+    ZlibCompressor.cc
+  )
+endif()

 add_library(ceph_zlib SHARED ${zlib_sources})
 target_link_libraries(ceph_zlib ZLIB::ZLIB compressor $<$<PLATFORM_ID:Windows>:ceph-common>)
--- a/src/compressor/zlib/CompressionPluginZlib.h
+++ b/src/compressor/zlib/CompressionPluginZlib.h
@ -42,6 +42,11 @@ public:
      ceph_arch_probe();
      isal = (ceph_arch_intel_pclmul && ceph_arch_intel_sse41);
    }
+#elif defined(__aarch64__)
+    if (cct->_conf->compressor_zlib_isal) {
+      ceph_arch_probe();
+      isal = (ceph_arch_aarch64_pmull && ceph_arch_neon);
+    }
 #endif
    if (compressor == 0 || has_isal != isal) {
      compressor = std::make_shared<ZlibCompressor>(cct, isal);
--- a/src/compressor/zlib/ZlibCompressor.cc
+++ b/src/compressor/zlib/ZlibCompressor.cc
@ -113,7 +113,7 @@ int ZlibCompressor::zlib_compress(const bufferlist &in, bufferlist &out, std::op
  return 0;
 }

-#if __x86_64__ && defined(HAVE_NASM_X64_AVX2)
+#if (__x86_64__ && defined(HAVE_NASM_X64_AVX2)) || defined(__aarch64__)
 int ZlibCompressor::isal_compress(const bufferlist &in, bufferlist &out, std::optional<int32_t> &compressor_message)
 {
  int ret;
@ -174,7 +174,7 @@ int ZlibCompressor::compress(const bufferlist &in, bufferlist &out, std::optiona
  if (qat_enabled)
    return qat_accel.compress(in, out, compressor_message);
 #endif
-#if __x86_64__ && defined(HAVE_NASM_X64_AVX2)
+#if (__x86_64__ && defined(HAVE_NASM_X64_AVX2)) || defined(__aarch64__)
  if (isal_enabled)
    return isal_compress(in, out, compressor_message);
  else
--- a/src/test/compressor/test_compression.cc
+++ b/src/test/compressor/test_compression.cc
@ -378,7 +378,7 @@ INSTANTIATE_TEST_SUITE_P(
 #ifdef HAVE_LZ4
    "lz4",
 #endif
-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(__aarch64__)
    "zlib/isal",
 #endif
    "zlib/noisal",
@ -388,7 +388,7 @@ INSTANTIATE_TEST_SUITE_P(
 #endif
    "zstd"));

-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(__aarch64__)

 TEST(ZlibCompressor, zlib_isal_compatibility)
 {
@ -453,7 +453,7 @@ TEST(CompressionPlugin, all)
  }
 }

-#ifdef __x86_64__
+#if defined(__x86_64__) || defined(__aarch64__)

 TEST(ZlibCompressor, isal_compress_zlib_decompress_random)
 {