Skip to content

qat #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 54 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
e23f96c
add qat zstd dependency
yyang52 Jul 24, 2023
0248f3d
add option and qat zstd
klmckeig Jul 24, 2023
05da54a
add zstdqatcompressor class
klmckeig Jul 24, 2023
6c0b6ac
rebase
klmckeig Oct 23, 2023
6236367
rebase
klmckeig Oct 23, 2023
f522a5e
fix typo
klmckeig Jul 24, 2023
a562692
rebase
klmckeig Oct 23, 2023
18645e2
add test
klmckeig Jul 26, 2023
a25eeb8
add qatzstd benchmark executable
klmckeig Jul 26, 2023
10ffc6c
fix qat zstd test build
klmckeig Jul 26, 2023
aa2eb95
add header
klmckeig Jul 27, 2023
504c936
rebase
klmckeig Oct 23, 2023
5177486
fix format
klmckeig Jul 27, 2023
f669fff
fix format
klmckeig Jul 27, 2023
a6a6e6d
rebase
klmckeig Oct 23, 2023
53abb03
rebase
klmckeig Oct 23, 2023
806824f
quick fix
yyang52 Jul 31, 2023
4bb9b48
add velox DwrfWriterBenchmark
yyang52 Aug 1, 2023
e4d86d1
rebase
klmckeig Oct 23, 2023
818425f
parquet writer bm
klmckeig Aug 1, 2023
fd1ad73
rebase
klmckeig Oct 23, 2023
44f6815
rebase
klmckeig Oct 23, 2023
06292a6
use suspender in dwrf and parquet writer benchmark
klmckeig Aug 2, 2023
60428d6
fuzzer attempt
klmckeig Aug 2, 2023
56638f6
formatfix
klmckeig Aug 2, 2023
0d42b2a
add flag in test
klmckeig Aug 3, 2023
a4b307b
simplify bm
klmckeig Aug 3, 2023
9dcc9b5
simplify bm
klmckeig Aug 3, 2023
d402ebd
update
yyang52 Aug 3, 2023
1511938
build command
klmckeig Aug 4, 2023
626c413
build command
klmckeig Aug 4, 2023
d80f4a5
build command
klmckeig Aug 4, 2023
c6c2b9f
build command
klmckeig Aug 4, 2023
f2c5624
fix qat build
yyang52 Aug 4, 2023
a22c6c5
fix build
yyang52 Aug 4, 2023
9a4b828
rebase
klmckeig Oct 23, 2023
0f2f572
rebase
klmckeig Oct 23, 2023
fe29768
rebase
klmckeig Oct 23, 2023
76151a0
rebase
klmckeig Oct 23, 2023
22b6cc1
Update CMake/resolve_dependency_modules/qatzstd.cmake
yyang52 Aug 8, 2023
01855df
Update CMake/resolve_dependency_modules/qatzstd.cmake
yyang52 Aug 8, 2023
29db7b0
Update CMakeLists.txt
yyang52 Aug 8, 2023
6420c86
update QAT url to apply some patch to fix known issue
yyang52 Aug 8, 2023
9195c81
update
klmckeig Aug 8, 2023
1c197ef
rebase
klmckeig Oct 23, 2023
9756077
rebase
klmckeig Oct 23, 2023
4199848
rebase
klmckeig Oct 23, 2023
43982be
rebase
klmckeig Oct 23, 2023
ceee5af
update qatplugin version
klmckeig Oct 23, 2023
c285d7b
update benchmark based on new compression class
klmckeig Oct 23, 2023
0e09f35
remove unused benchmark
klmckeig Oct 23, 2023
9638a22
formatfix
klmckeig Oct 23, 2023
94aa029
update
klmckeig Oct 24, 2023
a9c7814
update
klmckeig Oct 24, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rebase
  • Loading branch information
klmckeig committed Oct 23, 2023
commit 0f2f5727549f446b309aee8eef2846a28ef55115
33 changes: 1 addition & 32 deletions velox/dwio/common/compression/Compression.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@

DEFINE_bool(
VELOX_ENABLE_QAT_ZSTD_OT,
true,
false,
"if to use qat for zstd compression");
>>>>>>> c5224428f (flags):velox/dwio/dwrf/common/Compression.cpp

Expand Down Expand Up @@ -85,11 +85,8 @@ class ZstdQatCompressor : public Compressor {
zc = ZSTD_createCCtx();
QZSTD_startQatDevice();
sequenceProducerState = QZSTD_createSeqProdState();
<<<<<<< HEAD:velox/dwio/common/compression/Compression.cpp
<<<<<<< HEAD:velox/dwio/common/compression/Compression.cpp
ZSTD_registerSequenceProducer(
zc, sequenceProducerState, qatSequenceProducer);
<<<<<<< HEAD:velox/dwio/common/compression/Compression.cpp
ZSTD_CCtx_setParameter(
zc, ZSTD_c_enableSeqProducerFallback, FLAGS_ENABLE_QAT_ZSTD_FALLBACK);
ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, level);
Expand All @@ -100,21 +97,6 @@ class ZstdQatCompressor : public Compressor {
if (sequenceProducerState) {
QZSTD_freeSeqProdState(sequenceProducerState);
}
=======
ZSTD_registerSequenceProducer(zc, sequenceProducerState, qatSequenceProducer);
=======
ZSTD_registerSequenceProducer(
<<<<<<< HEAD:velox/dwio/common/compression/Compression.cpp
zc, sequenceProducerState, qatSequenceProducer);
>>>>>>> e1806c43f (fix benchmark build error):velox/dwio/dwrf/common/Compression.cpp
=======
zc, sequenceProducerState, qatSequenceProducer);
>>>>>>> 61b3e358d (gen test data):velox/dwio/dwrf/common/Compression.cpp
ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 1);
>>>>>>> 714f09ad8 (fix format):velox/dwio/dwrf/common/Compression.cpp
=======
ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, level);
>>>>>>> c5224428f (flags):velox/dwio/dwrf/common/Compression.cpp
}

ZSTD_CCtx* zc;
Expand All @@ -128,24 +110,11 @@ ZstdQatCompressor::compress(const void* src, void* dest, uint64_t length) {
XLOG_FIRST_N(INFO, 1) << fmt::format(
"qat fallback {}", FLAGS_ENABLE_QAT_ZSTD_FALLBACK);
size_t ret = ZSTD_compress2(zc, dest, length, src, length);
<<<<<<< HEAD:velox/dwio/common/compression/Compression.cpp
<<<<<<< HEAD:velox/dwio/common/compression/Compression.cpp
if ((int)ret <= 0) {
if (ZSTD_getErrorCode(ret) == ZSTD_ErrorCode::ZSTD_error_dstSize_tooSmall) {
return length;
}
=======
if ((int)ret <= 0) {
>>>>>>> 61b3e358d (gen test data):velox/dwio/dwrf/common/Compression.cpp
DWIO_RAISE("ZSTD QAT returned an error: ", ZSTD_getErrorName(ret));
=======
if ((int)ret <= 0) {
<<<<<<< HEAD:velox/dwio/common/compression/Compression.cpp
DWIO_RAISE("ZSTD QAT returned an error: ", ZSTD_getErrorName(ret));
>>>>>>> 714f09ad8 (fix format):velox/dwio/dwrf/common/Compression.cpp
=======
DWIO_RAISE("ZSTD QAT returned an error: ", ZSTD_getErrorName(ret));
>>>>>>> e1806c43f (fix benchmark build error):velox/dwio/dwrf/common/Compression.cpp
}
return ret;
}
Expand Down