diff --git a/.files b/.files index c29f2f0..4d84b8f 100644 Binary files a/.files and b/.files differ diff --git a/.rev b/.rev index 16ee0e6..a9350a3 100644 --- a/.rev +++ b/.rev @@ -2288,4 +2288,12 @@ See bugzilla 1167537 for the gory details. update to rpm-4.17.0 923520 + + 9e1623dd316765f3905f186b49c9c32f + 4.17.0 + + dimstar_suse + Switch to threaded zstd compression with a pool of 8 threads + 928134 + diff --git a/rpm.changes b/rpm.changes index 1ff8bb2..76b25cf 100644 --- a/rpm.changes +++ b/rpm.changes @@ -1,4 +1,12 @@ ------------------------------------------------------------------- +Thu Oct 14 18:16:22 CEST 2021 - mls@suse.de + +- Add support for using a thread pool for threaded zstd compression + new patch: zstdpool.diff +- Switch to threaded zstd compression with a pool of 8 threads + new patch: zstdthreaded.diff + +------------------------------------------------------------------- Thu Sep 23 20:57:45 CEST 2021 - mls@suse.de - update to rpm-4.17.0 diff --git a/rpm.spec b/rpm.spec index 05c4620..5003e08 100644 --- a/rpm.spec +++ b/rpm.spec @@ -115,6 +115,8 @@ Patch122: db_conversion.diff Patch123: nextiteratorheaderblob.diff Patch131: posttrans.diff Patch132: verbosearg.diff +Patch133: zstdpool.diff +Patch134: zstdthreaded.diff Patch200: finddebuginfo.diff Patch201: finddebuginfo-absolute-links.diff Patch202: debugsubpkg.diff @@ -242,7 +244,7 @@ rm -rf sqlite %patch -P 93 -P 94 -P 99 %patch -P 100 -P 102 -P 103 %patch -P 117 -%patch -P 122 -P 123 -P 131 -P 132 +%patch -P 122 -P 123 -P 131 -P 132 -P 133 -P 134 # debugedit patches pushd debugedit-5.0 diff --git a/zstdpool.diff b/zstdpool.diff new file mode 100644 index 0000000..b4d7ff5 --- /dev/null +++ b/zstdpool.diff @@ -0,0 +1,63 @@ +--- ./rpmio/rpmio.c.orig 2021-10-14 12:34:22.865316722 +0000 ++++ ./rpmio/rpmio.c 2021-10-14 13:37:56.835119919 +0000 +@@ -8,6 +8,7 @@ + #include + #include + #include ++#include + #if defined(__linux__) + #include + #endif +@@ -1034,6 +1035,7 @@ static const FDIO_t lzdio = &lzdio_s; + /* Support for ZSTD library. */ + #ifdef HAVE_ZSTD + ++#define ZSTD_STATIC_LINKING_ONLY + #include + + typedef struct rpmzstd_s { +@@ -1048,6 +1050,27 @@ typedef struct rpmzstd_s { + ZSTD_outBuffer zob; /*!< ZSTD_outBuffer */ + } * rpmzstd; + ++#if ZSTD_VERSION_NUMBER >= 10407 ++ ++static pthread_once_t zstdThreadPoolCreated = PTHREAD_ONCE_INIT; ++static ZSTD_threadPool *zstdThreadPool; ++ ++static void zstdCreateThreadPool(void) ++{ ++ int numthreads = rpmExpandNumeric("%{?_zstd_pool_threads}%{?!_zstd_pool_threads:-1}"); ++ if (numthreads >= 0) ++ numthreads = get_compression_threads(numthreads > 0 ? numthreads : -1); ++ if (numthreads > 0) { ++ zstdThreadPool = ZSTD_createThreadPool(numthreads); ++ if (!zstdThreadPool) ++ rpmlog(RPMLOG_WARNING, "Could not create zstd thread pool for %d threads\n", numthreads); ++ else ++ rpmlog(RPMLOG_DEBUG, "Created zstd thread pool for %d threads\n", numthreads); ++ } ++} ++ ++#endif ++ + static rpmzstd rpmzstdNew(int fdno, const char *fmode) + { + int flags = 0; +@@ -1133,8 +1156,15 @@ static rpmzstd rpmzstdNew(int fdno, cons + + threads = get_compression_threads(threads); + if (threads > 0) { +- if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads))) ++ if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads))) { + rpmlog(RPMLOG_DEBUG, "zstd library does not support multi-threading\n"); ++ } else { ++#if ZSTD_VERSION_NUMBER >= 10407 ++ pthread_once(&zstdThreadPoolCreated, zstdCreateThreadPool); ++ if (zstdThreadPool) ++ ZSTD_CCtx_refThreadPool(_stream, zstdThreadPool); ++#endif ++ } + } + + nb = ZSTD_CStreamOutSize(); diff --git a/zstdthreaded.diff b/zstdthreaded.diff new file mode 100644 index 0000000..0c942b2 --- /dev/null +++ b/zstdthreaded.diff @@ -0,0 +1,14 @@ +--- macros.in.orig 2021-10-15 07:40:15.601628187 +0000 ++++ macros.in 2021-10-15 07:42:23.557282503 +0000 +@@ -374,7 +374,10 @@ package or when debugging this package.\ + # "w.ufdio" uncompressed + # + #%_source_payload w9.gzdio +-%_binary_payload w19.zstdio ++%_binary_payload w19T0.zstdio ++ ++# use a pool with 8 threads for threaded zstd compression ++%_zstd_pool_threads 8 + + # Algorithm to use for generating file checksum digests on build. + # If not specified or 0, MD5 is used.