CMakeLists.txt                         |  2 ++
 cmake/modules/Finduring.cmake          |  2 +-
 cmake/modules/Findzstd.cmake           | 25 +++++++++++++++++
 src/common/options.cc                  |  2 +-
 src/compressor/zstd/CMakeLists.txt     | 25 +----------------
 src/compressor/zstd/ZstdCompressor.h   |  2 +-
 src/include/buffer.h                   |  4 ++-
 src/include/rados/librados.h           |  6 ----
 src/init-ceph.in                       |  1 -
 src/kv/rocksdb_cache/BinnedLRUCache.cc | 28 ++++++++++++++++---
 src/kv/rocksdb_cache/BinnedLRUCache.h  | 18 ++++++++----
 src/kv/rocksdb_cache/ShardedCache.cc   | 29 ++++++++++++++++++--
 src/kv/rocksdb_cache/ShardedCache.h    | 50 ++++++++++++++++++++++------------
 13 files changed, 130 insertions(+), 64 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78cae841bb6..ab25ce8743e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -334,6 +334,8 @@ if(WITH_LZ4)
   set(HAVE_LZ4 ${LZ4_FOUND})
 endif(WITH_LZ4)
 
+find_package(zstd REQUIRED)
+
 option(WITH_CEPH_DEBUG_MUTEX "Use debug ceph::mutex with lockdep" OFF)
 
 #if allocator is set on command line make sure it matches below strings
diff --git a/cmake/modules/Finduring.cmake b/cmake/modules/Finduring.cmake
index 10c8de42550..8e11c273a2d 100644
--- a/cmake/modules/Finduring.cmake
+++ b/cmake/modules/Finduring.cmake
@@ -5,7 +5,7 @@
 # uring_FOUND - True if uring found.
 
 find_path(URING_INCLUDE_DIR liburing.h)
-find_library(URING_LIBRARIES liburing.a liburing)
+find_library(URING_LIBRARIES liburing.so liburing)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(uring DEFAULT_MSG URING_LIBRARIES URING_INCLUDE_DIR)
diff --git a/cmake/modules/Findzstd.cmake b/cmake/modules/Findzstd.cmake
new file mode 100644
index 00000000000..696bb7603e2
--- /dev/null
+++ b/cmake/modules/Findzstd.cmake
@@ -0,0 +1,25 @@
+# Try to find liblz4
+#
+# Once done, this will define
+#
+# ZSTD_FOUND
+# ZSTD_INCLUDE_DIR
+# ZSTD_LIBRARY
+
+find_path(ZSTD_INCLUDE_DIR zstd.h)
+
+find_library(ZSTD_LIBRARY zstd)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(zstd
+  DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY)
+
+if(ZSTD_FOUND AND NOT (TARGET zstd::zstd))
+  add_library(zstd::zstd UNKNOWN IMPORTED)
+  set_target_properties(zstd::zstd PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${ZSTD_LIBRARY}")
+endif()
diff --git a/src/common/options.cc b/src/common/options.cc
index b0239e24aa4..bff4154decd 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5543,7 +5543,7 @@ std::vector<Option> get_global_options() {
     .add_see_also("mgr_module_path"),
 
     Option("mgr_initial_modules", Option::TYPE_STR, Option::LEVEL_BASIC)
-    .set_default("restful iostat nfs")
+    .set_default("iostat")
     .set_flag(Option::FLAG_NO_MON_UPDATE)
     .set_flag(Option::FLAG_CLUSTER_CREATE)
     .add_service("mon")
diff --git a/src/compressor/zstd/CMakeLists.txt b/src/compressor/zstd/CMakeLists.txt
index b53b0d94448..2b60137d7e2 100644
--- a/src/compressor/zstd/CMakeLists.txt
+++ b/src/compressor/zstd/CMakeLists.txt
@@ -1,34 +1,11 @@
 # zstd
 
-# libzstd - build it statically
-set(ZSTD_C_FLAGS "-fPIC -Wno-unused-variable -O3")
-
-include(ExternalProject)
-ExternalProject_Add(zstd_ext
-  SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/zstd/build/cmake
-  CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-             -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-             -DCMAKE_C_FLAGS=${ZSTD_C_FLAGS}
-             -DCMAKE_AR=${CMAKE_AR}
-             -DCMAKE_POSITION_INDEPENDENT_CODE=${ENABLE_SHARED}
-             -G${CMAKE_GENERATOR}
-  BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/libzstd
-  BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target libzstd_static
-  BUILD_BYPRODUCTS "${CMAKE_CURRENT_BINARY_DIR}/libzstd/lib/libzstd.a"
-  INSTALL_COMMAND "true")
-
-add_library(zstd STATIC IMPORTED)
-set_target_properties(zstd PROPERTIES
-  INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/src/zstd/lib"
-  IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/libzstd/lib/libzstd.a")
-add_dependencies(zstd zstd_ext)
-
 set(zstd_sources
   CompressionPluginZstd.cc
 )
 
 add_library(ceph_zstd SHARED ${zstd_sources})
-target_link_libraries(ceph_zstd PRIVATE zstd $<$<PLATFORM_ID:Windows>:ceph-common>)
+target_link_libraries(ceph_zstd PRIVATE zstd::zstd)
 set_target_properties(ceph_zstd PROPERTIES
   VERSION 2.0.0
   SOVERSION 2
diff --git a/src/compressor/zstd/ZstdCompressor.h b/src/compressor/zstd/ZstdCompressor.h
index 95b492deb87..f4b66a14dcc 100644
--- a/src/compressor/zstd/ZstdCompressor.h
+++ b/src/compressor/zstd/ZstdCompressor.h
@@ -16,7 +16,7 @@
 #define CEPH_ZSTDCOMPRESSOR_H
 
 #define ZSTD_STATIC_LINKING_ONLY
-#include "zstd/lib/zstd.h"
+#include <zstd.h>
 
 #include "include/buffer.h"
 #include "include/encoding.h"
diff --git a/src/include/buffer.h b/src/include/buffer.h
index 71cb0193500..118af14accf 100644
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -864,7 +864,9 @@ struct error_code;
 	if (first_round) {
 	  impl_f(first_round);
 	}
-	if (const auto second_round = len - first_round; second_round) {
+	// no C++17 for the sake of the C++11 guarantees of librados, sorry.
+	const auto second_round = len - first_round;
+	if (second_round) {
 	  _refill(second_round);
 	  impl_f(second_round);
 	}
diff --git a/src/include/rados/librados.h b/src/include/rados/librados.h
index d9e39eceb1f..a27fb1002b5 100644
--- a/src/include/rados/librados.h
+++ b/src/include/rados/librados.h
@@ -3995,12 +3995,6 @@ CEPH_RADOS_API int rados_pg_command(rados_t cluster, const char *pgstr,
 		                    char **outbuf, size_t *outbuflen,
 		                    char **outs, size_t *outslen);
 
-CEPH_RADOS_API int rados_mgr_command(rados_t cluster,
-                                     const char **cmd, size_t cmdlen,
-		                     const char *inbuf, size_t inbuflen,
-		                     char **outbuf, size_t *outbuflen,
-		                     char **outs, size_t *outslen);
-
 /*
  * This is not a doxygen comment leadin, because doxygen breaks on
  * a typedef with function params and returns, and I can't figure out
diff --git a/src/init-ceph.in b/src/init-ceph.in
index 76f0d59c1d8..5ca53954f54 100755
--- a/src/init-ceph.in
+++ b/src/init-ceph.in
@@ -174,7 +174,6 @@ use_default_conf=1
 
 ## set variables like cluster or conf
 [ -e /etc/sysconfig/ceph ] && . /etc/sysconfig/ceph
-[ -e /etc/default/ceph ] && . /etc/default/ceph
 
 
 while echo $1 | grep -q '^-'; do     # FIXME: why not '^-'?
diff --git a/src/kv/rocksdb_cache/BinnedLRUCache.cc b/src/kv/rocksdb_cache/BinnedLRUCache.cc
index 0d657883e92..47c56e2ddd7 100644
--- a/src/kv/rocksdb_cache/BinnedLRUCache.cc
+++ b/src/kv/rocksdb_cache/BinnedLRUCache.cc
@@ -151,13 +151,20 @@ void BinnedLRUCacheShard::EraseUnRefEntries() {
   }
 }
 
-void BinnedLRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                           bool thread_safe) {
+void BinnedLRUCacheShard::ApplyToAllCacheEntries(
+  const std::function<void(const rocksdb::Slice& key,
+                           void* value,
+                           size_t charge,
+                           DeleterFn)>& callback,
+  bool thread_safe)
+{
   if (thread_safe) {
     mutex_.lock();
   }
   table_.ApplyToAllCacheEntries(
-      [callback](BinnedLRUHandle* h) { callback(h->value, h->charge); });
+    [callback](BinnedLRUHandle* h) {
+      callback(h->key(), h->value, h->charge, h->deleter);
+    });
   if (thread_safe) {
     mutex_.unlock();
   }
@@ -345,7 +352,7 @@ bool BinnedLRUCacheShard::Release(rocksdb::Cache::Handle* handle, bool force_era
 
 rocksdb::Status BinnedLRUCacheShard::Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
                              size_t charge,
-                             void (*deleter)(const rocksdb::Slice& key, void* value),
+                             DeleterFn deleter,
                              rocksdb::Cache::Handle** handle, rocksdb::Cache::Priority priority) {
   auto e = new BinnedLRUHandle();
   rocksdb::Status s;
@@ -464,6 +471,12 @@ std::string BinnedLRUCacheShard::GetPrintableOptions() const {
   return std::string(buffer);
 }
 
+DeleterFn BinnedLRUCacheShard::GetDeleter(rocksdb::Cache::Handle* h) const
+{
+  auto* handle = reinterpret_cast<BinnedLRUHandle*>(h);
+  return handle->deleter;
+}
+
 BinnedLRUCache::BinnedLRUCache(CephContext *c, 
                                size_t capacity, 
                                int num_shard_bits,
@@ -519,6 +532,13 @@ void BinnedLRUCache::DisownData() {
 #endif  // !__SANITIZE_ADDRESS__
 }
 
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+DeleterFn BinnedLRUCache::GetDeleter(Handle* handle) const
+{
+  return reinterpret_cast<const BinnedLRUHandle*>(handle)->deleter;
+}
+#endif
+
 size_t BinnedLRUCache::TEST_GetLRUSize() {
   size_t lru_size_of_all_shards = 0;
   for (int i = 0; i < num_shards_; i++) {
diff --git a/src/kv/rocksdb_cache/BinnedLRUCache.h b/src/kv/rocksdb_cache/BinnedLRUCache.h
index 85608be0e57..88bf4502e89 100644
--- a/src/kv/rocksdb_cache/BinnedLRUCache.h
+++ b/src/kv/rocksdb_cache/BinnedLRUCache.h
@@ -56,7 +56,7 @@ std::shared_ptr<rocksdb::Cache> NewBinnedLRUCache(
 
 struct BinnedLRUHandle {
   void* value;
-  void (*deleter)(const rocksdb::Slice&, void* value);
+  DeleterFn deleter;
   BinnedLRUHandle* next_hash;
   BinnedLRUHandle* next;
   BinnedLRUHandle* prev;
@@ -189,7 +189,7 @@ class alignas(CACHE_LINE_SIZE) BinnedLRUCacheShard : public CacheShard {
   // Like Cache methods, but with an extra "hash" parameter.
   virtual rocksdb::Status Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
                         size_t charge,
-                        void (*deleter)(const rocksdb::Slice& key, void* value),
+                        DeleterFn deleter,
                         rocksdb::Cache::Handle** handle,
                         rocksdb::Cache::Priority priority) override;
   virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, uint32_t hash) override;
@@ -205,13 +205,19 @@ class alignas(CACHE_LINE_SIZE) BinnedLRUCacheShard : public CacheShard {
   virtual size_t GetUsage() const override;
   virtual size_t GetPinnedUsage() const override;
 
-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) override;
+  virtual void ApplyToAllCacheEntries(
+    const std::function<void(const rocksdb::Slice& key,
+                             void* value,
+                             size_t charge,
+                             DeleterFn)>& callback,
+    bool thread_safe) override;
 
   virtual void EraseUnRefEntries() override;
 
   virtual std::string GetPrintableOptions() const override;
 
+  virtual DeleterFn GetDeleter(rocksdb::Cache::Handle* handle) const override;
+
   void TEST_GetLRUList(BinnedLRUHandle** lru, BinnedLRUHandle** lru_low_pri);
 
   //  Retrieves number of elements in LRU, for unit test purpose only
@@ -304,7 +310,9 @@ class BinnedLRUCache : public ShardedCache {
   virtual size_t GetCharge(Handle* handle) const override;
   virtual uint32_t GetHash(Handle* handle) const override;
   virtual void DisownData() override;
-
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+  virtual DeleterFn GetDeleter(Handle* handle) const override;
+#endif
   //  Retrieves number of elements in LRU, for unit test purpose only
   size_t TEST_GetLRUSize();
   // Sets the high pri pool ratio
diff --git a/src/kv/rocksdb_cache/ShardedCache.cc b/src/kv/rocksdb_cache/ShardedCache.cc
index 367140a94d8..6cbd89ad647 100644
--- a/src/kv/rocksdb_cache/ShardedCache.cc
+++ b/src/kv/rocksdb_cache/ShardedCache.cc
@@ -44,7 +44,7 @@ void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
 }
 
 rocksdb::Status ShardedCache::Insert(const rocksdb::Slice& key, void* value, size_t charge,
-                            void (*deleter)(const rocksdb::Slice& key, void* value),
+                            DeleterFn deleter,
                             rocksdb::Cache::Handle** handle, Priority priority) {
   uint32_t hash = HashSlice(key);
   return GetShard(Shard(hash))
@@ -109,13 +109,36 @@ size_t ShardedCache::GetPinnedUsage() const {
   return usage;
 }
 
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+DeleterFn ShardedCache::GetDeleter(Handle* handle) const
+{
+  uint32_t hash = GetHash(handle);
+  return GetShard(Shard(hash))->GetDeleter(handle);
+}
+
+void ShardedCache::ApplyToAllEntries(
+    const std::function<void(const rocksdb::Slice& key, void* value, size_t charge,
+                             DeleterFn deleter)>& callback,
+    const ApplyToAllEntriesOptions& opts)
+{
+  int num_shards = 1 << num_shard_bits_;
+  for (int s = 0; s < num_shards; s++) {
+    GetShard(s)->ApplyToAllCacheEntries(callback, true /* thread_safe */);
+  }
+}
+#else
 void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
                                           bool thread_safe) {
   int num_shards = 1 << num_shard_bits_;
   for (int s = 0; s < num_shards; s++) {
-    GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
+    GetShard(s)->ApplyToAllCacheEntries(
+      [callback](const rocksdb::Slice&, void* value, size_t charge, DeleterFn) {
+        callback(value, charge);
+      },
+      thread_safe);
   }
 }
+#endif
 
 void ShardedCache::EraseUnRefEntries() {
   int num_shards = 1 << num_shard_bits_;
@@ -131,7 +154,7 @@ std::string ShardedCache::GetPrintableOptions() const {
   char buffer[kBufferSize];
   {
     std::lock_guard<std::mutex> l(capacity_mutex_);
-    snprintf(buffer, kBufferSize, "    capacity : %" ROCKSDB_PRIszt "\n",
+    snprintf(buffer, kBufferSize, "    capacity : %zu\n",
              capacity_);
     ret.append(buffer);
     snprintf(buffer, kBufferSize, "    num_shard_bits : %d\n", num_shard_bits_);
diff --git a/src/kv/rocksdb_cache/ShardedCache.h b/src/kv/rocksdb_cache/ShardedCache.h
index 4d64893ab1c..f98421a09a3 100644
--- a/src/kv/rocksdb_cache/ShardedCache.h
+++ b/src/kv/rocksdb_cache/ShardedCache.h
@@ -14,6 +14,7 @@
 #include <string>
 #include <mutex>
 
+#include "rocksdb/version.h"
 #include "rocksdb/cache.h"
 #include "include/ceph_hash.h"
 #include "common/PriorityCache.h"
@@ -22,10 +23,11 @@
 #ifndef CACHE_LINE_SIZE
 #define CACHE_LINE_SIZE 64 // XXX arch-specific define 
 #endif
-#define ROCKSDB_PRIszt "zu"
 
 namespace rocksdb_cache {
 
+using DeleterFn = void (*)(const rocksdb::Slice& key, void* value);
+
 // Single cache shard interface.
 class CacheShard {
  public:
@@ -34,7 +36,7 @@ class CacheShard {
 
   virtual rocksdb::Status Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
                                  size_t charge,
-                                 void (*deleter)(const rocksdb::Slice& key, void* value),
+                                 DeleterFn deleter,
                                  rocksdb::Cache::Handle** handle, rocksdb::Cache::Priority priority) = 0;
   virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, uint32_t hash) = 0;
   virtual bool Ref(rocksdb::Cache::Handle* handle) = 0;
@@ -44,10 +46,15 @@ class CacheShard {
   virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
   virtual size_t GetUsage() const = 0;
   virtual size_t GetPinnedUsage() const = 0;
-  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                                      bool thread_safe) = 0;
+  virtual void ApplyToAllCacheEntries(
+    const std::function<void(const rocksdb::Slice& key,
+                             void* value,
+                             size_t charge,
+                             DeleterFn)>& callback,
+    bool thread_safe) = 0;
   virtual void EraseUnRefEntries() = 0;
   virtual std::string GetPrintableOptions() const { return ""; }
+  virtual DeleterFn GetDeleter(rocksdb::Cache::Handle* handle) const = 0;
 };
 
 // Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
@@ -57,34 +64,43 @@ class ShardedCache : public rocksdb::Cache, public PriorityCache::PriCache {
  public:
   ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
   virtual ~ShardedCache() = default;
+  // rocksdb::Cache
   virtual const char* Name() const override = 0;
-  virtual CacheShard* GetShard(int shard) = 0;
-  virtual const CacheShard* GetShard(int shard) const = 0;
-  virtual void* Value(Handle* handle) override = 0;
-  virtual size_t GetCharge(Handle* handle) const = 0;
-  virtual uint32_t GetHash(Handle* handle) const = 0;
-  virtual void DisownData() override = 0;
-
-  virtual void SetCapacity(size_t capacity) override;
-  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
-
   virtual rocksdb::Status Insert(const rocksdb::Slice& key, void* value, size_t charge,
-                                 void (*deleter)(const rocksdb::Slice& key, void* value),
+                                 DeleterFn,
                                  rocksdb::Cache::Handle** handle, Priority priority) override;
   virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, rocksdb::Statistics* stats) override;
   virtual bool Ref(rocksdb::Cache::Handle* handle) override;
   virtual bool Release(rocksdb::Cache::Handle* handle, bool force_erase = false) override;
+  virtual void* Value(Handle* handle) override = 0;
   virtual void Erase(const rocksdb::Slice& key) override;
   virtual uint64_t NewId() override;
-  virtual size_t GetCapacity() const override;
+  virtual void SetCapacity(size_t capacity) override;
+  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
   virtual bool HasStrictCapacityLimit() const override;
+  virtual size_t GetCapacity() const override;
   virtual size_t GetUsage() const override;
   virtual size_t GetUsage(rocksdb::Cache::Handle* handle) const override;
   virtual size_t GetPinnedUsage() const override;
+  virtual size_t GetCharge(Handle* handle) const = 0;
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+  virtual DeleterFn GetDeleter(Handle* handle) const override;
+#endif
+  virtual void DisownData() override = 0;
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+  virtual void ApplyToAllEntries(
+      const std::function<void(const rocksdb::Slice& key, void* value, size_t charge,
+                               DeleterFn deleter)>& callback,
+      const ApplyToAllEntriesOptions& opts) override;
+#else
   virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
                                       bool thread_safe) override;
+#endif
   virtual void EraseUnRefEntries() override;
   virtual std::string GetPrintableOptions() const override;
+  virtual CacheShard* GetShard(int shard) = 0;
+  virtual const CacheShard* GetShard(int shard) const = 0;
+  virtual uint32_t GetHash(Handle* handle) const = 0;
 
   int GetNumShardBits() const { return num_shard_bits_; }
 
@@ -120,7 +136,7 @@ class ShardedCache : public rocksdb::Cache, public PriorityCache::PriCache {
 //    return Hash(s.data(), s.size(), 0);
   }
 
-  uint32_t Shard(uint32_t hash) {
+  uint32_t Shard(uint32_t hash) const {
     // Note, hash >> 32 yields hash in gcc, not the zero we expect!
     return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
   }