Merge pull request eBay#468 from eBay/enable_thin_provision

Enable thin provision

conanfile.py

@@ -2,7 +2,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "3.7.1"
+    version = "3.8.1"
 
     homepage = "https://github.corp.ebay.com/SDS/homestore"
     description = "HomeStore"

src/.clang-format

@@ -18,7 +18,6 @@ AlignOperands:   false
 AlignTrailingComments: true
 AllowShortBlocksOnASingleLine: true
 AllowShortIfStatementsOnASingleLine: true
-AllowShortBlocksOnASingleLine: true
 AllowShortCaseLabelsOnASingleLine: false
 # AllowShortFunctionsOnASingleLine: InlineOnly
 # AllowShortLoopsOnASingleLine: false

src/api/vol_interface.hpp

@@ -114,6 +114,22 @@ struct vol_interface_req : public sisl::ObjLifeCounter< vol_interface_req > {
     bool is_write() const { return op_type == Op_type::WRITE; }
     bool is_unmap() const { return op_type == Op_type::UNMAP; }
 
+    bool is_zero_request(const uint64_t page_size) {
+        if (iovecs.empty()) {
+            return !buffer || hs_utils::is_buf_zero(static_cast< uint8_t* >(buffer), nlbas * page_size);
+        }
+        return is_iovec_zero();
+    }
+
+    bool is_iovec_zero() {
+        for (const auto& iovec : iovecs) {
+            auto data = static_cast< uint8_t* >(iovec.iov_base);
+            const size_t size = iovec.iov_len;
+            if (!hs_utils::is_buf_zero(data, size)) { return false; }
+        }
+        return true;
+    }
+
     friend void intrusive_ptr_add_ref(vol_interface_req* req) { req->refcount.increment(1); }
 
     friend void intrusive_ptr_release(vol_interface_req* req) {
@@ -316,7 +332,7 @@ class VolInterface {
 
     virtual const char* get_name(const VolumePtr& vol) = 0;
     virtual uint64_t get_size(const VolumePtr& vol) = 0;
-    virtual std::map<boost::uuids::uuid, uint64_t> get_used_size(const VolumePtr& vol) = 0;
+    virtual std::map< boost::uuids::uuid, uint64_t > get_used_size(const VolumePtr& vol) = 0;
     virtual uint64_t get_page_size(const VolumePtr& vol) = 0;
     virtual boost::uuids::uuid get_uuid(std::shared_ptr< Volume > vol) = 0;
     virtual sisl::blob at_offset(const boost::intrusive_ptr< BlkBuffer >& buf, uint32_t offset) = 0;

src/engine/common/homestore_config.fbs

@@ -142,6 +142,8 @@ table Generic {
     // percentage of cache used to create indx mempool. It should be more than 100 to 
     // take into account some floating buffers in writeback cache.
     indx_mempool_percent : uint32 = 110;
+
+    enable_zero_padding: bool = true;
 }
 
 table ResourceLimits {

src/engine/common/homestore_utils.cpp

@@ -18,6 +18,7 @@
 #include <boost/archive/iterators/base64_from_binary.hpp>
 #include <boost/archive/iterators/binary_from_base64.hpp>
 #include <boost/archive/iterators/transform_width.hpp>
+#include <isa-l/crc.h>
 
 namespace homestore {
 uint8_t* hs_utils::iobuf_alloc(const size_t size, const sisl::buftag tag, const size_t alignment) {
@@ -76,6 +77,42 @@ sisl::byte_array hs_utils::extract_byte_array(const sisl::byte_view& b, const bo
     return (is_aligned_needed) ? b.extract(alignment) : b.extract(0);
 };
 
+constexpr unsigned long long operator"" _KB(unsigned long long x) { return x * 1024; }
+
+constexpr std::array< size_t, 7 > predefined_sizes = {4_KB, 8_KB, 16_KB, 32_KB, 64_KB, 128_KB, 256_KB};
+
+// Function to initialize the CRC map with predefined sizes
+void initialize_crc_map(std::map< size_t, uint16_t >& crc_map) {
+    std::vector< uint8_t > zero_buf;
+    for (auto s : predefined_sizes) {
+        zero_buf.resize(s, 0); // Resize buffer to the required size, filling with zeros
+        crc_map[s] = crc16_t10dif(init_crc_16, zero_buf.data(), s);
+    }
+}
+
+uint16_t hs_utils::crc_zero(const size_t size) {
+    static std::map< size_t, uint16_t > crc_map;
+    static std::once_flag init_flag;
+
+    // Thread-safe initialization of the CRC map
+    std::call_once(init_flag, initialize_crc_map, std::ref(crc_map));
+
+    // Check if the size is already in the map
+    if (auto it = crc_map.find(size); it != crc_map.end()) { return it->second; }
+
+    std::vector< uint8_t > zero_buf(size, 0);
+    return crc16_t10dif(init_crc_16, zero_buf.data(), size);
+}
+
+bool hs_utils::is_buf_zero(const uint8_t* buf, size_t size) {
+    // TODO: subsample the buffer to detect zero request instead of working on the whole buffer to achieve constant
+    //  processing time for large buffer size requests. Needs to investigate the performance impact of this change
+    //  in end2end testing.
+    auto zero_crc = crc_zero(size);
+    const auto crc = crc16_t10dif(init_crc_16, buf, size);
+    return (crc == zero_crc) ? (buf[0] == 0 && !std::memcmp(buf, buf + 1, size - 1)) : false;
+}
+
 std::string hs_utils::encodeBase64(const uint8_t* first, std::size_t size) {
     using Base64FromBinary = boost::archive::iterators::base64_from_binary<
         boost::archive::iterators::transform_width< const char*, // sequence of chars
@@ -90,15 +127,12 @@ std::string hs_utils::encodeBase64(const uint8_t* first, std::size_t size) {
     return encoded.append(bytes_to_pad, '=');
 }
 
-std::string hs_utils::encodeBase64(const sisl::byte_view& b){
-    return encodeBase64(b.bytes(), b.size());
-}
+std::string hs_utils::encodeBase64(const sisl::byte_view& b) { return encodeBase64(b.bytes(), b.size()); }
 
-template <typename T>
-void hs_utils::decodeBase64(const std::string &encoded_data, T out)
-{
+template < typename T >
+void hs_utils::decodeBase64(const std::string& encoded_data, T out) {
     using BinaryFromBase64 = boost::archive::iterators::transform_width<
-        boost::archive::iterators::binary_from_base64<std::string::const_iterator>,
+        boost::archive::iterators::binary_from_base64< std::string::const_iterator >,
         8, // get a view of 8 bit
         6  // from a sequence of 6 bit
         >;
@@ -107,14 +141,13 @@ void hs_utils::decodeBase64(const std::string &encoded_data, T out)
     std::replace(begin(unpadded_data), end(unpadded_data), '=', 'A'); // A_64 == \0
 
     std::string decoded_data{BinaryFromBase64{begin(unpadded_data)},
-                        BinaryFromBase64{begin(unpadded_data) + unpadded_data.length()}};
+                             BinaryFromBase64{begin(unpadded_data) + unpadded_data.length()}};
 
     decoded_data.erase(end(decoded_data) - bytes_to_pad, end(decoded_data));
     std::copy(begin(decoded_data), end(decoded_data), out);
 }
 
-std::string hs_utils::decodeBase64(const std::string &encoded_data)
-{
+std::string hs_utils::decodeBase64(const std::string& encoded_data) {
     std::string rv;
     decodeBase64(encoded_data, std::back_inserter(rv));
     return rv;

src/engine/common/homestore_utils.hpp

@@ -38,6 +38,8 @@ class hs_utils {
     static sisl::byte_array make_byte_array(const uint64_t size, const bool is_aligned_needed, const sisl::buftag tag,
                                             const size_t alignment);
     static hs_uuid_t gen_system_uuid();
+    static uint16_t crc_zero(const size_t size);
+    static bool is_buf_zero(const uint8_t* buf, size_t size);
     static std::string encodeBase64(const uint8_t* first, std::size_t size);
     static std::string encodeBase64(const sisl::byte_view& b);
     template <typename T> static void decodeBase64(const std::string &encoded_data, T out);

src/homeblks/homeblks_http_server.cpp

@@ -155,12 +155,13 @@ void HomeBlksHttpServer::set_log_level(const Pistache::Rest::Request& request,
 
     response.send(Pistache::Http::Code::Ok, resp);
 }
-void HomeBlksHttpServer::get_utilization(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response)
-{
-    const std::string vol_uuid = request.hasParam(":volumeUUID") ? request.param(":volumeUUID").as<std::string>():"";
+void HomeBlksHttpServer::get_utilization(const Pistache::Rest::Request& request,
+                                         Pistache::Http::ResponseWriter response) {
+    const std::string vol_uuid =
+        request.hasParam(":volumeUUID") ? request.param(":volumeUUID").as< std::string >() : "";
 
     VolumePtr vol = nullptr;
-    if (vol_uuid.length() != 0) {
+    if (vol_uuid.length()) {
         boost::uuids::string_generator gen;
         boost::uuids::uuid uuid = gen(vol_uuid);
         vol = VolInterface::get_instance()->lookup_volume(uuid);
@@ -170,10 +171,14 @@ void HomeBlksHttpServer::get_utilization(const Pistache::Rest::Request& request,
         }
     }
     nlohmann::json resp;
-    const auto total_data_size = VolInterface::get_instance()->get_system_capacity().initial_total_data_meta_size;
+    nlohmann::json partitions = nlohmann::json::array();
     for (auto [uuid, vol_used] : VolInterface::get_instance()->get_used_size(vol)) {
-        resp[boost::uuids::to_string(uuid)] = std::to_string(static_cast<double> (vol_used)/ total_data_size);
+        nlohmann::json partition;
+        partition["id"] = boost::uuids::to_string(uuid);
+        partition["usedCapacity"] = vol_used;
+        partitions.push_back(partition);
     }
+    resp["partitions"] = partitions;
     response.send(Pistache::Http::Code::Ok, resp.dump());
 }
 void HomeBlksHttpServer::get_log_level(const Pistache::Rest::Request& request,