Merge pull request #10366

baa631f Improve unpack reserve handling (Lee Clagett)

src/serialization/container.h

@@ -30,11 +30,15 @@
 
 #pragma once
 
+#include <algorithm>
 #include <cstdint>
 #include <cstddef>
+#include <limits>
 #include <string>
 #include <type_traits>
 
+#include "serialization/serialization.h"
+
 namespace serialization
 {
   namespace detail
@@ -76,7 +80,26 @@ namespace serialization
     template<typename... C>
     void do_reserve(const C&...) {}
     template<typename C>
-    auto do_reserve(C &c, std::size_t N) -> decltype(c.reserve(N)) { return c.reserve(N); }
+    auto do_reserve(C &c, std::size_t N, std::size_t B) -> decltype(c.reserve(N))
+    {
+      using T = typename C::value_type;
+
+      static constexpr std::size_t max_compression_ratio =
+        is_blob_type<T>::type::value ? 1 :
+        use_container_varint<T>() ? sizeof(T) :
+        (std::is_same<T, char>::value || std::is_same<T, unsigned char>::value) ? 1:
+        4; // default
+
+      // max compression ratio for upfront memory usage
+      B /= sizeof(T);
+      B = std::max(std::size_t(1), B);
+      if (std::numeric_limits<std::size_t>::max() / max_compression_ratio <= B)
+        B = std::numeric_limits<std::size_t>::max();
+      else
+        B *= max_compression_ratio;
+
+      return c.reserve(std::min(N, B));
+    }
 
     // The value_type of STL map-like containers come in the form std::pair<const K, V>.
     // Since we can't {de}serialize const types in this lib, we must convert this to std::pair<K, V>
@@ -104,7 +127,7 @@ bool do_serialize_container(Archive<false> &ar, C &v)
     return false;
   }
 
-  ::serialization::detail::do_reserve(v, cnt);
+  ::serialization::detail::do_reserve(v, cnt, ar.remaining_bytes());
 
   for (size_t i = 0; i < cnt; i++) {
     if (i > 0)

tests/unit_tests/serialization.cpp

@@ -372,6 +372,20 @@ TEST(Serialization, serializes_vector_int64_as_fixed_int)
   ASSERT_EQ(57, blob.size());
 }
 
+TEST(Serialization, deserializes_vector_reserve)
+{
+  std::vector<int64_t> v;
+  string blob;
+
+  tools::write_varint(std::back_inserter(blob), unsigned(100));
+  blob.append(std::string(100, 0));
+
+  ASSERT_LT(v.capacity(), 20);
+  ASSERT_FALSE(serialization::parse_binary(blob, v));
+  ASSERT_LT(v.capacity(), 100); // could fail if lib allocates more in reserve call
+}
+
+
 namespace
 {
   template<typename T>