Merge pull request libbitcoin#147 from evoskuil/version3

v3.3: add median time past caching in headers and txs.

configure.ac

@@ -13,7 +13,7 @@
 AC_PREREQ([2.65])
 
 # Process command-line arguments and perform initialization and verification.
-AC_INIT([libbitcoin-database], [3.2.0], [eric@voskuil.org])
+AC_INIT([libbitcoin-database], [3.3.0], [eric@voskuil.org])
 
 # Do compilation tests.
 AC_LANG(C++)
@@ -133,10 +133,10 @@ AS_CASE([${with_tests}], [yes],
      AC_MSG_NOTICE([boost_unit_test_framework_LIBS : ${boost_unit_test_framework_LIBS}])],
     [AC_SUBST([boost_unit_test_framework_LIBS], [])])
 
-# Require bitcoin of at least version 3.2.0 and output ${bitcoin_CPPFLAGS/LIBS/PKG}.
+# Require bitcoin of at least version 3.3.0 and output ${bitcoin_CPPFLAGS/LIBS/PKG}.
 #------------------------------------------------------------------------------
-PKG_CHECK_MODULES([bitcoin], [libbitcoin >= 3.2.0])
-AC_SUBST([bitcoin_PKG], ['libbitcoin >= 3.2.0'])
+PKG_CHECK_MODULES([bitcoin], [libbitcoin >= 3.3.0])
+AC_SUBST([bitcoin_PKG], ['libbitcoin >= 3.3.0'])
 AC_SUBST([bitcoin_CPPFLAGS], [${bitcoin_CFLAGS}])
 AC_MSG_NOTICE([bitcoin_CPPFLAGS : ${bitcoin_CPPFLAGS}])
 AC_MSG_NOTICE([bitcoin_LIBS : ${bitcoin_LIBS}])

include/bitcoin/database/data_base.hpp

@@ -142,7 +142,7 @@ class BCD_API data_base
     // ------------------------------------------------------------------------
 
     bool push_transactions(const chain::block& block, size_t height,
-        size_t bucket=0, size_t buckets=1);
+        uint32_t median_time_past, size_t bucket=0, size_t buckets=1);
     bool push_heights(const chain::block& block, size_t height);
     void push_inputs(const hash_digest& tx_hash, size_t height,
         const inputs& inputs);
@@ -164,10 +164,12 @@ class BCD_API data_base
     void push_next(const code& ec, block_const_ptr_list_const_ptr blocks,
         size_t index, size_t height, dispatcher& dispatch,
         result_handler handler);
-    void do_push(block_const_ptr block, size_t height, dispatcher& dispatch,
+    void do_push(block_const_ptr block, size_t height,
+        uint32_t median_time_past, dispatcher& dispatch,
         result_handler handler);
     void do_push_transactions(block_const_ptr block, size_t height,
-        size_t bucket, size_t buckets, result_handler handler);
+        uint32_t median_time_past, size_t bucket, size_t buckets,
+        result_handler handler);
     void handle_push_transactions(const code& ec, block_const_ptr block,
         size_t height, result_handler handler);
 

include/bitcoin/database/databases/transaction_database.hpp

@@ -69,11 +69,13 @@ class BCD_API transaction_database
 
     /// Get the output at the specified index within the transaction.
     bool get_output(chain::output& out_output, size_t& out_height,
-        bool& out_coinbase, const chain::output_point& point,
-        size_t fork_height, bool require_confirmed) const;
+        uint32_t& out_median_time_past, bool& out_coinbase,
+        const chain::output_point& point, size_t fork_height,
+        bool require_confirmed) const;
 
     /// Store a transaction in the database.
-    void store(const chain::transaction& tx, size_t height, size_t position);
+    void store(const chain::transaction& tx, size_t height,
+        uint32_t median_time_past, size_t position);
 
     /// Update the spender height of the output in the tx store.
     bool spend(const chain::output_point& point, size_t spender_height);
@@ -82,7 +84,8 @@ class BCD_API transaction_database
     bool unspend(const chain::output_point& point);
 
     /// Promote an unconfirmed tx (not including its indexes).
-    bool confirm(const hash_digest& hash, size_t height, size_t position);
+    bool confirm(const hash_digest& hash, size_t height,
+        uint32_t median_time_past, size_t position);
 
     /// Demote the transaction (not including its indexes).
     bool unconfirm(const hash_digest& hash);

include/bitcoin/database/result/transaction_result.hpp

@@ -35,9 +35,9 @@ class BCD_API transaction_result
     transaction_result();
     transaction_result(const memory_ptr slab);
     transaction_result(const memory_ptr slab, hash_digest&& hash,
-        uint32_t height, uint16_t position);
+        uint32_t height, uint32_t median_time_past, uint16_t position);
     transaction_result(const memory_ptr slab, const hash_digest& hash,
-        uint32_t height, uint16_t position);
+        uint32_t height, uint32_t median_time_past, uint16_t position);
 
     /// True if this transaction result is valid (found).
     operator bool() const;
@@ -54,6 +54,9 @@ class BCD_API transaction_result
     /// The ordinal position of the transaction within its block.
     size_t position() const;
 
+    /// The median time past of the block which includes the transaction.
+    uint32_t median_time_past() const;
+
     /// True if all transaction outputs are spent at or below fork_height.
     bool is_spent(size_t fork_height) const;
 
@@ -66,6 +69,7 @@ class BCD_API transaction_result
 private:
     memory_ptr slab_;
     const uint32_t height_;
+    const uint32_t median_time_past_;
     const uint16_t position_;
     const hash_digest hash_;
 };

include/bitcoin/database/unspent_outputs.hpp

@@ -55,7 +55,7 @@ class BCD_API unspent_outputs
 
     /// Add a set of outputs to the cache (purges older entry).
     void add(const chain::transaction& transaction, size_t height,
-        bool confirmed);
+        uint32_t median_time_past, bool confirmed);
 
     /// Remove a set of outputs from the cache (has been reorganized out).
     void remove(const hash_digest& tx_hash);
@@ -64,7 +64,8 @@ class BCD_API unspent_outputs
     void remove(const chain::output_point& point);
 
     /// Determine if the output is unspent (otherwise fall back to the store).
-    bool get(chain::output& out_output, size_t& out_height, bool& out_coinbase,
+    bool get(chain::output& out_output, size_t& out_height,
+        uint32_t& out_median_time_past, bool& out_coinbase,
         const chain::output_point& point, size_t fork_height,
         bool require_confirmed) const;
 

include/bitcoin/database/unspent_transaction.hpp

@@ -45,10 +45,11 @@ class BCD_API unspent_transaction
     explicit unspent_transaction(const hash_digest& hash);
     explicit unspent_transaction(const chain::output_point& point);
     explicit unspent_transaction(const chain::transaction& tx, size_t height,
-        bool confirmed);
+        uint32_t median_time_past, bool confirmed);
 
     /// Properties.
     size_t height() const;
+    uint32_t median_time_past() const;
     bool is_coinbase() const;
     bool is_confirmed() const;
     const hash_digest& hash() const;
@@ -65,6 +66,7 @@ class BCD_API unspent_transaction
 
     // These are thread safe (non-const only for assignment operator).
     size_t height_;
+    uint32_t median_time_past_;
     bool is_coinbase_;
     bool is_confirmed_;
     hash_digest hash_;

include/bitcoin/database/version.hpp

@@ -12,9 +12,9 @@
  * For interpretation of the versioning scheme see: http://semver.org
  */
 
-#define LIBBITCOIN_DATABASE_VERSION "3.2.0"
+#define LIBBITCOIN_DATABASE_VERSION "3.3.0"
 #define LIBBITCOIN_DATABASE_MAJOR_VERSION 3
-#define LIBBITCOIN_DATABASE_MINOR_VERSION 2
+#define LIBBITCOIN_DATABASE_MINOR_VERSION 3
 #define LIBBITCOIN_DATABASE_PATCH_VERSION 0
 
 #endif

libbitcoin-database.pc.in

@@ -25,7 +25,7 @@ Version: @PACKAGE_VERSION@
 #==============================================================================
 # Dependencies that publish package configuration.
 #------------------------------------------------------------------------------
-Requires: libbitcoin >= 3.2.0
+Requires: libbitcoin >= 3.3.0
 
 # Include directory and any other required compiler flags.
 #------------------------------------------------------------------------------

src/data_base.cpp

@@ -334,7 +334,10 @@ code data_base::insert(const chain::block& block, size_t height)
     if (ec)
         return ec;
 
-    if (!push_transactions(block, height) || !push_heights(block, height))
+    const auto median_time_past = block.header().validation.median_time_past;
+
+    if (!push_transactions(block, height, median_time_past) ||
+        !push_heights(block, height))
         return error::operation_failed;
 
     blocks_->store(block, height);
@@ -361,7 +364,7 @@ code data_base::push(const chain::transaction& tx, uint32_t forks)
         return error::operation_failed;
 
     // When position is unconfirmed, height is used to store validation forks.
-    transactions_->store(tx, forks, transaction_database::unconfirmed);
+    transactions_->store(tx, forks, 0, transaction_database::unconfirmed);
     transactions_->synchronize();
 
     return end_write() ? error::success : error::operation_failed;
@@ -388,7 +391,10 @@ code data_base::push(const block& block, size_t height)
     if (!begin_write())
         return error::operation_failed;
 
-    if (!push_transactions(block, height) || !push_heights(block, height))
+    const auto median_time_past = block.header().validation.median_time_past;
+
+    if (!push_transactions(block, height, median_time_past) ||
+        !push_heights(block, height))
         return error::operation_failed;
 
     blocks_->store(block, height);
@@ -402,7 +408,7 @@ code data_base::push(const block& block, size_t height)
 
 // To push in order call with bucket = 0 and buckets = 1 (defaults).
 bool data_base::push_transactions(const chain::block& block, size_t height,
-    size_t bucket, size_t buckets)
+    uint32_t median_time_past, size_t bucket, size_t buckets)
 {
     BITCOIN_ASSERT(bucket < buckets);
     const auto& txs = block.transactions();
@@ -412,7 +418,7 @@ bool data_base::push_transactions(const chain::block& block, size_t height,
         position = ceiling_add(position, buckets))
     {
         const auto& tx = txs[position];
-        transactions_->store(tx, height, position);
+        transactions_->store(tx, height, median_time_past, position);
 
         if (height < settings_.index_start_height)
             continue;
@@ -523,6 +529,7 @@ void data_base::push_stealth(const hash_digest& tx_hash, size_t height,
 bool data_base::pop(block& out_block)
 {
     size_t height;
+    const auto start_time = asio::steady_clock::now();
 
     // The blockchain is empty (nothing to pop, not even genesis).
     if (!blocks_->top(height))
@@ -570,8 +577,10 @@ bool data_base::pop(block& out_block)
     // Synchronise everything that was changed.
     synchronize();
 
-    // Return the block.
+    // Return the block (with header/block metadata and pop start time).
     out_block = chain::block(block.header(), std::move(transactions));
+    out_block.validation.error = error::success;
+    out_block.validation.start_pop = start_time;
     return true;
 }
 
@@ -648,6 +657,7 @@ void data_base::push_all(block_const_ptr_list_const_ptr in_blocks,
     push_next(error::success, in_blocks, 0, first_height, dispatch, handler);
 }
 
+// TODO: resolve inconsistency with height and median_time_past passing.
 void data_base::push_next(const code& ec,
     block_const_ptr_list_const_ptr blocks, size_t index, size_t height,
     dispatcher& dispatch, result_handler handler)
@@ -660,6 +670,7 @@ void data_base::push_next(const code& ec,
     }
 
     const auto block = (*blocks)[index];
+    const auto median_time_past = block->header().validation.median_time_past;
 
     // Set push start time for the block.
     block->validation.start_push = asio::steady_clock::now();
@@ -671,11 +682,11 @@ void data_base::push_next(const code& ec,
 
     // This is the beginning of the block sub-sequence.
     dispatch.concurrent(&data_base::do_push,
-        this, block, height, std::ref(dispatch), next);
+        this, block, height, median_time_past, std::ref(dispatch), next);
 }
 
 void data_base::do_push(block_const_ptr block, size_t height,
-    dispatcher& dispatch, result_handler handler)
+    uint32_t median_time_past, dispatcher& dispatch, result_handler handler)
 {
     result_handler block_complete =
         std::bind(&data_base::handle_push_transactions,
@@ -699,13 +710,16 @@ void data_base::do_push(block_const_ptr block, size_t height,
 
     for (size_t bucket = 0; bucket < buckets; ++bucket)
         dispatch.concurrent(&data_base::do_push_transactions,
-            this, block, height, bucket, buckets, join_handler);
+            this, block, height, median_time_past, bucket, buckets,
+                join_handler);
 }
 
 void data_base::do_push_transactions(block_const_ptr block, size_t height,
-    size_t bucket, size_t buckets, result_handler handler)
+    uint32_t median_time_past, size_t bucket, size_t buckets,
+    result_handler handler)
 {
-    const auto result = push_transactions(*block, height, bucket, buckets);
+    const auto result = push_transactions(*block, height, median_time_past,
+        bucket, buckets);
     handler(result ? error::success : error::operation_failed);
 }
 
@@ -771,7 +785,6 @@ void data_base::pop_above(block_const_ptr_list_ptr out_blocks,
     for (size_t height = top; height > fork; --height)
     {
         message::block next;
-        const auto start_time = asio::steady_clock::now();
 
         // TODO: parallelize pop of transactions within each block.
         if (!pop(next))
@@ -782,13 +795,6 @@ void data_base::pop_above(block_const_ptr_list_ptr out_blocks,
 
         BITCOIN_ASSERT(next.is_valid());
         auto block = std::make_shared<const message::block>(std::move(next));
-
-        // Mark the blocks as validated for their respective heights.
-        block->header().validation.height = height;
-        block->validation.error = error::success;
-        block->validation.start_pop = start_time;
-
-        // TODO: optimize.
         out_blocks->insert(out_blocks->begin(), block);
     }
 

src/databases/block_database.cpp

@@ -35,15 +35,17 @@ const file_offset block_database::empty = 0;
 static constexpr auto index_header_size = 0u;
 static constexpr auto index_record_size = sizeof(file_offset);
 
-// Record format:
+// Record format (median_time_past added in v3.3):
 //  [ header:80 ]
+//  [ median_time_past:4 ]
 //  [ height:4 ]
 //  TODO: [ checksum:4 ] (store all zeros if not computed).
 //  [ tx_count:1-2 ]
 //  [ [ tx_hash:32 ]... ]
 
 // See block result.cpp.
-static constexpr auto height_offset = 80u;
+static constexpr auto median_time_past_size = sizeof(uint32_t);
+static constexpr auto height_offset = 80u + median_time_past_size;
 
 // Blocks uses a hash table and an array index, both O(1).
 block_database::block_database(const path& map_filename,
@@ -186,8 +188,8 @@ void block_database::store(const block& block, size_t height)
     // Write block data.
     const auto write = [&](serializer<uint8_t*>& serial)
     {
-        // WRITE THE BLOCK HEADER AND TX HASHES
-        block.header().to_data(serial);
+        // WRITE THE BLOCK HEADER (including median_time_past metadata).
+        block.header().to_data(serial, false);
 
         ///////////////////////////////////////////////////////////////////////
         // Critical Section
@@ -202,8 +204,9 @@ void block_database::store(const block& block, size_t height)
             serial.write_hash(tx.hash());
     };
 
-    const auto key = block.header().hash();
-    const auto size = header::satoshi_fixed_size() + sizeof(height32) +
+    const auto& header = block.header();
+    const auto key = header.hash();
+    const auto size = header.serialized_size(false) + sizeof(height32) +
         message::variable_uint_size(tx_count) + (tx_count * hash_size);
 
     const auto position = lookup_map_.store(key, write, size);