coretests: test for miner tx with non-empty extra_nonce before and after HF4 (#532)

* try check extra nonce in miner_tx

* update c1

* write extra nonces, create pow pos block, checkers for exnonce, bug with HF4

* success check pos block

* correct pos/pos extra nonce validation for hf3+

* clean code, separate to methods, add template test, pos/pow extra nonce check in block

* delete logs

* back normal naming

* fix review comments

tests/core_tests/chaingen.cpp

@@ -940,7 +940,7 @@ bool test_generator::construct_block(int64_t manual_timestamp_adjustment,
                                               const crypto::hash& prev_id/* = crypto::hash()*/, const wide_difficulty_type& diffic/* = 1*/,
                                               const transaction& miner_tx/* = transaction()*/,
                                               const std::vector<crypto::hash>& tx_hashes/* = std::vector<crypto::hash>()*/,
-                                              size_t txs_sizes/* = 0*/)
+                                              size_t txs_sizes/* = 0*/, currency::blobdata extra_nonce/* = blobdata()*/)
 {
   size_t height = get_block_height(prev_block) + 1;
   blk.major_version = actual_params & bf_major_ver ? major_ver : m_hardforks.get_block_major_version_by_height(height);
@@ -966,7 +966,7 @@ bool test_generator::construct_block(int64_t manual_timestamp_adjustment,
     size_t current_block_size = txs_sizes + get_object_blobsize(blk.miner_tx);
     // TODO: This will work, until size of constructed block is less then CURRENCY_BLOCK_GRANTED_FULL_REWARD_ZONE
     if (!construct_miner_tx(height, misc_utils::median(block_sizes), already_generated_coins, current_block_size, 0,
-      miner_acc.get_public_address(), miner_acc.get_public_address(), blk.miner_tx, base_block_reward, block_reward, tx_version, tx_hardfork_id, blobdata(), 1))
+      miner_acc.get_public_address(), miner_acc.get_public_address(), blk.miner_tx, base_block_reward, block_reward, tx_version, tx_hardfork_id, extra_nonce, 1))
       return false;
   }
 
@@ -2498,6 +2498,102 @@ uint64_t decode_native_output_amount_or_throw(const account_base& acc, const tra
   return amount;
 }
 
+bool generate_pos_block_with_extra_nonce(test_generator& generator, const std::vector<test_event_entry>& events, const currency::account_base& miner, const currency::account_base& recipient, const currency::block& prev_block, const currency::transaction& stake_tx, const currency::blobdata& pos_nonce, currency::block& result)
+{
+  // get params for PoS
+  crypto::hash prev_id = get_block_hash(prev_block);
+  wide_difficulty_type pos_diff{};
+  crypto::hash last_pow_block_hash{}, last_pos_block_kernel_hash{};
+  bool r = generator.get_params_for_next_pos_block(
+    prev_id, pos_diff, last_pow_block_hash, last_pos_block_kernel_hash
+  );
+  CHECK_AND_ASSERT_MES(r, false, "get_params_for_next_pos_block failed");
+
+  // tx key and key image for stake out
+  crypto::public_key stake_pk = get_tx_pub_key_from_extra(stake_tx);
+  keypair kp;
+  crypto::key_image ki;
+  size_t stake_output_idx = 0;
+  generate_key_image_helper(miner.get_keys(), stake_pk, stake_output_idx, kp, ki);
+
+  // glob index for stake out
+  uint64_t stake_output_gidx = UINT64_MAX;
+  r = find_global_index_for_output(events, prev_id, stake_tx, stake_output_idx, stake_output_gidx);
+  CHECK_AND_ASSERT_MES(r, false, "find_global_index_for_output failed");
+
+  pos_block_builder pb;
+  uint64_t height = get_block_height(prev_block) + 1;
+  pb.step1_init_header(generator.get_hardforks(), height, prev_id);
+  pb.step2_set_txs({});
+
+  if (generator.get_hardforks().is_hardfork_active_for_height(ZANO_HARDFORK_04_ZARCANUM, height))
+  {
+    std::vector<tx_source_entry> sources;
+    bool ok = fill_tx_sources(
+      sources, events, prev_block, miner.get_keys(),
+      UINT64_MAX, 0, false, false, true
+    );
+
+    auto it = std::find_if(sources.begin(), sources.end(),
+      [&](const tx_source_entry &e){
+        return e.real_out_tx_key == stake_pk
+            && e.real_output_in_tx_index == stake_output_idx;
+      });
+    CHECK_AND_ASSERT_MES(it != sources.end(), false, "source entry not found");
+    const tx_source_entry& se = *it;
+
+    pb.step3a(pos_diff, last_pow_block_hash, last_pos_block_kernel_hash);
+    pb.step3b(
+      se.amount, ki,
+      se.real_out_tx_key, se.real_output_in_tx_index,
+      se.real_out_amount_blinding_mask,
+      miner.get_keys().view_secret_key,
+      stake_output_gidx,
+      prev_block.timestamp,
+      POS_SCAN_WINDOW, POS_SCAN_STEP
+    );
+
+    // insert extra_nonce
+    pb.step4_generate_coinbase_tx(
+      generator.get_timestamps_median(prev_id),
+      generator.get_already_generated_coins(prev_block),
+      recipient.get_public_address(),
+      pos_nonce,
+      CURRENCY_MINER_TX_MAX_OUTS
+    );
+
+    pb.step5_sign(se, miner.get_keys());
+  }
+  else // HF3: SLSAG
+  {
+    uint64_t amount = boost::get<tx_out_bare>(stake_tx.vout[stake_output_idx]).amount;
+    pb.step3_build_stake_kernel(
+      amount,
+      stake_output_gidx,
+      ki,
+      pos_diff,
+      last_pow_block_hash,
+      last_pos_block_kernel_hash,
+      prev_block.timestamp
+    );
+
+    // insert extra_nonce
+    pb.step4_generate_coinbase_tx(
+      generator.get_timestamps_median(prev_id),
+      generator.get_already_generated_coins(prev_block),
+      recipient.get_public_address(),
+      pos_nonce,
+      CURRENCY_MINER_TX_MAX_OUTS
+    );
+
+    crypto::public_key out_pk = boost::get<txout_to_key>(boost::get<tx_out_bare>(stake_tx.vout[stake_output_idx]).target).key;
+    pb.step5_sign(stake_pk, stake_output_idx, out_pk, miner);
+  }
+
+  result = pb.m_block;
+  return true;
+}
+
 bool generate_pos_block_with_given_coinstake(test_generator& generator, const std::vector<test_event_entry> &events, const currency::account_base& miner, const currency::block& prev_block,
   const currency::transaction& stake_tx, size_t stake_output_idx, currency::block& result, uint64_t stake_output_gidx /* = UINT64_MAX */)
 {

tests/core_tests/chaingen.h

@@ -570,7 +570,8 @@ public:
     const currency::account_base& miner_acc, int actual_params = bf_none, uint8_t major_ver = 0,
     uint8_t minor_ver = 0, uint64_t timestamp = 0, const crypto::hash& prev_id = crypto::hash(),
     const currency::wide_difficulty_type& diffic = 1, const currency::transaction& miner_tx = currency::transaction(),
-    const std::vector<crypto::hash>& tx_hashes = std::vector<crypto::hash>(), size_t txs_sizes = 0);
+    const std::vector<crypto::hash>& tx_hashes = std::vector<crypto::hash>(), size_t txs_sizes = 0, 
+    currency::blobdata extra_nonce = currency::blobdata());
   bool construct_block_manually_tx(currency::block& blk, const currency::block& prev_block,
     const currency::account_base& miner_acc, const std::vector<crypto::hash>& tx_hashes, size_t txs_size);
   bool find_nounce(currency::block& blk, std::vector<const block_info*>& blocks, currency::wide_difficulty_type dif, uint64_t height) const;
@@ -828,6 +829,7 @@ uint64_t get_last_block_of_type(bool looking_for_pos, const test_generator::bloc
 bool decode_output_amount_and_asset_id(const currency::account_base& acc, const currency::transaction& tx, size_t output_index, uint64_t &amount, crypto::public_key* p_asset_id = nullptr);
 uint64_t decode_native_output_amount_or_throw(const currency::account_base& acc, const currency::transaction& tx, size_t output_index);
 
+bool generate_pos_block_with_extra_nonce(test_generator& generator, const std::vector<test_event_entry>& events, const currency::account_base& miner, const currency::account_base& recipient, const currency::block& prev_block, const currency::transaction& stake_tx, const currency::blobdata& pos_nonce, currency::block& result);
 bool generate_pos_block_with_given_coinstake(test_generator& generator, const std::vector<test_event_entry> &events, const currency::account_base& miner, const currency::block& prev_block, const currency::transaction& stake_tx, size_t stake_output_idx, currency::block& result, uint64_t stake_output_gidx = UINT64_MAX);
 bool check_ring_signature_at_gen_time(const std::vector<test_event_entry>& events, const crypto::hash& last_block_id, const currency::txin_to_key& in_t_k,
   const crypto::hash& hash_for_sig, const std::vector<crypto::signature> &sig);

tests/core_tests/chaingen_main.cpp

@@ -1118,7 +1118,7 @@ int main(int argc, char* argv[])
     GENERATE_AND_PLAY(gen_pos_coinstake_already_spent);
     GENERATE_AND_PLAY(gen_pos_incorrect_timestamp);
     GENERATE_AND_PLAY(gen_pos_too_early_pos_block);
-    GENERATE_AND_PLAY(gen_pos_extra_nonce);
+    GENERATE_AND_PLAY_HF(gen_pos_extra_nonce, "3-*");
     GENERATE_AND_PLAY(gen_pos_min_allowed_height);
     GENERATE_AND_PLAY(gen_pos_invalid_coinbase);
     // GENERATE_AND_PLAY(pos_wallet_minting_same_amount_diff_outs); // Long test! Takes ~10 hours to simulate 6000 blocks on 2015 middle-end computer

tests/core_tests/pos_validation.cpp

@@ -217,56 +217,110 @@ bool gen_pos_too_early_pos_block::configure_core(currency::core& c, size_t ev_in
 
 //------------------------------------------------------------------
 
-bool gen_pos_extra_nonce::generate(std::vector<test_event_entry>& events) const
+bool gen_pos_extra_nonce::configure_core(currency::core& c, size_t ev_index, const std::vector<test_event_entry>&)
 {
-  uint64_t ts = time(NULL);
+  currency::core_runtime_config pc = c.get_blockchain_storage().get_core_runtime_config();
-
+  pc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE;
-  GENERATE_ACCOUNT(miner);
+  pc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH;
-  GENERATE_ACCOUNT(alice);
+  pc.hf4_minimum_mixins = 0;
-  MAKE_GENESIS_BLOCK(events, blk_0, miner, ts);
+  pc.hard_forks = m_hardforks;
-  DO_CALLBACK(events, "configure_core");
+  c.get_blockchain_storage().set_core_runtime_config(pc);
-  REWIND_BLOCKS(events, blk_0r, blk_0, miner);
-
-  // Legend: (n) - PoW block, [m] - PoS block
-  //  0     10    11                 <-- blockchain height  (assuming CURRENCY_MINED_MONEY_UNLOCK_WINDOW == 10)
-  // (0 )--(0r)--[1 ]                main chain
-
-  // make a PoS block manually with incorrect timestamp
-  crypto::hash prev_id = get_block_hash(blk_0r);
-  size_t height = CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 1;
-  currency::wide_difficulty_type diff = generator.get_difficulty_for_next_block(prev_id, false);
-
-  const transaction& stake = blk_0.miner_tx;
-  crypto::public_key stake_tx_pub_key = get_tx_pub_key_from_extra(stake);
-  size_t stake_output_idx = 0;
-  size_t stake_output_gidx = 0;
-  uint64_t stake_output_amount =boost::get<currency::tx_out_bare>( stake.vout[stake_output_idx]).amount;
-  crypto::key_image stake_output_key_image;
-  keypair kp;
-  generate_key_image_helper(miner.get_keys(), stake_tx_pub_key, stake_output_idx, kp, stake_output_key_image);
-  crypto::public_key stake_output_pubkey = boost::get<txout_to_key>(boost::get<currency::tx_out_bare>(stake.vout[stake_output_idx]).target).key;
-
-  pos_block_builder pb;
-  pb.step1_init_header(generator.get_hardforks(), height, prev_id);
-  pb.step2_set_txs(std::vector<transaction>());
-  pb.step3_build_stake_kernel(stake_output_amount, stake_output_gidx, stake_output_key_image, diff, prev_id, null_hash, blk_0r.timestamp);
-
-  // use biggest possible extra nonce (255 bytes) + largest alias
-  currency::blobdata extra_nonce(255, 'x');
-  //currency::extra_alias_entry alias = AUTO_VAL_INIT(alias); // TODO: this alias entry was ignored for a long time, now I commented it out, make sure it's okay -- sowle 
-  //alias.m_alias = std::string(255, 'a');
-  //alias.m_address = miner.get_keys().account_address;
-  //alias.m_text_comment = std::string(255, 'y');
-  pb.step4_generate_coinbase_tx(generator.get_timestamps_median(prev_id), generator.get_already_generated_coins(blk_0r), alice.get_public_address(), extra_nonce, CURRENCY_MINER_TX_MAX_OUTS);
-  pb.step5_sign(stake_tx_pub_key, stake_output_idx, stake_output_pubkey, miner);
-  block blk_1 = pb.m_block;
-
-  // EXPECTED: blk_1 is accepted
-  events.push_back(blk_1);
-
   return true;
 }
 
+gen_pos_extra_nonce::gen_pos_extra_nonce()
+{
+  REGISTER_CALLBACK_METHOD(gen_pos_extra_nonce, configure_core);
+  REGISTER_CALLBACK_METHOD(gen_pos_extra_nonce, request_pow_template_with_nonce);
+  REGISTER_CALLBACK_METHOD(gen_pos_extra_nonce, check_pow_nonce);
+  REGISTER_CALLBACK_METHOD(gen_pos_extra_nonce, check_pos_nonce);
+}
+
+// Test: verify custom extra_nonce in blocks and templates
+/*
+ * Scenarios:
+ * 1. PoW block contains m_pow_nonce
+ * 2. PoS block contains m_pos_nonce
+ * 3. PoW mining template contains m_pow_template_nonce
+ */
+bool gen_pos_extra_nonce::generate(std::vector<test_event_entry>& events) const
+{
+  GENERATE_ACCOUNT(miner);
+  GENERATE_ACCOUNT(alice);
+  m_accounts.push_back(miner);
+  m_accounts.push_back(alice);
+  m_pow_nonce = currency::blobdata(254, 'w');
+  m_pos_nonce = currency::blobdata(255, 's');
+  m_pow_template_nonce = "POW_TEMPLATE123";
+
+  uint64_t ts = test_core_time::get_time();
+  MAKE_GENESIS_BLOCK(events, blk_0, miner, ts);
+  DO_CALLBACK(events, "configure_core");
+  MAKE_NEXT_BLOCK(events, blk_1, blk_0, miner);
+
+  block blk_2 = AUTO_VAL_INIT(blk_2);
+  ts = blk_2.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN / 2; // to increase main chain difficulty
+  bool r = generator.construct_block_manually(blk_2, blk_1, miner, test_generator::bf_timestamp, 0, 0, 
+    ts, crypto::hash(), 1, transaction(), std::vector<crypto::hash>(), 0, m_pow_nonce);
+  CHECK_AND_ASSERT_MES(r, false, "construct_block_manually failed");
+  events.push_back(blk_2);
+
+  DO_CALLBACK(events, "check_pow_nonce");
+  REWIND_BLOCKS(events, blk_0r, blk_2, miner);
+
+  // setup params for PoS
+  const currency::transaction& stake = blk_2.miner_tx;
+  
+  currency::block new_pos_block;
+  bool ok = generate_pos_block_with_extra_nonce(generator, events, miner, alice, blk_0r, stake, m_pos_nonce, new_pos_block); 
+  CHECK_AND_ASSERT_MES(ok, false, "generate_pos_block_with_extra_nonce failed");
+
+  events.push_back(new_pos_block);
+  DO_CALLBACK(events, "check_pos_nonce");
+  DO_CALLBACK(events, "request_pow_template_with_nonce");
+  return true;
+}
+
+bool gen_pos_extra_nonce::request_pow_template_with_nonce(currency::core& c, size_t ev_index, const std::vector<test_event_entry>& events)
+{
+  block bl;
+  wide_difficulty_type diff;
+  uint64_t height;
+  bool ok = c.get_block_template(bl, m_accounts[0].get_public_address(), m_accounts[0].get_public_address(), diff, height, m_pow_template_nonce);
+  CHECK_AND_ASSERT_MES(ok, false, "get_block_template failed");
+  CHECK_AND_ASSERT_MES(has_extra_nonce(bl, m_pow_template_nonce), false, "PoW extra_nonce not found");
+  return true;
+}
+
+bool gen_pos_extra_nonce::check_pow_nonce(currency::core& c, size_t ev_index, const std::vector<test_event_entry>& events)
+{
+  block top;
+  bool ok = c.get_blockchain_storage().get_top_block(top);
+  CHECK_AND_ASSERT_MES(ok, false, "get_top_block failed");
+  CHECK_AND_ASSERT_MES(has_extra_nonce(top, m_pow_nonce), false, "PoW extra_nonce not found");
+  return true;
+}
+
+bool gen_pos_extra_nonce::check_pos_nonce(currency::core& c, size_t ev_index, const std::vector<test_event_entry>& events)
+{
+  block top;
+  bool ok = c.get_blockchain_storage().get_top_block(top);
+  CHECK_AND_ASSERT_MES(ok, false, "get_top_block failed");
+  CHECK_AND_ASSERT_MES(has_extra_nonce(top, m_pos_nonce), false, "PoS extra_nonce not found");
+  return true;
+}
+
+bool gen_pos_extra_nonce::has_extra_nonce(currency::block& blk, const std::string& expected_nonce)
+{
+  if (auto const* ud = get_type_in_variant_container<extra_user_data>(blk.miner_tx.extra)) {
+    LOG_PRINT_L0("Found extra nonce: '" << ud->buff << "' expected: '" << expected_nonce << "'");
+    return ud->buff == expected_nonce;
+  }
+  return false;
+}
+
+//------------------------------------------------------------------
+
 gen_pos_min_allowed_height::gen_pos_min_allowed_height()
 {
   REGISTER_CALLBACK_METHOD(gen_pos_min_allowed_height, configure_core);

tests/core_tests/pos_validation.h

@@ -71,7 +71,21 @@ struct gen_pos_too_early_pos_block : public pos_validation
 
 struct gen_pos_extra_nonce : public pos_validation
 {
+  gen_pos_extra_nonce();
+  bool configure_core(currency::core& c, size_t, const std::vector<test_event_entry>& events);
+  bool request_pow_template_with_nonce(currency::core& c, size_t, const std::vector<test_event_entry>& events);
+  bool check_pos_nonce(currency::core& c, size_t, const std::vector<test_event_entry>& events);
+  bool check_pow_nonce(currency::core& c, size_t, const std::vector<test_event_entry>& events);
   bool generate(std::vector<test_event_entry>& events) const;
+
+private:
+  bool has_extra_nonce(currency::block& blk, const std::string& expected_nonce);
+  
+private:
+  mutable currency::blobdata m_pow_nonce;
+  mutable currency::blobdata m_pos_nonce;
+  mutable currency::blobdata m_pow_template_nonce;
+  mutable std::vector<currency::account_base> m_accounts;
 };
 
 struct gen_pos_min_allowed_height : public pos_validation