Merge pull request #1930

d1530f06 update version to 0.10.3.1 tag (Riccardo Spagni)

.dockerignore

@@ -0,0 +1 @@
+*

CMakeLists.txt

@@ -228,8 +228,8 @@ if(STATIC)
   endif()
 endif()
 
-# default database:
-# should be lmdb for testing, memory for production still
+# Set default blockchain storage location:
+# memory was the default in Cryptonote before Monero implimented LMDB, it still works but is unneccessary.
 # set(DATABASE memory)
 set(DATABASE lmdb)
 
@@ -559,7 +559,7 @@ else()
   endif()
 
   if(NOT DEFINED USE_LTO_DEFAULT)
-    set(USE_LTO_DEFAULT true)
+    set(USE_LTO_DEFAULT false)
   endif()
   set(USE_LTO ${USE_LTO_DEFAULT} CACHE BOOL "Use Link-Time Optimization (Release mode only)")
 

Dockerfile

@@ -1,40 +1,36 @@
-FROM debian:testing
-MAINTAINER eiabea <developer@eiabea.com>
+FROM ubuntu:16.04
 
-# Install clone dependencies
-RUN set -e && \
-  apt-get update -q && \
-  apt-get install -q -y --no-install-recommends ca-certificates git && \
-  git clone https://github.com/monero-project/monero.git src && \
-  apt-get purge -y git && \
-  apt-get clean -q -y && \
-  apt-get autoclean -q -y && \
-  apt-get autoremove -q -y
+ENV SRC_DIR /usr/local/src/monero
 
-WORKDIR /src
+RUN set -x \
+  && buildDeps=' \
+      ca-certificates \
+      cmake \
+      g++ \
+      git \
+      libboost1.58-all-dev \
+      libssl-dev \
+      make \
+      pkg-config \
+  ' \
+  && apt-get -qq update \
+  && apt-get -qq --no-install-recommends install $buildDeps
 
-# Install make dependencies
-RUN set -e && \
-  apt-get update -q && \
-  apt-get install -q -y --no-install-recommends build-essential ca-certificates g++ gcc cmake \
-  pkg-config libunbound2 libevent-2.0-5 libgtest-dev libboost-all-dev libdb5.3++-dev libdb5.3-dev libssl1.0-dev && \
-  make -j 4 && \
-  apt-get purge -y g++ gcc cmake pkg-config && \
-  apt-get clean -q -y && \
-  apt-get autoclean -q -y && \
-  apt-get autoremove -q -y && \
-  mkdir /monero && \
-  mv /src/build/release/bin/* /monero && \
-  rm -rf /src
+RUN git clone https://github.com/monero-project/monero.git $SRC_DIR
+WORKDIR $SRC_DIR
+RUN make -j$(nproc) release-static
 
-WORKDIR /monero
+RUN cp build/release/bin/* /usr/local/bin/ \
+  \
+  && rm -r $SRC_DIR \
+  && apt-get -qq --auto-remove purge $buildDeps
 
 # Contains the blockchain
 VOLUME /root/.bitmonero
 
 # Generate your wallet via accessing the container and run:
 # cd /wallet
-# /./bitmonero/monero-wallet-cli
+# monero-wallet-cli
 VOLUME /wallet
 
 ENV LOG_LEVEL 0
@@ -46,4 +42,4 @@ ENV RPC_BIND_PORT 18081
 EXPOSE 18080
 EXPOSE 18081
 
-CMD ./monerod --log-level=$LOG_LEVEL --p2p-bind-ip=$P2P_BIND_IP --p2p-bind-port=$P2P_BIND_PORT --rpc-bind-ip=$RPC_BIND_IP --rpc-bind-port=$RPC_BIND_PORT
+CMD monerod --log-level=$LOG_LEVEL --p2p-bind-ip=$P2P_BIND_IP --p2p-bind-port=$P2P_BIND_PORT --rpc-bind-ip=$RPC_BIND_IP --rpc-bind-port=$RPC_BIND_PORT

Makefile

@@ -78,7 +78,7 @@ release-static-linux-armv7:
 
 release-static-android:
 	mkdir -p build/release
-	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android"../.. && $(MAKE)
+	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android" ../.. && $(MAKE)
 
 release-static-linux-armv8:
 	mkdir -p build/release

README.md

@@ -80,19 +80,18 @@ There are also several mining pools that kindly donate a portion of their fees,
 
 See [LICENSE](LICENSE).
 
-## Monero software updates and consensus protocol changes (hard forking)
+## Monero software updates and consensus protocol changes (hard fork schedule)
 
-Monero uses a hardforking mechanism to implement new features which requires that 
-users of Monero software run current versions and update their software on a 
-regular schedule. Here is the current schedule, versions, and compatibility.
-Dates are provided in the format YYYYMMDD. 
+Monero uses a fixed-schedule hard fork mechanism to implement new features. This means that users of Monero (end users and service providers) need to run current versions and update their software on a regular schedule. Here is the current schedule, versions, and compatibility.
+Dates are provided in the format YYYY-MM-DD. 
 
 
-| Date              | Consensus version | Minimum Monero Version | Recommended Monero Version | Details            |  
+| Fork Date              | Consensus version | Minimum Monero Version | Recommended Monero Version | Details            |  
 | ----------------- | ----------------- | ---------------------- | -------------------------- | ------------------ |
 | 2016-09-21        | v3                | v0.9.4                 | v0.10.0                    | Splits coinbase into denominations  |
-| 2017-01-05        | v4                | v0.10.1                 | v0.10.1                   | Allow normal and RingCT transactions | 
-| 2017-09-21        | v5                | v0.10.1                | v0.10.1                    | Allow only RingCT transactions      |
+| 2017-01-05        | v4                | v0.10.1                 | v0.10.2.1                   | Allow normal and RingCT transactions |
+| 2017-04-15        | v5                | v0.10.2.1               | v0.10.3                    | Adjusted minimum blocksize and fee algorithm      |
+| 2017-09-21        | v6                | Not determined as of 2017-03-06                | Not determined as of 2017-03-06                    | Allow only RingCT transactions      |
 
 ## Installing Monero from a Package
 
@@ -138,7 +137,6 @@ library archives (`.a`).
 | CMake          | 3.0.0         | NO       | `cmake`            | `cmake`        | NO       |                |
 | pkg-config     | any           | NO       | `pkg-config`       | `base-devel`   | NO       |                |
 | Boost          | 1.58          | NO       | `libboost-all-dev` | `boost`        | NO       | C++ libraries  |
-| libevent       | 2.0           | NO       | `libevent-dev`     | `libevent`     | NO       | Network IO     |
 | OpenSSL        | basically any | NO       | `libssl-dev`       | `openssl`      | NO       | sha256 sum     |
 | libunbound     | 1.4.16        | YES      | `libunbound-dev`   | `unbound`      | NO       | DNS resolver   |
 | libminiupnpc   | 2.0           | YES      | `libminiupnpc-dev` | `miniupnpc`    | YES      | NAT punching   |
@@ -291,7 +289,7 @@ application.
 
 ### On FreeBSD:
 
-The project can be built from scratch by following instructions for Linux above.
+The project can be built from scratch by following instructions for Linux above. If you are running monero in a jail you need to add the flag: `allow.sysvipc=1` to your jail configuration, otherwise lmdb will throw the error message: `Failed to open lmdb environment: Function not implemented`.
 
 We expect to add Monero into the ports tree in the near future, which will aid in managing installations using ports or packages.
 

contrib/epee/include/md5_l.inl

@@ -58,14 +58,11 @@ These notices must be retained in any copies of any part of this
 documentation and/or software.
 */
 
-/* do i need all of this just for htonl()? damn. */
-//#include <sys/types.h>
-//#include <sys/param.h>
-//#include <sys/socket.h>
-//#include <netinet/in.h>
-
-
-
+#ifdef _WIN32
+# include <winsock2.h>
+#else
+# include <arpa/inet.h>
+#endif
 #include "md5global.h"
 #include "md5_l.h"
 #include "hmac-md5.h"

contrib/epee/include/net/http_client.h

@@ -336,6 +336,11 @@ using namespace std;
         piece_of_transfer.clear();
 				return true;
 			}
+			//---------------------------------------------------------------------------
+			virtual bool on_header(const http_response_info &headers)
+      {
+        return true;
+      }
 			//---------------------------------------------------------------------------
 			inline 
 				bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
@@ -505,6 +510,12 @@ using namespace std;
 					m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end());
 
 					analize_cached_header_and_invoke_state();
+          if (!on_header(m_response_info))
+          {
+            MDEBUG("Connection cancelled by on_header");
+            m_state = reciev_machine_state_done;
+            return false;
+          }
 					m_header_cache.clear();
 					if(!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done))
 						need_more_data = true;
@@ -527,7 +538,11 @@ using namespace std;
 				}
 				CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()");
 				m_len_in_remain -= recv_buff.size();
-				m_pcontent_encoding_handler->update_in(recv_buff);
+				if (!m_pcontent_encoding_handler->update_in(recv_buff))
+				{
+					m_state = reciev_machine_state_done;
+					return false;
+				}
 
 				if(m_len_in_remain == 0)
 					m_state = reciev_machine_state_done;
@@ -700,7 +715,11 @@ using namespace std;
 								m_len_in_remain = 0;
 							}
 
-							m_pcontent_encoding_handler->update_in(chunk_body);
+							if (!m_pcontent_encoding_handler->update_in(chunk_body))
+							{
+								m_state = reciev_machine_state_error;
+								return false;
+							}
 
 							if(!m_len_in_remain)
 								m_chunked_state = http_chunked_state_chunk_head;

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -25,6 +25,7 @@
 // 
 
 #pragma once
+#include <boost/asio/deadline_timer.hpp>
 #include <boost/uuid/uuid_generators.hpp>
 #include <boost/unordered_map.hpp>
 #include <boost/interprocess/detail/atomic.hpp>

contrib/epee/include/net/net_utils_base.h

@@ -29,6 +29,7 @@
 #ifndef _NET_UTILS_BASE_H_
 #define _NET_UTILS_BASE_H_
 
+#include <boost/asio/io_service.hpp>
 #include <boost/uuid/uuid.hpp>
 #include "string_tools.h"
 #include "misc_log_ex.h"

contrib/epee/include/storages/http_abstract_invoke.h

@@ -38,7 +38,7 @@ namespace epee
   namespace net_utils
   {
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_json(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(5), const boost::string_ref method = "GET")
+    bool invoke_http_json(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "GET")
     {
       std::string req_param;
       if(!serialization::store_t_to_json(out_struct, req_param))
@@ -69,7 +69,7 @@ namespace epee
 
 
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(5), const boost::string_ref method = "GET")
+    bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "GET")
     {
       std::string req_param;
       if(!serialization::store_t_to_binary(out_struct, req_param))
@@ -98,7 +98,7 @@ namespace epee
     }
 
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(5), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
+    bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
     {
       epee::json_rpc::request<t_request> req_t = AUTO_VAL_INIT(req_t);
       req_t.jsonrpc = "2.0";
@@ -120,7 +120,7 @@ namespace epee
     }
 
     template<class t_command, class t_transport>
-    bool invoke_http_json_rpc(const boost::string_ref uri, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(5), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
+    bool invoke_http_json_rpc(const boost::string_ref uri, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
     {
       return invoke_http_json_rpc(uri, t_command::methodname(), out_struct, result_struct, transport, timeout, http_method, req_id);
     }

contrib/epee/include/string_tools.h

@@ -29,17 +29,21 @@
 #ifndef _STRING_TOOLS_H_
 #define _STRING_TOOLS_H_
 
+// Previously pulled in by ASIO, further cleanup still required ...
+#ifdef _WIN32
+# include <winsock2.h>
+# include <windows.h>
+#endif
+
 //#include <objbase.h>
 #include <locale>
 #include <cstdlib>
 #include <iomanip>
+#include <map>
 #include <type_traits>
-//#include <strsafe.h>
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/uuid_io.hpp>
 #include <boost/lexical_cast.hpp>
-#include <boost/asio.hpp>
-#include <boost/algorithm/string/compare.hpp>
 #include <boost/algorithm/string.hpp>
 #include "warnings.h"
 
@@ -327,26 +331,9 @@ POP_WARNINGS
   }
 
 	//----------------------------------------------------------------------------
-//#ifdef _WINSOCK2API_
-	inline std::string get_ip_string_from_int32(uint32_t ip)
-	{
-		in_addr adr;
-		adr.s_addr = ip;
-		const char* pbuf = inet_ntoa(adr);
-		if(pbuf)
-			return pbuf;
-		else
-			return "[failed]";
-	}
+	std::string get_ip_string_from_int32(uint32_t ip);
 	//----------------------------------------------------------------------------
-	inline bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str)
-	{
-		ip = inet_addr(ip_str.c_str());
-		if(INADDR_NONE == ip)
-			return false;
-
-		return true;
-	}
+	bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str);
   //----------------------------------------------------------------------------
   inline bool parse_peer_from_string(uint32_t& ip, uint32_t& port, const std::string& addres)
   {
@@ -376,7 +363,6 @@ POP_WARNINGS
     return true;
   }
 
-//#endif
 	//----------------------------------------------------------------------------
   template<typename t>
 	inline std::string get_t_as_hex_nwidth(const t& v, std::streamsize w = 8)

contrib/epee/src/CMakeLists.txt

@@ -26,8 +26,7 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-add_library(epee STATIC http_auth.cpp mlog.cpp)
-
+add_library(epee STATIC http_auth.cpp mlog.cpp string_tools.cpp)
 # Build and install libepee if we're building for GUI
 if (BUILD_GUI_DEPS)
     if(IOS)

contrib/epee/src/mlog.cpp

@@ -88,7 +88,7 @@ static const char *get_default_categories(int level)
   switch (level)
   {
     case 0:
-      categories = "*:WARNING,net:FATAL,net.p2p:FATAL,global:INFO,verify:FATAL,stacktrace:INFO";
+      categories = "*:WARNING,net:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,stacktrace:INFO";
       break;
     case 1:
       categories = "*:WARNING,global:INFO,stacktrace:INFO";

contrib/epee/src/string_tools.cpp

@@ -0,0 +1,61 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+#include "string_tools.h"
+
+#ifdef _WIN32
+# include <winsock2.h>
+#else
+# include <arpa/inet.h>
+# include <netinet/in.h>
+#endif
+
+namespace epee
+{
+namespace string_tools
+{
+  std::string get_ip_string_from_int32(uint32_t ip)
+  {
+    in_addr adr;
+    adr.s_addr = ip;
+    const char* pbuf = inet_ntoa(adr);
+    if(pbuf)
+      return pbuf;
+    else
+      return "[failed]";
+  }
+  //----------------------------------------------------------------------------
+  bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str)
+  {
+    ip = inet_addr(ip_str.c_str());
+    if(INADDR_NONE == ip)
+      return false;
+
+    return true;
+  }
+}
+}
+

contrib/snap/daemon.bash

@@ -1,12 +0,0 @@
-#!/bin/bash -e
-
-export LD_LIBRARY_PATH=${SNAP_LIBRARY_PATH}:${SNAP}/usr/lib/x86_64-linux-gnu
-export HOME=${SNAP_DATA}
-cd ${SNAP_DATA}
-
-ARGS=
-if [ -e "${SNAP_DATA}/etc/monerod.conf" ]; then
-	ARGS="--config-file ${SNAP_DATA}/etc/monerod.conf"
-fi
-
-exec ${SNAP}/bin/monerod --detach $ARGS

contrib/snap/log.bash

@@ -1,3 +0,0 @@
-#!/bin/bash -e
-
-exec tail -c +0 -F ${SNAP_DATA}/.bitmonero/bitmonero.log

contrib/snap/monerod-wrapper

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [ ! -d "$SNAP_USER_DATA/etc" ]; then
+  mkdir $SNAP_USER_DATA/etc/
+  cp -R $SNAP/etc/monerod.conf $SNAP_USER_DATA/etc/monerod.conf
+fi
+
+exec "$SNAP/bin/monerod" "$@"

contrib/snap/monerod.conf

@@ -0,0 +1,9 @@
+# Configuration for monerod
+# Syntax: any command line option may be specified as 'clioptionname=value'.
+# See 'monerod --help' for all available options.
+
+# Overrided by snap:
+# data-dir=/var/lib/monero
+# log-file=/var/log/monero/monero.log
+
+log-level=0

contrib/snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: monero
-version: 0.10.1 # Current stable version
+version: 0.10.2-1
 summary: "Monero: the secure, private, untraceable cryptocurrency https://getmonero.org"
 description: |
     Monero is a private, secure, untraceable, decentralised digital currency.
@@ -12,20 +12,20 @@ apps:
     monerod:
         daemon: forking
         command: |
-          monerod --detach --data-dir ${SNAP_DATA}
+          monerod-wrapper --detach --data-dir ${SNAP_COMMON} --config-file ${SNAP_USER_DATA}/etc/monerod.conf
         plugs:
             - network
             - network-bind
     monero-wallet-rpc:
         command: |
-            monero-wallet-rpc --log-file ${SNAP_USER_DATA}
+          monero-wallet-rpc --log-file ${SNAP_USER_DATA}
         plugs:
             - home
             - network
             - network-bind
     monero-wallet-cli:
         command: |
-            monero-wallet-cli --log-file ${SNAP_USER_DATA}
+          monero-wallet-cli --log-file ${SNAP_USER_DATA}
         plugs:
             - home
             - network
@@ -63,3 +63,13 @@ parts:
             - usr/lib/
             - -usr/lib/gcc
             - -usr/share
+
+    dist-files:
+        plugin: dump
+        source: .
+        organize:
+            contrib/snap/monerod.conf: etc/monerod.conf
+            contrib/snap/monerod-wrapper: bin/monerod-wrapper
+        prime:
+            - etc
+            - bin

contrib/snap/wallet.bash

@@ -1,7 +0,0 @@
-#!/bin/bash -e
-
-export LD_LIBRARY_PATH=${SNAP_LIBRARY_PATH}:${SNAP}/usr/lib/x86_64-linux-gnu
-export HOME=${SNAP_USER_DATA}
-cd ${SNAP_USER_DATA}
-
-exec ${SNAP}/usr/bin/rlwrap ${SNAP}/bin/monero-wallet-cli "$@"

snap

@@ -0,0 +1 @@
+contrib/snap

src/blockchain_db/blockchain_db.h

@@ -768,7 +768,7 @@ public:
    *
    * @return the block requested
    */
-  block get_block(const crypto::hash& h) const;
+  virtual block get_block(const crypto::hash& h) const;
 
   /**
    * @brief gets the height of the block with a given hash
@@ -821,7 +821,7 @@ public:
    *
    * @return the block
    */
-  block get_block_from_height(const uint64_t& height) const;
+  virtual block get_block_from_height(const uint64_t& height) const;
 
   /**
    * @brief fetch a block's timestamp
@@ -1041,7 +1041,7 @@ public:
    *
    * @return the transaction with the given hash
    */
-  transaction get_tx(const crypto::hash& h) const;
+  virtual transaction get_tx(const crypto::hash& h) const;
 
   /**
    * @brief fetches the transaction with the given hash
@@ -1052,7 +1052,7 @@ public:
    *
    * @return true iff the transaction was found
    */
-  bool get_tx(const crypto::hash& h, transaction &tx) const;
+  virtual bool get_tx(const crypto::hash& h, transaction &tx) const;
 
   /**
    * @brief fetches the transaction blob with the given hash

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -749,7 +749,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
   uint64_t m_height = height();
 
   int result;
-  uint64_t tx_id = num_txs();
+  uint64_t tx_id = get_tx_count();
 
   CURSOR(txs)
   CURSOR(tx_indices)
@@ -1741,20 +1741,6 @@ uint64_t BlockchainLMDB::height() const
   return db_stats.ms_entries;
 }
 
-uint64_t BlockchainLMDB::num_txs() const
-{
-  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-  check_open();
-  TXN_PREFIX_RDONLY();
-  int result;
-
-  // get current height
-  MDB_stat db_stats;
-  if ((result = mdb_stat(m_txn, m_txs, &db_stats)))
-    throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str()));
-  return db_stats.ms_entries;
-}
-
 uint64_t BlockchainLMDB::num_outputs() const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -1900,10 +1886,11 @@ uint64_t BlockchainLMDB::get_tx_count() const
   check_open();
 
   TXN_PREFIX_RDONLY();
+  int result;
 
   MDB_stat db_stats;
-  if (mdb_stat(m_txn, m_tx_indices, &db_stats))
-    throw0(DB_ERROR("Failed to query m_tx_indices"));
+  if ((result = mdb_stat(m_txn, m_txs, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str()));
 
   TXN_POSTFIX_RDONLY();
 
@@ -2516,7 +2503,8 @@ void BlockchainLMDB::block_txn_start(bool readonly)
       throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str()));
     }
     memset(&m_wcursors, 0, sizeof(m_wcursors));
-  }
+  } else if (m_writer != boost::this_thread::get_id())
+    throw0(DB_ERROR_TXN_START((std::string("Attempted to start new write txn when batch txn already exists in ")+__FUNCTION__).c_str()));
 }
 
 void BlockchainLMDB::block_txn_stop()

src/blockchain_db/lmdb/db_lmdb.h

@@ -310,7 +310,6 @@ private:
 
   virtual void remove_spent_key(const crypto::key_image& k_image);
 
-  uint64_t num_txs() const;
   uint64_t num_outputs() const;
 
   // Hard fork

src/blockchain_utilities/CMakeLists.txt

@@ -119,6 +119,7 @@ monero_add_executable(cn_deserialize
 target_link_libraries(cn_deserialize
   LINK_PRIVATE
     cryptonote_core
+    blockchain_db
 	p2p
     epee
     ${CMAKE_THREAD_LIBS_INIT})

src/blockchain_utilities/blockchain_export.cpp

@@ -57,6 +57,8 @@ std::string join_set_strings(const std::unordered_set<std::string>& db_types_all
 
 int main(int argc, char* argv[])
 {
+  TRY_ENTRY();
+
   epee::string_tools::set_module_name_and_folder(argv[0]);
 
   std::string default_db_type = "lmdb";
@@ -80,7 +82,7 @@ int main(int argc, char* argv[])
   po::options_description desc_cmd_only("Command line options");
   po::options_description desc_cmd_sett("Command line options and settings options");
   const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true};
-  const command_line::arg_descriptor<uint32_t> arg_log_level  = {"log-level",  "", log_level};
+  const command_line::arg_descriptor<std::string> arg_log_level  = {"log-level",  "0-4 or categories", ""};
   const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
   const command_line::arg_descriptor<bool>     arg_testnet_on = {
     "testnet"
@@ -124,11 +126,13 @@ int main(int argc, char* argv[])
     return 1;
   }
 
-  log_level    = command_line::get_arg(vm, arg_log_level);
+  mlog_configure(mlog_get_default_log_path("monero-blockchain-export.log"), true);
+  if (!vm["log-level"].defaulted())
+    mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
+  else
+    mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
   block_stop = command_line::get_arg(vm, arg_block_stop);
 
-  mlog_configure(mlog_get_default_log_path("monero-blockchain-export.log"), true);
-  mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
   LOG_PRINT_L0("Starting...");
 
   bool opt_testnet = command_line::get_arg(vm, arg_testnet_on);
@@ -226,4 +230,7 @@ int main(int argc, char* argv[])
   }
   CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data");
   LOG_PRINT_L0("Blockchain raw data exported OK");
+  return 0;
+
+  CATCH_ENTRY("Export error", 1);
 }

src/blockchain_utilities/blockchain_import.cpp

@@ -596,6 +596,8 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
 
 int main(int argc, char* argv[])
 {
+  TRY_ENTRY();
+
   epee::string_tools::set_module_name_and_folder(argv[0]);
 
   std::string default_db_type = "lmdb";
@@ -622,7 +624,7 @@ int main(int argc, char* argv[])
   po::options_description desc_cmd_only("Command line options");
   po::options_description desc_cmd_sett("Command line options and settings options");
   const command_line::arg_descriptor<std::string> arg_input_file = {"input-file", "Specify input file", "", true};
-  const command_line::arg_descriptor<uint32_t> arg_log_level   = {"log-level",  "", log_level};
+  const command_line::arg_descriptor<std::string> arg_log_level   = {"log-level",  "0-4 or categories", ""};
   const command_line::arg_descriptor<uint64_t> arg_block_stop  = {"block-stop", "Stop at block number", block_stop};
   const command_line::arg_descriptor<uint64_t> arg_batch_size  = {"batch-size", "", db_batch_size};
   const command_line::arg_descriptor<uint64_t> arg_pop_blocks  = {"pop-blocks", "Remove blocks from end of blockchain", num_blocks};
@@ -682,7 +684,6 @@ int main(int argc, char* argv[])
   if (! r)
     return 1;
 
-  log_level     = command_line::get_arg(vm, arg_log_level);
   opt_verify    = command_line::get_arg(vm, arg_verify);
   opt_batch     = command_line::get_arg(vm, arg_batch);
   opt_resume    = command_line::get_arg(vm, arg_resume);
@@ -725,7 +726,11 @@ int main(int argc, char* argv[])
   db_arg_str = command_line::get_arg(vm, arg_database);
 
   mlog_configure(mlog_get_default_log_path("monero-blockchain-import.log"), true);
-  mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+  if (!vm["log-level"].defaulted())
+    mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
+  else
+    mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+
   MINFO("Starting...");
 
   boost::filesystem::path fs_import_file_path;
@@ -762,7 +767,11 @@ int main(int argc, char* argv[])
     return 1;
   }
 
-  if ((db_type == "lmdb") || (db_type == "berkeley"))
+  if ((db_type == "lmdb")
+#if defined(BERKELEY_DB)
+   || (db_type == "berkeley")
+#endif
+  )
   {
     db_engine_compiled = "blockchain_db";
   }
@@ -799,13 +808,11 @@ int main(int argc, char* argv[])
   // properties to do so. Both ways work, but fake core isn't necessary in that
   // circumstance.
 
-  // for multi_db_runtime:
-  if (db_type == "lmdb" || db_type == "berkeley")
-  {
-    fake_core_db simple_core(m_config_folder, opt_testnet, opt_batch, db_type, db_flags);
-    import_from_file(simple_core, import_file_path, block_stop);
-  }
-  else
+  if (db_type != "lmdb"
+#if defined(BERKELEY_DB)
+  && db_type != "berkeley"
+#endif
+  )
   {
     std::cerr << "database type unrecognized" << ENDL;
     return 1;
@@ -850,4 +857,6 @@ int main(int argc, char* argv[])
   // calls delete on its BlockchainDB derived class' object, which closes its
   // files.
   return 0;
+
+  CATCH_ENTRY("Import error", 1);
 }

src/blocks/CMakeLists.txt

@@ -30,8 +30,8 @@ if(APPLE)
     add_library(blocks STATIC blockexports.c)
     set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C)
 else()
-    add_custom_command(OUTPUT blocks.o COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocks.o blocks.dat)
-    add_custom_command(OUTPUT testnet_blocks.o COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/testnet_blocks.o testnet_blocks.dat)
+    add_custom_command(OUTPUT blocks.o MAIN_DEPENDENCY blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocks.o blocks.dat)
+    add_custom_command(OUTPUT testnet_blocks.o MAIN_DEPENDENCY testnet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/testnet_blocks.o testnet_blocks.dat)
     add_library(blocks STATIC blocks.o testnet_blocks.o blockexports.c)
     set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C)
 endif()

src/common/command_line.h

@@ -168,7 +168,7 @@ namespace command_line
     {
       return parser();
     }
-    catch (std::exception& e)
+    catch (const std::exception& e)
     {
       std::cerr << "Failed to parse arguments: " << e.what() << std::endl;
       std::cerr << desc << std::endl;

src/common/dns_utils.cpp

@@ -405,21 +405,23 @@ std::vector<std::string> addresses_from_url(const std::string& url, bool& dnssec
   return addresses;
 }
 
-std::string get_account_address_as_str_from_url(const std::string& url, bool& dnssec_valid)
+std::string get_account_address_as_str_from_url(const std::string& url, bool& dnssec_valid, bool cli_confirm)
 {
   // attempt to get address from dns query
   auto addresses = addresses_from_url(url, dnssec_valid);
   if (addresses.empty())
   {
-    std::cout << tr("wrong address: ") << url;
+    LOG_ERROR("wrong address: " << url);
     return {};
   }
   // for now, move on only if one address found
   if (addresses.size() > 1)
   {
-    std::cout << tr("not yet supported: Multiple Monero addresses found for given URL: ") << url;
+    LOG_ERROR("not yet supported: Multiple Monero addresses found for given URL: " << url);
     return {};
   }
+  if (!cli_confirm)
+    return addresses[0];
   // prompt user for confirmation.
   // inform user of DNSSEC validation status as well.
   std::string dnssec_str;

src/common/dns_utils.h

@@ -163,7 +163,7 @@ namespace dns_utils
 std::string address_from_txt_record(const std::string& s);
 std::vector<std::string> addresses_from_url(const std::string& url, bool& dnssec_valid);
 
-std::string get_account_address_as_str_from_url(const std::string& url, bool& dnssec_valid);
+std::string get_account_address_as_str_from_url(const std::string& url, bool& dnssec_valid, bool cli_confirm = true);
 
 bool load_txt_records_from_dns(std::vector<std::string> &records, const std::vector<std::string> &dns_urls);
 

src/common/download.cpp

@@ -27,6 +27,7 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <string>
+#include <atomic>
 #include <boost/filesystem.hpp>
 #include <boost/asio.hpp>
 #include <boost/thread/thread.hpp>
@@ -40,27 +41,82 @@
 
 namespace tools
 {
-  static bool download_thread(const std::string &path, const std::string &url)
+  struct download_thread_control
   {
+    const std::string path;
+    const std::string uri;
+    std::function<void(const std::string&, const std::string&, bool)> result_cb;
+    std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb;
+    bool stop;
+    bool stopped;
+    bool success;
+    boost::thread thread;
+    boost::mutex mutex;
+
+    download_thread_control(const std::string &path, const std::string &uri, std::function<void(const std::string&, const std::string&, bool)> result_cb, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb):
+        path(path), uri(uri), result_cb(result_cb), progress_cb(progress_cb), stop(false), stopped(false), success(false) {}
+    ~download_thread_control() { if (thread.joinable()) thread.detach(); }
+  };
+
+  static void download_thread(download_async_handle control)
+  {
+    static std::atomic<unsigned int> thread_id(0);
+
+    MLOG_SET_THREAD_NAME("DL" + std::to_string(thread_id++));
+
+    struct stopped_setter
+    {
+      stopped_setter(const download_async_handle &control): control(control) {}
+      ~stopped_setter() { control->stopped = true; }
+      download_async_handle control;
+    } stopped_setter(control);
+
     try
     {
-      MINFO("Downloading " << url << " to " << path);
+      boost::unique_lock<boost::mutex> lock(control->mutex);
+      MINFO("Downloading " << control->uri << " to " << control->path);
       std::ofstream f;
-      f.open(path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
+      f.open(control->path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
       if (!f.good()) {
-        MERROR("Failed to open file " << path);
-        return false;
+        MERROR("Failed to open file " << control->path);
+        control->result_cb(control->path, control->uri, control->success);
+        return;
       }
       class download_client: public epee::net_utils::http::http_simple_client
       {
       public:
-        download_client(std::ofstream &f): f(f) {}
+        download_client(download_async_handle control, std::ofstream &f):
+          control(control), f(f), content_length(-1), total(0) {}
         virtual ~download_client() { f.close(); }
+        virtual bool on_header(const epee::net_utils::http::http_response_info &headers)
+        {
+          ssize_t length;
+          if (epee::string_tools::get_xtype_from_string(length, headers.m_header_info.m_content_length) && length >= 0)
+          {
+            MINFO("Content-Length: " << length);
+            content_length = length;
+            boost::filesystem::path path(control->path);
+            boost::filesystem::space_info si = boost::filesystem::space(path);
+            if (si.available < (size_t)content_length)
+            {
+              const uint64_t avail = (si.available + 1023) / 1024, needed = (content_length + 1023) / 1024;
+              MERROR("Not enough space to download " << needed << " kB to " << path << " (" << avail << " kB available)");
+              return false;
+            }
+          }
+          return true;
+        }
         virtual bool handle_target_data(std::string &piece_of_transfer)
         {
           try
           {
+            boost::lock_guard<boost::mutex> lock(control->mutex);
+            if (control->stop)
+              return false;
             f << piece_of_transfer;
+            total += piece_of_transfer.size();
+            if (control->progress_cb && !control->progress_cb(control->path, control->uri, total, content_length))
+              return false;
             return f.good();
           }
           catch (const std::exception &e)
@@ -70,69 +126,145 @@ namespace tools
           }
         }
       private:
+        download_async_handle control;
         std::ofstream &f;
-      } client(f);
+        ssize_t content_length;
+        size_t total;
+      } client(control, f);
       epee::net_utils::http::url_content u_c;
-      if (!epee::net_utils::parse_url(url, u_c))
+      if (!epee::net_utils::parse_url(control->uri, u_c))
       {
-        MWARNING("Failed to parse URL " << url);
-        return false;
+        MERROR("Failed to parse URL " << control->uri);
+        control->result_cb(control->path, control->uri, control->success);
+        return;
       }
       if (u_c.host.empty())
       {
-        MWARNING("Failed to determine address from URL " << url);
-        return false;
+        MERROR("Failed to determine address from URL " << control->uri);
+        control->result_cb(control->path, control->uri, control->success);
+        return;
       }
+
+      lock.unlock();
+
       uint16_t port = u_c.port ? u_c.port : 80;
       MDEBUG("Connecting to " << u_c.host << ":" << port);
       client.set_server(u_c.host, std::to_string(port), boost::none);
       if (!client.connect(std::chrono::seconds(30)))
       {
-        MERROR("Failed to connect to " << url);
-        return false;
+        boost::lock_guard<boost::mutex> lock(control->mutex);
+        MERROR("Failed to connect to " << control->uri);
+        control->result_cb(control->path, control->uri, control->success);
+        return;
       }
       MDEBUG("GETting " << u_c.uri);
       const epee::net_utils::http::http_response_info *info = NULL;
       if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info))
       {
-        MERROR("Failed to connect to " << url);
+        boost::lock_guard<boost::mutex> lock(control->mutex);
+        MERROR("Failed to connect to " << control->uri);
         client.disconnect();
-        return false;
+        control->result_cb(control->path, control->uri, control->success);
+        return;
+      }
+      if (control->stop)
+      {
+        boost::lock_guard<boost::mutex> lock(control->mutex);
+        MDEBUG("Download cancelled");
+        client.disconnect();
+        control->result_cb(control->path, control->uri, control->success);
+        return;
       }
       if (!info)
       {
-        MERROR("Failed invoking GET command to " << url << ", no status info returned");
+        boost::lock_guard<boost::mutex> lock(control->mutex);
+        MERROR("Failed invoking GET command to " << control->uri << ", no status info returned");
         client.disconnect();
-        return false;
+        control->result_cb(control->path, control->uri, control->success);
+        return;
       }
       MDEBUG("response code: " << info->m_response_code);
+      MDEBUG("response length: " << info->m_header_info.m_content_length);
       MDEBUG("response comment: " << info->m_response_comment);
       MDEBUG("response body: " << info->m_body);
       for (const auto &f: info->m_additional_fields)
         MDEBUG("additional field: " << f.first << ": " << f.second);
       if (info->m_response_code != 200)
       {
+        boost::lock_guard<boost::mutex> lock(control->mutex);
         MERROR("Status code " << info->m_response_code);
         client.disconnect();
-        return false;
+        control->result_cb(control->path, control->uri, control->success);
+        return;
       }
       client.disconnect();
       f.close();
       MDEBUG("Download complete");
-      return true;
+      lock.lock();
+      control->success = true;
+      control->result_cb(control->path, control->uri, control->success);
+      return;
     }
     catch (const std::exception &e)
     {
       MERROR("Exception in download thread: " << e.what());
-      return false;
+      // fall through and call result_cb not from the catch block to avoid another exception
     }
+    boost::lock_guard<boost::mutex> lock(control->mutex);
+    control->result_cb(control->path, control->uri, control->success);
   }
 
-  bool download(const std::string &path, const std::string &url)
+  bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> cb)
   {
-    bool success;
-    std::unique_ptr<boost::thread> thread(new boost::thread([&](){ success = download_thread(path, url); }));
-    thread->join();
+    bool success = false;
+    download_async_handle handle = download_async(path, url, [&success](const std::string&, const std::string&, bool result) {success = result;}, cb);
+    download_wait(handle);
     return success;
   }
+
+  download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress)
+  {
+    download_async_handle control = std::make_shared<download_thread_control>(path, url, result, progress);
+    control->thread = boost::thread([control](){ download_thread(control); });
+    return control;
+  }
+
+  bool download_finished(const download_async_handle &control)
+  {
+    CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
+    boost::lock_guard<boost::mutex> lock(control->mutex);
+    return control->stopped;
+  }
+
+  bool download_error(const download_async_handle &control)
+  {
+    CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
+    boost::lock_guard<boost::mutex> lock(control->mutex);
+    return !control->success;
+  }
+
+  bool download_wait(const download_async_handle &control)
+  {
+    CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
+    {
+      boost::lock_guard<boost::mutex> lock(control->mutex);
+      if (control->stopped)
+        return true;
+    }
+    control->thread.join();
+    return true;
+  }
+
+  bool download_cancel(const download_async_handle &control)
+  {
+    CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
+    {
+      boost::lock_guard<boost::mutex> lock(control->mutex);
+      if (control->stopped)
+        return true;
+      control->stop = true;
+    }
+    control->thread.join();
+    return true;
+  }
 }

src/common/download.h

@@ -32,5 +32,13 @@
 
 namespace tools
 {
-  bool download(const std::string &path, const std::string &url);
+  struct download_thread_control;
+  typedef std::shared_ptr<download_thread_control> download_async_handle;
+
+  bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress = NULL);
+  download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress = NULL);
+  bool download_error(const download_async_handle &h);
+  bool download_finished(const download_async_handle &h);
+  bool download_wait(const download_async_handle &h);
+  bool download_cancel(const download_async_handle &h);
 }

src/common/util.cpp

@@ -587,6 +587,18 @@ std::string get_nix_version_display_string()
     return 0;
   }
 
+  bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash)
+  {
+    SHA256_CTX ctx;
+    if (!SHA256_Init(&ctx))
+      return false;
+    if (!SHA256_Update(&ctx, data, len))
+      return false;
+    if (!SHA256_Final((unsigned char*)hash.data, &ctx))
+      return false;
+    return true;
+  }
+
   bool sha256sum(const std::string &filename, crypto::hash &hash)
   {
     if (!epee::file_io_utils::is_file_exist(filename))

src/common/util.h

@@ -185,5 +185,6 @@ namespace tools
   bool is_local_address(const std::string &address);
   int vercmp(const char *v0, const char *v1); // returns < 0, 0, > 0, similar to strcmp, but more human friendly than lexical - does not attempt to validate
 
+  bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash);
   bool sha256sum(const std::string &filename, crypto::hash &hash);
 }

src/crypto/slow-hash.c

@@ -494,7 +494,7 @@ void slow_hash_free_state(void)
  * buffer of pseudorandom data by hashing the supplied data.  It then uses this
  * random data to fill a large 2MB buffer with pseudorandom data by iteratively
  * encrypting it using 10 rounds of AES per entry.  After this initialization,
- * it executes 500,000 rounds of mixing through the random 2MB buffer using
+ * it executes 524,288 rounds of mixing through the random 2MB buffer using
  * AES (typically provided in hardware on modern CPUs) and a 64 bit multiply.
  * Finally, it re-mixes this large buffer back into
  * the 200 byte "text" buffer, and then hashes this buffer using one of four
@@ -530,7 +530,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash)
 
     size_t i, j;
     uint64_t *p = NULL;
-    oaes_ctx *aes_ctx;
+    oaes_ctx *aes_ctx = NULL;
     int useAes = !force_software_aes() && check_aes_hw();
 
     static void (*const extra_hashes[4])(const void *, size_t, char *) =
@@ -578,8 +578,8 @@ void cn_slow_hash(const void *data, size_t length, char *hash)
     U64(b)[0] = U64(&state.k[16])[0] ^ U64(&state.k[48])[0];
     U64(b)[1] = U64(&state.k[16])[1] ^ U64(&state.k[48])[1];
 
-    /* CryptoNight Step 3:  Bounce randomly 1 million times through the mixing buffer,
-     * using 500,000 iterations of the following mixing function.  Each execution
+    /* CryptoNight Step 3:  Bounce randomly 1,048,576 times (1<<20) through the mixing buffer,
+     * using 524,288 iterations of the following mixing function.  Each execution
      * performs two reads and writes from the mixing buffer.
      */
 
@@ -895,8 +895,8 @@ void cn_slow_hash(const void *data, size_t length, char *hash)
     U64(b)[0] = U64(&state.k[16])[0] ^ U64(&state.k[48])[0];
     U64(b)[1] = U64(&state.k[16])[1] ^ U64(&state.k[48])[1];
 
-    /* CryptoNight Step 3:  Bounce randomly 1 million times through the mixing buffer,
-     * using 500,000 iterations of the following mixing function.  Each execution
+    /* CryptoNight Step 3:  Bounce randomly 1,048,576 times (1<<20) through the mixing buffer,
+     * using 524,288 iterations of the following mixing function.  Each execution
      * performs two reads and writes from the mixing buffer.
      */
 

src/cryptonote_basic/CMakeLists.txt

@@ -32,8 +32,8 @@ set(cryptonote_basic_sources
   cryptonote_basic_impl.cpp
   cryptonote_format_utils.cpp
   difficulty.cpp
-  miner.cpp
-  hardfork.cpp)
+  hardfork.cpp
+  miner.cpp)
 
 set(cryptonote_basic_headers)
 
@@ -48,10 +48,10 @@ set(cryptonote_basic_private_headers
   cryptonote_format_utils.h
   cryptonote_stat_info.h
   difficulty.h
+  hardfork.h
   miner.h
   tx_extra.h
-  verification_context.h
-  hardfork.h)
+  verification_context.h)
 
 monero_private_headers(cryptonote_basic
   ${crypto_private_headers})

src/cryptonote_basic/cryptonote_basic.h

@@ -35,6 +35,7 @@
 #include <vector>
 #include <cstring>  // memcmp
 #include <sstream>
+#include <atomic>
 #include "serialization/serialization.h"
 #include "serialization/variant.h"
 #include "serialization/vector.h"
@@ -186,15 +187,37 @@ namespace cryptonote
 
   class transaction: public transaction_prefix
   {
+  private:
+    // hash cash
+    mutable std::atomic<bool> hash_valid;
+    mutable std::atomic<bool> blob_size_valid;
+
   public:
     std::vector<std::vector<crypto::signature> > signatures; //count signatures  always the same as inputs count
     rct::rctSig rct_signatures;
 
+    // hash cash
+    mutable crypto::hash hash;
+    mutable size_t blob_size;
+
     transaction();
+    transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } }
+    transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } return *this; }
     virtual ~transaction();
     void set_null();
+    void invalidate_hashes();
+    bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
+    void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
+    bool is_blob_size_valid() const { return hash_valid.load(std::memory_order_acquire); }
+    void set_blob_size_valid(bool v) const { blob_size_valid.store(v,std::memory_order_release); }
 
     BEGIN_SERIALIZE_OBJECT()
+      if (!typename Archive<W>::is_saving())
+      {
+        set_hash_valid(false);
+        set_blob_size_valid(false);
+      }
+
       FIELDS(*static_cast<transaction_prefix *>(this))
 
       if (version == 1)
@@ -250,6 +273,28 @@ namespace cryptonote
       }
     END_SERIALIZE()
 
+    template<bool W, template <bool> class Archive>
+    bool serialize_base(Archive<W> &ar)
+    {
+      FIELDS(*static_cast<transaction_prefix *>(this))
+
+      if (version == 1)
+      {
+      }
+      else
+      {
+        ar.tag("rct_signatures");
+        if (!vin.empty())
+        {
+          ar.begin_object();
+          bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size());
+          if (!r || !ar.stream().good()) return false;
+          ar.end_object();
+        }
+      }
+      return true;
+    }
+
   private:
     static size_t get_signature_size(const txin_v& tx_in);
   };
@@ -277,6 +322,15 @@ namespace cryptonote
     extra.clear();
     signatures.clear();
     rct_signatures.type = rct::RCTTypeNull;
+    set_hash_valid(false);
+    set_blob_size_valid(false);
+  }
+
+  inline
+  void transaction::invalidate_hashes()
+  {
+    set_hash_valid(false);
+    set_blob_size_valid(false);
   }
 
   inline
@@ -317,10 +371,28 @@ namespace cryptonote
 
   struct block: public block_header
   {
+  private:
+    // hash cash
+    mutable std::atomic<bool> hash_valid;
+
+  public:
+    block(): block_header(), hash_valid(false) {}
+    block(const block &b): block_header(b), hash_valid(false), miner_tx(b.miner_tx), tx_hashes(b.tx_hashes) { if (b.is_hash_valid()) { hash = b.hash; set_hash_valid(true); } }
+    block &operator=(const block &b) { block_header::operator=(b); hash_valid = false; miner_tx = b.miner_tx; tx_hashes = b.tx_hashes; if (b.is_hash_valid()) { hash = b.hash; set_hash_valid(true); } return *this; }
+    void invalidate_hashes() { set_hash_valid(false); }
+    bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
+    void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
+
     transaction miner_tx;
     std::vector<crypto::hash> tx_hashes;
 
+    // hash cash
+    mutable crypto::hash hash;
+
     BEGIN_SERIALIZE_OBJECT()
+      if (!typename Archive<W>::is_saving())
+        set_hash_valid(false);
+
       FIELDS(*static_cast<block_header *>(this))
       FIELD(miner_tx)
       FIELD(tx_hashes)

src/cryptonote_basic/cryptonote_basic_impl.cpp

@@ -67,6 +67,15 @@ namespace cryptonote {
   /* Cryptonote helper functions                                          */
   /************************************************************************/
   //-----------------------------------------------------------------------------------------------
+  size_t get_min_block_size(uint8_t version)
+  {
+    if (version < 2)
+      return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1;
+    if (version < 5)
+      return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2;
+    return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5;
+  }
+  //-----------------------------------------------------------------------------------------------
   size_t get_max_block_size()
   {
     return CRYPTONOTE_MAX_BLOCK_SIZE;
@@ -89,7 +98,7 @@ namespace cryptonote {
       base_reward = FINAL_SUBSIDY_PER_MINUTE*target_minutes;
     }
 
-    uint64_t full_reward_zone = version < 2 ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 : CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2;
+    uint64_t full_reward_zone = get_min_block_size(version);
 
     //make it soft
     if (median_size < full_reward_zone) {
@@ -299,12 +308,13 @@ namespace cryptonote {
     , crypto::hash8& payment_id
     , bool testnet
     , const std::string& str_or_url
+    , bool cli_confirm
     )
   {
     if (get_account_integrated_address_from_str(address, has_payment_id, payment_id, testnet, str_or_url))
       return true;
     bool dnssec_valid;
-    std::string address_str = tools::dns_utils::get_account_address_as_str_from_url(str_or_url, dnssec_valid);
+    std::string address_str = tools::dns_utils::get_account_address_as_str_from_url(str_or_url, dnssec_valid, cli_confirm);
     return !address_str.empty() &&
       get_account_integrated_address_from_str(address, has_payment_id, payment_id, testnet, address_str);
   }
@@ -313,11 +323,12 @@ namespace cryptonote {
       cryptonote::account_public_address& address
     , bool testnet
     , const std::string& str_or_url
+    , bool cli_confirm
     )
   {
     bool has_payment_id;
     crypto::hash8 payment_id;
-    return get_account_address_from_str_or_url(address, has_payment_id, payment_id, testnet, str_or_url);
+    return get_account_address_from_str_or_url(address, has_payment_id, payment_id, testnet, str_or_url, cli_confirm);
   }
   //--------------------------------------------------------------------------------
   bool operator ==(const cryptonote::transaction& a, const cryptonote::transaction& b) {

src/cryptonote_basic/cryptonote_basic_impl.h

@@ -69,6 +69,7 @@ namespace cryptonote {
   /************************************************************************/
   /* Cryptonote helper functions                                          */
   /************************************************************************/
+  size_t get_min_block_size(uint8_t version);
   size_t get_max_block_size();
   size_t get_max_tx_size();
   bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version);
@@ -106,12 +107,14 @@ namespace cryptonote {
     , crypto::hash8& payment_id
     , bool testnet
     , const std::string& str_or_url
+    , bool cli_confirm = true
     );
 
   bool get_account_address_from_str_or_url(
       cryptonote::account_public_address& address
     , bool testnet
     , const std::string& str_or_url
+    , bool cli_confirm = true
     );
 
   bool is_coinbase(const transaction& tx);

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -31,6 +31,7 @@
 #include "include_base_utils.h"
 using namespace epee;
 
+#include <atomic>
 #include "cryptonote_format_utils.h"
 #include "cryptonote_config.h"
 #include "crypto/crypto.h"
@@ -42,6 +43,8 @@ using namespace epee;
 
 #define ENCRYPTED_PAYMENT_ID_TAIL 0x8d
 
+// #define ENABLE_HASH_CASH_INTEGRITY_CHECK
+
 static const uint64_t valid_decomposed_outputs[] = {
   (uint64_t)1, (uint64_t)2, (uint64_t)3, (uint64_t)4, (uint64_t)5, (uint64_t)6, (uint64_t)7, (uint64_t)8, (uint64_t)9, // 1 piconero
   (uint64_t)10, (uint64_t)20, (uint64_t)30, (uint64_t)40, (uint64_t)50, (uint64_t)60, (uint64_t)70, (uint64_t)80, (uint64_t)90,
@@ -65,6 +68,13 @@ static const uint64_t valid_decomposed_outputs[] = {
   (uint64_t)10000000000000000000ull
 };
 
+static std::atomic<unsigned int> default_decimal_point(CRYPTONOTE_DISPLAY_DECIMAL_POINT);
+
+static std::atomic<uint64_t> tx_hashes_calculated_count(0);
+static std::atomic<uint64_t> tx_hashes_cached_count(0);
+static std::atomic<uint64_t> block_hashes_calculated_count(0);
+static std::atomic<uint64_t> block_hashes_cached_count(0);
+
 namespace cryptonote
 {
   //---------------------------------------------------------------
@@ -90,6 +100,17 @@ namespace cryptonote
     binary_archive<false> ba(ss);
     bool r = ::serialization::serialize(ba, tx);
     CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
+    tx.invalidate_hashes();
+    return true;
+  }
+  //---------------------------------------------------------------
+  bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx)
+  {
+    std::stringstream ss;
+    ss << tx_blob;
+    binary_archive<false> ba(ss);
+    bool r = tx.serialize_base(ba);
+    CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
     return true;
   }
   //---------------------------------------------------------------
@@ -100,6 +121,7 @@ namespace cryptonote
     binary_archive<false> ba(ss);
     bool r = ::serialization::serialize(ba, tx);
     CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
+    tx.invalidate_hashes();
     //TODO: validate tx
 
     get_transaction_hash(tx, tx_hash);
@@ -142,12 +164,12 @@ namespace cryptonote
     if (std::string::npos != point_index)
     {
       fraction_size = str_amount.size() - point_index - 1;
-      while (CRYPTONOTE_DISPLAY_DECIMAL_POINT < fraction_size && '0' == str_amount.back())
+      while (default_decimal_point < fraction_size && '0' == str_amount.back())
       {
         str_amount.erase(str_amount.size() - 1, 1);
         --fraction_size;
       }
-      if (CRYPTONOTE_DISPLAY_DECIMAL_POINT < fraction_size)
+      if (default_decimal_point < fraction_size)
         return false;
       str_amount.erase(point_index, 1);
     }
@@ -159,9 +181,9 @@ namespace cryptonote
     if (str_amount.empty())
       return false;
 
-    if (fraction_size < CRYPTONOTE_DISPLAY_DECIMAL_POINT)
+    if (fraction_size < default_decimal_point)
     {
-      str_amount.append(CRYPTONOTE_DISPLAY_DECIMAL_POINT - fraction_size, '0');
+      str_amount.append(default_decimal_point - fraction_size, '0');
     }
 
     return string_tools::get_xtype_from_string(amount, str_amount);
@@ -504,14 +526,59 @@ namespace cryptonote
     cn_fast_hash(blob.data(), blob.size(), res);
   }
   //---------------------------------------------------------------
-  std::string print_money(uint64_t amount)
+  void set_default_decimal_point(unsigned int decimal_point)
   {
-    std::string s = std::to_string(amount);
-    if(s.size() < CRYPTONOTE_DISPLAY_DECIMAL_POINT+1)
+    switch (decimal_point)
     {
-      s.insert(0, CRYPTONOTE_DISPLAY_DECIMAL_POINT+1 - s.size(), '0');
+      case 12:
+      case 9:
+      case 6:
+      case 3:
+      case 0:
+        default_decimal_point = decimal_point;
+        break;
+      default:
+        ASSERT_MES_AND_THROW("Invalid decimal point specification: " << decimal_point);
     }
-    s.insert(s.size() - CRYPTONOTE_DISPLAY_DECIMAL_POINT, ".");
+  }
+  //---------------------------------------------------------------
+  unsigned int get_default_decimal_point()
+  {
+    return default_decimal_point;
+  }
+  //---------------------------------------------------------------
+  std::string get_unit(unsigned int decimal_point)
+  {
+    if (decimal_point == (unsigned int)-1)
+      decimal_point = default_decimal_point;
+    switch (std::atomic_load(&default_decimal_point))
+    {
+      case 12:
+        return "monero";
+      case 9:
+        return "millinero";
+      case 6:
+        return "micronero";
+      case 3:
+        return "nanonero";
+      case 0:
+        return "piconero";
+      default:
+        ASSERT_MES_AND_THROW("Invalid decimal point specification: " << default_decimal_point);
+    }
+  }
+  //---------------------------------------------------------------
+  std::string print_money(uint64_t amount, unsigned int decimal_point)
+  {
+    if (decimal_point == (unsigned int)-1)
+      decimal_point = default_decimal_point;
+    std::string s = std::to_string(amount);
+    if(s.size() < decimal_point+1)
+    {
+      s.insert(0, decimal_point+1 - s.size(), '0');
+    }
+    if (decimal_point > 0)
+      s.insert(s.size() - decimal_point, ".");
     return s;
   }
   //---------------------------------------------------------------
@@ -534,7 +601,7 @@ namespace cryptonote
     return get_transaction_hash(t, res, NULL);
   }
   //---------------------------------------------------------------
-  bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
+  bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
   {
     // v1 transactions hash the entire blob
     if (t.version == 1)
@@ -589,6 +656,43 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
+  bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
+  {
+    return calculate_transaction_hash(t, res, blob_size);
+#if 0
+    if (t.is_hash_valid())
+    {
+#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK
+      CHECK_AND_ASSERT_THROW_MES(!calculate_transaction_hash(t, res, blob_size) || t.hash == res, "tx hash cash integrity failure");
+#endif
+      res = t.hash;
+      if (blob_size)
+      {
+        if (!t.is_blob_size_valid())
+        {
+          t.blob_size = get_object_blobsize(t);
+          t.set_blob_size_valid(true);
+        }
+        *blob_size = t.blob_size;
+      }
+      ++tx_hashes_cached_count;
+      return true;
+    }
+    ++tx_hashes_calculated_count;
+    bool ret = calculate_transaction_hash(t, res, blob_size);
+    if (!ret)
+      return false;
+    t.hash = res;
+    t.set_hash_valid(true);
+    if (blob_size)
+    {
+      t.blob_size = *blob_size;
+      t.set_blob_size_valid(true);
+    }
+    return true;
+#endif
+  }
+  //---------------------------------------------------------------
   bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size)
   {
     return get_transaction_hash(t, res, &blob_size);
@@ -603,7 +707,7 @@ namespace cryptonote
     return blob;
   }
   //---------------------------------------------------------------
-  bool get_block_hash(const block& b, crypto::hash& res)
+  bool calculate_block_hash(const block& b, crypto::hash& res)
   {
     // EXCEPTION FOR BLOCK 202612
     const std::string correct_blob_hash_202612 = "3a8a2b3a29b50fc86ff73dd087ea43c6f0d6b8f936c849194d5c84c737903966";
@@ -630,6 +734,29 @@ namespace cryptonote
     return hash_result;
   }
   //---------------------------------------------------------------
+  bool get_block_hash(const block& b, crypto::hash& res)
+  {
+    return calculate_block_hash(b, res);
+#if 0
+    if (b.is_hash_valid())
+    {
+#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK
+      CHECK_AND_ASSERT_THROW_MES(!calculate_block_hash(b, res) || b.hash == res, "block hash cash integrity failure");
+#endif
+      res = b.hash;
+      ++block_hashes_cached_count;
+      return true;
+    }
+    ++block_hashes_calculated_count;
+    bool ret = calculate_block_hash(b, res);
+    if (!ret)
+      return false;
+    b.hash = res;
+    b.set_hash_valid(true);
+    return true;
+#endif
+  }
+  //---------------------------------------------------------------
   crypto::hash get_block_hash(const block& b)
   {
     crypto::hash p = null_hash;
@@ -646,7 +773,6 @@ namespace cryptonote
       string_tools::hex_to_pod(longhash_202612, res);
       return true;
     }
-    block b_local = b; //workaround to avoid const errors with do_serialize
     blobdata bd = get_block_hashing_blob(b);
     crypto::cn_slow_hash(bd.data(), bd.size(), res);
     return true;
@@ -686,6 +812,8 @@ namespace cryptonote
     binary_archive<false> ba(ss);
     bool r = ::serialization::serialize(ba, b);
     CHECK_AND_ASSERT_MES(r, false, "Failed to parse block from blob");
+    b.invalidate_hashes();
+    b.miner_tx.invalidate_hashes();
     return true;
   }
   //---------------------------------------------------------------
@@ -740,4 +868,11 @@ namespace cryptonote
     return std::binary_search(begin, end, amount);
   }
   //---------------------------------------------------------------
+  void get_hash_stats(uint64_t &tx_hashes_calculated, uint64_t &tx_hashes_cached, uint64_t &block_hashes_calculated, uint64_t & block_hashes_cached)
+  {
+    tx_hashes_calculated = tx_hashes_calculated_count;
+    tx_hashes_cached = tx_hashes_cached_count;
+    block_hashes_calculated = block_hashes_calculated_count;
+    block_hashes_cached = block_hashes_cached_count;
+  }
 }

src/cryptonote_basic/cryptonote_format_utils.h

@@ -44,6 +44,7 @@ namespace cryptonote
   crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx);
   bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash);
   bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx);
+  bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx);
   bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key);
   bool decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key);
 
@@ -84,7 +85,9 @@ namespace cryptonote
   bool get_transaction_hash(const transaction& t, crypto::hash& res);
   bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size);
   bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size);
+  bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size);
   blobdata get_block_hashing_blob(const block& b);
+  bool calculate_block_hash(const block& b, crypto::hash& res);
   bool get_block_hash(const block& b, crypto::hash& res);
   crypto::hash get_block_hash(const block& b);
   bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height);
@@ -102,7 +105,10 @@ namespace cryptonote
   uint64_t get_block_height(const block& b);
   std::vector<uint64_t> relative_output_offsets_to_absolute(const std::vector<uint64_t>& off);
   std::vector<uint64_t> absolute_output_offsets_to_relative(const std::vector<uint64_t>& off);
-  std::string print_money(uint64_t amount);
+  void set_default_decimal_point(unsigned int decimal_point = CRYPTONOTE_DISPLAY_DECIMAL_POINT);
+  unsigned int get_default_decimal_point();
+  std::string get_unit(unsigned int decimal_point = -1);
+  std::string print_money(uint64_t amount, unsigned int decimal_point = -1);
   //---------------------------------------------------------------
   template<class t_object>
   bool t_serializable_object_to_blob(const t_object& to, blobdata& b_blob)
@@ -205,6 +211,7 @@ namespace cryptonote
   crypto::hash get_tx_tree_hash(const std::vector<crypto::hash>& tx_hashes);
   crypto::hash get_tx_tree_hash(const block& b);
   bool is_valid_decomposed_amount(uint64_t amount);
+  void get_hash_stats(uint64_t &tx_hashes_calculated, uint64_t &tx_hashes_cached, uint64_t &block_hashes_calculated, uint64_t & block_hashes_cached);
 
 #define CHECKED_GET_SPECIFIC_VARIANT(variant_var, specific_type, variable_name, fail_return_val) \
   CHECK_AND_ASSERT_MES(variant_var.type() == typeid(specific_type), fail_return_val, "wrong variant type: " << variant_var.type().name() << ", expected " << typeid(specific_type).name()); \

src/cryptonote_basic/hardfork.cpp

@@ -29,7 +29,7 @@
 #include <algorithm>
 #include <cstdio>
 
-#include "cryptonote_basic.h"
+#include "cryptonote_basic/cryptonote_basic.h"
 #include "blockchain_db/blockchain_db.h"
 #include "hardfork.h"
 
@@ -222,7 +222,7 @@ bool HardFork::reorganize_from_block_height(uint64_t height)
     return false;
 
   db.set_batch_transactions(true);
-  db.batch_start();
+  bool stop_batch = db.batch_start();
 
   versions.clear();
 
@@ -250,7 +250,8 @@ bool HardFork::reorganize_from_block_height(uint64_t height)
     add(db.get_block_from_height(h), h);
   }
 
-  db.batch_stop();
+  if (stop_batch)
+    db.batch_stop();
 
   return true;
 }

src/cryptonote_basic/hardfork.h

@@ -29,7 +29,7 @@
 #pragma once
 
 #include "syncobj.h"
-#include "cryptonote_basic.h"
+#include "cryptonote_basic/cryptonote_basic.h"
 
 namespace cryptonote
 {

src/cryptonote_basic/miner.cpp

@@ -355,9 +355,11 @@ namespace cryptonote
 
       if(check_hash(h, diffic))
       {
+        bl.invalidate_hashes();
         return true;
       }
     }
+    bl.invalidate_hashes();
     return false;
   }
   //-----------------------------------------------------------------------------------------------------

src/cryptonote_config.h

@@ -57,6 +57,7 @@
 #define CRYPTONOTE_REWARD_BLOCKS_WINDOW                 100
 #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2    60000 //size of block (bytes) after which reward for block calculated using block size
 #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1    20000 //size of block (bytes) after which reward for block calculated using block size - before first fork
+#define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5    300000 //size of block (bytes) after which reward for block calculated using block size - second change, from v5
 #define CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE          600
 #define CRYPTONOTE_DISPLAY_DECIMAL_POINT                12
 // COIN - number of smallest units in one coin
@@ -66,6 +67,7 @@
 #define FEE_PER_KB                                      ((uint64_t)2000000000) // 2 * pow(10, 9)
 #define DYNAMIC_FEE_PER_KB_BASE_FEE                     ((uint64_t)2000000000) // 2 * pow(10,9)
 #define DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD            ((uint64_t)10000000000000) // 10 * pow(10,12)
+#define DYNAMIC_FEE_PER_KB_BASE_FEE_V5                  ((uint64_t)2000000000 * (uint64_t)CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 / CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5)
 
 #define ORPHANED_BLOCKS_MAX_COUNT                       100
 
@@ -128,6 +130,10 @@
 #define THREAD_STACK_SIZE                       5 * 1024 * 1024
 
 #define HF_VERSION_DYNAMIC_FEE                  4
+#define HF_VERSION_MIN_MIXIN_4                  6
+#define HF_VERSION_ENFORCE_RCT                  6
+
+#define PER_KB_FEE_QUANTIZATION_DECIMALS        8
 
 // New constants are intended to go here
 namespace config

src/cryptonote_core/blockchain.cpp

@@ -97,8 +97,8 @@ static const struct {
   // version 4 starts from block 1220516, which is on or around the 5th of January, 2017. Fork time finalised on 2016-09-18.
   { 4, 1220516, 0, 1483574400 },
   
-  // version 5 starts from block 1406997, which is on or around the 20th of September, 2017. Fork time finalised on 2016-09-18.
-  { 5, 1406997, 0, 1505865600 },  
+  // version 5 starts from block 1288616, which is on or around the 15th of April, 2017. Fork time finalised on 2017-03-14.
+  { 5, 1288616, 0, 1489520158 },  
 };
 static const uint64_t mainnet_hard_fork_version_1_till = 1009826;
 
@@ -117,7 +117,7 @@ static const struct {
   // versions 3-5 were passed in rapid succession from September 18th, 2016
   { 3, 800500, 0, 1472415034 },
   { 4, 801219, 0, 1472415035 },
-  { 5, 802660, 0, 1472415036 },
+  { 5, 802660, 0, 1472415036 + 86400*180 }, // add 5 months on testnet to shut the update warning up since there's a large gap to v6
 };
 static const uint64_t testnet_hard_fork_version_1_till = 624633;
 
@@ -2426,7 +2426,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
   {
     size_t n_unmixable = 0, n_mixable = 0;
     size_t mixin = std::numeric_limits<size_t>::max();
-    const size_t min_mixin = hf_version >= 5 ? 4 : 2;
+    const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2;
     for (const auto& txin : tx.vin)
     {
       // non txin_to_key inputs will be rejected below
@@ -2479,7 +2479,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
       tvc.m_verifivation_failed = true;
       return false;
     }
-    const size_t min_tx_version = (n_unmixable > 0 ? 1 : (hf_version >= 5) ? 2 : 1);
+    const size_t min_tx_version = (n_unmixable > 0 ? 1 : (hf_version >= HF_VERSION_ENFORCE_RCT) ? 2 : 1);
     if (tx.version < min_tx_version)
     {
       MERROR_VER("transaction version " << (unsigned)tx.version << " is lower than min accepted version " << min_tx_version);
@@ -2796,21 +2796,43 @@ void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const
 }
 
 //------------------------------------------------------------------
-uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size)
+static uint64_t get_fee_quantization_mask()
 {
-  if (median_block_size < CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2)
-    median_block_size = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2;
+  static uint64_t mask = 0;
+  if (mask == 0)
+  {
+    mask = 1;
+    for (size_t n = PER_KB_FEE_QUANTIZATION_DECIMALS; n < CRYPTONOTE_DISPLAY_DECIMAL_POINT; ++n)
+      mask *= 10;
+  }
+  return mask;
+}
 
-  uint64_t unscaled_fee_per_kb = (DYNAMIC_FEE_PER_KB_BASE_FEE * CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 / median_block_size);
+//------------------------------------------------------------------
+uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version)
+{
+  const uint64_t min_block_size = get_min_block_size(version);
+  const uint64_t fee_per_kb_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE;
+
+  if (median_block_size < min_block_size)
+    median_block_size = min_block_size;
+
+  uint64_t unscaled_fee_per_kb = (fee_per_kb_base * min_block_size / median_block_size);
   uint64_t hi, lo = mul128(unscaled_fee_per_kb, block_reward, &hi);
   static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD % 1000000 == 0, "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD must be divisible by 1000000");
   static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000 <= std::numeric_limits<uint32_t>::max(), "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD is too large");
+
   // divide in two steps, since the divisor must be 32 bits, but DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD isn't
   div128_32(hi, lo, DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000, &hi, &lo);
   div128_32(hi, lo, 1000000, &hi, &lo);
   assert(hi == 0);
 
-  return lo;
+  // quantize fee up to 8 decimals
+  uint64_t mask = get_fee_quantization_mask();
+  uint64_t qlo = (lo + mask - 1) / mask * mask;
+  MDEBUG("lo " << print_money(lo) << ", qlo " << print_money(qlo) << ", mask " << mask);
+
+  return qlo;
 }
 
 //------------------------------------------------------------------
@@ -2830,7 +2852,7 @@ bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const
     uint64_t base_reward;
     if (!get_block_reward(median, 1, already_generated_coins, base_reward, version))
       return false;
-    fee_per_kb = get_dynamic_per_kb_fee(base_reward, median);
+    fee_per_kb = get_dynamic_per_kb_fee(base_reward, median, version);
   }
   MDEBUG("Using " << print_money(fee) << "/kB fee");
 
@@ -2838,7 +2860,7 @@ bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const
   needed_fee += (blob_size % 1024) ? 1 : 0;
   needed_fee *= fee_per_kb;
 
-  if (fee < needed_fee)
+  if (fee < needed_fee * 0.98) // keep a little buffer on acceptance
   {
     MERROR_VER("transaction fee is not enough: " << print_money(fee) << ", minimum fee: " << print_money(needed_fee));
     return false;
@@ -2857,14 +2879,15 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons
   if (grace_blocks >= CRYPTONOTE_REWARD_BLOCKS_WINDOW)
     grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1;
 
+  const uint64_t min_block_size = get_min_block_size(version);
   std::vector<size_t> sz;
   get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks);
   for (size_t i = 0; i < grace_blocks; ++i)
-    sz.push_back(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2);
+    sz.push_back(min_block_size);
 
   uint64_t median = epee::misc_utils::median(sz);
-  if(median <= CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2)
-    median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2;
+  if(median <= min_block_size)
+    median = min_block_size;
 
   uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0;
   uint64_t base_reward;
@@ -2874,7 +2897,7 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons
     base_reward = BLOCK_REWARD_OVERESTIMATE;
   }
 
-  uint64_t fee = get_dynamic_per_kb_fee(base_reward, median);
+  uint64_t fee = get_dynamic_per_kb_fee(base_reward, median, version);
   MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/kB");
   return fee;
 }
@@ -3393,7 +3416,7 @@ leave:
 //------------------------------------------------------------------
 bool Blockchain::update_next_cumulative_size_limit()
 {
-  uint64_t full_reward_zone = get_current_hard_fork_version() < 2 ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 : CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2;
+  uint64_t full_reward_zone = get_min_block_size(get_current_hard_fork_version());
 
   LOG_PRINT_L3("Blockchain::" << __func__);
   std::vector<size_t> sz;
@@ -3995,10 +4018,37 @@ void Blockchain::cancel()
 }
 
 #if defined(PER_BLOCK_CHECKPOINT)
+static const char expected_block_hashes_hash[] = "23d8a8c73de7b2383c72a016d9a6034e69d62dd48077d1c414e064ceab6daa94";
 void Blockchain::load_compiled_in_block_hashes()
 {
-  if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr)
+  if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr && get_blocks_dat_size(m_testnet) > 0)
   {
+    MINFO("Loading precomputed blocks (" << get_blocks_dat_size(m_testnet) << " bytes)");
+
+    if (!m_testnet)
+    {
+      // first check hash
+      crypto::hash hash;
+      if (!tools::sha256sum(get_blocks_dat_start(m_testnet), get_blocks_dat_size(m_testnet), hash))
+      {
+        MERROR("Failed to hash precomputed blocks data");
+        return;
+      }
+      MINFO("precomputed blocks hash: " << hash << ", expected " << expected_block_hashes_hash);
+      cryptonote::blobdata expected_hash_data;
+      if (!epee::string_tools::parse_hexstr_to_binbuff(std::string(expected_block_hashes_hash), expected_hash_data) || expected_hash_data.size() != sizeof(crypto::hash))
+      {
+        MERROR("Failed to parse expected block hashes hash");
+        return;
+      }
+      const crypto::hash expected_hash = *reinterpret_cast<const crypto::hash*>(expected_hash_data.data());
+      if (hash != expected_hash)
+      {
+        MERROR("Block hash data does not match expected hash");
+        return;
+      }
+    }
+
     if (get_blocks_dat_size(m_testnet) > 4)
     {
       const unsigned char *p = get_blocks_dat_start(m_testnet);
@@ -4006,7 +4056,6 @@ void Blockchain::load_compiled_in_block_hashes()
       const size_t size_needed = 4 + nblocks * sizeof(crypto::hash);
       if(nblocks > 0 && nblocks > m_db->height() && get_blocks_dat_size(m_testnet) >= size_needed)
       {
-        MINFO("Loading precomputed blocks: " << nblocks);
         p += sizeof(uint32_t);
         for (uint32_t i = 0; i < nblocks; i++)
         {
@@ -4015,6 +4064,7 @@ void Blockchain::load_compiled_in_block_hashes()
           p += sizeof(hash.data);
           m_blocks_hash_check.push_back(hash);
         }
+        MINFO(nblocks << " block hashes loaded");
 
         // FIXME: clear tx_pool because the process might have been
         // terminated and caused it to store txs kept by blocks.

src/cryptonote_core/blockchain.h

@@ -29,6 +29,7 @@
 // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
 
 #pragma once
+#include <boost/asio/io_service.hpp>
 #include <boost/serialization/serialization.hpp>
 #include <boost/serialization/version.hpp>
 #include <boost/serialization/list.hpp>
@@ -520,10 +521,11 @@ namespace cryptonote
      *
      * @param block_reward the current block reward
      * @param median_block_size the median blob's size in the past window
+     * @param version hard fork version for rules and constants to use
      *
      * @return the per kB fee
      */
-    static uint64_t get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size);
+    static uint64_t get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version);
 
     /**
      * @brief get dynamic per kB fee estimate for the next few blocks

src/cryptonote_core/cryptonote_core.cpp

@@ -60,6 +60,8 @@ DISABLE_VS_WARNINGS(4355)
 
 #define MERROR_VER(x) MCERROR("verify", x)
 
+#define BAD_SEMANTICS_TXES_MAX_SIZE 100
+
 namespace cryptonote
 {
 
@@ -73,7 +75,8 @@ namespace cryptonote
               m_target_blockchain_height(0),
               m_checkpoints_path(""),
               m_last_dns_checkpoints_update(0),
-              m_last_json_checkpoints_update(0)
+              m_last_json_checkpoints_update(0),
+              m_update_download(0)
   {
     set_cryptonote_protocol(pprotocol);
   }
@@ -132,6 +135,15 @@ namespace cryptonote
   void core::stop()
   {
     m_blockchain_storage.cancel();
+
+    tools::download_async_handle handle;
+    {
+      boost::lock_guard<boost::mutex> lock(m_update_mutex);
+      handle = m_update_download;
+      m_update_download = 0;
+    }
+    if (handle)
+      tools::download_cancel(handle);
   }
   //-----------------------------------------------------------------------------------
   void core::init_options(boost::program_options::options_description& desc)
@@ -496,11 +508,14 @@ namespace cryptonote
     }
     //std::cout << "!"<< tx.vin.size() << std::endl;
 
-    if (bad_semantics_txes.find(tx_hash) != bad_semantics_txes.end())
+    for (int idx = 0; idx < 2; ++idx)
     {
-      LOG_PRINT_L1("Transaction already seen with bad semantics, rejected");
-      tvc.m_verifivation_failed = true;
-      return false;
+      if (bad_semantics_txes[idx].find(tx_hash) != bad_semantics_txes[idx].end())
+      {
+        LOG_PRINT_L1("Transaction already seen with bad semantics, rejected");
+        tvc.m_verifivation_failed = true;
+        return false;
+      }
     }
 
     uint8_t version = m_blockchain_storage.get_current_hard_fork_version();
@@ -551,8 +566,13 @@ namespace cryptonote
     if(!check_tx_semantic(tx, keeped_by_block))
     {
       LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected");
-      bad_semantics_txes.insert(tx_hash);
       tvc.m_verifivation_failed = true;
+      bad_semantics_txes[0].insert(tx_hash);
+      if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE)
+      {
+        std::swap(bad_semantics_txes[0], bad_semantics_txes[1]);
+        bad_semantics_txes[0].clear();
+      }
       return false;
     }
 
@@ -776,7 +796,7 @@ namespace cryptonote
   {
     // we attempt to relay txes that should be relayed, but were not
     std::list<std::pair<crypto::hash, cryptonote::transaction>> txs;
-    if (m_mempool.get_relayable_transactions(txs))
+    if (m_mempool.get_relayable_transactions(txs) && !txs.empty())
     {
       cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context);
       tx_verification_context tvc = AUTO_VAL_INIT(tvc);
@@ -867,11 +887,28 @@ namespace cryptonote
     m_miner.resume();
   }
   //-----------------------------------------------------------------------------------------------
+  block_complete_entry get_block_complete_entry(block& b, tx_memory_pool &pool)
+  {
+    block_complete_entry bce;
+    bce.block = cryptonote::block_to_blob(b);
+    for (const auto &tx_hash: b.tx_hashes)
+    {
+      cryptonote::transaction tx;
+      CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, tx), "Transaction not found in pool");
+      bce.txs.push_back(cryptonote::tx_to_blob(tx));
+    }
+    return bce;
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::handle_block_found(block& b)
   {
     block_verification_context bvc = boost::value_initialized<block_verification_context>();
     m_miner.pause();
+    std::list<block_complete_entry> blocks;
+    blocks.push_back(get_block_complete_entry(b, m_mempool));
+    prepare_handle_incoming_blocks(blocks);
     m_blockchain_storage.add_new_block(b, bvc);
+    cleanup_handle_incoming_blocks(true);
     //anyway - update miner template
     update_miner_block_template();
     m_miner.resume();
@@ -999,7 +1036,13 @@ namespace cryptonote
     m_mempool.get_transactions(txs);
     return true;
   }
-  //-----------------------------------------------------------------------------------------------  
+  //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_transaction_hashes(std::vector<crypto::hash>& txs) const
+  {
+    m_mempool.get_transaction_hashes(txs);
+    return true;
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transaction(const crypto::hash &id, transaction& tx) const
   {
     return m_mempool.get_transaction(id, tx);
@@ -1108,7 +1151,7 @@ namespace cryptonote
       return true;
 
     std::string url = tools::get_update_url(software, subdir, buildtag, version, true);
-    MGINFO("Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash);
+    MCLOG_CYAN(el::Level::Info, "global", "Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash);
 
     if (check_updates_level == UPDATES_NOTIFY)
       return true;
@@ -1123,32 +1166,55 @@ namespace cryptonote
     boost::filesystem::path path(epee::string_tools::get_current_module_folder());
     path /= filename;
 
+    boost::unique_lock<boost::mutex> lock(m_update_mutex);
+
+    if (m_update_download != 0)
+    {
+      MCDEBUG("updates", "Already downloading update");
+      return true;
+    }
+
     crypto::hash file_hash;
     if (!tools::sha256sum(path.string(), file_hash) || (hash != epee::string_tools::pod_to_hex(file_hash)))
     {
       MCDEBUG("updates", "We don't have that file already, downloading");
-      if (!tools::download(path.string(), url))
-      {
-        MCERROR("updates", "Failed to download " << url);
-        return false;
-      }
-      if (!tools::sha256sum(path.string(), file_hash))
-      {
-        MCERROR("updates", "Failed to hash " << path);
-        return false;
-      }
-      if (hash != epee::string_tools::pod_to_hex(file_hash))
-      {
-        MCERROR("updates", "Download from " << url << " does not match the expected hash");
-        return false;
-      }
-      MGINFO("New version downloaded to " << path);
+      m_last_update_length = 0;
+      m_update_download = tools::download_async(path.string(), url, [this, hash](const std::string &path, const std::string &uri, bool success) {
+        if (success)
+        {
+          crypto::hash file_hash;
+          if (!tools::sha256sum(path, file_hash))
+          {
+            MCERROR("updates", "Failed to hash " << path);
+          }
+          if (hash != epee::string_tools::pod_to_hex(file_hash))
+          {
+            MCERROR("updates", "Download from " << uri << " does not match the expected hash");
+          }
+          MCLOG_CYAN(el::Level::Info, "updates", "New version downloaded to " << path);
+        }
+        else
+        {
+          MCERROR("updates", "Failed to download " << uri);
+        }
+        boost::unique_lock<boost::mutex> lock(m_update_mutex);
+        m_update_download = 0;
+      }, [this](const std::string &path, const std::string &uri, size_t length, ssize_t content_length) {
+        if (length >= m_last_update_length + 1024 * 1024 * 10)
+        {
+          m_last_update_length = length;
+          MCDEBUG("updates", "Downloaded " << length << "/" << (content_length ? std::to_string(content_length) : "unknown"));
+        }
+        return true;
+      });
     }
     else
     {
       MCDEBUG("updates", "We already have " << path << " with expected hash");
     }
 
+    lock.unlock();
+
     if (check_updates_level == UPDATES_DOWNLOAD)
       return true;
 

src/cryptonote_core/cryptonote_core.h

@@ -39,6 +39,7 @@
 #include "p2p/net_node_common.h"
 #include "cryptonote_protocol/cryptonote_protocol_handler_common.h"
 #include "storages/portable_storage_template_helper.h"
+#include "common/download.h"
 #include "tx_pool.h"
 #include "blockchain.h"
 #include "cryptonote_basic/miner.h"
@@ -395,6 +396,13 @@ namespace cryptonote
       */
      bool get_pool_transactions(std::list<transaction>& txs) const;
      
+     /**
+      * @copydoc tx_memory_pool::get_transactions
+      *
+      * @note see tx_memory_pool::get_transactions
+      */
+     bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs) const;
+
      /**
       * @copydoc tx_memory_pool::get_transaction
       *
@@ -818,7 +826,6 @@ namespace cryptonote
      epee::math_helper::once_a_time_seconds<60*2, false> m_txpool_auto_relayer; //!< interval for checking re-relaying txpool transactions
      epee::math_helper::once_a_time_seconds<60*60*12, true> m_check_updates_interval; //!< interval for checking for new versions
 
-     friend class tx_validate_inputs;
      std::atomic<bool> m_starter_message_showed; //!< has the "daemon will sync now" message been shown?
 
      uint64_t m_target_blockchain_height; //!< blockchain height target
@@ -833,13 +840,11 @@ namespace cryptonote
 
      std::atomic_flag m_checkpoints_updating; //!< set if checkpoints are currently updating to avoid multiple threads attempting to update at once
 
-     boost::interprocess::file_lock db_lock; //!< a lock object for a file lock in the db directory
-
      size_t block_sync_size;
 
      time_t start_time;
 
-     std::unordered_set<crypto::hash> bad_semantics_txes;
+     std::unordered_set<crypto::hash> bad_semantics_txes[2];
 
      enum {
        UPDATES_DISABLED,
@@ -847,6 +852,10 @@ namespace cryptonote
        UPDATES_DOWNLOAD,
        UPDATES_UPDATE,
      } check_updates_level;
+
+     tools::download_async_handle m_update_download;
+     size_t m_last_update_length;
+     boost::mutex m_update_mutex;
    };
 }
 

src/cryptonote_core/cryptonote_tx_utils.cpp

@@ -132,6 +132,9 @@ namespace cryptonote
     //lock
     tx.unlock_time = height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW;
     tx.vin.push_back(in);
+
+    tx.invalidate_hashes();
+
     //LOG_PRINT("MINER_TX generated ok, block_reward=" << print_money(block_reward) << "("  << print_money(block_reward - fee) << "+" << print_money(fee)
     //  << "), current_block_size=" << current_block_size << ", already_generated_coins=" << already_generated_coins << ", tx_id=" << get_transaction_hash(tx), LOG_LEVEL_2);
     return true;
@@ -451,6 +454,8 @@ namespace cryptonote
       MCINFO("construct_tx", "transaction_created: " << get_transaction_hash(tx) << ENDL << obj_to_json_str(tx) << ENDL);
     }
 
+    tx.invalidate_hashes();
+
     return true;
   }
   //---------------------------------------------------------------
@@ -487,6 +492,7 @@ namespace cryptonote
     bl.timestamp = 0;
     bl.nonce = nonce;
     miner::find_nonce_for_given_block(bl, 1, 0);
+    bl.invalidate_hashes();
     return true;
   }
   //---------------------------------------------------------------

src/cryptonote_core/tx_pool.cpp

@@ -59,8 +59,6 @@ namespace cryptonote
     //      codebase.  As it stands, it is at best nontrivial to test
     //      whether or not changing these parameters (or adding new)
     //      will work correctly.
-    size_t const TRANSACTION_SIZE_LIMIT_V1 = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE);
-    size_t const TRANSACTION_SIZE_LIMIT_V2 = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE);
     time_t const MIN_RELAY_TIME = (60 * 5); // only start re-relaying transactions after that many seconds
     time_t const MAX_RELAY_TIME = (60 * 60 * 4); // at most that many seconds between resends
     float const ACCEPT_THRESHOLD = 1.0f;
@@ -78,6 +76,11 @@ namespace cryptonote
     {
       return amount * ACCEPT_THRESHOLD;
     }
+
+    uint64_t get_transaction_size_limit(uint8_t version)
+    {
+      return get_min_block_size(version) * 125 / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
+    }
   }
   //---------------------------------------------------------------------------------
   //---------------------------------------------------------------------------------
@@ -149,7 +152,7 @@ namespace cryptonote
       return false;
     }
 
-    size_t tx_size_limit = (version < 2 ? TRANSACTION_SIZE_LIMIT_V1 : TRANSACTION_SIZE_LIMIT_V2);
+    size_t tx_size_limit = get_transaction_size_limit(version);
     if (!kept_by_block && blob_size >= tx_size_limit)
     {
       LOG_PRINT_L1("transaction is too big: " << blob_size << " bytes, maximum size: " << tx_size_limit);
@@ -420,6 +423,13 @@ namespace cryptonote
       txs.push_back(tx_vt.second.tx);
   }
   //------------------------------------------------------------------
+  void tx_memory_pool::get_transaction_hashes(std::vector<crypto::hash>& txs) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    for(const auto& tx_vt: m_transactions)
+      txs.push_back(get_transaction_hash(tx_vt.second.tx));
+  }
+  //------------------------------------------------------------------
   //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const
   {
@@ -611,7 +621,7 @@ namespace cryptonote
 
     CRITICAL_REGION_LOCAL(m_transactions_lock);
 
-    uint64_t best_coinbase = 0;
+    uint64_t best_coinbase = 0, coinbase = 0;
     total_size = 0;
     fee = 0;
     
@@ -619,11 +629,9 @@ namespace cryptonote
     get_block_reward(median_size, total_size, already_generated_coins, best_coinbase, version);
 
 
-#if 1
-    size_t max_total_size = (130 * median_size) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
-#else
-    size_t max_total_size = 2 * median_size - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
-#endif
+    size_t max_total_size_pre_v5 = (130 * median_size) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
+    size_t max_total_size_v5 = 2 * median_size - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
+    size_t max_total_size = version >= 5 ? max_total_size_v5 : max_total_size_pre_v5;
     std::unordered_set<crypto::key_image> k_images;
 
     LOG_PRINT_L2("Filling block template, median size " << median_size << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool");
@@ -641,39 +649,49 @@ namespace cryptonote
         continue;
       }
 
-#if 1
-      // If we've exceeded the penalty free size,
-      // stop including more tx
-      if (total_size > median_size)
+      // start using the optimal filling algorithm from v5
+      if (version >= 5)
       {
-        LOG_PRINT_L2("  would exceed median block size");
-        break;
+        // If we're getting lower coinbase tx,
+        // stop including more tx
+        uint64_t block_reward;
+        if(!get_block_reward(median_size, total_size + tx_it->second.blob_size, already_generated_coins, block_reward, version))
+        {
+          LOG_PRINT_L2("  would exceed maximum block size");
+          sorted_it++;
+          continue;
+        }
+        coinbase = block_reward + fee + tx_it->second.fee;
+        if (coinbase < template_accept_threshold(best_coinbase))
+        {
+          LOG_PRINT_L2("  would decrease coinbase to " << print_money(coinbase));
+          sorted_it++;
+          continue;
+        }
       }
-#else
-      // If we're getting lower coinbase tx,
-      // stop including more tx
-      uint64_t block_reward;
-      if(!get_block_reward(median_size, total_size + tx_it->second.blob_size, already_generated_coins, block_reward, version))
+      else
       {
-        LOG_PRINT_L2("  would exceed maximum block size");
-        sorted_it++;
-        continue;
+        // If we've exceeded the penalty free size,
+        // stop including more tx
+        if (total_size > median_size)
+        {
+          LOG_PRINT_L2("  would exceed median block size");
+          break;
+        }
       }
-      uint64_t coinbase = block_reward + fee + tx_it->second.fee;
-      if (coinbase < template_accept_threshold(best_coinbase))
-      {
-        LOG_PRINT_L2("  would decrease coinbase to " << print_money(coinbase));
-        sorted_it++;
-        continue;
-      }
-#endif
 
       // Skip transactions that are not ready to be
       // included into the blockchain or that are
       // missing key images
-      if (!is_transaction_ready_to_go(tx_it->second) || have_key_images(k_images, tx_it->second.tx))
+      if (!is_transaction_ready_to_go(tx_it->second))
       {
-        LOG_PRINT_L2("  not ready to go, or key images already seen");
+        LOG_PRINT_L2("  not ready to go");
+        sorted_it++;
+        continue;
+      }
+      if (have_key_images(k_images, tx_it->second.tx))
+      {
+        LOG_PRINT_L2("  key images already seen");
         sorted_it++;
         continue;
       }
@@ -681,9 +699,7 @@ namespace cryptonote
       bl.tx_hashes.push_back(tx_it->first);
       total_size += tx_it->second.blob_size;
       fee += tx_it->second.fee;
-#if 0
       best_coinbase = coinbase;
-#endif
       append_key_images(k_images, tx_it->second.tx);
       sorted_it++;
       LOG_PRINT_L2("  added, new block size " << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase));
@@ -699,15 +715,24 @@ namespace cryptonote
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     size_t n_removed = 0;
-    size_t tx_size_limit = (version < 2 ? TRANSACTION_SIZE_LIMIT_V1 : TRANSACTION_SIZE_LIMIT_V2);
+    size_t tx_size_limit = get_transaction_size_limit(version);
     for (auto it = m_transactions.begin(); it != m_transactions.end(); ) {
+      bool remove = false;
+      const crypto::hash &txid = get_transaction_hash(it->second.tx);
       if (it->second.blob_size >= tx_size_limit) {
-        LOG_PRINT_L1("Transaction " << get_transaction_hash(it->second.tx) << " is too big (" << it->second.blob_size << " bytes), removing it from pool");
+        LOG_PRINT_L1("Transaction " << txid << " is too big (" << it->second.blob_size << " bytes), removing it from pool");
+        remove = true;
+      }
+      else if (m_blockchain.have_tx(txid)) {
+        LOG_PRINT_L1("Transaction " << txid << " is in the blockchain, removing it from pool");
+        remove = true;
+      }
+      if (remove) {
         remove_transaction_keyimages(it->second.tx);
-        auto sorted_it = find_tx_in_sorted_container(it->first);
+        auto sorted_it = find_tx_in_sorted_container(txid);
         if (sorted_it == m_txs_by_fee_and_receive_time.end())
         {
-          LOG_PRINT_L1("Removing tx " << it->first << " from tx pool, but it was not found in the sorted txs container!");
+          LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
         }
         else
         {

src/cryptonote_core/tx_pool.h

@@ -233,6 +233,13 @@ namespace cryptonote
      */
     void get_transactions(std::list<transaction>& txs) const;
 
+    /**
+     * @brief get a list of all transaction hashes in the pool
+     *
+     * @param txs return-by-reference the list of transactions
+     */
+    void get_transaction_hashes(std::vector<crypto::hash>& txs) const;
+
     /**
      * @brief get information about all transactions and key images in the pool
      *

src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp

@@ -122,7 +122,7 @@ cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() {
 void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list<crypto::hash>& ids) {
 	using namespace epee::net_utils;
 	MDEBUG("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size());
-	MWARNING("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)");
+	MDEBUG("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)");
 	// TODO
 }
 

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -266,6 +266,9 @@ namespace cryptonote
       return true;
 
     uint64_t target = m_core.get_target_blockchain_height();
+    if (target == 0)
+      target = m_core.get_current_blockchain_height();
+
     if(m_core.have_block(hshd.top_id))
     {
       context.m_state = cryptonote_connection_context::state_normal;
@@ -280,7 +283,6 @@ namespace cryptonote
     I prefer pushing target height to the core at the same time it is pushed to the user.
     Nz. */
     m_core.set_target_blockchain_height(static_cast<int64_t>(hshd.current_height));
-
     int64_t diff = static_cast<int64_t>(hshd.current_height) - static_cast<int64_t>(m_core.get_current_blockchain_height());
 	int64_t max_block_height = max(static_cast<int64_t>(hshd.current_height),static_cast<int64_t>(m_core.get_current_blockchain_height()));
 	int64_t last_block_v1 = 1009826;
@@ -546,7 +548,16 @@ namespace cryptonote
           tx_ids.push_back(tx_hash);
           if (m_core.get_transactions(tx_ids, txes, missing) && missing.empty())
           {
-            have_tx.push_back(tx_to_blob(tx));
+            if (txes.size() == 1)
+            {
+              have_tx.push_back(tx_to_blob(txes.front()));
+            }
+            else
+            {
+              MERROR("1 tx requested, none not found, but " << txes.size() << " returned");
+              m_core.resume_mine();
+              return 1;
+            }
           }
           else
           {
@@ -603,7 +614,7 @@ namespace cryptonote
           NOTIFY_NEW_BLOCK::request reg_arg = AUTO_VAL_INIT(reg_arg);
           reg_arg.hop = arg.hop;
           reg_arg.current_blockchain_height = arg.current_blockchain_height;
-          reg_arg.b.block = b.block;
+          reg_arg.b = b;
           relay_block(reg_arg, context);
         }
         else if( bvc.m_marked_as_orphaned )

src/daemon/CMakeLists.txt

@@ -29,9 +29,9 @@
 set(blocksdat "")
 if(PER_BLOCK_CHECKPOINT)
   if(APPLE)
-    add_custom_command(OUTPUT blocksdat.o COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*)
+    add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*)
   else()
-    add_custom_command(OUTPUT blocksdat.o COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat)
+    add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat)
   endif()
   set(blocksdat "blocksdat.o")
 endif()

src/daemon/command_parser_executor.cpp

@@ -51,6 +51,13 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>&
   return m_executor.print_peer_list();
 }
 
+bool t_command_parser_executor::print_peer_list_stats(const std::vector<std::string>& args)
+{
+  if (!args.empty()) return false;
+
+  return m_executor.print_peer_list_stats();
+}
+
 bool t_command_parser_executor::save_blockchain(const std::vector<std::string>& args)
 {
   if (!args.empty()) return false;
@@ -545,4 +552,15 @@ bool t_command_parser_executor::print_blockchain_dynamic_stats(const std::vector
   return m_executor.print_blockchain_dynamic_stats(nblocks);
 }
 
+bool t_command_parser_executor::update(const std::vector<std::string>& args)
+{
+  if(args.size() != 1)
+  {
+    std::cout << "Exactly one parameter is needed: check, download, or update" << std::endl;
+    return false;
+  }
+
+  return m_executor.update(args.front());
+}
+
 } // namespace daemonize

src/daemon/command_parser_executor.h

@@ -59,6 +59,8 @@ public:
 
   bool print_peer_list(const std::vector<std::string>& args);
 
+  bool print_peer_list_stats(const std::vector<std::string>& args);
+
   bool save_blockchain(const std::vector<std::string>& args);
 
   bool show_hash_rate(const std::vector<std::string>& args);
@@ -128,6 +130,8 @@ public:
   bool alt_chain_info(const std::vector<std::string>& args);
 
   bool print_blockchain_dynamic_stats(const std::vector<std::string>& args);
+
+  bool update(const std::vector<std::string>& args);
 };
 
 } // namespace daemonize

src/daemon/command_server.cpp

@@ -68,6 +68,11 @@ t_command_server::t_command_server(
     , std::bind(&t_command_parser_executor::print_peer_list, &m_parser, p::_1)
     , "Print peer list"
     );
+  m_command_lookup.set_handler(
+      "print_pl_stats"
+    , std::bind(&t_command_parser_executor::print_peer_list_stats, &m_parser, p::_1)
+    , "Print peer list stats"
+    );
   m_command_lookup.set_handler(
       "print_cn"
     , std::bind(&t_command_parser_executor::print_connections, &m_parser, p::_1)
@@ -238,6 +243,11 @@ t_command_server::t_command_server(
     , std::bind(&t_command_parser_executor::print_blockchain_dynamic_stats, &m_parser, p::_1)
     , "Print information about current blockchain dynamic state"
     );
+    m_command_lookup.set_handler(
+      "update"
+    , std::bind(&t_command_parser_executor::update, &m_parser, p::_1)
+    , "subcommands: check (check if an update is available), download (download it is there is), update (not implemented)"
+    );
 }
 
 bool t_command_server::process_command_str(const std::string& cmd)

src/daemon/executor.cpp

@@ -63,6 +63,13 @@ namespace daemonize
     return t_daemon{vm};
   }
 
+  bool t_executor::run_non_interactive(
+      boost::program_options::variables_map const & vm
+    )
+  {
+    return t_daemon{vm}.run(false);
+  }
+
   bool t_executor::run_interactive(
       boost::program_options::variables_map const & vm
     )

src/daemon/executor.h

@@ -56,6 +56,10 @@ namespace daemonize
         boost::program_options::variables_map const & vm
       );
 
+    bool run_non_interactive(
+        boost::program_options::variables_map const & vm
+      );
+
     bool run_interactive(
         boost::program_options::variables_map const & vm
       );

src/daemon/main.cpp

@@ -168,7 +168,6 @@ int main(int argc, char const * argv[])
     // Create data dir if it doesn't exist
     boost::filesystem::path data_dir = boost::filesystem::absolute(
         command_line::get_arg(vm, data_dir_arg));
-    tools::create_directories_if_necessary(data_dir.string());
 
     // FIXME: not sure on windows implementation default, needs further review
     //bf::path relative_path_base = daemonizer::get_relative_path_base(vm);
@@ -216,6 +215,9 @@ int main(int argc, char const * argv[])
       mlog_set_log(command_line::get_arg(vm, daemon_args::arg_log_level).c_str());
     }
 
+    // after logs initialized
+    tools::create_directories_if_necessary(data_dir.string());
+
     // If there are positional options, we're running a daemon command
     {
       auto command = command_line::get_arg(vm, daemon_args::arg_command);

src/daemon/protocol.h

@@ -79,7 +79,6 @@ public:
       m_protocol.deinit();
       m_protocol.set_p2p_endpoint(nullptr);
       MGINFO("Cryptonote protocol stopped successfully");
-      tools::success_msg_writer() << "Daemon stopped successfully";
     } catch (...) {
       LOG_ERROR("Failed to stop cryptonote protocol!");
     }

src/daemon/rpc_command_executor.cpp

@@ -91,6 +91,13 @@ namespace {
       s = boost::lexical_cast<std::string>(dt/(3600*24)) + " days";
     return s + " " + (t > now ? "in the future" : "ago");
   }
+
+  std::string make_error(const std::string &base, const std::string &status)
+  {
+    if (status == CORE_RPC_STATUS_OK)
+      return base;
+    return base + " -- " + status;
+  }
 }
 
 t_rpc_command_executor::t_rpc_command_executor(
@@ -162,6 +169,34 @@ bool t_rpc_command_executor::print_peer_list() {
   return true;
 }
 
+bool t_rpc_command_executor::print_peer_list_stats() {
+  cryptonote::COMMAND_RPC_GET_PEER_LIST::request req;
+  cryptonote::COMMAND_RPC_GET_PEER_LIST::response res;
+
+  std::string failure_message = "Couldn't retrieve peer list";
+  if (m_is_rpc)
+  {
+    if (!m_rpc_client->rpc_request(req, res, "/get_peer_list", failure_message.c_str()))
+    {
+      return false;
+    }
+  }
+  else
+  {
+    if (!m_rpc_server->on_get_peer_list(req, res) || res.status != CORE_RPC_STATUS_OK)
+    {
+      tools::fail_msg_writer() << failure_message;
+      return false;
+    }
+  }
+
+  tools::msg_writer()
+    << "White list size: " << res.white_list.size() << "/" << P2P_LOCAL_WHITE_PEERLIST_LIMIT << " (" << res.white_list.size() *  100.0 / P2P_LOCAL_WHITE_PEERLIST_LIMIT << "%)" << std::endl
+    << "Gray list size: " << res.gray_list.size() << "/" << P2P_LOCAL_GRAY_PEERLIST_LIMIT << " (" << res.gray_list.size() *  100.0 / P2P_LOCAL_GRAY_PEERLIST_LIMIT << "%)";
+
+  return true;
+}
+
 bool t_rpc_command_executor::save_blockchain() {
   cryptonote::COMMAND_RPC_SAVE_BC::request req;
   cryptonote::COMMAND_RPC_SAVE_BC::response res;
@@ -179,7 +214,7 @@ bool t_rpc_command_executor::save_blockchain() {
   {
     if (!m_rpc_server->on_save_bc(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -207,7 +242,7 @@ bool t_rpc_command_executor::show_hash_rate() {
   {
     if (!m_rpc_server->on_set_log_hash_rate(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
     }
   }
 
@@ -234,7 +269,7 @@ bool t_rpc_command_executor::hide_hash_rate() {
   {
     if (!m_rpc_server->on_set_log_hash_rate(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -261,7 +296,7 @@ bool t_rpc_command_executor::show_difficulty() {
   {
     if (!m_rpc_server->on_get_info(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message.c_str(), res.status);
       return true;
     }
   }
@@ -345,12 +380,12 @@ bool t_rpc_command_executor::show_status() {
   {
     if (!m_rpc_server->on_get_info(ireq, ires) || ires.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, ires.status);
       return true;
     }
     if (!m_rpc_server->on_hard_fork_info(hfreq, hfres, error_resp) || hfres.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, hfres.status);
       return true;
     }
     if (!m_rpc_server->on_mining_status(mreq, mres))
@@ -365,7 +400,7 @@ bool t_rpc_command_executor::show_status() {
     }
     else if (mres.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, mres.status);
       return true;
     }
   }
@@ -411,7 +446,7 @@ bool t_rpc_command_executor::print_connections() {
   {
     if (!m_rpc_server->on_get_connections(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -476,7 +511,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u
   {
     if (!m_rpc_server->on_get_block_headers_range(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -516,7 +551,7 @@ bool t_rpc_command_executor::set_log_level(int8_t level) {
   {
     if (!m_rpc_server->on_set_log_level(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -544,7 +579,7 @@ bool t_rpc_command_executor::set_log_categories(const std::string &categories) {
   {
     if (!m_rpc_server->on_set_log_categories(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -571,7 +606,7 @@ bool t_rpc_command_executor::print_height() {
   {
     if (!m_rpc_server->on_get_height(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -601,7 +636,7 @@ bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) {
   {
     if (!m_rpc_server->on_get_block(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -632,7 +667,7 @@ bool t_rpc_command_executor::print_block_by_height(uint64_t height) {
   {
     if (!m_rpc_server->on_get_block(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -661,7 +696,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) {
   {
     if (!m_rpc_server->on_get_transactions(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -724,7 +759,7 @@ bool t_rpc_command_executor::is_key_image_spent(const crypto::key_image &ki) {
   {
     if (!m_rpc_server->on_is_key_image_spent(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -759,7 +794,7 @@ bool t_rpc_command_executor::print_transaction_pool_long() {
   {
     if (!m_rpc_server->on_get_transaction_pool(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -842,7 +877,7 @@ bool t_rpc_command_executor::print_transaction_pool_short() {
   {
     if (!m_rpc_server->on_get_transaction_pool(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -890,7 +925,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() {
   {
     if (!m_rpc_server->on_get_transaction_pool(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -950,7 +985,7 @@ bool t_rpc_command_executor::start_mining(cryptonote::account_public_address add
   {
     if (!m_rpc_server->on_start_mining(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -975,7 +1010,7 @@ bool t_rpc_command_executor::stop_mining() {
   {
     if (!m_rpc_server->on_stop_mining(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -1016,7 +1051,7 @@ bool t_rpc_command_executor::stop_daemon()
   {
     if (!m_rpc_server->on_stop_daemon(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -1114,7 +1149,7 @@ bool t_rpc_command_executor::out_peers(uint64_t limit)
 	{
 		if (!m_rpc_server->on_out_peers(req, res) || res.status != CORE_RPC_STATUS_OK)
 		{
-			tools::fail_msg_writer() << fail_message.c_str();
+			tools::fail_msg_writer() << make_error(fail_message, res.status);
 			return true;
 		}
 	}
@@ -1142,7 +1177,7 @@ bool t_rpc_command_executor::start_save_graph()
     {
 		if (!m_rpc_server->on_start_save_graph(req, res) || res.status != CORE_RPC_STATUS_OK)
 		{
-			tools::fail_msg_writer() << fail_message.c_str();
+			tools::fail_msg_writer() << make_error(fail_message, res.status);
 			return true;
 		}
 	}
@@ -1168,7 +1203,7 @@ bool t_rpc_command_executor::stop_save_graph()
     {
 		if (!m_rpc_server->on_stop_save_graph(req, res) || res.status != CORE_RPC_STATUS_OK)
 		{
-			tools::fail_msg_writer() << fail_message.c_str();
+			tools::fail_msg_writer() << make_error(fail_message, res.status);
 			return true;
 		}
 	}
@@ -1195,7 +1230,7 @@ bool t_rpc_command_executor::hard_fork_info(uint8_t version)
     {
         if (!m_rpc_server->on_hard_fork_info(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
         {
-            tools::fail_msg_writer() << fail_message.c_str();
+            tools::fail_msg_writer() << make_error(fail_message, res.status);
             return true;
         }
     }
@@ -1226,7 +1261,7 @@ bool t_rpc_command_executor::print_bans()
     {
         if (!m_rpc_server->on_get_bans(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
         {
-            tools::fail_msg_writer() << fail_message.c_str();
+            tools::fail_msg_writer() << make_error(fail_message, res.status);
             return true;
         }
     }
@@ -1268,7 +1303,7 @@ bool t_rpc_command_executor::ban(const std::string &ip, time_t seconds)
     {
         if (!m_rpc_server->on_set_bans(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
         {
-            tools::fail_msg_writer() << fail_message.c_str();
+            tools::fail_msg_writer() << make_error(fail_message, res.status);
             return true;
         }
     }
@@ -1304,7 +1339,7 @@ bool t_rpc_command_executor::unban(const std::string &ip)
     {
         if (!m_rpc_server->on_set_bans(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
         {
-            tools::fail_msg_writer() << fail_message.c_str();
+            tools::fail_msg_writer() << make_error(fail_message, res.status);
             return true;
         }
     }
@@ -1331,9 +1366,9 @@ bool t_rpc_command_executor::flush_txpool(const std::string &txid)
     }
     else
     {
-        if (!m_rpc_server->on_flush_txpool(req, res, error_resp))
+        if (!m_rpc_server->on_flush_txpool(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
         {
-            tools::fail_msg_writer() << fail_message.c_str();
+            tools::fail_msg_writer() << make_error(fail_message, res.status);
             return true;
         }
     }
@@ -1362,9 +1397,9 @@ bool t_rpc_command_executor::output_histogram(uint64_t min_count, uint64_t max_c
     }
     else
     {
-        if (!m_rpc_server->on_get_output_histogram(req, res, error_resp))
+        if (!m_rpc_server->on_get_output_histogram(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
         {
-            tools::fail_msg_writer() << fail_message.c_str();
+            tools::fail_msg_writer() << make_error(fail_message, res.status);
             return true;
         }
     }
@@ -1399,9 +1434,9 @@ bool t_rpc_command_executor::print_coinbase_tx_sum(uint64_t height, uint64_t cou
   }
   else
   {
-    if (!m_rpc_server->on_get_coinbase_tx_sum(req, res, error_resp))
+    if (!m_rpc_server->on_get_coinbase_tx_sum(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -1416,6 +1451,8 @@ bool t_rpc_command_executor::print_coinbase_tx_sum(uint64_t height, uint64_t cou
 
 bool t_rpc_command_executor::alt_chain_info()
 {
+  cryptonote::COMMAND_RPC_GET_INFO::request ireq;
+  cryptonote::COMMAND_RPC_GET_INFO::response ires;
   cryptonote::COMMAND_RPC_GET_ALTERNATE_CHAINS::request req;
   cryptonote::COMMAND_RPC_GET_ALTERNATE_CHAINS::response res;
   epee::json_rpc::error error_resp;
@@ -1424,6 +1461,10 @@ bool t_rpc_command_executor::alt_chain_info()
 
   if (m_is_rpc)
   {
+    if (!m_rpc_client->rpc_request(ireq, ires, "/getinfo", fail_message.c_str()))
+    {
+      return true;
+    }
     if (!m_rpc_client->json_rpc_request(req, res, "get_alternate_chains", fail_message.c_str()))
     {
       return true;
@@ -1431,9 +1472,14 @@ bool t_rpc_command_executor::alt_chain_info()
   }
   else
   {
+    if (!m_rpc_server->on_get_info(ireq, ires) || ires.status != CORE_RPC_STATUS_OK)
+    {
+      tools::fail_msg_writer() << make_error(fail_message, ires.status);
+      return true;
+    }
     if (!m_rpc_server->on_get_alternate_chains(req, res, error_resp))
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
   }
@@ -1441,8 +1487,9 @@ bool t_rpc_command_executor::alt_chain_info()
   tools::msg_writer() << boost::lexical_cast<std::string>(res.chains.size()) << " alternate chains found:";
   for (const auto chain: res.chains)
   {
-    tools::msg_writer() << chain.length << " blocks long, branching at height " << (chain.height - chain.length + 1)
-        << ", difficulty " << chain.difficulty << ": " << chain.block_hash;
+    uint64_t start_height = (chain.height - chain.length + 1);
+    tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1)
+        << " deep), diff " << chain.difficulty << ": " << chain.block_hash;
   }
   return true;
 }
@@ -1466,7 +1513,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks)
     {
       return true;
     }
-    if (!m_rpc_client->rpc_request(fereq, feres, "/get_fee_estimate", fail_message.c_str()))
+    if (!m_rpc_client->json_rpc_request(fereq, feres, "get_fee_estimate", fail_message.c_str()))
     {
       return true;
     }
@@ -1475,12 +1522,12 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks)
   {
     if (!m_rpc_server->on_get_info(ireq, ires) || ires.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, ires.status);
       return true;
     }
     if (!m_rpc_server->on_get_per_kb_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK)
     {
-      tools::fail_msg_writer() << fail_message.c_str();
+      tools::fail_msg_writer() << make_error(fail_message, feres.status);
       return true;
     }
   }
@@ -1497,7 +1544,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks)
     bhreq.end_height = ires.height - 1;
     if (m_is_rpc)
     {
-      if (!m_rpc_client->rpc_request(bhreq, bhres, "/getblockheadersrange", fail_message.c_str()))
+      if (!m_rpc_client->json_rpc_request(bhreq, bhres, "getblockheadersrange", fail_message.c_str()))
       {
         return true;
       }
@@ -1506,7 +1553,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks)
     {
       if (!m_rpc_server->on_get_block_headers_range(bhreq, bhres, error_resp) || bhres.status != CORE_RPC_STATUS_OK)
       {
-        tools::fail_msg_writer() << fail_message.c_str();
+        tools::fail_msg_writer() << make_error(fail_message, bhres.status);
         return true;
       }
     }
@@ -1552,4 +1599,51 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks)
   return true;
 }
 
+bool t_rpc_command_executor::update(const std::string &command)
+{
+  cryptonote::COMMAND_RPC_UPDATE::request req;
+  cryptonote::COMMAND_RPC_UPDATE::response res;
+  epee::json_rpc::error error_resp;
+
+  std::string fail_message = "Problem fetching info";
+
+  req.command = command;
+  if (m_is_rpc)
+  {
+    if (!m_rpc_client->rpc_request(req, res, "/update", fail_message.c_str()))
+    {
+      return true;
+    }
+  }
+  else
+  {
+    if (!m_rpc_server->on_update(req, res) || res.status != CORE_RPC_STATUS_OK)
+    {
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
+      return true;
+    }
+  }
+
+  if (!res.update)
+  {
+    tools::msg_writer() << "No update available";
+    return true;
+  }
+
+  tools::msg_writer() << "Update available: v" << res.version << ": " << res.user_uri << ", hash " << res.hash;
+  if (command == "check")
+    return true;
+
+  if (!res.path.empty())
+    tools::msg_writer() << "Update downloaded to: " << res.path;
+  else
+    tools::msg_writer() << "Update download failed: " << res.status;
+  if (command == "download")
+    return true;
+
+  tools::msg_writer() << "'update' not implemented yet";
+
+  return true;
+}
+
 }// namespace daemonize

src/daemon/rpc_command_executor.h

@@ -72,6 +72,8 @@ public:
 
   bool print_peer_list();
 
+  bool print_peer_list_stats();
+
   bool save_blockchain();
 
   bool show_hash_rate();
@@ -149,6 +151,8 @@ public:
   bool alt_chain_info();
 
   bool print_blockchain_dynamic_stats(uint64_t nblocks);
+
+  bool update(const std::string &command);
 };
 
 } // namespace daemonize

src/daemonizer/posix_daemonizer.inl

@@ -43,6 +43,10 @@ namespace daemonizer
       "detach"
     , "Run as daemon"
     };
+    const command_line::arg_descriptor<bool> arg_non_interactive = {
+      "non-interactive"
+    , "Run non-interactive"
+    };
   }
 
   inline void init_options(
@@ -51,6 +55,7 @@ namespace daemonizer
     )
   {
     command_line::add_arg(normal_options, arg_detach);
+    command_line::add_arg(normal_options, arg_non_interactive);
   }
 
   inline boost::filesystem::path get_default_data_dir()
@@ -79,6 +84,10 @@ namespace daemonizer
       auto daemon = executor.create_daemon(vm);
       return daemon.run();
     }
+    else if (command_line::has_arg(vm, arg_non_interactive))
+    {
+      return executor.run_non_interactive(vm);
+    }
     else
     {
       //LOG_PRINT_L0("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL);

src/mnemonics/CMakeLists.txt

@@ -34,6 +34,8 @@ set(mnemonics_headers)
 set(mnemonics_private_headers
   electrum-words.h
   english.h
+  dutch.h
+  french.h
   german.h
   italian.h
   japanese.h

src/mnemonics/electrum-words.cpp

@@ -52,6 +52,8 @@
 #include <boost/algorithm/string/join.hpp>
 
 #include "english.h"
+#include "dutch.h"
+#include "french.h"
 #include "italian.h"
 #include "german.h"
 #include "spanish.h"
@@ -83,6 +85,8 @@ namespace
     // If there's a new language added, add an instance of it here.
     std::vector<Language::Base*> language_instances({
       Language::Singleton<Language::English>::instance(),
+      Language::Singleton<Language::Dutch>::instance(),
+      Language::Singleton<Language::French>::instance(),
       Language::Singleton<Language::Spanish>::instance(),
       Language::Singleton<Language::German>::instance(),
       Language::Singleton<Language::Italian>::instance(),
@@ -312,6 +316,14 @@ namespace crypto
       {
         language = Language::Singleton<Language::English>::instance();
       }
+      else if (language_name == "Dutch")
+      {
+        language = Language::Singleton<Language::Dutch>::instance();
+      }
+      else if (language_name == "French")
+      {
+        language = Language::Singleton<Language::French>::instance();
+      }
       else if (language_name == "Spanish")
       {
         language = Language::Singleton<Language::Spanish>::instance();
@@ -382,6 +394,8 @@ namespace crypto
     {
       std::vector<Language::Base*> language_instances({
         Language::Singleton<Language::English>::instance(),
+        Language::Singleton<Language::Dutch>::instance(),
+        Language::Singleton<Language::French>::instance(),
         Language::Singleton<Language::Spanish>::instance(),
         Language::Singleton<Language::German>::instance(),
         Language::Singleton<Language::Italian>::instance(),

src/mnemonics/english.h

@@ -49,9 +49,7 @@ namespace Language
   class English: public Base
   {
   public:
-    English()
-    {
-      word_list = new std::vector<std::string>({
+    English(): Base("English", std::vector<std::string>({
         "abbey",
         "abducts",
         "ability",
@@ -1678,11 +1676,8 @@ namespace Language
         "zombie",
         "zones",
         "zoom"
-      });
-      unique_prefix_length = 3;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "English";
+      }), 3)
+    {
       populate_maps();
     }
   };

src/mnemonics/german.h

@@ -51,9 +51,7 @@ namespace Language
   class German: public Base
   {
   public:
-    German()
-    {
-      word_list = new std::vector<std::string>({
+    German(): Base("German", std::vector<std::string>({
         "Abakus",
         "Abart",
         "abbilden",
@@ -1680,11 +1678,8 @@ namespace Language
         "Zündung",
         "Zweck",
         "Zyklop"
-      });
-      unique_prefix_length = 4;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "German";
+      }), 4)
+    {
       populate_maps();
     }
   };

src/mnemonics/italian.h

@@ -51,9 +51,7 @@ namespace Language
   class Italian: public Base
   {
   public:
-    Italian()
-    {
-      word_list = new std::vector<std::string>({
+    Italian(): Base("Italian", std::vector<std::string>({
         "abbinare",
         "abbonato",
         "abisso",
@@ -1680,11 +1678,8 @@ namespace Language
         "zolfo",
         "zombie",
         "zucchero"
-      });
-      unique_prefix_length = 4;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Italian";
+      }), 4)
+    {
       populate_maps();
     }
   };

src/mnemonics/japanese.h

@@ -51,9 +51,7 @@ namespace Language
   class Japanese: public Base
   {
   public:
-    Japanese()
-    {
-      word_list = new std::vector<std::string>({
+    Japanese(): Base("Japanese", std::vector<std::string>({
         "あいこくしん",
         "あいさつ",
         "あいだ",
@@ -1680,11 +1678,8 @@ namespace Language
         "ひさん",
         "びじゅつかん",
         "ひしょ"
-      });
-      unique_prefix_length = 3;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Japanese";
+      }), 3)
+    {
       populate_maps();
     }
   };

src/mnemonics/language_base.h

@@ -38,6 +38,7 @@
 #include <vector>
 #include <unordered_map>
 #include <string>
+#include "misc_log_ex.h"
 
 /*!
  * \namespace Language
@@ -73,44 +74,62 @@ namespace Language
   class Base
   {
   protected:
-    std::vector<std::string> *word_list; /*!< A pointer to the array of words */
-    std::unordered_map<std::string, uint32_t> *word_map; /*!< hash table to find word's index */
-    std::unordered_map<std::string, uint32_t> *trimmed_word_map; /*!< hash table to find word's trimmed index */
+    enum {
+      ALLOW_SHORT_WORDS = 1<<0,
+      ALLOW_DUPLICATE_PREFIXES = 1<<1,
+    };
+    const std::vector<std::string> word_list; /*!< A pointer to the array of words */
+    std::unordered_map<std::string, uint32_t> word_map; /*!< hash table to find word's index */
+    std::unordered_map<std::string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
     std::string language_name; /*!< Name of language */
     uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */
     /*!
      * \brief Populates the word maps after the list is ready.
      */
-    void populate_maps()
+    void populate_maps(uint32_t flags = 0)
     {
       int ii;
-      std::vector<std::string>::iterator it;
-      for (it = word_list->begin(), ii = 0; it != word_list->end(); it++, ii++)
+      std::vector<std::string>::const_iterator it;
+      if (word_list.size () != 1626)
+        throw std::runtime_error("Wrong word list length for " + language_name);
+      for (it = word_list.begin(), ii = 0; it != word_list.end(); it++, ii++)
       {
-        (*word_map)[*it] = ii;
+        word_map[*it] = ii;
+        if ((*it).size() < unique_prefix_length)
+        {
+          if (flags & ALLOW_SHORT_WORDS)
+            MWARNING(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
+          else
+            throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
+        }
+        std::string trimmed;
         if (it->length() > unique_prefix_length)
         {
-          (*trimmed_word_map)[utf8prefix(*it, unique_prefix_length)] = ii;
+          trimmed = utf8prefix(*it, unique_prefix_length);
         }
         else
         {
-          (*trimmed_word_map)[*it] = ii;
+          trimmed = *it;
         }
+        if (trimmed_word_map.find(trimmed) != trimmed_word_map.end())
+        {
+          if (flags & ALLOW_DUPLICATE_PREFIXES)
+            MWARNING("Duplicate prefix in " << language_name << " word list: " << trimmed);
+          else
+            throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + trimmed);
+        }
+        trimmed_word_map[trimmed] = ii;
       }
     }
   public:
-    Base()
+    Base(const char *language_name, const std::vector<std::string> &words, uint32_t prefix_length):
+      word_list(words),
+      unique_prefix_length(prefix_length),
+      language_name(language_name)
     {
-      word_list = new std::vector<std::string>;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      unique_prefix_length = 4;
     }
     virtual ~Base()
     {
-      delete word_list;
-      delete word_map;
-      delete trimmed_word_map;
     }
     /*!
      * \brief Returns a pointer to the word list.
@@ -118,7 +137,7 @@ namespace Language
      */
     const std::vector<std::string>& get_word_list() const
     {
-      return *word_list;
+      return word_list;
     }
     /*!
      * \brief Returns a pointer to the word map.
@@ -126,7 +145,7 @@ namespace Language
      */
     const std::unordered_map<std::string, uint32_t>& get_word_map() const
     {
-      return *word_map;
+      return word_map;
     }
     /*!
      * \brief Returns a pointer to the trimmed word map.
@@ -134,13 +153,13 @@ namespace Language
      */
     const std::unordered_map<std::string, uint32_t>& get_trimmed_word_map() const
     {
-      return *trimmed_word_map;
+      return trimmed_word_map;
     }
     /*!
      * \brief Returns the name of the language.
      * \return Name of the language.
      */
-    std::string get_language_name() const
+    const std::string &get_language_name() const
     {
       return language_name;
     }

src/mnemonics/old_english.h

@@ -51,9 +51,7 @@ namespace Language
   class OldEnglish: public Base
   {
   public:
-    OldEnglish()
-    {
-      word_list = new std::vector<std::string>({
+    OldEnglish(): Base("OldEnglish", std::vector<std::string>({
         "like",
         "just",
         "love",
@@ -1680,12 +1678,9 @@ namespace Language
         "unseen",
         "weapon",
         "weary"
-      });
-      unique_prefix_length = 4;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "OldEnglish";
-      populate_maps();
+      }), 4)
+    {
+      populate_maps(ALLOW_DUPLICATE_PREFIXES | ALLOW_SHORT_WORDS);
     }
   };
 }

src/mnemonics/portuguese.h

@@ -49,9 +49,7 @@ namespace Language
   class Portuguese: public Base
   {
   public:
-    Portuguese()
-    {
-      word_list = new std::vector<std::string>({
+    Portuguese(): Base("Portuguese", std::vector<std::string>({
         "abaular",
         "abdominal",
         "abeto",
@@ -1678,11 +1676,8 @@ namespace Language
         "zeloso",
         "zenite",
         "zumbi"
-      });
-      unique_prefix_length = 4;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Portuguese";
+      }), 4)
+    {
       populate_maps();
     }
   };

src/mnemonics/russian.h

@@ -51,9 +51,7 @@ namespace Language
   class Russian: public Base
   {
   public:
-    Russian()
-    {
-      word_list = new std::vector<std::string>({
+    Russian(): Base("Russian", std::vector<std::string>({
         "абажур",
         "абзац",
         "абонент",
@@ -1680,11 +1678,8 @@ namespace Language
         "яхта",
         "ячейка",
         "ящик"
-      });
-      unique_prefix_length = 4;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Russian";
+      }), 4)
+    {
       populate_maps();
     }
   };

src/mnemonics/spanish.h

@@ -51,9 +51,7 @@ namespace Language
   class Spanish: public Base
   {
   public:
-    Spanish()
-    {
-      word_list = new std::vector<std::string>({
+    Spanish(): Base("Spanish", std::vector<std::string>({
         "ábaco",
         "abdomen",
         "abeja",
@@ -1680,12 +1678,9 @@ namespace Language
         "risa",
         "ritmo",
         "rito"
-      });
-      unique_prefix_length = 4;
-      word_map = new std::unordered_map<std::string, uint32_t>;
-      trimmed_word_map = new std::unordered_map<std::string, uint32_t>;
-      language_name = "Spanish";
-      populate_maps();
+      }), 4)
+    {
+      populate_maps(ALLOW_SHORT_WORDS);
     }
   };
 }

src/p2p/net_node.h

@@ -191,7 +191,6 @@ namespace nodetool
     bool parse_peer_from_string(nodetool::net_address& pe, const std::string& node_addr);
     bool handle_command_line(
         const boost::program_options::variables_map& vm
-      , bool testnet
       );
     bool idle_worker();
     bool handle_remote_peerlist(const std::list<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context);
@@ -218,6 +217,7 @@ namespace nodetool
     void cache_connect_fail_info(const net_address& addr);
     bool is_addr_recently_failed(const net_address& addr);
     bool is_priority_node(const net_address& na);
+    std::set<std::string> get_seed_nodes(bool testnet) const;
 
     template <class Container>
     bool connect_to_peerlist(const Container& peers);
@@ -321,6 +321,8 @@ namespace nodetool
 
     epee::critical_section m_ip_fails_score_lock;
     std::map<uint32_t, uint64_t> m_ip_fails_score;
+
+    bool m_testnet;
   };
 }
 

src/p2p/net_node.inl

@@ -65,6 +65,7 @@
 
 #define NET_MAKE_IP(b1,b2,b3,b4)  ((LPARAM)(((DWORD)(b1)<<24)+((DWORD)(b2)<<16)+((DWORD)(b3)<<8)+((DWORD)(b4))))
 
+#define MIN_WANTED_SEED_NODES 12
 
 namespace nodetool
 {
@@ -287,10 +288,9 @@ namespace nodetool
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::handle_command_line(
       const boost::program_options::variables_map& vm
-    , bool testnet
     )
   {
-    auto p2p_bind_arg = testnet ? arg_testnet_p2p_bind_port : arg_p2p_bind_port;
+    auto p2p_bind_arg = m_testnet ? arg_testnet_p2p_bind_port : arg_p2p_bind_port;
 
     m_bind_ip = command_line::get_arg(vm, arg_p2p_bind_ip);
     m_port = command_line::get_arg(vm, p2p_bind_arg);
@@ -309,7 +309,7 @@ namespace nodetool
         bool r = parse_peer_from_string(pe.adr, pr_str);
         CHECK_AND_ASSERT_MES(r, false, "Failed to parse address from string: " << pr_str);
         if (pe.adr.port == 0)
-          pe.adr.port = testnet ? ::config::testnet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT;
+          pe.adr.port = m_testnet ? ::config::testnet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT;
         m_command_line_peers.push_back(pe);
       }
     }
@@ -398,14 +398,11 @@ namespace nodetool
 
   //-----------------------------------------------------------------------------------
   template<class t_payload_net_handler>
-  bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm)
+  std::set<std::string> node_server<t_payload_net_handler>::get_seed_nodes(bool testnet) const
   {
     std::set<std::string> full_addrs;
-    bool testnet = command_line::get_arg(vm, command_line::arg_testnet_on);
-
     if (testnet)
     {
-      memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16);
       full_addrs.insert("212.83.175.67:28080");
       full_addrs.insert("5.9.100.248:28080");
       full_addrs.insert("163.172.182.165:28080");
@@ -413,6 +410,32 @@ namespace nodetool
       full_addrs.insert("212.83.172.165:28080");
     }
     else
+    {
+      full_addrs.insert("107.152.130.98:18080");
+      full_addrs.insert("212.83.175.67:18080");
+      full_addrs.insert("5.9.100.248:18080");
+      full_addrs.insert("163.172.182.165:18080");
+      full_addrs.insert("161.67.132.39:18080");
+      full_addrs.insert("198.74.231.92:18080");
+      full_addrs.insert("195.154.123.123:28080");
+      full_addrs.insert("212.83.172.165:28080");
+    }
+    return full_addrs;
+  }
+
+  //-----------------------------------------------------------------------------------
+  template<class t_payload_net_handler>
+  bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm)
+  {
+    std::set<std::string> full_addrs;
+    m_testnet = command_line::get_arg(vm, command_line::arg_testnet_on);
+
+    if (m_testnet)
+    {
+      memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16);
+      full_addrs = get_seed_nodes(true);
+    }
+    else
     {
       memcpy(&m_network_id, &::config::NETWORK_ID, 16);
       // for each hostname in the seed nodes list, attempt to DNS resolve and
@@ -483,18 +506,16 @@ namespace nodetool
         ++i;
       }
 
-      if (!full_addrs.size())
+      // append the fallback nodes if we have too few seed nodes to start with
+      if (full_addrs.size() < MIN_WANTED_SEED_NODES)
       {
-        MINFO("DNS seed node lookup either timed out or failed, falling back to defaults");
+        if (full_addrs.empty())
+          MINFO("DNS seed node lookup either timed out or failed, falling back to defaults");
+        else
+          MINFO("Not enough DNS seed nodes found, using fallback defaults too");
 
-        full_addrs.insert("107.152.130.98:18080");
-        full_addrs.insert("212.83.175.67:18080");
-        full_addrs.insert("5.9.100.248:18080");
-        full_addrs.insert("163.172.182.165:18080");
-        full_addrs.insert("161.67.132.39:18080");
-        full_addrs.insert("198.74.231.92:18080");
-        full_addrs.insert("195.154.123.123:28080");
-        full_addrs.insert("212.83.172.165:28080");
+        for (const auto &peer: get_seed_nodes(false))
+          full_addrs.insert(peer);
       }
     }
 
@@ -505,14 +526,14 @@ namespace nodetool
     }
     MDEBUG("Number of seed nodes: " << m_seed_nodes.size());
 
-    bool res = handle_command_line(vm, testnet);
+    bool res = handle_command_line(vm);
     CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line");
 
-    auto config_arg = testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
+    auto config_arg = m_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
     m_config_folder = command_line::get_arg(vm, config_arg);
 
-    if ((!testnet && m_port != std::to_string(::config::P2P_DEFAULT_PORT))
-        || (testnet && m_port != std::to_string(::config::testnet::P2P_DEFAULT_PORT))) {
+    if ((!m_testnet && m_port != std::to_string(::config::P2P_DEFAULT_PORT))
+        || (m_testnet && m_port != std::to_string(::config::testnet::P2P_DEFAULT_PORT))) {
       m_config_folder = m_config_folder + "/" + m_port;
     }
 
@@ -1023,7 +1044,14 @@ namespace nodetool
     while(rand_count < (max_random_index+1)*3 &&  try_count < 10 && !m_net_server.is_stop_signal_sent())
     {
       ++rand_count;
-      size_t random_index = get_random_index_with_fixed_probability(max_random_index);
+      size_t random_index;
+
+      if (use_white_list) {
+        random_index = get_random_index_with_fixed_probability(max_random_index);
+      } else {
+        random_index = crypto::rand<size_t>() % m_peerlist.get_gray_peers_count();
+      }
+
       CHECK_AND_ASSERT_MES(random_index < local_peers_count, false, "random_starter_index < peers_local.size() failed!!");
 
       if(tried_peers.count(random_index))
@@ -1075,6 +1103,7 @@ namespace nodetool
     {
       size_t try_count = 0;
       size_t current_index = crypto::rand<size_t>()%m_seed_nodes.size();
+      bool fallback_nodes_added = false;
       while(true)
       {
         if(m_net_server.is_stop_signal_sent())
@@ -1084,8 +1113,22 @@ namespace nodetool
           break;
         if(++try_count > m_seed_nodes.size())
         {
-          MWARNING("Failed to connect to any of seed peers, continuing without seeds");
-          break;
+          if (!fallback_nodes_added)
+          {
+            MWARNING("Failed to connect to any of seed peers, trying fallback seeds");
+            for (const auto &peer: get_seed_nodes(m_testnet))
+            {
+              MDEBUG("Fallback seed node: " << peer);
+              append_net_address(m_seed_nodes, peer);
+            }
+            fallback_nodes_added = true;
+            // continue for another few cycles
+          }
+          else
+          {
+            MWARNING("Failed to connect to any of seed peers, continuing without seeds");
+            break;
+          }
         }
         if(++current_index >= m_seed_nodes.size())
           current_index = 0;
@@ -1647,7 +1690,6 @@ namespace nodetool
   bool node_server<t_payload_net_handler>::parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container)
   {
     std::vector<std::string> perrs = command_line::get_arg(vm, arg);
-    bool testnet = command_line::get_arg(vm, command_line::arg_testnet_on);
 
     for(const std::string& pr_str: perrs)
     {
@@ -1655,7 +1697,7 @@ namespace nodetool
       bool r = parse_peer_from_string(na, pr_str);
       CHECK_AND_ASSERT_MES(r, false, "Failed to parse address from string: " << pr_str);
       if (na.port == 0)
-        na.port = testnet ? ::config::testnet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT;
+        na.port = m_testnet ? ::config::testnet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT;
       container.push_back(na);
     }
 

src/p2p/net_peerlist.h

@@ -408,13 +408,10 @@ namespace nodetool
       return false;
     }
 
-    size_t x = crypto::rand<size_t>() % (m_peers_gray.size() + 1);
-    size_t res = (x * x * x) / (m_peers_gray.size() * m_peers_gray.size()); //parabola \/
-
-    LOG_PRINT_L3("Random gray peer index=" << res << "(x="<< x << ", max_index=" << m_peers_gray.size() << ")");
+    size_t random_index = crypto::rand<size_t>() % m_peers_gray.size();
 
     peers_indexed::index<by_time>::type& by_time_index = m_peers_gray.get<by_time>();
-    pe = *epee::misc_utils::move_it_backward(--by_time_index.end(), res);
+    pe = *epee::misc_utils::move_it_backward(--by_time_index.end(), random_index);
 
     return true;
 

src/ringct/rctOps.cpp

@@ -36,6 +36,8 @@ using namespace std;
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "ringct"
 
+#define CHECK_AND_ASSERT_THROW_MES_L1(expr, message) {if(!(expr)) {MWARNING(message); throw std::runtime_error(message);}}
+
 namespace rct {
 
     //Various key initialization functions
@@ -175,7 +177,7 @@ namespace rct {
     void scalarmultKey(key & aP, const key &P, const key &a) {
         ge_p3 A;
         ge_p2 R;
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&A, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_scalarmult(&R, a.bytes, &A);
         ge_tobytes(aP.bytes, &R);
     }
@@ -184,7 +186,7 @@ namespace rct {
     key scalarmultKey(const key & P, const key & a) {
         ge_p3 A;
         ge_p2 R;
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&A, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_scalarmult(&R, a.bytes, &A);
         key aP;
         ge_tobytes(aP.bytes, &R);
@@ -196,7 +198,7 @@ namespace rct {
     key scalarmultH(const key & a) {
         ge_p3 A;
         ge_p2 R;
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&A, H.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A, H.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_scalarmult(&R, a.bytes, &A);
         key aP;
         ge_tobytes(aP.bytes, &R);
@@ -208,8 +210,8 @@ namespace rct {
     //for curve points: AB = A + B
     void addKeys(key &AB, const key &A, const key &B) {
         ge_p3 B2, A2;
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&B2, B.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&A2, A.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&B2, B.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A2, A.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_cached tmp2;
         ge_p3_to_cached(&tmp2, &B2);
         ge_p1p1 tmp3;
@@ -231,7 +233,7 @@ namespace rct {
     void addKeys2(key &aGbB, const key &a, const key &b, const key & B) {
         ge_p2 rv;
         ge_p3 B2;
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&B2, B.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&B2, B.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_double_scalarmult_base_vartime(&rv, b.bytes, &B2, a.bytes);
         ge_tobytes(aGbB.bytes, &rv);
     }
@@ -240,7 +242,7 @@ namespace rct {
     // input B a curve point and output a ge_dsmp which has precomputation applied
     void precomp(ge_dsmp rv, const key & B) {
         ge_p3 B2;
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&B2, B.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&B2, B.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_dsm_precomp(rv, &B2);
     }
 
@@ -250,7 +252,7 @@ namespace rct {
     void addKeys3(key &aAbB, const key &a, const key &A, const key &b, const ge_dsmp B) {
         ge_p2 rv;
         ge_p3 A2;
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&A2, A.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A2, A.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_double_scalarmult_precomp_vartime(&rv, a.bytes, &A2, b.bytes, B);
         ge_tobytes(aAbB.bytes, &rv);
     }
@@ -260,8 +262,8 @@ namespace rct {
     //AB = A - B where A, B are curve points
     void subKeys(key & AB, const key &A, const key &B) {
         ge_p3 B2, A2;
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&B2, B.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&A2, A.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&B2, B.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A2, A.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_cached tmp2;
         ge_p3_to_cached(&tmp2, &B2);
         ge_p1p1 tmp3;
@@ -381,7 +383,7 @@ namespace rct {
         ge_p2 point;
         ge_p3 res;
         key h = cn_fast_hash(hh); 
-        CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&res, h.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&res, h.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
         ge_p3_to_p2(&point, &res);
         ge_mul8(&point2, &point);
         ge_p1p1_to_p3(&res, &point2);

src/ringct/rctSigs.cpp

@@ -873,8 +873,7 @@ namespace rct {
     //   must know the destination private key to find the correct amount, else will return a random number    
     xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask) {
         CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "decodeRct called on non-full rctSig");
-        CHECK_AND_ASSERT_THROW_MES(rv.p.rangeSigs.size() > 0, "Empty rv.p.rangeSigs");
-        CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.p.rangeSigs.size(), "Mismatched sizes of rv.outPk and rv.p.rangeSigs");
+        CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo");
         CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index");
 
         //mask amount and mask
@@ -902,8 +901,7 @@ namespace rct {
 
     xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask) {
         CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple, false, "decodeRct called on non simple rctSig");
-        CHECK_AND_ASSERT_THROW_MES(rv.p.rangeSigs.size() > 0, "Empty rv.p.rangeSigs");
-        CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.p.rangeSigs.size(), "Mismatched sizes of rv.outPk and rv.p.rangeSigs");
+        CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo");
         CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index");
 
         //mask amount and mask

src/rpc/core_rpc_server.cpp

@@ -33,6 +33,9 @@ using namespace epee;
 
 #include "core_rpc_server.h"
 #include "common/command_line.h"
+#include "common/updates.h"
+#include "common/download.h"
+#include "common/util.h"
 #include "cryptonote_basic/cryptonote_format_utils.h"
 #include "cryptonote_basic/account.h"
 #include "cryptonote_basic/cryptonote_basic_impl.h"
@@ -41,6 +44,9 @@ using namespace epee;
 #include "rpc/rpc_args.h"
 #include "core_rpc_server_error_codes.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "daemon.rpc"
+
 #define MAX_RESTRICTED_FAKE_OUTS_COUNT 40
 #define MAX_RESTRICTED_GLOBAL_FAKE_OUTS_COUNT 500
 
@@ -148,6 +154,23 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  static cryptonote::blobdata get_pruned_tx_blob(const cryptonote::blobdata &blobdata)
+  {
+    cryptonote::transaction tx;
+
+    if (!cryptonote::parse_and_validate_tx_from_blob(blobdata, tx))
+    {
+      MERROR("Failed to parse and validate tx from blob");
+      return blobdata;
+    }
+
+    std::stringstream ss;
+    binary_archive<true> ba(ss);
+    bool r = tx.serialize_base(ba);
+    CHECK_AND_ASSERT_MES(r, blobdata, "Failed to serialize rct signatures base");
+    return ss.str();
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res)
   {
     CHECK_CORE_BUSY();
@@ -159,10 +182,13 @@ namespace cryptonote
       return false;
     }
 
+    size_t pruned_size = 0, unpruned_size = 0, ntxes = 0;
     for(auto& bd: bs)
     {
       res.blocks.resize(res.blocks.size()+1);
       res.blocks.back().block = bd.first;
+      pruned_size += bd.first.size();
+      unpruned_size += bd.first.size();
       res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
       res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
       block b;
@@ -178,9 +204,15 @@ namespace cryptonote
         return false;
       }
       size_t txidx = 0;
+      ntxes += bd.second.size();
       for(const auto& t: bd.second)
       {
-        res.blocks.back().txs.push_back(t);
+        if (req.prune)
+          res.blocks.back().txs.push_back(get_pruned_tx_blob(t));
+        else
+          res.blocks.back().txs.push_back(t);
+        pruned_size += res.blocks.back().txs.back().size();
+        unpruned_size += t.size();
         res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
         bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices);
         if (!r)
@@ -191,6 +223,7 @@ namespace cryptonote
       }
     }
 
+    MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, pruned size " << pruned_size << ", unpruned size " << unpruned_size);
     res.status = CORE_RPC_STATUS_OK;
     return true;
   }
@@ -665,7 +698,7 @@ namespace cryptonote
   //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res)
   {
-    CHECK_CORE_READY();
+    CHECK_CORE_BUSY();
 
     const miner& lMiner = m_core.get_miner();
     res.active = lMiner.is_mining();
@@ -756,6 +789,14 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res)
+  {
+    CHECK_CORE_BUSY();
+    m_core.get_pool_transaction_hashes(res.tx_hashes);
+    res.status = CORE_RPC_STATUS_OK;
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res)
   {
     // FIXME: replace back to original m_p2p.send_stop_signal() after
@@ -1455,6 +1496,98 @@ namespace cryptonote
 	  return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res)
+  {
+    static const char software[] = "monero";
+#ifdef BUILD_TAG
+    static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG);
+#else
+    static const char buildtag[] = "source";
+#endif
+
+    if (req.command != "check" && req.command != "download" && req.command != "update")
+    {
+      res.status = std::string("unknown command: '") + req.command + "'";
+      return true;
+    }
+
+    std::string version, hash;
+    if (!tools::check_updates(software, buildtag, version, hash))
+    {
+      res.status = "Error checking for updates";
+      return true;
+    }
+    if (tools::vercmp(version.c_str(), MONERO_VERSION) <= 0)
+    {
+      res.update = false;
+      res.status = CORE_RPC_STATUS_OK;
+      return true;
+    }
+    res.update = true;
+    res.version = version;
+    res.user_uri = tools::get_update_url(software, "cli", buildtag, version, true);
+    res.auto_uri = tools::get_update_url(software, "cli", buildtag, version, false);
+    res.hash = hash;
+    if (req.command == "check")
+    {
+      res.status = CORE_RPC_STATUS_OK;
+      return true;
+    }
+
+    boost::filesystem::path path;
+    if (req.path.empty())
+    {
+      std::string filename;
+      const char *slash = strrchr(res.auto_uri.c_str(), '/');
+      if (slash)
+        filename = slash + 1;
+      else
+        filename = std::string(software) + "-update-" + version;
+      path = epee::string_tools::get_current_module_folder();
+      path /= filename;
+    }
+    else
+    {
+      path = req.path;
+    }
+
+    crypto::hash file_hash;
+    if (!tools::sha256sum(path.string(), file_hash) || (hash != epee::string_tools::pod_to_hex(file_hash)))
+    {
+      MDEBUG("We don't have that file already, downloading");
+      if (!tools::download(path.string(), res.auto_uri))
+      {
+        MERROR("Failed to download " << res.auto_uri);
+        return false;
+      }
+      if (!tools::sha256sum(path.string(), file_hash))
+      {
+        MERROR("Failed to hash " << path);
+        return false;
+      }
+      if (hash != epee::string_tools::pod_to_hex(file_hash))
+      {
+        MERROR("Download from " << res.auto_uri << " does not match the expected hash");
+        return false;
+      }
+      MINFO("New version downloaded to " << path);
+    }
+    else
+    {
+      MDEBUG("We already have " << path << " with expected hash");
+    }
+    res.path = path.string();
+
+    if (req.command == "download")
+    {
+      res.status = CORE_RPC_STATUS_OK;
+      return true;
+    }
+
+    res.status = "'update' not implemented yet";
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
 
   const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_port = {
       "rpc-bind-port"

src/rpc/core_rpc_server.h

@@ -92,12 +92,14 @@ namespace cryptonote
       MAP_URI_AUTO_JON2_IF("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL, !m_restricted)
       MAP_URI_AUTO_JON2_IF("/set_log_categories", on_set_log_categories, COMMAND_RPC_SET_LOG_CATEGORIES, !m_restricted)
       MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL)
+      MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES)
       MAP_URI_AUTO_JON2_IF("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON, !m_restricted)
       MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO)
       MAP_URI_AUTO_JON2_IF("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS, !m_restricted)
       MAP_URI_AUTO_JON2_IF("/start_save_graph", on_start_save_graph, COMMAND_RPC_START_SAVE_GRAPH, !m_restricted)
       MAP_URI_AUTO_JON2_IF("/stop_save_graph", on_stop_save_graph, COMMAND_RPC_STOP_SAVE_GRAPH, !m_restricted)
       MAP_URI_AUTO_JON2("/get_outs", on_get_outs, COMMAND_RPC_GET_OUTPUTS)      
+      MAP_URI_AUTO_JON2_IF("/update", on_update, COMMAND_RPC_UPDATE, !m_restricted)
       BEGIN_JSON_RPC_MAP("/json_rpc")
         MAP_JON_RPC("getblockcount",             on_getblockcount,              COMMAND_RPC_GETBLOCKCOUNT)
         MAP_JON_RPC_WE("on_getblockhash",        on_getblockhash,               COMMAND_RPC_GETBLOCKHASH)
@@ -144,10 +146,12 @@ namespace cryptonote
     bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res);
     bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res);
     bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res);
+    bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res);
     bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res);
     bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res);
     bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res);
     bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res);
+    bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res);
     
     //json_rpc
     bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res);

src/rpc/core_rpc_server_commands_defs.h

@@ -49,7 +49,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 1
-#define CORE_RPC_VERSION_MINOR 6
+#define CORE_RPC_VERSION_MINOR 8
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
 
@@ -80,9 +80,11 @@ namespace cryptonote
     {
       std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */
       uint64_t    start_height;
+      bool        prune;
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids)
         KV_SERIALIZE(start_height)
+        KV_SERIALIZE(prune)
       END_KV_SERIALIZE_MAP()
     };
 
@@ -1023,6 +1025,26 @@ namespace cryptonote
     };
   };
 
+  struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES
+  {
+    struct request
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::string status;
+      std::vector<crypto::hash> tx_hashes;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(status)
+        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(tx_hashes)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
   struct COMMAND_RPC_GET_CONNECTIONS
   {
     struct request
@@ -1438,4 +1460,39 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
   };
+
+  struct COMMAND_RPC_UPDATE
+  {
+    struct request
+    {
+      std::string command;
+      std::string path;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(command);
+        KV_SERIALIZE(path);
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::string status;
+      bool update;
+      std::string version;
+      std::string user_uri;
+      std::string auto_uri;
+      std::string hash;
+      std::string path;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(status)
+        KV_SERIALIZE(update)
+        KV_SERIALIZE(version)
+        KV_SERIALIZE(user_uri)
+        KV_SERIALIZE(auto_uri)
+        KV_SERIALIZE(hash)
+        KV_SERIALIZE(path)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
 }

src/rpc/rpc_args.cpp

@@ -37,7 +37,7 @@ namespace cryptonote
   rpc_args::descriptors::descriptors()
      : rpc_bind_ip({"rpc-bind-ip", rpc_args::tr("Specify ip to bind rpc server"), "127.0.0.1"})
      , rpc_login({"rpc-login", rpc_args::tr("Specify username[:password] required for RPC server"), "", true})
-     , confirm_external_bind({"confirm-external-bind", rpc_args::tr("Confirm rcp-bind-ip value is NOT a loopback (local) IP")})
+     , confirm_external_bind({"confirm-external-bind", rpc_args::tr("Confirm rpc-bind-ip value is NOT a loopback (local) IP")})
   {}
 
   const char* rpc_args::tr(const char* str) { return i18n_translate(str, "cryptonote::rpc_args"); }

src/simplewallet/simplewallet.cpp

@@ -380,9 +380,9 @@ bool simple_wallet::change_password(const std::vector<std::string> &args)
     return false;
   }
 
-  // prompts for a new password, this is not a new wallet so pass in false.
-  const auto pwd_container = tools::wallet2::password_prompt(false);
-  
+  // prompts for a new password, pass true to verify the password
+  const auto pwd_container = tools::wallet2::password_prompt(true);
+
   try
   {
     m_wallet->rewrite(m_wallet_file, pwd_container->password());
@@ -482,16 +482,11 @@ bool simple_wallet::set_default_mixin(const std::vector<std::string> &args/* = s
 bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   int priority = 0;
-  if (m_wallet->watch_only())
-  {
-    fail_msg_writer() << tr("wallet is watch-only and cannot transfer");
-    return true;
-  }
   try
   {
     if (strchr(args[1].c_str(), '-'))
     {
-      fail_msg_writer() << tr("priority must be 0, 1, 2, or 3 ");
+      fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4 ");
       return true;
     }
     if (args[1] == "0")
@@ -501,9 +496,9 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/*
     else
     {
       priority = boost::lexical_cast<int>(args[1]);
-      if (priority != 1 && priority != 2 && priority != 3)
+      if (priority < 1 || priority > 4)
       {
-        fail_msg_writer() << tr("priority must be 0, 1, 2, or 3");
+        fail_msg_writer() << tr("priority must be 0, 1, 2, 3,or 4");
         return true;
       }
     }
@@ -518,7 +513,7 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/*
   }
   catch(const boost::bad_lexical_cast &)
   {
-    fail_msg_writer() << tr("priority must be 0, 1, 2 or 3");
+    fail_msg_writer() << tr("priority must be 0, 1, 2 3,or 4");
     return true;
   }
   catch(...)
@@ -590,6 +585,83 @@ bool simple_wallet::set_ask_password(const std::vector<std::string> &args/* = st
   return true;
 }
 
+bool simple_wallet::set_unit(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  const std::string &unit = args[1];
+  unsigned int decimal_point = CRYPTONOTE_DISPLAY_DECIMAL_POINT;
+
+  if (unit == "monero")
+    decimal_point = CRYPTONOTE_DISPLAY_DECIMAL_POINT;
+  else if (unit == "millinero")
+    decimal_point = CRYPTONOTE_DISPLAY_DECIMAL_POINT - 3;
+  else if (unit == "micronero")
+    decimal_point = CRYPTONOTE_DISPLAY_DECIMAL_POINT - 6;
+  else if (unit == "nanonero")
+    decimal_point = CRYPTONOTE_DISPLAY_DECIMAL_POINT - 9;
+  else if (unit == "piconero")
+    decimal_point = 0;
+  else
+  {
+    fail_msg_writer() << tr("invalid unit");
+    return true;
+  }
+
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    m_wallet->set_default_decimal_point(decimal_point);
+    m_wallet->rewrite(m_wallet_file, pwd_container->password());
+  }
+  return true;
+}
+
+bool simple_wallet::set_min_output_count(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  uint32_t count;
+  if (!string_tools::get_xtype_from_string(count, args[1]))
+  {
+    fail_msg_writer() << tr("invalid count: must be an unsigned integer");
+    return true;
+  }
+
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    m_wallet->set_min_output_count(count);
+    m_wallet->rewrite(m_wallet_file, pwd_container->password());
+  }
+  return true;
+}
+
+bool simple_wallet::set_min_output_value(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  uint64_t value;
+  if (!cryptonote::parse_amount(value, args[1]))
+  {
+    fail_msg_writer() << tr("invalid value");
+    return true;
+  }
+
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    m_wallet->set_min_output_value(value);
+    m_wallet->rewrite(m_wallet_file, pwd_container->password());
+  }
+  return true;
+}
+
+bool simple_wallet::set_merge_destinations(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    m_wallet->merge_destinations(is_it_true(args[1]));
+    m_wallet->rewrite(m_wallet_file, pwd_container->password());
+  }
+  return true;
+}
+
 bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   success_msg_writer() << get_commands_str();
@@ -612,11 +684,11 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("incoming_transfers", boost::bind(&simple_wallet::show_incoming_transfers, this, _1), tr("incoming_transfers [available|unavailable] - Show incoming transfers, all or filtered by availability"));
   m_cmd_binder.set_handler("payments", boost::bind(&simple_wallet::show_payments, this, _1), tr("payments <PID_1> [<PID_2> ... <PID_N>] - Show payments for given payment ID[s]"));
   m_cmd_binder.set_handler("bc_height", boost::bind(&simple_wallet::show_blockchain_height, this, _1), tr("Show blockchain height"));
-  m_cmd_binder.set_handler("transfer_original", boost::bind(&simple_wallet::transfer, this, _1), tr("transfer [<mixin_count>] <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of extra inputs to include for untraceability (from 2 to maximum available)"));
-  m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer_new, this, _1), tr("Same as transfer_original, but using a new transaction building algorithm"));
+  m_cmd_binder.set_handler("transfer_original", boost::bind(&simple_wallet::transfer, this, _1), tr("Same as transfer, but using an older transaction building algorithm"));
+  m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer_new, this, _1), tr("transfer [<mixin_count>] <address> <amount> [<payment_id>] - Transfer <amount> to <address>. <mixin_count> is the number of extra inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)"));
   m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), tr("locked_transfer [<mixin_count>] <addr> <amount> <lockblocks>(Number of blocks to lock the transaction for, max 1000000) [<payment_id>]"));
   m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with mixin 0"));
-  m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1), tr("sweep_all [mixin] address [payment_id] - Send all unlocked balance an address"));
+  m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1), tr("sweep_all [mixin] address [payment_id] - Send all unlocked balance to an address"));
   m_cmd_binder.set_handler("donate", boost::bind(&simple_wallet::donate, this, _1), tr("donate [<mixin_count>] <amount> [payment_id] - Donate <amount> to the development team (donate.getmonero.org)"));
   m_cmd_binder.set_handler("sign_transfer", boost::bind(&simple_wallet::sign_transfer, this, _1), tr("Sign a transaction from a file"));
   m_cmd_binder.set_handler("submit_transfer", boost::bind(&simple_wallet::submit_transfer, this, _1), tr("Submit a signed transaction from a file"));
@@ -629,7 +701,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("viewkey", boost::bind(&simple_wallet::viewkey, this, _1), tr("Display private view key"));
   m_cmd_binder.set_handler("spendkey", boost::bind(&simple_wallet::spendkey, this, _1), tr("Display private spend key"));
   m_cmd_binder.set_handler("seed", boost::bind(&simple_wallet::seed, this, _1), tr("Display Electrum-style mnemonic seed"));
-  m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr("Available options: seed language - set wallet seed language; always-confirm-transfers <1|0> - whether to confirm unsplit txes; print-ring-members <1|0> - whether to print detailed information about ring members during confirmation; store-tx-info <1|0> - whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference; default-mixin <n> - set default mixin (default is 4); auto-refresh <1|0> - whether to automatically sync new blocks from the daemon; refresh-type <full|optimize-coinbase|no-coinbase|default> - set wallet refresh behaviour; priority [1|2|3] - normal/elevated/priority fee; confirm-missing-payment-id <1|0>"));
+  m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr("Available options: seed language - set wallet seed language; always-confirm-transfers <1|0> - whether to confirm unsplit txes; print-ring-members <1|0> - whether to print detailed information about ring members during confirmation; store-tx-info <1|0> - whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference; default-mixin <n> - set default mixin (default is 4); auto-refresh <1|0> - whether to automatically sync new blocks from the daemon; refresh-type <full|optimize-coinbase|no-coinbase|default> - set wallet refresh behaviour; priority [0|1|2|3|4] - default/unimportant/normal/elevated/priority fee; confirm-missing-payment-id <1|0>; ask-password <1|0>; unit <monero|millinero|micronero|nanonero|piconero> - set default monero (sub-)unit; min-output-count [n] - try to keep at least that many outputs of value at least min-output-value; min-output-value [n] - try to keep at least min-output-count outputs of at least that value - merge-destinations <1|0> - whether to merge multiple payments to the same destination address"));
   m_cmd_binder.set_handler("rescan_spent", boost::bind(&simple_wallet::rescan_spent, this, _1), tr("Rescan blockchain for spent outputs"));
   m_cmd_binder.set_handler("get_tx_key", boost::bind(&simple_wallet::get_tx_key, this, _1), tr("Get transaction key (r) for a given <txid>"));
   m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), tr("Check amount going to <address> in <txid>"));
@@ -646,6 +718,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("export_outputs", boost::bind(&simple_wallet::export_outputs, this, _1), tr("Export a set of outputs owned by this wallet"));
   m_cmd_binder.set_handler("import_outputs", boost::bind(&simple_wallet::import_outputs, this, _1), tr("Import set of outputs owned by this wallet"));
   m_cmd_binder.set_handler("show_transfer", boost::bind(&simple_wallet::show_transfer, this, _1), tr("Show information about a transfer to/from this address"));
+  m_cmd_binder.set_handler("password", boost::bind(&simple_wallet::change_password, this, _1), tr("Change wallet password"));
   m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), tr("Show this help"));
 }
 //----------------------------------------------------------------------------------------------------
@@ -663,6 +736,10 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     success_msg_writer() << "priority = " << m_wallet->get_default_priority();
     success_msg_writer() << "confirm-missing-payment-id = " << m_wallet->confirm_missing_payment_id();
     success_msg_writer() << "ask-password = " << m_wallet->ask_password();
+    success_msg_writer() << "unit = " << cryptonote::get_unit(m_wallet->get_default_decimal_point());
+    success_msg_writer() << "min-outputs-count = " << m_wallet->get_min_output_count();
+    success_msg_writer() << "min-outputs-value = " << cryptonote::print_money(m_wallet->get_min_output_value());
+    success_msg_writer() << "merge-destinations = " << m_wallet->merge_destinations();
     return true;
   }
   else
@@ -763,7 +840,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     {
       if (args.size() <= 1)
       {
-        fail_msg_writer() << tr("set priority: needs an argument: 0, 1, 2, or 3");
+        fail_msg_writer() << tr("set priority: needs an argument: 0, 1, 2, 3, or 4");
         return true;
       }
       else
@@ -798,6 +875,58 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
         return true;
       }
     }
+    else if (args[0] == "unit")
+    {
+      if (args.size() <= 1)
+      {
+        fail_msg_writer() << tr("set unit: needs an argument (monero, millinero, micronero, nanop, piconero)");
+        return true;
+      }
+      else
+      {
+        set_unit(args);
+        return true;
+      }
+    }
+    else if (args[0] == "min-outputs-count")
+    {
+      if (args.size() <= 1)
+      {
+        fail_msg_writer() << tr("set min-outputs-count: needs an argument (unsigned integer)");
+        return true;
+      }
+      else
+      {
+        set_min_output_count(args);
+        return true;
+      }
+    }
+    else if (args[0] == "min-outputs-value")
+    {
+      if (args.size() <= 1)
+      {
+        fail_msg_writer() << tr("set min-outputs-value: needs an argument (unsigned integer)");
+        return true;
+      }
+      else
+      {
+        set_min_output_value(args);
+        return true;
+      }
+    }
+    else if (args[0] == "merge-destinations")
+    {
+      if (args.size() <= 1)
+      {
+        fail_msg_writer() << tr("set merge-destinations: needs an argument (0 or 1)");
+        return true;
+      }
+      else
+      {
+        set_merge_destinations(args);
+        return true;
+      }
+    }
   }
   fail_msg_writer() << tr("set: unrecognized argument(s)");
   return true;
@@ -811,21 +940,7 @@ bool simple_wallet::set_log(const std::vector<std::string> &args)
     fail_msg_writer() << tr("usage: set_log <log_level_number_0-4> | <categories>");
     return true;
   }
-  uint16_t l = 0;
-  if(epee::string_tools::get_xtype_from_string(l, args[0]))
-  {
-    if(4 < l)
-    {
-      fail_msg_writer() << tr("wrong number range, use: set_log <log_level_number_0-4>");
-      return true;
-    }
-
-    mlog_set_log_level(l);
-  }
-  else
-  {
-    mlog_set_categories(args.front().c_str());
-  }
+  mlog_set_log(args[0].c_str());
   return true;
 }
 //----------------------------------------------------------------------------------------------------
@@ -1282,7 +1397,7 @@ std::string simple_wallet::get_mnemonic_language()
         fail_msg_writer() << tr("invalid language choice passed. Please try again.\n");
       }
     }
-    catch (std::exception &e)
+    catch (const std::exception &e)
     {
       fail_msg_writer() << tr("invalid language choice passed. Please try again.\n");
     }
@@ -1638,11 +1753,11 @@ void simple_wallet::on_new_block(uint64_t height, const cryptonote::block& block
     m_refresh_progress_reporter.update(height, false);
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::on_money_received(uint64_t height, const cryptonote::transaction& tx, uint64_t amount)
+void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount)
 {
   message_writer(console_color_green, false) << "\r" <<
     tr("Height ") << height << ", " <<
-    tr("transaction ") << get_transaction_hash(tx) << ", " <<
+    tr("transaction ") << txid << ", " <<
     tr("received ") << print_money(amount);
   if (m_auto_refresh_refreshing)
     m_cmd_binder.print_prompt();
@@ -1650,16 +1765,16 @@ void simple_wallet::on_money_received(uint64_t height, const cryptonote::transac
     m_refresh_progress_reporter.update(height, true);
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::on_unconfirmed_money_received(uint64_t height, const cryptonote::transaction& tx, uint64_t amount)
+void simple_wallet::on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount)
 {
   // Not implemented in CLI wallet
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::on_money_spent(uint64_t height, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx)
+void simple_wallet::on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx)
 {
   message_writer(console_color_magenta, false) << "\r" <<
     tr("Height ") << height << ", " <<
-    tr("transaction ") << get_transaction_hash(spend_tx) << ", " <<
+    tr("transaction ") << txid << ", " <<
     tr("spent ") << print_money(amount);
   if (m_auto_refresh_refreshing)
     m_cmd_binder.print_prompt();
@@ -1667,11 +1782,11 @@ void simple_wallet::on_money_spent(uint64_t height, const cryptonote::transactio
     m_refresh_progress_reporter.update(height, true);
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::on_skip_transaction(uint64_t height, const cryptonote::transaction& tx)
+void simple_wallet::on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx)
 {
   message_writer(console_color_red, true) << "\r" <<
     tr("Height ") << height << ", " <<
-    tr("transaction ") << get_transaction_hash(tx) << ", " <<
+    tr("transaction ") << txid << ", " <<
     tr("unsupported transaction format");
   if (m_auto_refresh_refreshing)
     m_cmd_binder.print_prompt();
@@ -2253,6 +2368,12 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri
         break;
     }
 
+    if (ptx_vector.empty())
+    {
+      fail_msg_writer() << tr("No outputs found, or daemon is not ready");
+      return true;
+    }
+
     // if more than one tx necessary, prompt user to confirm
     if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1)
     {
@@ -2719,7 +2840,7 @@ bool simple_wallet::sweep_all(const std::vector<std::string> &args_)
 
     if (ptx_vector.empty())
     {
-      fail_msg_writer() << tr("No outputs found");
+      fail_msg_writer() << tr("No outputs found, or daemon is not ready");
       return true;
     }
 
@@ -4008,7 +4129,7 @@ bool simple_wallet::export_key_images(const std::vector<std::string> &args)
       return true;
     }
   }
-  catch (std::exception &e)
+  catch (const std::exception &e)
   {
     LOG_ERROR("Error exporting key images: " << e.what());
     fail_msg_writer() << "Error exporting key images: " << e.what();
@@ -4333,7 +4454,8 @@ int main(int argc, char* argv[])
    argc, argv,
    "monero-wallet-cli [--wallet-file=<file>|--generate-new-wallet=<file>] [<COMMAND>]",
     desc_params,
-    positional_options
+    positional_options,
+    "monero-wallet-cli.log"
   );
 
   if (!vm)

src/simplewallet/simplewallet.h

@@ -116,6 +116,10 @@ namespace cryptonote
     bool set_refresh_type(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_confirm_missing_payment_id(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_ask_password(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_unit(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_min_output_count(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_min_output_value(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_merge_destinations(const std::vector<std::string> &args = std::vector<std::string>());
     bool help(const std::vector<std::string> &args = std::vector<std::string>());
     bool start_mining(const std::vector<std::string> &args);
     bool stop_mining(const std::vector<std::string> &args);
@@ -190,10 +194,10 @@ namespace cryptonote
 
     //----------------- i_wallet2_callback ---------------------
     virtual void on_new_block(uint64_t height, const cryptonote::block& block);
-    virtual void on_money_received(uint64_t height, const cryptonote::transaction& tx, uint64_t amount);
-    virtual void on_unconfirmed_money_received(uint64_t height, const cryptonote::transaction& tx, uint64_t amount);
-    virtual void on_money_spent(uint64_t height, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx);
-    virtual void on_skip_transaction(uint64_t height, const cryptonote::transaction& tx);
+    virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount);
+    virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount);
+    virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx);
+    virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx);
     //----------------------------------------------------------
 
     friend class refresh_progress_reporter_t;

src/version.h.in

@@ -1,4 +1,4 @@
 #define MONERO_VERSION_TAG "@VERSIONTAG@"
-#define MONERO_VERSION "0.10.2.0"
+#define MONERO_VERSION "0.10.3.1"
 #define MONERO_RELEASE_NAME "Wolfram Warptangent"
 #define MONERO_VERSION_FULL MONERO_VERSION "-" MONERO_VERSION_TAG

src/wallet/api/pending_transaction.cpp

@@ -124,7 +124,7 @@ bool PendingTransactionImpl::commit(const std::string &filename, bool overwrite)
         m_errorString = writer.str();
         if (!reason.empty())
           m_errorString  += string(tr(". Reason: ")) + reason;
-    } catch (std::exception &e) {
+    } catch (const std::exception &e) {
         m_errorString = string(tr("Unknown exception: ")) + e.what();
         m_status = Status_Error;
     } catch (...) {

src/wallet/api/wallet.cpp

@@ -99,10 +99,10 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback
         }
     }
 
-    virtual void on_money_received(uint64_t height, const cryptonote::transaction& tx, uint64_t amount)
+    virtual void on_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount)
     {
 
-        std::string tx_hash =  epee::string_tools::pod_to_hex(get_transaction_hash(tx));
+        std::string tx_hash =  epee::string_tools::pod_to_hex(txid);
 
         LOG_PRINT_L3(__FUNCTION__ << ": money received. height:  " << height
                      << ", tx: " << tx_hash
@@ -114,10 +114,10 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback
         }
     }
 
-    virtual void on_unconfirmed_money_received(uint64_t height, const cryptonote::transaction& tx, uint64_t amount)
+    virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount)
     {
 
-        std::string tx_hash =  epee::string_tools::pod_to_hex(get_transaction_hash(tx));
+        std::string tx_hash =  epee::string_tools::pod_to_hex(txid);
 
         LOG_PRINT_L3(__FUNCTION__ << ": unconfirmed money received. height:  " << height
                      << ", tx: " << tx_hash
@@ -129,11 +129,11 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback
         }
     }
 
-    virtual void on_money_spent(uint64_t height, const cryptonote::transaction& in_tx, uint64_t amount,
+    virtual void on_money_spent(uint64_t height, const crypto::hash &txid, uint64_t amount,
                                 const cryptonote::transaction& spend_tx)
     {
         // TODO;
-        std::string tx_hash = epee::string_tools::pod_to_hex(get_transaction_hash(spend_tx));
+        std::string tx_hash = epee::string_tools::pod_to_hex(txid);
         LOG_PRINT_L3(__FUNCTION__ << ": money spent. height:  " << height
                      << ", tx: " << tx_hash
                      << ", amount: " << print_money(amount));
@@ -144,7 +144,7 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback
         }
     }
 
-    virtual void on_skip_transaction(uint64_t height, const cryptonote::transaction& tx)
+    virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid)
     {
         // TODO;
     }
@@ -656,9 +656,11 @@ string WalletImpl::keysFilename() const
     return m_wallet->get_keys_file();
 }
 
-bool WalletImpl::init(const std::string &daemon_address, uint64_t upper_transaction_size_limit)
+bool WalletImpl::init(const std::string &daemon_address, uint64_t upper_transaction_size_limit, const std::string &daemon_username, const std::string &daemon_password)
 {
     clearStatus();
+    if(daemon_username != "")
+        m_daemon_login.emplace(daemon_username, daemon_password);
     return doInit(daemon_address, upper_transaction_size_limit);
 }
 
@@ -831,7 +833,7 @@ bool WalletImpl::exportKeyImages(const string &filename)
       return false;
     }
   }
-  catch (std::exception &e)
+  catch (const std::exception &e)
   {
     LOG_ERROR("Error exporting key images: " << e.what());
     m_errorString = e.what();
@@ -1365,7 +1367,7 @@ bool WalletImpl::isNewWallet() const
 
 bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit)
 {
-    if (!m_wallet->init(daemon_address, boost::none, upper_transaction_size_limit))
+    if (!m_wallet->init(daemon_address, m_daemon_login, upper_transaction_size_limit))
        return false;
 
     // in case new wallet, this will force fast-refresh (pulling hashes instead of blocks)
@@ -1411,6 +1413,18 @@ bool WalletImpl::rescanSpent()
   }
   return true;
 }
+
+
+void WalletImpl::hardForkInfo(uint8_t &version, uint64_t &earliest_height) const
+{
+    m_wallet->get_hard_fork_info(version, earliest_height);
+}
+
+bool WalletImpl::useForkRules(uint8_t version, int64_t early_blocks) const 
+{
+    return m_wallet->use_fork_rules(version,early_blocks);
+}
+
 } // namespace
 
 namespace Bitmonero = Monero;

src/wallet/api/wallet.h

@@ -78,7 +78,7 @@ public:
     bool store(const std::string &path);
     std::string filename() const;
     std::string keysFilename() const;
-    bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0);
+    bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "");
     bool connectToDaemon();
     ConnectionStatus connected() const;
     void setTrustedDaemon(bool arg);
@@ -99,7 +99,8 @@ public:
     bool watchOnly() const;
     bool rescanSpent();
     bool testnet() const {return m_wallet->testnet();}
-
+    void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const;
+    bool useForkRules(uint8_t version, int64_t early_blocks) const;
 
     PendingTransaction * createTransaction(const std::string &dst_addr, const std::string &payment_id,
                                         optional<uint64_t> amount, uint32_t mixin_count,
@@ -170,6 +171,7 @@ private:
     std::atomic<bool>   m_rebuildWalletCache;
     // cache connection status to avoid unnecessary RPC calls
     mutable std::atomic<bool>   m_is_connected;
+    boost::optional<epee::net_utils::http::login> m_daemon_login{};
 };
 
 

src/wallet/api/wallet_manager.cpp

@@ -377,26 +377,6 @@ double WalletManagerImpl::miningHashRate() const
     return mres.speed;
 }
 
-void WalletManagerImpl::hardForkInfo(uint8_t &version, uint64_t &earliest_height) const
-{
-    epee::json_rpc::request<cryptonote::COMMAND_RPC_HARD_FORK_INFO::request> req_t = AUTO_VAL_INIT(req_t);
-    epee::json_rpc::response<cryptonote::COMMAND_RPC_HARD_FORK_INFO::response, std::string> resp_t = AUTO_VAL_INIT(resp_t);
-
-    version = 0;
-    earliest_height = 0;
-
-    epee::net_utils::http::http_simple_client http_client;
-    req_t.jsonrpc = "2.0";
-    req_t.id = epee::serialization::storage_entry(0);
-    req_t.method = "hard_fork_info";
-    req_t.params.version = 0;
-    bool r = connect_and_invoke(m_daemonAddress, "/json_rpc", req_t, resp_t);
-    if (!r || resp_t.result.status != CORE_RPC_STATUS_OK)
-        return;
-    version = resp_t.result.version;
-    earliest_height = resp_t.result.earliest_height;
-}
-
 uint64_t WalletManagerImpl::blockTarget() const
 {
     cryptonote::COMMAND_RPC_GET_INFO::request ireq;
@@ -417,13 +397,15 @@ bool WalletManagerImpl::isMining() const
     return mres.active;
 }
 
-bool WalletManagerImpl::startMining(const std::string &address, uint32_t threads)
+bool WalletManagerImpl::startMining(const std::string &address, uint32_t threads, bool background_mining, bool ignore_battery)
 {
     cryptonote::COMMAND_RPC_START_MINING::request mreq;
     cryptonote::COMMAND_RPC_START_MINING::response mres;
 
     mreq.miner_address = address;
     mreq.threads_count = threads;
+    mreq.ignore_battery = ignore_battery;
+    mreq.do_background_mining = background_mining;
 
     if (!connect_and_invoke(m_daemonAddress, "/start_mining", mreq, mres))
       return false;
@@ -492,7 +474,7 @@ void WalletManagerFactory::setLogLevel(int level)
 
 void WalletManagerFactory::setLogCategories(const std::string &categories)
 {
-    mlog_set_categories(categories.c_str());
+    mlog_set_log(categories.c_str());
 }
 
 

src/wallet/api/wallet_manager.h

@@ -59,10 +59,9 @@ public:
     uint64_t blockchainTargetHeight() const;
     uint64_t networkDifficulty() const;
     double miningHashRate() const;
-    void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const;
     uint64_t blockTarget() const;
     bool isMining() const;
-    bool startMining(const std::string &address, uint32_t threads = 1);
+    bool startMining(const std::string &address, uint32_t threads = 1, bool background_mining = false, bool ignore_battery = true);
     bool stopMining();
     std::string resolveOpenAlias(const std::string &address, bool &dnssec_valid) const;
 

src/wallet/node_rpc_proxy.cpp

@@ -36,6 +36,8 @@ using namespace epee;
 namespace tools
 {
 
+static const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30);
+
 NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, boost::mutex &mutex)
   : m_http_client(http_client)
   , m_daemon_rpc_mutex(mutex)
@@ -45,8 +47,43 @@ NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::http_simple_client &http_clien
   , m_dynamic_per_kb_fee_estimate(0)
   , m_dynamic_per_kb_fee_estimate_cached_height(0)
   , m_dynamic_per_kb_fee_estimate_grace_blocks(0)
+  , m_rpc_version(0)
 {}
 
+void NodeRPCProxy::invalidate()
+{
+  m_height = 0;
+  m_height_time = 0;
+  for (size_t n = 0; n < 256; ++n)
+    m_earliest_height[n] = 0;
+  m_dynamic_per_kb_fee_estimate = 0;
+  m_dynamic_per_kb_fee_estimate_cached_height = 0;
+  m_dynamic_per_kb_fee_estimate_grace_blocks = 0;
+  m_rpc_version = 0;
+}
+
+boost::optional<std::string> NodeRPCProxy::get_rpc_version(uint32_t &rpc_version)
+{
+  const time_t now = time(NULL);
+  if (m_rpc_version == 0)
+  {
+    epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_VERSION::request> req_t = AUTO_VAL_INIT(req_t);
+    epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_VERSION::response, std::string> resp_t = AUTO_VAL_INIT(resp_t);
+    req_t.jsonrpc = "2.0";
+    req_t.id = epee::serialization::storage_entry(0);
+    req_t.method = "get_version";
+    m_daemon_rpc_mutex.lock();
+    bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout);
+    m_daemon_rpc_mutex.unlock();
+    CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon");
+    CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, resp_t.result.status, "Failed to connect to daemon");
+    CHECK_AND_ASSERT_MES(resp_t.result.status == CORE_RPC_STATUS_OK, resp_t.result.status, "Failed to get daemon RPC version");
+    m_rpc_version = resp_t.result.version;
+  }
+  rpc_version = m_rpc_version;
+  return boost::optional<std::string>();
+}
+
 boost::optional<std::string> NodeRPCProxy::get_height(uint64_t &height)
 {
   const time_t now = time(NULL);
@@ -56,7 +93,7 @@ boost::optional<std::string> NodeRPCProxy::get_height(uint64_t &height)
     cryptonote::COMMAND_RPC_GET_HEIGHT::response res = AUTO_VAL_INIT(res);
 
     m_daemon_rpc_mutex.lock();
-    bool r = net_utils::invoke_http_json("/getheight", req, res, m_http_client);
+    bool r = net_utils::invoke_http_json("/getheight", req, res, m_http_client, rpc_timeout);
     m_daemon_rpc_mutex.unlock();
     CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon");
     CHECK_AND_ASSERT_MES(res.status != CORE_RPC_STATUS_BUSY, res.status, "Failed to connect to daemon");
@@ -85,7 +122,7 @@ boost::optional<std::string> NodeRPCProxy::get_earliest_height(uint8_t version,
     req_t.id = epee::serialization::storage_entry(0);
     req_t.method = "hard_fork_info";
     req_t.params.version = version;
-    bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client);
+    bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout);
     m_daemon_rpc_mutex.unlock();
     CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon");
     CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, resp_t.result.status, "Failed to connect to daemon");
@@ -115,7 +152,7 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6
     req_t.id = epee::serialization::storage_entry(0);
     req_t.method = "get_fee_estimate";
     req_t.params.grace_blocks = grace_blocks;
-    bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client);
+    bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout);
     m_daemon_rpc_mutex.unlock();
     CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon");
     CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, resp_t.result.status, "Failed to connect to daemon");

src/wallet/node_rpc_proxy.h

@@ -41,6 +41,9 @@ class NodeRPCProxy
 public:
   NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, boost::mutex &mutex);
 
+  void invalidate();
+
+  boost::optional<std::string> get_rpc_version(uint32_t &version);
   boost::optional<std::string> get_height(uint64_t &height);
   void set_height(uint64_t h);
   boost::optional<std::string> get_earliest_height(uint8_t version, uint64_t &earliest_height);
@@ -56,6 +59,7 @@ private:
   uint64_t m_dynamic_per_kb_fee_estimate;
   uint64_t m_dynamic_per_kb_fee_estimate_cached_height;
   uint64_t m_dynamic_per_kb_fee_estimate_grace_blocks;
+  uint32_t m_rpc_version;
 };
 
 }

src/wallet/wallet2.cpp

@@ -452,10 +452,10 @@ void wallet2::init_options(boost::program_options::options_description& desc_par
   command_line::add_arg(desc_params, opts.restricted);
 }
 
-boost::optional<password_container> wallet2::password_prompt(const bool is_new_wallet)
+boost::optional<password_container> wallet2::password_prompt(const bool new_password)
 {
   auto pwd_container = tools::password_container::prompt(
-    is_new_wallet, (is_new_wallet ? tr("Enter a password for your new wallet") : tr("Wallet password"))
+    new_password, (new_password ? tr("Enter new wallet password") : tr("Wallet password"))
   );
   if (!pwd_container)
   {
@@ -633,28 +633,13 @@ bool wallet2::wallet_generate_key_image_helper(const cryptonote::account_keys& a
   return true;
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::process_new_transaction(const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool)
+void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool)
 {
-  class lazy_txid_getter
-  {
-    const cryptonote::transaction &tx;
-    crypto::hash lazy_txid;
-    bool computed;
-    public:
-    lazy_txid_getter(const transaction &tx): tx(tx), computed(false) {}
-    const crypto::hash &operator()()
-    {
-      if (!computed)
-      {
-        lazy_txid = cryptonote::get_transaction_hash(tx);
-        computed = true;
-      }
-      return lazy_txid;
-    }
-  } txid(tx);
+  // In this function, tx (probably) only contains the base information
+  // (that is, the prunable stuff may or may not be included)
 
   if (!miner_tx)
-    process_unconfirmed(tx, height);
+    process_unconfirmed(txid, tx, height);
   std::vector<size_t> outs;
   uint64_t tx_money_got_in_outs = 0;
   crypto::public_key tx_pub_key = null_pkey;
@@ -663,7 +648,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
   if(!parse_tx_extra(tx.extra, tx_extra_fields))
   {
     // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key
-    LOG_PRINT_L0("Transaction extra has unsupported format: " << txid());
+    LOG_PRINT_L0("Transaction extra has unsupported format: " << txid);
   }
 
   // Don't try to extract tx public key if tx has no ouputs
@@ -677,9 +662,9 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
     {
       if (pk_index > 1)
         break;
-      LOG_PRINT_L0("Public key wasn't found in the transaction extra. Skipping transaction " << txid());
+      LOG_PRINT_L0("Public key wasn't found in the transaction extra. Skipping transaction " << txid);
       if(0 != m_callback)
-	m_callback->on_skip_transaction(height, tx);
+	m_callback->on_skip_transaction(height, txid, tx);
       return;
     }
 
@@ -879,7 +864,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
 	    td.m_internal_output_index = o;
 	    td.m_global_output_index = o_indices[o];
 	    td.m_tx = (const cryptonote::transaction_prefix&)tx;
-	    td.m_txid = txid();
+	    td.m_txid = txid;
             td.m_key_image = ki[o];
             td.m_key_image_known = !m_watch_only;
             td.m_amount = tx.vout[o].amount;
@@ -903,9 +888,9 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
 	    set_unspent(m_transfers.size()-1);
 	    m_key_images[td.m_key_image] = m_transfers.size()-1;
 	    m_pub_keys[in_ephemeral[o].pub] = m_transfers.size()-1;
-	    LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid());
+	    LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid);
 	    if (0 != m_callback)
-	      m_callback->on_money_received(height, tx, td.m_amount);
+	      m_callback->on_money_received(height, txid, tx, td.m_amount);
           }
         }
 	else if (m_transfers[kit->second].m_spent || m_transfers[kit->second].amount() >= tx.vout[o].amount)
@@ -930,7 +915,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
 	    td.m_internal_output_index = o;
 	    td.m_global_output_index = o_indices[o];
 	    td.m_tx = (const cryptonote::transaction_prefix&)tx;
-	    td.m_txid = txid();
+	    td.m_txid = txid;
             td.m_amount = tx.vout[o].amount;
             td.m_pk_index = pk_index - 1;
             if (td.m_amount == 0)
@@ -952,9 +937,9 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
             THROW_WALLET_EXCEPTION_IF(td.get_public_key() != in_ephemeral[o].pub, error::wallet_internal_error, "Inconsistent public keys");
 	    THROW_WALLET_EXCEPTION_IF(td.m_spent, error::wallet_internal_error, "Inconsistent spent status");
 
-	    LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid());
+	    LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid);
 	    if (0 != m_callback)
-	      m_callback->on_money_received(height, tx, td.m_amount);
+	      m_callback->on_money_received(height, txid, tx, td.m_amount);
           }
         }
       }
@@ -982,17 +967,17 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
       tx_money_spent_in_ins += amount;
       if (!pool)
       {
-        LOG_PRINT_L0("Spent money: " << print_money(amount) << ", with tx: " << txid());
+        LOG_PRINT_L0("Spent money: " << print_money(amount) << ", with tx: " << txid);
         set_spent(it->second, height);
         if (0 != m_callback)
-          m_callback->on_money_spent(height, tx, amount, tx);
+          m_callback->on_money_spent(height, txid, tx, amount, tx);
       }
     }
   }
 
   if (tx_money_spent_in_ins > 0)
   {
-    process_outgoing(tx, height, ts, tx_money_spent_in_ins, tx_money_got_in_outs);
+    process_outgoing(txid, tx, height, ts, tx_money_spent_in_ins, tx_money_got_in_outs);
   }
 
   uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0;
@@ -1038,7 +1023,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
     }
 
     payment_details payment;
-    payment.m_tx_hash      = txid();
+    payment.m_tx_hash      = txid;
     payment.m_amount       = received;
     payment.m_block_height = height;
     payment.m_unlock_time  = tx.unlock_time;
@@ -1046,7 +1031,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
     if (pool) {
       m_unconfirmed_payments.emplace(payment_id, payment);
       if (0 != m_callback)
-        m_callback->on_unconfirmed_money_received(height, tx, payment.m_amount);
+        m_callback->on_unconfirmed_money_received(height, txid, tx, payment.m_amount);
     }
     else
       m_payments.emplace(payment_id, payment);
@@ -1054,12 +1039,11 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, const s
   }
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::process_unconfirmed(const cryptonote::transaction& tx, uint64_t height)
+void wallet2::process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height)
 {
   if (m_unconfirmed_txs.empty())
     return;
 
-  crypto::hash txid = get_transaction_hash(tx);
   auto unconf_it = m_unconfirmed_txs.find(txid);
   if(unconf_it != m_unconfirmed_txs.end()) {
     if (store_tx_info()) {
@@ -1075,9 +1059,8 @@ void wallet2::process_unconfirmed(const cryptonote::transaction& tx, uint64_t he
   }
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received)
+void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::transaction &tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received)
 {
-  crypto::hash txid = get_transaction_hash(tx);
   std::pair<std::unordered_map<crypto::hash, confirmed_transfer_details>::iterator, bool> entry = m_confirmed_txs.insert(std::make_pair(txid, confirmed_transfer_details()));
   // fill with the info we know, some info might already be there
   if (entry.second)
@@ -1120,16 +1103,19 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry
   if(b.timestamp + 60*60*24 > m_account.get_createtime() && height >= m_refresh_from_block_height)
   {
     TIME_MEASURE_START(miner_tx_handle_time);
-    process_new_transaction(b.miner_tx, o_indices.indices[txidx++].indices, height, b.timestamp, true, false);
+    process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, o_indices.indices[txidx++].indices, height, b.timestamp, true, false);
     TIME_MEASURE_FINISH(miner_tx_handle_time);
 
     TIME_MEASURE_START(txs_handle_time);
-    for(auto& txblob: bche.txs)
+    THROW_WALLET_EXCEPTION_IF(bche.txs.size() != b.tx_hashes.size(), error::wallet_internal_error, "Wrong amount of transactions for block");
+    size_t idx = 0;
+    for (const auto& txblob: bche.txs)
     {
       cryptonote::transaction tx;
-      bool r = parse_and_validate_tx_from_blob(txblob, tx);
+      bool r = parse_and_validate_tx_base_from_blob(txblob, tx);
       THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob);
-      process_new_transaction(tx, o_indices.indices[txidx++].indices, height, b.timestamp, false, false);
+      process_new_transaction(b.tx_hashes[idx], tx, o_indices.indices[txidx++].indices, height, b.timestamp, false, false);
+      ++idx;
     }
     TIME_MEASURE_FINISH(txs_handle_time);
     LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " <<  miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms");
@@ -1185,6 +1171,34 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height,
   cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res);
   req.block_ids = short_chain_history;
 
+  uint32_t rpc_version;
+  boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version);
+  // no error
+  if (!!result)
+  {
+    // empty string -> not connection
+    THROW_WALLET_EXCEPTION_IF(result->empty(), tools::error::no_connection_to_daemon, "getversion");
+    THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion");
+    if (*result != CORE_RPC_STATUS_OK)
+    {
+      MDEBUG("Cannot determined daemon RPC version, not asking for pruned blocks");
+      req.prune = false; // old daemon
+    }
+  }
+  else
+  {
+    if (rpc_version >= MAKE_CORE_RPC_VERSION(1, 7))
+    {
+      MDEBUG("Daemon is recent enough, asking for pruned blocks");
+      req.prune = true;
+    }
+    else
+    {
+      MDEBUG("Daemon is too old, not asking for pruned blocks");
+      req.prune = false;
+    }
+  }
+
   req.start_height = start_height;
   m_daemon_rpc_mutex.lock();
   bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, m_http_client, rpc_timeout);
@@ -1367,25 +1381,28 @@ void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_hei
 //----------------------------------------------------------------------------------------------------
 void wallet2::update_pool_state()
 {
+  MDEBUG("update_pool_state start");
+
   // get the pool state
-  cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::request req;
-  cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::response res;
+  cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request req;
+  cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response res;
   m_daemon_rpc_mutex.lock();
-  bool r = epee::net_utils::invoke_http_json("/get_transaction_pool", req, res, m_http_client, rpc_timeout);
+  bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout);
   m_daemon_rpc_mutex.unlock();
-  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool");
-  THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool");
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool_hashes.bin");
+  THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool_hashes.bin");
   THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error);
+  MDEBUG("update_pool_state got pool");
 
   // remove any pending tx that's not in the pool
   std::unordered_map<crypto::hash, wallet2::unconfirmed_transfer_details>::iterator it = m_unconfirmed_txs.begin();
   while (it != m_unconfirmed_txs.end())
   {
-    const std::string txid = epee::string_tools::pod_to_hex(it->first);
+    const crypto::hash &txid = it->first;
     bool found = false;
-    for (auto it2: res.transactions)
+    for (const auto &it2: res.tx_hashes)
     {
-      if (it2.id_hash == txid)
+      if (it2 == txid)
       {
         found = true;
         break;
@@ -1430,16 +1447,17 @@ void wallet2::update_pool_state()
       }
     }
   }
+  MDEBUG("update_pool_state done first loop");
 
   // remove pool txes to us that aren't in the pool anymore
   std::unordered_map<crypto::hash, wallet2::payment_details>::iterator uit = m_unconfirmed_payments.begin();
   while (uit != m_unconfirmed_payments.end())
   {
-    const std::string txid = string_tools::pod_to_hex(uit->first);
+    const crypto::hash &txid = uit->first;
     bool found = false;
-    for (auto it2: res.transactions)
+    for (const auto &it2: res.tx_hashes)
     {
-      if (it2.id_hash == txid)
+      if (it2 == txid)
       {
         found = true;
         break;
@@ -1448,102 +1466,122 @@ void wallet2::update_pool_state()
     auto pit = uit++;
     if (!found)
     {
+      MDEBUG("Removing " << txid << " from unconfirmed payments, not found in pool");
       m_unconfirmed_payments.erase(pit);
     }
   }
+  MDEBUG("update_pool_state done second loop");
 
-  // add new pool txes to us
-  for (auto it: res.transactions)
+  // gather txids of new pool txes to us
+  std::vector<crypto::hash> txids;
+  for (const auto &txid: res.tx_hashes)
   {
-    cryptonote::blobdata txid_data;
-    if(epee::string_tools::parse_hexstr_to_binbuff(it.id_hash, txid_data) && txid_data.size() == sizeof(crypto::hash))
+    if (m_scanned_pool_txs[0].find(txid) != m_scanned_pool_txs[0].end() || m_scanned_pool_txs[1].find(txid) != m_scanned_pool_txs[1].end())
     {
-      const crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
-      if (m_unconfirmed_payments.find(txid) == m_unconfirmed_payments.end())
+      LOG_PRINT_L2("Already seen " << txid << ", skipped");
+      continue;
+    }
+    if (m_unconfirmed_payments.find(txid) == m_unconfirmed_payments.end())
+    {
+      LOG_PRINT_L1("Found new pool tx: " << txid);
+      bool found = false;
+      for (const auto &i: m_unconfirmed_txs)
       {
-        LOG_PRINT_L1("Found new pool tx: " << txid);
-        bool found = false;
-        for (const auto &i: m_unconfirmed_txs)
+        if (i.first == txid)
         {
-          if (i.first == txid)
-          {
-            found = true;
-            break;
-          }
-        }
-        if (!found)
-        {
-          // not one of those we sent ourselves
-          cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req;
-          cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res;
-          req.txs_hashes.push_back(it.id_hash);
-          req.decode_as_json = false;
-          m_daemon_rpc_mutex.lock();
-          bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
-          m_daemon_rpc_mutex.unlock();
-          if (r && res.status == CORE_RPC_STATUS_OK)
-          {
-            if (res.txs.size() == 1)
-            {
-              // might have just been put in a block
-              if (res.txs[0].in_pool)
-              {
-                cryptonote::transaction tx;
-                cryptonote::blobdata bd;
-                crypto::hash tx_hash, tx_prefix_hash;
-                if (epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd))
-                {
-                  if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash))
-                  {
-                    if (tx_hash == txid)
-                    {
-                      process_new_transaction(tx, std::vector<uint64_t>(), 0, time(NULL), false, true);
-                    }
-                    else
-                    {
-                      LOG_PRINT_L0("Mismatched txids when processing unconfimed txes from pool");
-                    }
-                  }
-                  else
-                  {
-                    LOG_PRINT_L0("failed to validate transaction from daemon");
-                  }
-                }
-                else
-                {
-                  LOG_PRINT_L0("Failed to parse tx " << txid);
-                }
-              }
-              else
-              {
-                LOG_PRINT_L1("Tx " << txid << " was in pool, but is no more");
-              }
-            }
-            else
-            {
-              LOG_PRINT_L0("Expected 1 tx, got " << res.txs.size());
-            }
-          }
-          else
-          {
-            LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status);
-          }
-        }
-        else
-        {
-          LOG_PRINT_L1("We sent that one");
+          found = true;
+          break;
         }
       }
+      if (!found)
+      {
+        // not one of those we sent ourselves
+        txids.push_back(txid);
+      }
       else
       {
-        LOG_PRINT_L1("Already saw that one");
+        LOG_PRINT_L1("We sent that one");
       }
     }
     else
     {
-      LOG_PRINT_L0("Failed to parse txid");
+      LOG_PRINT_L1("Already saw that one, it's for us");
     }
   }
+
+  // get those txes
+  if (!txids.empty())
+  {
+    cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req;
+    cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res;
+    for (const auto &txid: txids)
+      req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
+    MDEBUG("asking for " << txids.size() << " transactions");
+    req.decode_as_json = false;
+    m_daemon_rpc_mutex.lock();
+    bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
+    m_daemon_rpc_mutex.unlock();
+    MDEBUG("Got " << r << " and " << res.status);
+    if (r && res.status == CORE_RPC_STATUS_OK)
+    {
+      if (res.txs.size() == txids.size())
+      {
+        size_t n = 0;
+        for (const auto &txid: txids)
+        {
+          // might have just been put in a block
+          if (res.txs[n].in_pool)
+          {
+            cryptonote::transaction tx;
+            cryptonote::blobdata bd;
+            crypto::hash tx_hash, tx_prefix_hash;
+            if (epee::string_tools::parse_hexstr_to_binbuff(res.txs[n].as_hex, bd))
+            {
+              if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash))
+              {
+                if (tx_hash == txid)
+                {
+                  process_new_transaction(txid, tx, std::vector<uint64_t>(), 0, time(NULL), false, true);
+                  m_scanned_pool_txs[0].insert(txid);
+                  if (m_scanned_pool_txs[0].size() > 5000)
+                  {
+                    std::swap(m_scanned_pool_txs[0], m_scanned_pool_txs[1]);
+                    m_scanned_pool_txs[0].clear();
+                  }
+                }
+                else
+                {
+                  LOG_PRINT_L0("Mismatched txids when processing unconfimed txes from pool");
+                }
+              }
+              else
+              {
+                LOG_PRINT_L0("failed to validate transaction from daemon");
+              }
+            }
+            else
+            {
+              LOG_PRINT_L0("Failed to parse tx " << txid);
+            }
+          }
+          else
+          {
+            LOG_PRINT_L1("Tx " << txid << " was in pool, but is no more");
+          }
+          ++n;
+        }
+      }
+      else
+      {
+        LOG_PRINT_L0("Expected " << txids.size() << " tx(es), got " << res.txs.size());
+      }
+    }
+    else
+    {
+      LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status);
+    }
+  }
+  MDEBUG("update_pool_state end");
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history)
@@ -1880,6 +1918,18 @@ bool wallet2::store_keys(const std::string& keys_file_name, const std::string& p
   value2.SetInt(m_ask_password ? 1 :0);
   json.AddMember("ask_password", value2, json.GetAllocator());
 
+  value2.SetUint(m_min_output_count);
+  json.AddMember("min_output_count", value2, json.GetAllocator());
+
+  value2.SetUint64(m_min_output_value);
+  json.AddMember("min_output_value", value2, json.GetAllocator());
+
+  value2.SetInt(cryptonote::get_default_decimal_point());
+  json.AddMember("default_decimal_point", value2, json.GetAllocator());
+
+  value2.SetInt(m_merge_destinations ? 1 :0);
+  json.AddMember("merge_destinations", value2, json.GetAllocator());
+
   // Serialize the JSON object
   rapidjson::StringBuffer buffer;
   rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
@@ -1948,6 +1998,9 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa
     m_refresh_type = RefreshType::RefreshDefault;
     m_confirm_missing_payment_id = true;
     m_ask_password = true;
+    m_min_output_count = 0;
+    m_min_output_value = 0;
+    m_merge_destinations = false;
   }
   else
   {
@@ -2010,6 +2063,14 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa
     m_confirm_missing_payment_id = field_confirm_missing_payment_id;
     GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, ask_password, int, Int, false, true);
     m_ask_password = field_ask_password;
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, default_decimal_point, int, Int, false, CRYPTONOTE_DISPLAY_DECIMAL_POINT);
+    cryptonote::set_default_decimal_point(field_default_decimal_point);
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, min_output_count, uint32_t, Uint, false, 0);
+    m_min_output_count = field_min_output_count;
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, min_output_value, uint64_t, Uint64, false, 0);
+    m_min_output_value = field_min_output_value;
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, merge_destinations, int, Int, false, false);
+    m_merge_destinations = field_merge_destinations;
   }
 
   const cryptonote::account_keys& keys = m_account.get_keys();
@@ -2274,6 +2335,16 @@ bool wallet2::parse_payment_id(const std::string& payment_id_str, crypto::hash&
   return false;
 }
 //----------------------------------------------------------------------------------------------------
+void wallet2::set_default_decimal_point(unsigned int decimal_point)
+{
+  cryptonote::set_default_decimal_point(decimal_point);
+}
+//----------------------------------------------------------------------------------------------------
+unsigned int wallet2::get_default_decimal_point() const
+{
+  return cryptonote::get_default_decimal_point();
+}
+//----------------------------------------------------------------------------------------------------
 bool wallet2::prepare_file_names(const std::string& file_path)
 {
   do_prepare_file_names(file_path, m_keys_file, m_wallet_file);
@@ -2286,6 +2357,7 @@ bool wallet2::check_connection(uint32_t *version, uint32_t timeout)
 
   if(!m_http_client.is_connected())
   {
+    m_node_rpc_proxy.invalidate();
     if (!m_http_client.connect(std::chrono::milliseconds(timeout)))
       return false;
   }
@@ -2298,7 +2370,11 @@ bool wallet2::check_connection(uint32_t *version, uint32_t timeout)
     req_t.id = epee::serialization::storage_entry(0);
     req_t.method = "get_version";
     bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client);
-    if (!r || resp_t.result.status != CORE_RPC_STATUS_OK)
+    if(!r) {
+      *version = 0;
+      return false;
+    }
+    if (resp_t.result.status != CORE_RPC_STATUS_OK)
       *version = 0;
     else
       *version = resp_t.result.version;
@@ -2605,28 +2681,31 @@ void wallet2::get_unconfirmed_payments(std::list<std::pair<crypto::hash,wallet2:
 //----------------------------------------------------------------------------------------------------
 void wallet2::rescan_spent()
 {
-  std::vector<std::string> key_images;
-
-  // make a list of key images for all our outputs
-  for (size_t i = 0; i < m_transfers.size(); ++i)
+  // This is RPC call that can take a long time if there are many outputs,
+  // so we call it several times, in stripes, so we don't time out spuriously
+  std::vector<int> spent_status;
+  spent_status.reserve(m_transfers.size());
+  const size_t chunk_size = 1000;
+  for (size_t start_offset = 0; start_offset < m_transfers.size(); start_offset += chunk_size)
   {
-    const transfer_details& td = m_transfers[i];
-    key_images.push_back(string_tools::pod_to_hex(td.m_key_image));
+    const size_t n_outputs = std::min<size_t>(chunk_size, m_transfers.size() - start_offset);
+    MDEBUG("Calling is_key_image_spent on " << start_offset << " - " << (start_offset + n_outputs - 1) << ", out of " << m_transfers.size());
+    COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req);
+    COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp);
+    for (size_t n = start_offset; n < start_offset + n_outputs; ++n)
+      req.key_images.push_back(string_tools::pod_to_hex(m_transfers[n].m_key_image));
+    m_daemon_rpc_mutex.lock();
+    bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout);
+    m_daemon_rpc_mutex.unlock();
+    THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "is_key_image_spent");
+    THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "is_key_image_spent");
+    THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, daemon_resp.status);
+    THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != n_outputs, error::wallet_internal_error,
+      "daemon returned wrong response for is_key_image_spent, wrong amounts count = " +
+      std::to_string(daemon_resp.spent_status.size()) + ", expected " +  std::to_string(n_outputs));
+    std::copy(daemon_resp.spent_status.begin(), daemon_resp.spent_status.end(), std::back_inserter(spent_status));
   }
 
-  COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req);
-  COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp);
-  req.key_images = key_images;
-  m_daemon_rpc_mutex.lock();
-  bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout);
-  m_daemon_rpc_mutex.unlock();
-  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "is_key_image_spent");
-  THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "is_key_image_spent");
-  THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, daemon_resp.status);
-  THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != key_images.size(), error::wallet_internal_error,
-    "daemon returned wrong response for is_key_image_spent, wrong amounts count = " +
-    std::to_string(daemon_resp.spent_status.size()) + ", expected " +  std::to_string(key_images.size()));
-
   // update spent status
   for (size_t i = 0; i < m_transfers.size(); ++i)
   {
@@ -2634,7 +2713,7 @@ void wallet2::rescan_spent()
     // a view wallet may not know about key images
     if (!td.m_key_image_known)
       continue;
-    if (td.m_spent != (daemon_resp.spent_status[i] != COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT))
+    if (td.m_spent != (spent_status[i] != COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT))
     {
       if (td.m_spent)
       {
@@ -3286,20 +3365,35 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal
   return true;
 }
 //----------------------------------------------------------------------------------------------------
-uint64_t wallet2::get_fee_multiplier(uint32_t priority, bool use_new_fee) const
+uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const
 {
   static const uint64_t old_multipliers[3] = {1, 2, 3};
   static const uint64_t new_multipliers[3] = {1, 20, 166};
+  static const uint64_t newer_multipliers[4] = {1, 4, 20, 166};
 
-  // 0 -> default (here, x1)
+  // 0 -> default (here, x1 till fee algorithm 2, x4 from it)
   if (priority == 0)
     priority = m_default_priority;
   if (priority == 0)
-    priority = 1;
+  {
+    if (fee_algorithm >= 2)
+      priority = 2;
+    else
+      priority = 1;
+  }
 
-  // 1 to 3 are allowed as priorities
-  if (priority >= 1 && priority <= 3)
-    return (use_new_fee ? new_multipliers : old_multipliers)[priority-1];
+  // 1 to 3/4 are allowed as priorities
+  uint32_t max_priority = (fee_algorithm >= 2) ? 4 : 3;
+  if (priority >= 1 && priority <= max_priority)
+  {
+    switch (fee_algorithm)
+    {
+      case 0: return old_multipliers[priority-1];
+      case 1: return new_multipliers[priority-1];
+      case 2: return newer_multipliers[priority-1];
+      default: THROW_WALLET_EXCEPTION_IF (true, error::invalid_priority);
+    }
+  }
 
   THROW_WALLET_EXCEPTION_IF (false, error::invalid_priority);
   return 1;
@@ -3324,6 +3418,16 @@ uint64_t wallet2::get_per_kb_fee()
   return get_dynamic_per_kb_fee_estimate();
 }
 //----------------------------------------------------------------------------------------------------
+int wallet2::get_fee_algorithm()
+{
+  // changes at v3 and v5
+  if (use_fork_rules(5, 0))
+    return 2;
+  if (use_fork_rules(3, -720 * 14))
+   return 1;
+  return 0;
+}
+//----------------------------------------------------------------------------------------------------
 // separated the call(s) to wallet2::transfer into their own function
 //
 // this function will make multiple calls to wallet2::transfer if multiple
@@ -3332,9 +3436,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto
 {
   const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, true, true, trusted_daemon);
 
-  const bool use_new_fee  = use_fork_rules(3, -720 * 14);
   const uint64_t fee_per_kb  = get_per_kb_fee();
-  const uint64_t fee_multiplier = get_fee_multiplier(priority, use_new_fee);
+  const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm());
 
   // failsafe split attempt counter
   size_t attempt_count = 0;
@@ -3453,7 +3556,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
     req_t.params.amounts.resize(std::distance(req_t.params.amounts.begin(), end));
     req_t.params.unlocked = true;
     req_t.params.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE;
-    bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client);
+    bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout);
     m_daemon_rpc_mutex.unlock();
     THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected");
     THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram");
@@ -4059,7 +4162,7 @@ std::vector<size_t> wallet2::pick_preferred_rct_inputs(uint64_t needed_money) co
   return picks;
 }
 
-static bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices)
+bool wallet2::should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const
 {
   if (!use_rct)
     return false;
@@ -4067,9 +4170,52 @@ static bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const
     return false;
   if (unused_dust_indices.empty() && unused_transfers_indices.empty())
     return false;
+  // we want at least one free rct output to avoid a corner case where
+  // we'd choose a non rct output which doesn't have enough "siblings"
+  // value-wise on the chain, and thus can't be mixed
+  bool found = false;
+  for (auto i: unused_dust_indices)
+  {
+    if (m_transfers[i].is_rct())
+    {
+      found = true;
+      break;
+    }
+  }
+  if (!found) for (auto i: unused_transfers_indices)
+  {
+    if (m_transfers[i].is_rct())
+    {
+      found = true;
+      break;
+    }
+  }
+  if (!found)
+    return false;
   return true;
 }
 
+std::vector<size_t> wallet2::get_only_rct(const std::vector<size_t> &unused_dust_indices, const std::vector<size_t> &unused_transfers_indices) const
+{
+  std::vector<size_t> indices;
+  for (size_t n: unused_dust_indices)
+    if (m_transfers[n].is_rct())
+      indices.push_back(n);
+  for (size_t n: unused_transfers_indices)
+    if (m_transfers[n].is_rct())
+      indices.push_back(n);
+  return indices;
+}
+
+static uint32_t get_count_above(const std::vector<wallet2::transfer_details> &transfers, const std::vector<size_t> &indices, uint64_t threshold)
+{
+  uint32_t count = 0;
+  for (size_t idx: indices)
+    if (transfers[idx].amount() >= threshold)
+      ++count;
+  return count;
+}
+
 // Another implementation of transaction creation that is hopefully better
 // While there is anything left to pay, it goes through random outputs and tries
 // to fill the next destination/amount. If it fully fills it, it will use the
@@ -4098,10 +4244,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
     pending_tx ptx;
     size_t bytes;
 
-    void add(const account_public_address &addr, uint64_t amount) {
+    void add(const account_public_address &addr, uint64_t amount, bool merge_destinations) {
       std::vector<cryptonote::tx_destination_entry>::iterator i;
       i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &addr, sizeof(addr)); });
-      if (i == dsts.end())
+      if (!merge_destinations || i == dsts.end())
         dsts.push_back(tx_destination_entry(amount,addr));
       else
         i->amount += amount;
@@ -4113,9 +4259,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
   uint64_t upper_transaction_size_limit = get_upper_tranaction_size_limit();
   const bool use_rct = use_fork_rules(4, 0);
 
-  const bool use_new_fee  = use_fork_rules(3, -720 * 14);
   const uint64_t fee_per_kb  = get_per_kb_fee();
-  const uint64_t fee_multiplier = get_fee_multiplier(priority, use_new_fee);
+  const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm());
 
   // throw if attempting a transaction with no destinations
   THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination);
@@ -4135,8 +4280,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
   THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_destination);
 
   // gather all our dust and non dust outputs
-  const std::vector<size_t> unused_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, true, true, trusted_daemon);
-  for (size_t i: unused_indices)
+  for (size_t i = 0; i < m_transfers.size(); ++i)
   {
     const transfer_details& td = m_transfers[i];
     if (!td.m_spent && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td))
@@ -4219,16 +4363,27 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
     size_t idx;
     if ((dsts.empty() || dsts[0].amount == 0) && !adding_fee) {
       // the "make rct txes 2/2" case - we pick a small value output to "clean up" the wallet too
-      idx = pop_best_value(unused_dust_indices.empty() ? unused_transfers_indices : unused_dust_indices, tx.selected_transfers, true);
+      std::vector<size_t> indices = get_only_rct(unused_dust_indices, unused_transfers_indices);
+      idx = pop_best_value(indices, tx.selected_transfers, true);
+
+      // we might not want to add it if it's a large output and we don't have many left
+      if (m_transfers[idx].amount() >= m_min_output_value) {
+        if (get_count_above(m_transfers, unused_transfers_indices, m_min_output_value) < m_min_output_count) {
+          LOG_PRINT_L2("Second output was not strictly needed, and we're running out of outputs above " << print_money(m_min_output_value) << ", not adding");
+          break;
+        }
+      }
 
       // since we're trying to add a second output which is not strictly needed,
       // we only add it if it's unrelated enough to the first one
       float relatedness = get_output_relatedness(m_transfers[idx], m_transfers[tx.selected_transfers.front()]);
       if (relatedness > SECOND_OUTPUT_RELATEDNESS_THRESHOLD)
       {
-        LOG_PRINT_L2("Second outout was not strictly needed, and relatedness " << relatedness << ", not adding");
+        LOG_PRINT_L2("Second output was not strictly needed, and relatedness " << relatedness << ", not adding");
         break;
       }
+      pop_if_present(unused_transfers_indices, idx);
+      pop_if_present(unused_dust_indices, idx);
     } else if (!prefered_inputs.empty()) {
       idx = pop_back(prefered_inputs);
       pop_if_present(unused_transfers_indices, idx);
@@ -4259,7 +4414,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
         // we can fully pay that destination
         LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_testnet, dsts[0].addr) <<
           " for " << print_money(dsts[0].amount));
-        tx.add(dsts[0].addr, dsts[0].amount);
+        tx.add(dsts[0].addr, dsts[0].amount, m_merge_destinations);
         available_amount -= dsts[0].amount;
         dsts[0].amount = 0;
         pop_index(dsts, 0);
@@ -4269,7 +4424,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
         // we can partially fill that destination
         LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_testnet, dsts[0].addr) <<
           " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount));
-        tx.add(dsts[0].addr, available_amount);
+        tx.add(dsts[0].addr, available_amount, m_merge_destinations);
         dsts[0].amount -= available_amount;
         available_amount = 0;
       }
@@ -4440,9 +4595,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton
   std::vector<std::vector<get_outs_entry>> outs;
 
   const bool use_rct = fake_outs_count > 0 && use_fork_rules(4, 0);
-  const bool use_new_fee  = use_fork_rules(3, -720 * 14);
   const uint64_t fee_per_kb  = get_per_kb_fee();
-  const uint64_t fee_multiplier = get_fee_multiplier(priority, use_new_fee);
+  const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm());
 
   LOG_PRINT_L2("Starting with " << unused_transfers_indices.size() << " non-dust outputs and " << unused_dust_indices.size() << " dust outputs");
 
@@ -4603,7 +4757,7 @@ uint64_t wallet2::get_upper_tranaction_size_limit()
 {
   if (m_upper_transaction_size_limit > 0)
     return m_upper_transaction_size_limit;
-  uint64_t full_reward_zone = use_fork_rules(2, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 : CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1;
+  uint64_t full_reward_zone = use_fork_rules(5, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : use_fork_rules(2, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 : CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1;
   return ((full_reward_zone * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
 }
 //----------------------------------------------------------------------------------------------------
@@ -4653,7 +4807,7 @@ std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t co
   req_t.params.min_count = count;
   req_t.params.max_count = 0;
   req_t.params.unlocked = unlocked;
-  bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client);
+  bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout);
   m_daemon_rpc_mutex.unlock();
   THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "select_available_outputs_from_histogram");
   THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram");
@@ -4692,7 +4846,7 @@ uint64_t wallet2::get_num_rct_outputs()
   req_t.params.amounts.push_back(0);
   req_t.params.min_count = 0;
   req_t.params.max_count = 0;
-  bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client);
+  bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout);
   m_daemon_rpc_mutex.unlock();
   THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_num_rct_outputs");
   THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram");
@@ -4729,7 +4883,6 @@ std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bo
   const bool hf1_rules = use_fork_rules(2, 10); // first hard fork has version 2
   tx_dust_policy dust_policy(hf1_rules ? 0 : ::config::DEFAULT_DUST_THRESHOLD);
 
-  const bool use_new_fee  = use_fork_rules(3, -720 * 14);
   const uint64_t fee_per_kb  = get_per_kb_fee();
 
   // may throw
@@ -5424,7 +5577,7 @@ uint64_t wallet2::get_blockchain_height_by_date(uint16_t year, uint8_t month, ui
       height_mid,
       height_max
     };
-    bool r = net_utils::invoke_http_bin("/getblocks_by_height.bin", req, res, m_http_client);
+    bool r = net_utils::invoke_http_bin("/getblocks_by_height.bin", req, res, m_http_client, rpc_timeout);
     if (!r || res.status != CORE_RPC_STATUS_OK)
     {
       std::ostringstream oss;