Merge pull request #1775

700d218c fix broken test data (Riccardo Spagni)
0abddd64 fix broken checkpoints.dat (Riccardo Spagni)

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 #
 # All rights reserved.
 #
@@ -88,6 +88,14 @@ else()
   set(OPT_FLAGS_RELEASE "-Ofast")
 endif()
 
+# BUILD_TAG is used to select the build type to check for a new version
+if(BUILD_TAG)
+  message(STATUS "Building build tag ${BUILD_TAG}")
+  add_definitions("-DBUILD_TAG=${BUILD_TAG}")
+else()
+  message(STATUS "Building without build tag")
+endif()
+
 set(CMAKE_C_FLAGS_RELEASE "-DNDEBUG ${OPT_FLAGS_RELEASE}")
 set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG ${OPT_FLAGS_RELEASE}")
 
@@ -149,12 +157,15 @@ message(STATUS "Building for a ${ARCH_WIDTH}-bit system")
 
 # Check if we're on FreeBSD so we can exclude the local miniupnpc (it should be installed from ports instead)
 # CMAKE_SYSTEM_NAME checks are commonly known, but specifically taken from libsdl's CMakeLists
-if(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD.*")
-  set(FREEBSD TRUE)
-elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly.*|FreeBSD")
+if(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD.*|FreeBSD")
   set(FREEBSD TRUE)
 endif()
 
+# Check if we're on DragonFly BSD. See the README.md for build instructions.
+if(CMAKE_SYSTEM_NAME MATCHES "DragonFly.*")
+  set(DRAGONFLY TRUE)
+endif()
+
 # Check if we're on OpenBSD. See the README.md for build instructions.
 if(CMAKE_SYSTEM_NAME MATCHES "kOpenBSD.*|OpenBSD.*")
   set(OPENBSD TRUE)
@@ -167,10 +178,13 @@ endif()
 # elseif(CMAKE_SYSTEM_NAME MATCHES ".*BSDI.*")
 #   set(BSDI TRUE)
 
-include_directories(src contrib/epee/include external "${CMAKE_BINARY_DIR}/version")
+include_directories(external/easylogging++ src contrib/epee/include external "${CMAKE_BINARY_DIR}/version")
 
 if(APPLE)
   include_directories(SYSTEM /usr/include/malloc)
+  if(POLICY CMP0042)
+    cmake_policy(SET CMP0042 NEW)
+  endif()
 endif()
 
 if(MSVC OR MINGW)
@@ -188,6 +202,7 @@ endif()
 if (BUILD_SHARED_LIBS)
   message(STATUS "Building internal libraries with position independent code")
   set(PIC_FLAG "-fPIC")
+  add_definitions("-DBUILD_SHARED_LIBS")
 else()
   message(STATUS "Building internal libraries as static")
 endif()
@@ -267,14 +282,25 @@ endif()
 
 add_definitions("-DBLOCKCHAIN_DB=${BLOCKCHAIN_DB}")
 
-find_package(Libunwind)
 # Can't install hook in static build on OSX, because OSX linker does not support --wrap
 # On ARM, having libunwind package (with .so's only) installed breaks static link.
-if(LIBUNWIND_FOUND AND NOT (STATIC AND (APPLE OR ARM)))
-  set(DEFAULT_STACK_TRACE ON)
-else()
+if (APPLE)
   set(DEFAULT_STACK_TRACE OFF)
   set(LIBUNWIND_LIBRARIES "")
+elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT MINGW)
+  set(DEFAULT_STACK_TRACE ON)
+  set(LIBUNWIND_LIBRARIES "")
+elseif (ARM AND STATIC)
+  set(DEFAULT_STACK_TRACE OFF)
+  set(LIBUNWIND_LIBRARIES "")
+else()
+  find_package(Libunwind)
+  if(LIBUNWIND_FOUND)
+    set(DEFAULT_STACK_TRACE ON)
+  else()
+    set(DEFAULT_STACK_TRACE OFF)
+    set(LIBUNWIND_LIBRARIES "")
+  endif()
 endif()
 
 option(STACK_TRACE "Install a hook that dumps stack on exception" ${DEFAULT_STACK_TRACE})
@@ -285,6 +311,23 @@ else()
   message(STATUS "Stack trace on exception disabled")
 endif()
 
+# Handle OpenSSL, used for sha256sum on binary updates
+if (APPLE)
+  if (NOT OpenSSL_DIR)
+      EXECUTE_PROCESS(COMMAND brew --prefix openssl
+        OUTPUT_VARIABLE OPENSSL_ROOT_DIR
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
+    message(STATUS "Using OpenSSL found at ${OPENSSL_ROOT_DIR}")
+  endif()
+endif()
+
+find_package(OpenSSL REQUIRED)
+if(STATIC)
+  if(UNIX)
+    set(OPENSSL_LIBRARIES "${OPENSSL_LIBRARIES};${CMAKE_DL_LIBS}")
+  endif()
+endif()
+
 if (UNIX AND NOT APPLE)
   # Note that at the time of this writing the -Wstrict-prototypes flag added below will make this fail
   set(THREADS_PREFER_PTHREAD_FLAG ON)
@@ -344,7 +387,6 @@ else()
     set(WARNINGS_AS_ERRORS_FLAG "-Werror")
   endif()
   if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
-    set(WARNINGS "${WARNINGS} -Wno-deprecated-register -Wno-error=mismatched-tags -Wno-error=null-conversion -Wno-overloaded-shift-op-parentheses -Wno-error=shift-count-overflow -Wno-error=tautological-constant-out-of-range-compare -Wno-error=unused-private-field -Wno-error=unneeded-internal-declaration")
     if(ARM)
       set(WARNINGS "${WARNINGS} -Wno-error=inline-asm")
     endif()
@@ -372,6 +414,13 @@ else()
     set(STATIC_ASSERT_FLAG "-Dstatic_assert=_Static_assert")
   endif()
 
+  try_compile(STATIC_ASSERT_CPP_RES "${CMAKE_CURRENT_BINARY_DIR}/static-assert" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/test-static-assert.cpp" COMPILE_DEFINITIONS "-std=c++11")
+  if(STATIC_ASSERT_CPP_RES)
+    set(STATIC_ASSERT_CPP_FLAG "")
+  else()
+    set(STATIC_ASSERT_CPP_FLAG "-Dstatic_assert=_Static_assert")
+  endif()
+
   option(COVERAGE "Enable profiling for test coverage report" 0)
 
   if(COVERAGE)
@@ -407,7 +456,7 @@ else()
   endif()
 
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${ARCH_FLAG} ${COVERAGE_FLAGS} ${PIC_FLAG}")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG} ${COVERAGE_FLAGS} ${PIC_FLAG}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG} ${COVERAGE_FLAGS} ${PIC_FLAG}")
 
   # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that
   # is fixed in the code (Issue #847), force compiler to be conservative.
@@ -491,6 +540,13 @@ else()
 
   endif(ARM)
 
+  if(ANDROID AND NOT BUILD_GUI_DEPS STREQUAL "ON")
+    #From Android 5: "only position independent executables (PIE) are supported" 
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIE")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIE")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_CXX_FLAGS} -fPIE -pie")
+  endif()
+
   if(APPLE)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_TR1_TUPLE=0")
   endif()
@@ -519,7 +575,7 @@ else()
       set(RELEASE_FLAGS "${RELEASE_FLAGS} -ffat-lto-objects")
     endif()
     # Since gcc 4.9 the LTO format is non-standard (slim), so we need the gcc-specific ar and ranlib binaries
-    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9.0) AND NOT OPENBSD)
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9.0) AND NOT OPENBSD AND NOT DRAGONFLY)
       # When invoking cmake on distributions on which gcc's binaries are prefixed
       # with an arch-specific triplet, the user must specify -DCHOST=<prefix>
       if (DEFINED CHOST)
@@ -544,7 +600,7 @@ else()
       # On Windows, this is as close to fully-static as we get:
       # this leaves only deps on /c/Windows/system32/*.dll
       set(STATIC_FLAGS "-static")
-    elseif (NOT (APPLE OR FREEBSD OR OPENBSD))
+    elseif (NOT (APPLE OR FREEBSD OR OPENBSD OR DRAGONFLY))
       # On Linux, we don't support fully static build, but these can be static
       set(STATIC_FLAGS "-static-libgcc -static-libstdc++")
     endif()
@@ -577,8 +633,13 @@ endif()
 include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
 if(MINGW)
   set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi)
-elseif(APPLE OR FREEBSD OR OPENBSD)
+elseif(APPLE OR OPENBSD OR ANDROID)
   set(EXTRA_LIBRARIES "")
+elseif(FREEBSD)
+  set(EXTRA_LIBRARIES execinfo)
+elseif(DRAGONFLY)
+  find_library(COMPAT compat)
+  set(EXTRA_LIBRARIES execinfo ${COMPAT})
 elseif(NOT MSVC)
   find_library(RT rt)
   set(EXTRA_LIBRARIES ${RT})
@@ -586,6 +647,9 @@ endif()
 
 list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS})
 
+if(ANDROID)
+  set(ATOMIC libatomic.a)
+endif()
 if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND ARCH_WIDTH EQUAL "32")
   find_library(ATOMIC atomic)
   list(APPEND EXTRA_LIBRARIES ${ATOMIC})

Dockerfile

@@ -17,7 +17,7 @@ WORKDIR /src
 RUN set -e && \
   apt-get update -q && \
   apt-get install -q -y --no-install-recommends build-essential ca-certificates g++ gcc cmake \
-  pkg-config libunbound2 libevent-2.0-5 libgtest-dev libboost-all-dev libdb5.3++-dev libdb5.3-dev libssl-dev && \
+  pkg-config libunbound2 libevent-2.0-5 libgtest-dev libboost-all-dev libdb5.3++-dev libdb5.3-dev libssl1.0-dev && \
   make -j 4 && \
   apt-get purge -y g++ gcc cmake pkg-config && \
   apt-get clean -q -y && \
@@ -46,4 +46,4 @@ ENV RPC_BIND_PORT 18081
 EXPOSE 18080
 EXPOSE 18081
 
-CMD ./monerod --log-level=$LOG_LEVEL --p2p-bind-ip=$P2P_BIND_IP --p2p-bind-port=$P2P_BIND_PORT --rpc-bind-ip=$RPC_BIND_IP --rpc-bind-port=$RPC_BIND_PORT
+CMD ./monerod --log-level=$LOG_LEVEL --p2p-bind-ip=$P2P_BIND_IP --p2p-bind-port=$P2P_BIND_PORT --rpc-bind-ip=$RPC_BIND_IP --rpc-bind-port=$RPC_BIND_PORT

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2016, The Monero Project
+Copyright (c) 2014-2017, The Monero Project
 
 All rights reserved.
 

Makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 #
 # All rights reserved.
 #
@@ -58,39 +58,55 @@ release-all:
 	mkdir -p build/release
 	cd build/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
 
+release-static:
+	mkdir -p build/release
+	cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
+
 coverage:
 	mkdir -p build/debug
 	cd build/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug -D COVERAGE=ON ../.. && $(MAKE) && $(MAKE) test
 
-release-static-armv6:
-	mkdir -p build/release
-	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv6zk" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
+# Targets for specific prebuilt builds which will be advertised for updates by their build tag
 
-release-static-armv7:
+release-static-linux-armv6:
 	mkdir -p build/release
-	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
+	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv6zk" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv6" ../.. && $(MAKE)
 
-release-static-armv8:
+release-static-linux-armv7:
 	mkdir -p build/release
-	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
+	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv7" ../.. && $(MAKE)
 
-release-static: release-static-64
-
-release-static-64:
+release-static-android:
 	mkdir -p build/release
-	cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
+	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D INSTALL_VENDORED_LIBUNBOUND=ON -D BUILD_TAG="android"../.. && $(MAKE)
 
-release-static-32:
+release-static-linux-armv8:
 	mkdir -p build/release
-	cd build/release && cmake -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
+	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-armv8" ../.. && $(MAKE)
+
+release-static-linux-x86_64:
+	mkdir -p build/release
+	cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-x64" ../.. && $(MAKE)
+
+release-static-freebsd-x86_64:
+	mkdir -p build/release
+	cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="freebsd-x64" ../.. && $(MAKE)
+
+release-static-mac-x86_64:
+	mkdir -p build/release
+	cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="mac-x64" ../.. && $(MAKE)
+
+release-static-linux-i686:
+	mkdir -p build/release
+	cd build/release && cmake -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D BUILD_TAG="linux-x86" ../.. && $(MAKE)
 
 release-static-win64:
 	mkdir -p build/release
-	cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=../../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ../.. && $(MAKE)
+	cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x64" -D CMAKE_TOOLCHAIN_FILE=../../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ../.. && $(MAKE)
 
 release-static-win32:
 	mkdir -p build/release
-	cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=../../cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 ../.. && $(MAKE)
+	cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="win-x32" -D CMAKE_TOOLCHAIN_FILE=../../cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 ../.. && $(MAKE)
 
 clean:
 	@echo "WARNING: Back-up your wallet if it exists within ./build!" ; \

README.md

@@ -1,6 +1,6 @@
 # Monero
 
-Copyright (c) 2014-2016, The Monero Project
+Copyright (c) 2014-2017, The Monero Project
 
 ## Development Resources
 
@@ -21,7 +21,8 @@ Copyright (c) 2014-2016, The Monero Project
 | OSX 10.10             |  amd64   | [![OSX 10.10 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.10)](https://build.getmonero.org/builders/monero-static-osx-10.10)
 | OSX 10.11             |  amd64   | [![OSX 10.11 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.11)](https://build.getmonero.org/builders/monero-static-osx-10.11)
 | OSX 10.12             |  amd64   | [![OSX 10.12 amd64](https://build.getmonero.org/png?builder=monero-static-osx-10.12)](https://build.getmonero.org/builders/monero-static-osx-10.12)
-| FreeBSD 10.3          |  amd64   | [![FreeBSD 10.3 amd64](https://build.getmonero.org/png?builder=monero-static-freebsd64)](https://build.getmonero.org/builders/monero-static-freebsd64)
+| FreeBSD 11            |  amd64   | [![FreeBSD 11 amd64](https://build.getmonero.org/png?builder=monero-static-freebsd64)](https://build.getmonero.org/builders/monero-static-freebsd64)
+| DragonFly BSD 4.6     |  amd64   | [![DragonFly BSD amd64](https://build.getmonero.org/png?builder=monero-static-dragonflybsd-amd64)](https://build.getmonero.org/builders/monero-static-dragonflybsd-amd64)
 | Windows (MSYS2/MinGW) |  i686    | [![Windows (MSYS2/MinGW) i686](https://build.getmonero.org/png?builder=monero-static-win32)](https://build.getmonero.org/builders/monero-static-win32)
 | Windows (MSYS2/MinGW) |  amd64   | [![Windows (MSYS2/MinGW) amd64](https://build.getmonero.org/png?builder=monero-static-win64)](https://build.getmonero.org/builders/monero-static-win64)
 
@@ -61,6 +62,8 @@ The Monero donation address is: `44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft
 
 The Bitcoin donation address is: `1KTexdemPdxSBcG55heUuTjDRYqbC5ZL8H`
 
+*Note: you can easily donate XMR to the Monero donation address by using the `donate` command. Type `help` in the command-line wallet for details.*
+
 Core development funding and/or some supporting services are also graciously provided by sponsors:
 
 [<img width="80" src="https://static.getmonero.org/images/sponsors/mymonero.png"/>](https://mymonero.com)
@@ -81,7 +84,7 @@ See [LICENSE](LICENSE).
 
 Monero uses a hardforking mechanism to implement new features which requires that 
 users of Monero software run current versions and update their software on a 
-regular schedule. Here is the current schedule, versions, and compatability.
+regular schedule. Here is the current schedule, versions, and compatibility.
 Dates are provided in the format YYYYMMDD. 
 
 
@@ -95,7 +98,9 @@ Dates are provided in the format YYYYMMDD.
 
 Packages are available for
 
-* Arch Linux via AUR: [`bitmonero-git`](https://aur.archlinux.org/packages/bitmonero-git)
+* Arch Linux (via [AUR](https://aur.archlinux.org/)):
+  - Stable release: [`monero`](https://aur.archlinux.org/packages/monero)
+  - Bleeding edge: [`bitmonero-git`](https://aur.archlinux.org/packages/bitmonero-git)
 
 * OS X via [Homebrew](http://brew.sh)
 
@@ -132,17 +137,18 @@ library archives (`.a`).
 | GCC            | 4.7.3         | NO       | `build-essential`  | `base-devel`   | NO       |                |
 | CMake          | 3.0.0         | NO       | `cmake`            | `cmake`        | NO       |                |
 | pkg-config     | any           | NO       | `pkg-config`       | `base-devel`   | NO       |                |
-| Boost          | 1.58          | NO       | `libboost-all-dev` | `boost`        | NO       |                |
-| BerkeleyDB     | 4.8           | NO       | `libdb{,++}-dev`   | `db`           | NO       |                |
-| libevent       | 2.0           | NO       | `libevent-dev`     | `libevent`     | NO       |                |
-| libunbound     | 1.4.16        | YES      | `libunbound-dev`   | `unbound`      | NO       |                |
+| Boost          | 1.58          | NO       | `libboost-all-dev` | `boost`        | NO       | C++ libraries  |
+| libevent       | 2.0           | NO       | `libevent-dev`     | `libevent`     | NO       | Network IO     |
+| OpenSSL        | basically any | NO       | `libssl-dev`       | `openssl`      | NO       | sha256 sum     |
+| libunbound     | 1.4.16        | YES      | `libunbound-dev`   | `unbound`      | NO       | DNS resolver   |
 | libminiupnpc   | 2.0           | YES      | `libminiupnpc-dev` | `miniupnpc`    | YES      | NAT punching   |
-| libunwind      | any           | NO       | `libunwind8-dev`   | `libunwind`    | YES      | stack traces   |
-| ldns           | 1.6.17        | NO       | `libldns-dev`      | `ldns`         | YES      | ?              |
-| expat          | 1.1           | NO       | `libexpat1-dev`    | `expat`        | YES      | ?              |
-| GTest          | 1.5           | YES      | `libgtest-dev`^    | `gtest`        | YES      | test suite     |
-| Doxygen        | any           | NO       | `doxygen`          | `doxygen`      | YES      | documentation  |
-| Graphviz       | any           | NO       | `graphviz`         | `graphviz`     | YES      | documentation  |
+| libunwind      | any           | NO       | `libunwind8-dev`   | `libunwind`    | YES      | Stack traces   |
+| liblzma        | any           | NO       | `liblzma-dev`      | `xz`           | YES      | For libunwind  |
+| ldns           | 1.6.17        | NO       | `libldns-dev`      | `ldns`         | YES      | SSL toolkit    |
+| expat          | 1.1           | NO       | `libexpat1-dev`    | `expat`        | YES      | XML parsing    |
+| GTest          | 1.5           | YES      | `libgtest-dev`^    | `gtest`        | YES      | Test suite     |
+| Doxygen        | any           | NO       | `doxygen`          | `doxygen`      | YES      | Documentation  |
+| Graphviz       | any           | NO       | `graphviz`         | `graphviz`     | YES      | Documentation  |
 
 [^] On Debian/Ubuntu `libgtest-dev` only includes sources and headers. You must
 build the library binary manually. This can be done with the following command ```sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake . && sudo make && sudo mv libg* /usr/lib/ ```
@@ -382,6 +388,50 @@ If you want to help out, see CONTRIBUTING for a set of guidelines.
 
 This section contains general instructions for debugging failed installs or problems encountered with Monero. First ensure you are running the latest version built from the github repo.
 
+## Obtaining Stack Traces and Core Dumps on Unix Systems
+
+We generally use the tool `gdb` (GNU debugger) to provide stack trace functionality, and `ulimit` to provide core dumps in builds which crash or segfault.
+
+* To use gdb in order to obtain a stack trace for a build that has stalled:
+
+Run the build.
+
+Once it stalls, enter the following command:
+
+```
+gdb /path/to/monerod `pidof monerod` 
+```
+
+Type `thread apply all bt` within gdb in order to obtain the stack trace
+
+* If however the core dumps or segfaults:
+
+Enter `ulimit -c unlimited` on the command line to enable unlimited filesizes for core dumps
+
+Run the build.
+
+When it terminates with an output along the lines of "Segmentation fault (core dumped)", there should be a core dump file in the same directory as monerod.
+
+You can now analyse this core dump with `gdb` as follows:
+
+`gdb /path/to/monerod /path/to/dumpfile`
+
+Print the stack trace with `bt`
+
+* To run monero within gdb:
+
+Type `gdb /path/to/monerod`
+
+Pass command-line options with `--args` followed by the relevant arguments
+
+Type `run` to run monerod
+
+## Analysing Memory Corruption
+
+We use the tool `valgrind` for this.
+
+Run with `valgrind /path/to/monerod`. It will be slow.
+
 ## LMDB
 
 Instructions for debugging suspected blockchain corruption as per @HYC

cmake/32-bit-toolchain.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/64-bit-toolchain.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/FindLibunwind.cmake

@@ -17,6 +17,22 @@ find_path(LIBUNWIND_INCLUDE_DIR libunwind.h
 )
 
 find_library(LIBUNWIND_LIBRARIES NAMES unwind )
+if(NOT LIBUNWIND_LIBRARIES STREQUAL "LIBUNWIND_LIBRARIES-NOTFOUND")
+  if (CMAKE_COMPILER_IS_GNUCC)
+    set(LIBUNWIND_LIBRARIES "gcc_eh;${LIBUNWIND_LIBRARIES}")
+  endif()
+endif()
+
+# some versions of libunwind need liblzma, and we don't use pkg-config
+# so we just look whether liblzma is installed, and add it if it is.
+# It might not be actually needed, but doesn't hurt if it is not.
+# We don't need any headers, just the lib, as it's privately needed.
+message(STATUS "looking for liblzma")
+find_library(LIBLZMA_LIBRARIES lzma )
+if(NOT LIBLZMA_LIBRARIES STREQUAL "LIBLZMA_LIBRARIES-NOTFOUND")
+  message(STATUS "liblzma found")
+  set(LIBUNWIND_LIBRARIES "${LIBUNWIND_LIBRARIES};${LIBLZMA_LIBRARIES}")
+endif()
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(Libunwind "Could not find libunwind" LIBUNWIND_INCLUDE_DIR LIBUNWIND_LIBRARIES)

cmake/FindUnbound.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 # All rights reserved.
 # 
 # Redistribution and use in source and binary forms, with or without modification, are

cmake/test-static-assert.c

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 // 
 // All rights reserved.
 // 

cmake/test-static-assert.cpp

@@ -0,0 +1,34 @@
+// Copyright (c) 2014-2017, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <assert.h>
+
+static_assert(1, "FAIL");
+int main(int argc, char *argv[]) {
+	return 0;
+}

contrib/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 # 
 # All rights reserved.
 # 
@@ -26,5 +26,5 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-add_subdirectory(otshell_utils)
+add_subdirectory(epee)
 

contrib/epee/CMakeLists.txt

@@ -0,0 +1,30 @@
+# Copyright (c) 2014-2017, The Monero Project
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+# 
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+# 
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+add_subdirectory(src)
+

contrib/epee/demo/demo_http_server/demo_http_server.cpp

@@ -1,217 +0,0 @@
-// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-// * Neither the name of the Andrey N. Sabelnikov nor the
-// names of its contributors may be used to endorse or promote products
-// derived from this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
-// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-
-#include "stdafx.h"
-#include "console_handler.h"
-#include "demo_http_server.h"
-#include "net/http_client.h"
-#include "storages/http_abstract_invoke.h"
-
-
-template<class t_request, class t_response>
-bool communicate(const std::string url, t_request& req, t_response& rsp, const std::string& ip, const std::string& port, bool use_json, bool use_jrpc = false)
-{
-  epee::net_utils::http::http_simple_client http_client;
-  bool r = http_client.connect(ip, port, 1000);
-  CHECK_AND_ASSERT_MES(r, false, "failed to connect");
-  if(use_json)
-  {
-    if(use_jrpc)
-    {
-      epee::json_rpc::request<t_request> req_t = AUTO_VAL_INIT(req_t);
-      req_t.jsonrpc = "2.0";
-      req_t.id = epee::serialization::storage_entry(10);
-      req_t.method = "command_example_1";
-      req_t.params = req;
-      epee::json_rpc::response<t_response, std::string> resp_t = AUTO_VAL_INIT(resp_t);
-      if(!epee::net_utils::invoke_http_json_remote_command2("/request_json_rpc", req_t, resp_t, http_client))
-      {
-        return false;
-      }
-      rsp = resp_t.result;
-      return true;
-    }else
-      return epee::net_utils::invoke_http_json_remote_command2(url, req, rsp, http_client);
-  }
-  else   
-    return epee::net_utils::invoke_http_bin_remote_command2(url, req, rsp, http_client);
-}
-
-
-int main(int argc, char* argv[])
-{
-  TRY_ENTRY();
-  string_tools::set_module_name_and_folder(argv[0]);
-
-  //set up logging options
-  log_space::get_set_log_detalisation_level(true, LOG_LEVEL_2);
-  log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
-  log_space::log_singletone::add_logger(LOGGER_FILE, 
-    log_space::log_singletone::get_default_log_file().c_str(), 
-    log_space::log_singletone::get_default_log_folder().c_str());
-
-
-
-  LOG_PRINT("Demo server starting ...", LOG_LEVEL_0);
-
-
-  demo::demo_http_server srv;
-
-  start_default_console(&srv, "#");
-
-  std::string bind_param = "0.0.0.0";
-  std::string port = "83";
-
-  if(!srv.init(port, bind_param))
-  {
-    LOG_ERROR("Failed to initialize srv!");
-    return 1;
-  }
-
-  //log loop
-  srv.run();
-  size_t count = 0;
-  while (!srv.is_stop())
-  {
-   
-    demo::COMMAND_EXAMPLE_1::request req;
-    req.sub = demo::get_test_data();
-    demo::COMMAND_EXAMPLE_1::response rsp;
-    bool r = false;
-    if(count%2)
-    {//invoke json 
-      r = communicate("/request_api_json_1", req, rsp, "127.0.0.1", port, true, true);
-    }else{
-      r = communicate("/request_api_bin_1", req, rsp, "127.0.0.1", port, false);
-    }
-    CHECK_AND_ASSERT_MES(r, false, "failed to invoke http request");
-    CHECK_AND_ASSERT_MES(rsp.m_success, false, "wrong response");
-    CHECK_AND_ASSERT_MES(rsp.subs.size()==1, false, "wrong response");
-    CHECK_AND_ASSERT_MES(rsp.subs.front() == demo::get_test_data(), false, "wrong response");
-    //misc_utils::sleep_no_w(1000);
-    ++count;
-  }
-  bool r = srv.wait_stop();
-  CHECK_AND_ASSERT_MES(r, 1, "failed to wait server stop");
-  srv.deinit();
-
-  LOG_PRINT("Demo server stoped.", LOG_LEVEL_0);
-  return 1;
-
-  CATCH_ENTRY_L0("main", 1);
-}
-
-/************************************************************************/
-/*                                                                      */
-/************************************************************************/
-namespace demo
-{
-  bool demo_http_server::init(const std::string& bind_port, const std::string& bind_ip)
-  {
-
-
-    //set self as callback handler
-    m_net_server.get_config_object().m_phandler = this;
-
-    //here set folder for hosting reqests
-    m_net_server.get_config_object().m_folder = "";
-
-    LOG_PRINT_L0("Binding on " << bind_ip << ":" << bind_port);
-    return m_net_server.init_server(bind_port, bind_ip);
-  }
-
-  bool demo_http_server::run()
-  {
-    m_stop = false;
-    //here you can set worker threads count
-    int thrds_count = 4;
-
-    //go to loop
-    LOG_PRINT("Run net_service loop( " << thrds_count << " threads)...", LOG_LEVEL_0);
-    if(!m_net_server.run_server(thrds_count, false))
-    {
-      LOG_ERROR("Failed to run net tcp server!");
-    }
-
-    return true;
-  }
-
-  bool demo_http_server::deinit()
-  {
-    return m_net_server.deinit_server();
-  }
-
-  bool demo_http_server::send_stop_signal()
-  {
-    m_stop = true;
-    m_net_server.send_stop_signal();
-    return true;
-  }
- 
-  bool demo_http_server::on_requestr_uri_1(const net_utils::http::http_request_info& query_info, 
-    net_utils::http::http_response_info& response,
-    const net_utils::connection_context_base& m_conn_context)
-  {
-    return true;
-  }
-
-
-  bool demo_http_server::on_requestr_uri_2(const net_utils::http::http_request_info& query_info, 
-    net_utils::http::http_response_info& response,
-    const net_utils::connection_context_base& m_conn_context)
-  {
-    return true;
-  }
-
-
-  bool demo_http_server::on_hosting_request( const net_utils::http::http_request_info& query_info,
-    net_utils::http::http_response_info& response,
-    const net_utils::connection_context_base& m_conn_context)
-  {
-    //read file from filesystem here
-    return true;
-  }
-
-  bool demo_http_server::on_request_api_1(const COMMAND_EXAMPLE_1::request& req, COMMAND_EXAMPLE_1::response& res, connection_context& ctxt)
-  {
-    CHECK_AND_ASSERT_MES(req.sub == demo::get_test_data(), false, "wrong request");
-    res.m_success = true;
-    res.subs.push_back(req.sub);
-    return true;
-  }
-
-  bool demo_http_server::on_request_api_1_with_error(const COMMAND_EXAMPLE_1::request& req, COMMAND_EXAMPLE_1::response& res, epee::json_rpc::error& error_resp, connection_context& ctxt)
-  {
-    error_resp.code = 232432;
-    error_resp.message = "bla bla bla";
-    return false;
-  }
-
-  bool demo_http_server::on_request_api_2(const COMMAND_EXAMPLE_2::request& req, COMMAND_EXAMPLE_2::response& res, connection_context& ctxt)
-  {
-    return true;
-  }
-}

contrib/epee/demo/demo_http_server/demo_http_server.h

@@ -1,103 +0,0 @@
-// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-// * Neither the name of the Andrey N. Sabelnikov nor the
-// names of its contributors may be used to endorse or promote products
-// derived from this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
-// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-
-
-
-#pragma once
-#include <boost/thread.hpp>
-#include <boost/bind.hpp> 
-
-#include "net/http_server_cp2.h"
-#include "transport_defs.h"
-#include "net/http_server_handlers_map2.h"
-
-using namespace epee;
-
-namespace demo
-{
-
-  class demo_http_server: public net_utils::http::i_http_server_handler<epee::net_utils::connection_context_base>
-  {
-  public:
-	  typedef epee::net_utils::connection_context_base connection_context;
-
-    demo_http_server():m_stop(false){}
-    bool run();
-    bool init(const std::string& bind_port = "11112", const std::string& bind_ip = "0.0.0.0");
-    bool deinit();
-    bool send_stop_signal();
-    bool is_stop(){return m_stop;}
-    bool wait_stop(){return m_net_server.timed_wait_server_stop(100000);}
-  private: 
-
-
-    CHAIN_HTTP_TO_MAP2(connection_context); //forward http requests to uri map
-
-    BEGIN_URI_MAP2()
-      MAP_URI2("/requestr_uri_1", on_requestr_uri_1)
-      MAP_URI2("/requestr_uri_2", on_requestr_uri_1)
-      //MAP_URI_AUTO_XML2("/request_api_xml_1", on_request_api_1, COMMAND_EXAMPLE_1)
-      //MAP_URI_AUTO_XML2("/request_api_xml_2", on_request_api_2, COMMAND_EXAMPLE_2)
-      MAP_URI_AUTO_JON2("/request_api_json_1", on_request_api_1, COMMAND_EXAMPLE_1)
-      MAP_URI_AUTO_JON2("/request_api_json_2", on_request_api_2, COMMAND_EXAMPLE_2)
-      MAP_URI_AUTO_BIN2("/request_api_bin_1", on_request_api_1, COMMAND_EXAMPLE_1)
-      MAP_URI_AUTO_BIN2("/request_api_bin_2", on_request_api_2, COMMAND_EXAMPLE_2)      
-      BEGIN_JSON_RPC_MAP("/request_json_rpc")
-        MAP_JON_RPC("command_example_1", on_request_api_1, COMMAND_EXAMPLE_1)
-        MAP_JON_RPC("command_example_2", on_request_api_2, COMMAND_EXAMPLE_2)
-        MAP_JON_RPC_WE("command_example_1_we", on_request_api_1_with_error, COMMAND_EXAMPLE_1)
-      END_JSON_RPC_MAP()
-      CHAIN_URI_MAP2(on_hosting_request)
-    END_URI_MAP2()
-
-
-
-    bool on_requestr_uri_1(const net_utils::http::http_request_info& query_info, 
-                             net_utils::http::http_response_info& response,
-                             const net_utils::connection_context_base& m_conn_context);
-
-
-    bool on_requestr_uri_2(const net_utils::http::http_request_info& query_info, 
-                             net_utils::http::http_response_info& response,
-                             const net_utils::connection_context_base& m_conn_context);
-
-
-
-
-    bool on_hosting_request( const net_utils::http::http_request_info& query_info,
-                             net_utils::http::http_response_info& response,
-                             const net_utils::connection_context_base& m_conn_context);
-
-    bool on_request_api_1(const COMMAND_EXAMPLE_1::request& req, COMMAND_EXAMPLE_1::response& res, connection_context& ctxt);
-    bool on_request_api_2(const COMMAND_EXAMPLE_2::request& req, COMMAND_EXAMPLE_2::response& res, connection_context& ctxt);
-
-    bool on_request_api_1_with_error(const COMMAND_EXAMPLE_1::request& req, COMMAND_EXAMPLE_1::response& res, epee::json_rpc::error& error_resp, connection_context& ctxt);
-
-    net_utils::boosted_http_server_custum_handling m_net_server;
-    std::atomic<bool> m_stop;
-  };
-}
-

contrib/epee/demo/demo_levin_server/demo_levin_server.cpp

@@ -1,200 +0,0 @@
-// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-// * Neither the name of the Andrey N. Sabelnikov nor the
-// names of its contributors may be used to endorse or promote products
-// derived from this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
-// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-
-
-
-#include "stdafx.h"
-#include "demo_levin_server.h"
-#include "console_handler.h"
-
-
-template<class t_request>
-bool communicate(net_utils::boosted_levin_async_server& transport, int id, t_request& req, const std::string& ip, const std::string& port, bool use_async)
-{
-  if(use_async)
-  {
-    //IMPORTANT: do not pass local parameters from stack by reference! connect_async returns immediately, and callback will call in any thread later
-    transport.connect_async(ip, port, 10000, [&transport, id, req, ip, port](net_utils::connection_context_base& ctx, const boost::system::error_code& ec_)
-    {
-      if(!!ec_)
-      {
-        LOG_ERROR("Failed to connect to " << ip << ":" << port);
-      }else
-      {//connected ok!
-
-        epee::net_utils::async_invoke_remote_command2<demo::COMMAND_EXAMPLE_1::response>(ctx.m_connection_id, id, req, transport.get_config_object(), [&transport, ip, port](int res_code, demo::COMMAND_EXAMPLE_1::response& rsp, net_utils::connection_context_base& ctx)
-        {
-          if(res_code < 0)
-          {
-            LOG_ERROR("Failed to invoke to " << ip << ":" << port);
-          }else
-          {//invoked ok
-            CHECK_AND_ASSERT_MES(rsp.m_success, false, "wrong response");
-            CHECK_AND_ASSERT_MES(rsp.subs.size()==1, false, "wrong response");
-            CHECK_AND_ASSERT_MES(rsp.subs.front() == demo::get_test_data(), false, "wrong response");
-            LOG_PRINT_GREEN("Client COMMAND_EXAMPLE_1 async invoked ok", LOG_LEVEL_0);
-          }
-          transport.get_config_object().close(ctx.m_connection_id);
-          return true;
-        });
-        LOG_PRINT_GREEN("Client COMMAND_EXAMPLE_1 async invoke requested", LOG_LEVEL_0);
-      }
-    });
-  }else
-  {
-    net_utils::connection_context_base ctx = AUTO_VAL_INIT(ctx);
-    bool r = transport.connect(ip, port, 10000, ctx);
-    CHECK_AND_ASSERT_MES(r, false, "failed to connect to " << ip << ":" << port);
-    demo::COMMAND_EXAMPLE_1::response rsp = AUTO_VAL_INIT(rsp);
-    LOG_PRINT_GREEN("Client COMMAND_EXAMPLE_1 sync invoke requested", LOG_LEVEL_0);
-    r = epee::net_utils::invoke_remote_command2(ctx.m_connection_id, id, req, rsp, transport.get_config_object());
-    CHECK_AND_ASSERT_MES(r, false, "failed to invoke levin request");
-    CHECK_AND_ASSERT_MES(rsp.m_success, false, "wrong response");
-    CHECK_AND_ASSERT_MES(rsp.subs.size()==1, false, "wrong response");
-    CHECK_AND_ASSERT_MES(rsp.subs.front() == demo::get_test_data(), false, "wrong response");
-    transport.get_config_object().close(ctx.m_connection_id);
-    LOG_PRINT_GREEN("Client COMMAND_EXAMPLE_1 sync invoked ok", LOG_LEVEL_0);
-  }
-  return true;
-}
-
-
-int main(int argc, char* argv[])
-{
-  TRY_ENTRY();
-  string_tools::set_module_name_and_folder(argv[0]);
-
-  //set up logging options
-  log_space::get_set_log_detalisation_level(true, LOG_LEVEL_2);
-  log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
-  log_space::log_singletone::add_logger(LOGGER_FILE, 
-                      log_space::log_singletone::get_default_log_file().c_str(), 
-                      log_space::log_singletone::get_default_log_folder().c_str());
-
-
-
-  LOG_PRINT("Demo server starting ...", LOG_LEVEL_0);
-
-
-  demo::demo_levin_server srv;
-
-  start_default_console(&srv, "#");
-
-  std::string bind_param = "0.0.0.0";
-  std::string port = "12345";
-
-  if(!srv.init(port, bind_param))
-  {
-    LOG_ERROR("Failed to initialize srv!");
-    return 1;
-  }
-
-  srv.run();
-
-  size_t c = 1;
-  while (!srv.is_stop())
-  {
-
-    demo::COMMAND_EXAMPLE_1::request req;
-    req.sub = demo::get_test_data();
-    bool r = communicate(srv.get_server(), demo::COMMAND_EXAMPLE_1::ID, req, "127.0.0.1", port, (c%2 == 0));
-    misc_utils::sleep_no_w(1000);
-    ++c;
-  }
-  bool r = srv.wait_stop();
-  CHECK_AND_ASSERT_MES(r, 1, "failed to wait server stop");
-
-
-  srv.deinit();
-
-  LOG_PRINT("Demo server stoped.", LOG_LEVEL_0);
-  return 1;
-
-  CATCH_ENTRY_L0("main", 1);
-}
-
-/************************************************************************/
-/*                                                                      */
-/************************************************************************/
-namespace demo
-{
-  bool demo_levin_server::init(const std::string& bind_port, const std::string& bind_ip)
-  {
-    m_net_server.get_config_object().m_pcommands_handler = this;
-    LOG_PRINT_L0("Binding on " << bind_ip << ":" << bind_port);
-    return m_net_server.init_server(bind_port, bind_ip);
-  }
-
-  bool demo_levin_server::run()
-  {
-    m_stop = false;
-    //here you can set worker threads count
-    int thrds_count = 4;
-    m_net_server.get_config_object().m_invoke_timeout = 10000;
-    m_net_server.get_config_object().m_pcommands_handler = this;
-
-    //go to loop
-    LOG_PRINT("Run net_service loop( " << thrds_count << " threads)...", LOG_LEVEL_0);
-    if(!m_net_server.run_server(thrds_count, false))
-    {
-      LOG_ERROR("Failed to run net tcp server!");
-    }
-
-    LOG_PRINT("net_service loop stopped.", LOG_LEVEL_0);
-    return true;
-  }
-
-  bool demo_levin_server::deinit()
-  {
-    return m_net_server.deinit_server();
-  }
-
-  bool demo_levin_server::send_stop_signal()
-  {
-    m_net_server.send_stop_signal();
-    return true;
-  }
-  int demo_levin_server::handle_command_1(int command, COMMAND_EXAMPLE_1::request& arg, COMMAND_EXAMPLE_1::response& rsp, const net_utils::connection_context_base& context)
-  {
-    CHECK_AND_ASSERT_MES(arg.sub == demo::get_test_data(), false, "wrong request");
-    rsp.m_success = true;
-    rsp.subs.push_back(arg.sub);
-    LOG_PRINT_GREEN("Server COMMAND_EXAMPLE_1 ok", LOG_LEVEL_0);
-    return 1;
-  }
-  int demo_levin_server::handle_command_2(int command, COMMAND_EXAMPLE_2::request& arg, COMMAND_EXAMPLE_2::response& rsp, const net_utils::connection_context_base& context)
-  {
-    return 1;
-  }
-  int demo_levin_server::handle_notify_1(int command, COMMAND_EXAMPLE_1::request& arg, const net_utils::connection_context_base& context)
-  {
-    return 1;
-  }
-  int demo_levin_server::handle_notify_2(int command, COMMAND_EXAMPLE_2::request& arg, const net_utils::connection_context_base& context)
-  {
-    return 1;
-  }
-}

contrib/epee/demo/demo_levin_server/demo_levin_server.h

@@ -1,76 +0,0 @@
-// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-// * Neither the name of the Andrey N. Sabelnikov nor the
-// names of its contributors may be used to endorse or promote products
-// derived from this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
-// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-
-
-
-#pragma once
-#include <boost/thread.hpp>
-#include <boost/bind.hpp> 
-
-#include "net/levin_server_cp2.h"
-#include "transport_defs.h"
-#include "storages/levin_abstract_invoke2.h"
-
-using namespace epee;
-
-namespace demo
-{
-
-  class demo_levin_server: public levin::levin_commands_handler<>
-  {
-  public:
-    bool run();
-    bool init(const std::string& bind_port = "11112", const std::string& bind_ip = "0.0.0.0");
-    bool deinit();
-    bool send_stop_signal();
-    bool is_stop(){return m_stop;}
-    bool wait_stop(){return m_net_server.timed_wait_server_stop(100000);}
-    net_utils::boosted_levin_async_server& get_server(){return m_net_server;}
-  private: 
-
-
-    CHAIN_LEVIN_INVOKE_MAP(); //move levin_commands_handler interface invoke(...) callbacks into invoke map
-    CHAIN_LEVIN_NOTIFY_STUB(); //move levin_commands_handler interface notify(...) callbacks into nothing
-
-    BEGIN_INVOKE_MAP2(demo_levin_server)
-      HANDLE_INVOKE_T2(COMMAND_EXAMPLE_1, &demo_levin_server::handle_command_1)
-      HANDLE_INVOKE_T2(COMMAND_EXAMPLE_2, &demo_levin_server::handle_command_2)
-      HANDLE_NOTIFY_T2(COMMAND_EXAMPLE_1, &demo_levin_server::handle_notify_1)
-      HANDLE_NOTIFY_T2(COMMAND_EXAMPLE_2, &demo_levin_server::handle_notify_2)
-    END_INVOKE_MAP2()
-
-    //----------------- commands handlers ----------------------------------------------
-    int handle_command_1(int command, COMMAND_EXAMPLE_1::request& arg, COMMAND_EXAMPLE_1::response& rsp, const net_utils::connection_context_base& context);
-    int handle_command_2(int command, COMMAND_EXAMPLE_2::request& arg, COMMAND_EXAMPLE_2::response& rsp, const net_utils::connection_context_base& context);
-    int handle_notify_1(int command, COMMAND_EXAMPLE_1::request& arg, const net_utils::connection_context_base& context);
-    int handle_notify_2(int command, COMMAND_EXAMPLE_2::request& arg, const net_utils::connection_context_base& context);
-    //----------------------------------------------------------------------------------
-    net_utils::boosted_levin_async_server m_net_server;
-    std::atomic<bool> m_stop;
-
-  };
-}
-

contrib/epee/include/console_handler.h

@@ -133,7 +133,7 @@ namespace epee
     bool wait_stdin_data()
     {
 #if !defined(WIN32)
-      #ifdef __OpenBSD__
+      #if defined(__OpenBSD__) || defined(__ANDROID__)
       int stdin_fileno = fileno(stdin);
       #else
       int stdin_fileno = ::fileno(stdin);
@@ -277,11 +277,11 @@ namespace epee
     {
       if (!m_prompt.empty())
       {
-        epee::log_space::set_console_color(epee::log_space::console_color_yellow, true);
+        epee::set_console_color(epee::console_color_yellow, true);
         std::cout << m_prompt;
         if (' ' != m_prompt.back())
           std::cout << ' ';
-        epee::log_space::reset_console_color();
+        epee::reset_console_color();
         std::cout.flush();
       }
     }
@@ -310,7 +310,7 @@ namespace epee
           }
           if (!get_line_ret)
           {
-            LOG_PRINT("Failed to read line.", LOG_LEVEL_0);
+            MERROR("Failed to read line.");
           }
           string_tools::trim(command);
 

contrib/epee/include/md5_l.h

@@ -74,7 +74,7 @@ namespace md5
 	static void MD5Init(MD5_CTX * context);
 	static void MD5Update( MD5_CTX *context, const unsigned char *input, unsigned int inputLen );
 	static void MD5Final ( unsigned char digest[16], MD5_CTX *context );
-	static void hmac_md5(const unsigned char* text, int text_len, const unsigned char* key, int key_len, unsigned char *digest);
+	static inline void hmac_md5(const unsigned char* text, int text_len, const unsigned char* key, int key_len, unsigned char *digest);
 
 	
 	inline bool md5( unsigned char *input, int ilen, unsigned char output[16] )

contrib/epee/include/net/abstract_tcp_server.h

@@ -38,6 +38,9 @@
 
 #pragma comment(lib, "Ws2_32.lib")
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 namespace epee 
 {
 namespace net_utils

contrib/epee/include/net/abstract_tcp_server2.h

@@ -55,9 +55,11 @@
 #include "net_utils_base.h"
 #include "syncobj.h"
 #include "../../../../src/p2p/connection_basic.hpp"
-#include "../../../../contrib/otshell_utils/utils.hpp"
 #include "../../../../src/p2p/network_throttle-detail.hpp"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000
 
 namespace epee

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -51,9 +51,8 @@
 
 #include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal()
 
-#include "../../../../contrib/otshell_utils/utils.hpp"
-#include "../../../../src/p2p/data_logger.hpp"
-using namespace nOT::nUtils; // TODO
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
 
 #define CONNECTION_CLEANUP_TIME 30 // seconds
 
@@ -83,7 +82,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 		m_throttle_speed_in("speed_in", "throttle_speed_in"),
 		m_throttle_speed_out("speed_out", "throttle_speed_out")
   {
-    _info_c("net/sleepRPC", "test, connection constructor set m_connection_type="<<m_connection_type);
+    MINFO("test, connection constructor set m_connection_type="<<m_connection_type);
   }
 PRAGMA_WARNING_DISABLE_VS(4355)
   //---------------------------------------------------------------------------------
@@ -229,7 +228,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   {
     TRY_ENTRY();
     boost::shared_ptr<connection<t_protocol_handler> >  back_connection_copy;
-    LOG_PRINT_L4("[sock " << socket_.native_handle() << "] release");
+    LOG_TRACE_CC(context, "[sock " << socket_.native_handle() << "] release");
     CRITICAL_REGION_BEGIN(m_self_refs_lock);
     CHECK_AND_ASSERT_MES(m_self_refs.size(), false, "[sock " << socket_.native_handle() << "] m_self_refs empty at connection<t_protocol_handler>::release() call");
     //erasing from container without additional copy can cause start deleting object, including m_self_refs
@@ -266,8 +265,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
       address = endpoint.address().to_string();
       port = boost::lexical_cast<std::string>(endpoint.port());
     }
-    _mark_c("net/kind" ,
-        " connection type " << to_string( m_connection_type ) << " "
+    MINFO(" connection type " << to_string( m_connection_type ) << " "
         << socket_.local_endpoint().address().to_string() << ":" << socket_.local_endpoint().port()
         << " <--> " << address << ":" << port);
   }
@@ -306,7 +304,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 				delay *= 0.5;
 				if (delay > 0) {
 					long int ms = (long int)(delay * 100);
-					epee::net_utils::data_logger::get_instance().add_data("sleep_down", ms);
 					boost::this_thread::sleep_for(boost::chrono::milliseconds(ms));
 				}
 			} while(delay > 0);
@@ -397,14 +394,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         const t_safe chunksize_max = chunksize_good * 2 ;
 		const bool allow_split = (m_connection_type == e_connection_type_RPC) ? false : true; // do not split RPC data
 
-        ASRT(! (chunksize_max<0) ); // make sure it is unsigned before removin sign with cast:
+        CHECK_AND_ASSERT_MES(! (chunksize_max<0), false, "Negative chunksize_max" ); // make sure it is unsigned before removin sign with cast:
         long long unsigned int chunksize_max_unsigned = static_cast<long long unsigned int>( chunksize_max ) ;
 
         if (allow_split && (cb > chunksize_max_unsigned)) {
 			{ // LOCK: chunking
     		epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** 
 
-				_dbg3_c("net/out/size", "do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr);
+				MDEBUG("do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr);
 				t_safe all = cb; // all bytes to send 
 				t_safe pos = 0; // current sending position
 				// 01234567890 
@@ -419,39 +416,39 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 				while (pos < all) {
 					t_safe lenall = all-pos; // length from here to end
 					t_safe len = std::min( chunksize_good , lenall); // take a smaller part
-					ASRT(len<=chunksize_good);
+					CHECK_AND_ASSERT_MES(len<=chunksize_good, false, "len too large");
 					// pos=8; len=4; all=10;	len=3;
 
-                    ASRT(! (len<0) ); // check before we cast away sign:
+                    CHECK_AND_ASSERT_MES(! (len<0), false, "negative len"); // check before we cast away sign:
                     unsigned long long int len_unsigned = static_cast<long long int>( len );
-                    ASRT(len>0); // (redundand)
-                    ASRT(len_unsigned < std::numeric_limits<size_t>::max());   // yeap we want strong < then max size, to be sure
+                    CHECK_AND_ASSERT_MES(len>0, false, "len not strictly positive"); // (redundant)
+                    CHECK_AND_ASSERT_MES(len_unsigned < std::numeric_limits<size_t>::max(), false, "Invalid len_unsigned");   // yeap we want strong < then max size, to be sure
 					
 					void *chunk_start = ((char*)ptr) + pos;
-					_fact_c("net/out/size","chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos);
-					ASRT(chunk_start >= ptr); // not wrapped around address?
+					MDEBUG("chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos);
+					CHECK_AND_ASSERT_MES(chunk_start >= ptr, false, "Pointer wraparound"); // not wrapped around address?
 					//std::memcpy( (void*)buf, chunk_start, len);
 
-					_dbg3_c("net/out/size", "part of " << lenall << ": pos="<<pos << " len="<<len);
+					MDEBUG("part of " << lenall << ": pos="<<pos << " len="<<len);
 
 					bool ok = do_send_chunk(chunk_start, len); // <====== ***
 
 					all_ok = all_ok && ok;
 					if (!all_ok) {
-						_dbg1_c("net/out/size", "do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr);
-						_dbg1("do_send() SEND was aborted in middle of big package - this is mostly harmless "
+						MDEBUG("do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr);
+						MDEBUG("do_send() SEND was aborted in middle of big package - this is mostly harmless "
 							<< " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb);
 						return false; // partial failure in sending
 					}
-					pos = pos+len; ASRT(pos >0);
+					pos = pos+len;
+					CHECK_AND_ASSERT_MES(pos >0, false, "pos <= 0");
 
 					// (in catch block, or uniq pointer) delete buf;
 				} // each chunk
 
-				_dbg3_c("net/out/size", "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
-				_dbg3  (                "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
+				MDEBUG("do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
 
-                _info_c("net/sleepRPC", "do_send() m_connection_type = " << m_connection_type);
+                MDEBUG("do_send() m_connection_type = " << m_connection_type);
 
 				return all_ok; // done - e.g. queued - all the chunks of current do_send call
 			} // LOCK: chunking
@@ -505,15 +502,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         }*/
 
         long int ms = 250 + (rand()%50);
-        _info_c("net/sleep", "Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep
+        MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep
         m_send_que_lock.unlock();
         boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
         m_send_que_lock.lock();
         _dbg1("sleep for queue: " << ms);
 
         if (retry > retry_limit) {
-            _erro("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
-            //	_dbg1_c("net/sleep", "send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
+            MWARNING("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
             shutdown();
             return false;
         }
@@ -525,10 +521,10 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     if(m_send_que.size() > 1)
     { // active operation should be in progress, nothing to do, just wait last operation callback
         auto size_now = cb;
-        _info_c("net/out/size", "do_send() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size());
+        MDEBUG("do_send() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size());
         //do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function
       
-      LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
+      LOG_TRACE_CC(context, "[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size());
     }
     else
     { // no active operation
@@ -540,11 +536,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         }
 
         auto size_now = m_send_que.front().size();
-        _dbg1_c("net/out/size", "do_send() NOW SENSD: packet="<<size_now<<" B");
+        MDEBUG("do_send() NOW SENSD: packet="<<size_now<<" B");
         if (speed_limit_is_enabled())
 			do_send_handler_write( ptr , size_now ); // (((H)))
 
-        ASRT( size_now == m_send_que.front().size() );
+        CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), false, "Unexpected queue size");
         boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now ) ,
                                  //strand_.wrap(
                                  boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2)
@@ -602,7 +598,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   void connection<t_protocol_handler>::handle_write(const boost::system::error_code& e, size_t cb)
   {
     TRY_ENTRY();
-    LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async send calledback " << cb);
+    LOG_TRACE_CC(context, "[sock " << socket_.native_handle() << "] Async send calledback " << cb);
 
     if (e)
     {
@@ -635,10 +631,10 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     {
       //have more data to send
 		auto size_now = m_send_que.front().size();
-		_dbg1_c("net/out/size", "handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from  queue size="<<m_send_que.size());
+		MDEBUG("handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from  queue size="<<m_send_que.size());
 		if (speed_limit_is_enabled())
 			do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H)))
-		ASRT( size_now == m_send_que.front().size() );
+		CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), void(), "Unexpected queue size");
 		boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now) , 
         // strand_.wrap(
           boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2)
@@ -660,8 +656,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   void connection<t_protocol_handler>::setRpcStation()
   {
     m_connection_type = e_connection_type_RPC; 
-    _fact_c("net/sleepRPC", "set m_connection_type = RPC ");
-    _info_c("net/kind", "set m_connection_type = RPC ");
+    MDEBUG("set m_connection_type = RPC ");
   }
 
 
@@ -683,7 +678,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 	m_sock_count(0), m_sock_number(0), m_threads_count(0), 
 	m_pfilter(NULL), m_thread_index(0),
 		m_connection_type( connection_type ),
-    new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type))
+    new_connection_()
   {
     create_server_type_map();
     m_thread_name_prefix = "NET";
@@ -697,7 +692,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 		m_sock_count(0), m_sock_number(0), m_threads_count(0), 
 		m_pfilter(NULL), m_thread_index(0),
 		m_connection_type(connection_type),
-    new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, connection_type))
+    new_connection_()
   {
     create_server_type_map();
     m_thread_name_prefix = "NET";
@@ -735,7 +730,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     acceptor_.listen();
     boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint();
     m_port = binded_endpoint.port();
-    _fact_c("net/RPClog", "start accept");
+    MDEBUG("start accept");
+    new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type));
     acceptor_.async_accept(new_connection_->socket(),
       boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this,
       boost::asio::placeholders::error));
@@ -752,7 +748,7 @@ DISABLE_GCC_WARNING(maybe-uninitialized)
     uint32_t p = 0;
 
     if (port.size() && !string_tools::get_xtype_from_string(p, port)) {
-      LOG_ERROR("Failed to convert port no = " << port);
+      MERROR("Failed to convert port no = " << port);
       return false;
     }
     return this->init_server(p, address);
@@ -766,7 +762,7 @@ POP_WARNINGS
     uint32_t local_thr_index = boost::interprocess::ipcdetail::atomic_inc32(&m_thread_index); 
     std::string thread_name = std::string("[") + m_thread_name_prefix;
     thread_name += boost::to_string(local_thr_index) + "]";
-    log_space::log_singletone::set_thread_log_prefix(thread_name);
+    MLOG_SET_THREAD_NAME(thread_name);
     //   _fact("Thread name: " << m_thread_name_prefix);
     while(!m_stop_signal_sent)
     {
@@ -795,8 +791,7 @@ POP_WARNINGS
 		auto it = server_type_map.find(m_thread_name_prefix);
 		if (it==server_type_map.end()) throw std::runtime_error("Unknown prefix/server type:" + std::string(prefix_name));
     auto connection_type = it->second; // the value of type
-    _info_c("net/RPClog", "Set server type to: " << connection_type << " from name: " << m_thread_name_prefix);
-    _info_c("net/RPClog", "prefix_name = " << prefix_name);
+    MINFO("Set server type to: " << connection_type << " from name: " << m_thread_name_prefix << ", prefix_name = " << prefix_name);
   }
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
@@ -811,7 +806,7 @@ POP_WARNINGS
     TRY_ENTRY();
     m_threads_count = threads_count;
     m_main_thread_id = boost::this_thread::get_id();
-    log_space::log_singletone::set_thread_log_prefix("[SRV_MAIN]");
+    MLOG_SET_THREAD_NAME("[SRV_MAIN]");
     add_idle_handler(boost::bind(&boosted_tcp_server::cleanup_connections, this), 5000);
     while(!m_stop_signal_sent)
     {
@@ -932,13 +927,12 @@ POP_WARNINGS
   template<class t_protocol_handler>
   void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e)
   {
-	_fact_c("net/RPClog", "handle_accept");
+    MDEBUG("handle_accept");
     TRY_ENTRY();
     if (!e)
     {
 		if (m_connection_type == e_connection_type_RPC) {
-			_note_c("net/rpc", "New server for RPC connections");
-			_fact_c("net/RPClog", "New server for RPC connections");
+			MDEBUG("New server for RPC connections");
 			new_connection_->setRpcStation(); // hopefully this is not needed actually
 		}
 		connection_ptr conn(std::move(new_connection_));
@@ -964,7 +958,7 @@ POP_WARNINGS
     connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type) );
     connections_mutex.lock();
     connections_.push_back(std::make_pair(boost::get_system_time(), new_connection_l));
-    LOG_PRINT_L2("connections_ size now " << connections_.size());
+    MDEBUG("connections_ size now " << connections_.size());
     connections_mutex.unlock();
     boost::asio::ip::tcp::socket&  sock_ = new_connection_l->socket();
     
@@ -1051,7 +1045,7 @@ POP_WARNINGS
     }
     else
     {
-      _erro("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count);
+      _erro("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count);
     }
     
 	new_connection_l->save_dbg_log();
@@ -1068,7 +1062,7 @@ POP_WARNINGS
     connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type) );
     connections_mutex.lock();
     connections_.push_back(std::make_pair(boost::get_system_time(), new_connection_l));
-    LOG_PRINT_L2("connections_ size now " << connections_.size());
+    MDEBUG("connections_ size now " << connections_.size());
     connections_mutex.unlock();
     boost::asio::ip::tcp::socket&  sock_ = new_connection_l->socket();
     

contrib/epee/include/net/abstract_tcp_server_cp.h

@@ -42,6 +42,9 @@
 #include "net_utils_base.h"
 #include "pragma_comp_defs.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 #define LEVIN_DEFAULT_DATA_BUFF_SIZE       2000  
 
 namespace epee

contrib/epee/include/net/abstract_tcp_server_cp.inl

@@ -27,6 +27,9 @@
 
 #pragma comment(lib, "Ws2_32.lib")
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 namespace epee
 {
 namespace net_utils

contrib/epee/include/net/http_auth.h

@@ -0,0 +1,170 @@
+// Copyright (c) 2014-2017, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <boost/optional/optional.hpp>
+#include <boost/utility/string_ref.hpp>
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <utility>
+
+#include "http_base.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
+namespace epee
+{
+namespace net_utils
+{
+  namespace http
+  {
+    struct login
+    {
+      login() : username(), password() {}
+      login(std::string username_, std::string password_)
+        : username(std::move(username_)), password(std::move(password_))
+      {}
+
+      std::string username;
+      std::string password;
+    };
+
+    //! Implements RFC 2617 digest auth. Digests from RFC 7616 can be added.
+    class http_server_auth
+    {
+    public:
+      struct session
+      {
+        session(login credentials_)
+          : credentials(std::move(credentials_)), nonce(), counter(0)
+        {}
+
+        login credentials;
+        std::string nonce;
+        std::uint32_t counter;
+      };
+
+      http_server_auth() : user() {}
+      http_server_auth(login credentials);
+
+      //! \return Auth response, or `boost::none` iff `request` had valid auth.
+      boost::optional<http_response_info> get_response(const http_request_info& request)
+      {
+        if (user)
+          return do_get_response(request);
+        return boost::none;
+      }
+    private:
+      boost::optional<http_response_info> do_get_response(const http_request_info& request);
+
+      boost::optional<session> user;
+    };
+
+    //! Implements RFC 2617 digest auth. Digests from RFC 7616 can be added.
+    class http_client_auth
+    {
+    public:
+      enum status : std::uint8_t { kSuccess = 0, kBadPassword, kParseFailure };
+
+      struct session
+      {
+        session(login credentials_)
+          : credentials(std::move(credentials_)), server(), counter(0)
+        {}
+
+        struct keys
+        {
+          using algorithm =
+            std::function<std::string(const session&, boost::string_ref, boost::string_ref)>;
+
+          keys() : nonce(), opaque(), realm(), generator() {}
+          keys(std::string nonce_, std::string opaque_, std::string realm_, algorithm generator_)
+            : nonce(std::move(nonce_))
+            , opaque(std::move(opaque_))
+            , realm(std::move(realm_))
+            , generator(std::move(generator_))
+          {}
+
+          std::string nonce;
+          std::string opaque;
+          std::string realm;
+          algorithm generator;
+        };
+
+        login credentials;
+        keys server;
+        std::uint32_t counter;
+      };
+
+      http_client_auth() : user() {}
+      http_client_auth(login credentials);
+
+      /*!
+        Clients receiving a 401 response code from the server should call this
+        function to process the server auth. Then, before every client request,
+        `get_auth_field()` should be called to retrieve the newest
+        authorization request.
+
+        \return `kBadPassword` if client will never be able to authenticate,
+          `kParseFailure` if all server authentication responses were invalid,
+          and `kSuccess` if `get_auth_field` is ready to generate authorization
+          fields.
+      */
+      status handle_401(const http_response_info& response)
+      {
+        if (user)
+          return do_handle_401(response);
+        return kBadPassword;
+      }
+
+      /*!
+        After calling `handle_401`, clients should call this function to
+        generate an authentication field for every request.
+
+        \return A HTTP "Authorization" field if `handle_401(...)` previously
+          returned `kSuccess`.
+      */
+      boost::optional<std::pair<std::string, std::string>> get_auth_field(
+        const boost::string_ref method, const boost::string_ref uri)
+      {
+        if (user)
+          return do_get_auth_field(method, uri);
+        return boost::none;
+      }
+
+    private:
+      status do_handle_401(const http_response_info&);
+      boost::optional<std::pair<std::string, std::string>> do_get_auth_field(boost::string_ref, boost::string_ref);
+
+      boost::optional<session> user;
+    };
+  }
+}
+}

contrib/epee/include/net/http_base.h

@@ -29,8 +29,15 @@
 #pragma once
 #include <boost/lexical_cast.hpp>
 #include <boost/regex.hpp>
+#include <boost/utility/string_ref.hpp>
+#include <string>
+#include <utility>
 
 #include "string_tools.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 namespace epee
 {
 namespace net_utils
@@ -86,6 +93,15 @@ namespace net_utils
 			return std::string();
 		}
 
+		static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value)
+		{
+			out.append(name.data(), name.size()).append(": ");
+			out.append(value.data(), value.size()).append("\r\n");
+		}
+		static inline void add_field(std::string& out, const std::pair<std::string, std::string>& field)
+		{
+			add_field(out, field.first, field.second);
+		}
 
 
 		struct http_header_info

contrib/epee/include/net/http_client.h

@@ -30,6 +30,8 @@
 #include <boost/shared_ptr.hpp>
 #include <boost/regex.hpp>
 #include <boost/lexical_cast.hpp>
+#include <boost/optional/optional.hpp>
+#include <boost/utility/string_ref.hpp>
 //#include <mbstring.h>
 #include <algorithm>
 #include <cctype>
@@ -45,6 +47,7 @@
 #include "string_tools.h"
 #include "reg_exp_definer.h"
 #include "http_base.h" 
+#include "http_auth.h"
 #include "to_nonconst_iterator.h"
 #include "net_parse_helpers.h"
 
@@ -52,6 +55,9 @@
 
 //#pragma comment(lib, "shlwapi.lib")
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 extern epee::critical_section gregexp_lock;
 
 
@@ -101,14 +107,14 @@ using namespace std;
 	//---------------------------------------------------------------------------
 	static inline const char* get_hex_vals()
 	{
-		static char hexVals[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
+		static const char hexVals[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
 		return hexVals;
 	}
 
 	static inline const char* get_unsave_chars()
 	{
 		//static char unsave_chars[] = "\"<>%\\^[]`+$,@:;/!#?=&";
-		static char unsave_chars[] = "\"<>%\\^[]`+$,@:;!#&";
+		static const char unsave_chars[] = "\"<>%\\^[]`+$,@:;!#&";
 		return unsave_chars;
 	}
 
@@ -233,9 +239,6 @@ using namespace std;
 
 		class http_simple_client: public i_target_handler
 		{
-		public:
-			
-
 		private:
 			enum reciev_machine_state
 			{
@@ -260,7 +263,7 @@ using namespace std;
 			blocked_mode_client m_net_client;
 			std::string m_host_buff;
 			std::string m_port;
-			unsigned int m_timeout;
+			http_client_auth m_auth;
 			std::string m_header_cache;
 			http_response_info m_response_info;
 			size_t m_len_in_summary;
@@ -273,23 +276,45 @@ using namespace std;
 			critical_section m_lock;
 
 		public:
-			void set_host_name(const std::string& name)
+			explicit http_simple_client()
+				: i_target_handler()
+				, m_net_client()
+				, m_host_buff()
+				, m_port()
+				, m_auth()
+				, m_header_cache()
+				, m_response_info()
+				, m_len_in_summary(0)
+				, m_len_in_remain(0)
+				, m_pcontent_encoding_handler(nullptr)
+				, m_state()
+				, m_chunked_state()
+				, m_chunked_cache()
+				, m_lock()
+			{}
+
+			bool set_server(const std::string& address, boost::optional<login> user)
+			{
+				http::url_content parsed{};
+				const bool r = parse_url(address, parsed);
+				CHECK_AND_ASSERT_MES(r, false, "failed to parse url: " << address);
+				set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user));
+				return true;
+			}
+
+			void set_server(std::string host, std::string port, boost::optional<login> user)
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
-				m_host_buff = name;
+				disconnect();
+				m_host_buff = std::move(host);
+				m_port = std::move(port);
+                                m_auth = user ? http_client_auth{std::move(*user)} : http_client_auth{};
 			}
-      bool connect(const std::string& host, int port, unsigned int timeout)
-      {
-        return connect(host, std::to_string(port), timeout);
-      }
-      bool connect(const std::string& host, const std::string& port, unsigned int timeout)
+
+      bool connect(std::chrono::milliseconds timeout)
       {
         CRITICAL_REGION_LOCAL(m_lock);
-        m_host_buff = host;
-        m_port = port;
-        m_timeout = timeout;
-
-        return m_net_client.connect(host,  port, timeout, timeout);
+        return m_net_client.connect(m_host_buff, m_port, timeout);
       }
 			//---------------------------------------------------------------------------
 			bool disconnect()
@@ -313,58 +338,90 @@ using namespace std;
 			}
 			//---------------------------------------------------------------------------
 			inline 
-				bool invoke_get(const std::string& uri, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
+				bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
 			{
 					CRITICAL_REGION_LOCAL(m_lock);
-					return invoke(uri, "GET", body, ppresponse_info, additional_params);
+					return invoke(uri, "GET", body, timeout, ppresponse_info, additional_params);
 			}
 
 			//---------------------------------------------------------------------------
-			inline bool invoke(const std::string& uri, const std::string& method, const std::string& body, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
+			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				if(!is_connected())
 				{
-					LOG_PRINT("Reconnecting...", LOG_LEVEL_3);
-					if(!connect(m_host_buff, m_port, m_timeout))
+					MDEBUG("Reconnecting...");
+					if(!connect(timeout))
 					{
-						LOG_PRINT("Failed to connect to " << m_host_buff << ":" << m_port, LOG_LEVEL_3);
+						MDEBUG("Failed to connect to " << m_host_buff << ":" << m_port);
 						return false;
 					}
 				}
-				m_response_info.clear();
-				std::string req_buff = 	method + " ";
-				req_buff += uri + " HTTP/1.1\r\n" + 
-					"Host: "+ m_host_buff +"\r\n" +	"Content-Length: " + boost::lexical_cast<std::string>(body.size()) + "\r\n";
 
+				std::string req_buff{};
+				req_buff.reserve(2048);
+				req_buff.append(method.data(), method.size()).append(" ").append(uri.data(), uri.size()).append(" HTTP/1.1\r\n");
+				add_field(req_buff, "Host", m_host_buff);
+				add_field(req_buff, "Content-Length", std::to_string(body.size()));
 
 				//handle "additional_params"
-				for(fields_list::const_iterator it = additional_params.begin(); it!=additional_params.end(); it++)
-					req_buff += it->first + ": " + it->second + "\r\n";
-				req_buff += "\r\n";
-				//--
+				for(const auto& field : additional_params)
+					add_field(req_buff, field);
 
-				bool res = m_net_client.send(req_buff);
-				CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
-				if(body.size())
-					res = m_net_client.send(body);
-				CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+				for (unsigned sends = 0; sends < 2; ++sends)
+				{
+					const std::size_t initial_size = req_buff.size();
+					const auto auth = m_auth.get_auth_field(method, uri);
+					if (auth)
+						add_field(req_buff, *auth);
 
-				if(ppresponse_info)
-					*ppresponse_info = &m_response_info;
+					req_buff += "\r\n";
+					//--
 
-				m_state = reciev_machine_state_header;
-				return handle_reciev();
+					bool res = m_net_client.send(req_buff, timeout);
+					CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+					if(body.size())
+						res = m_net_client.send(body, timeout);
+					CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+
+
+					m_response_info.clear();
+					m_state = reciev_machine_state_header;
+					if (!handle_reciev(timeout))
+						return false;
+					if (m_response_info.m_response_code != 401)
+					{
+						if(ppresponse_info)
+							*ppresponse_info = std::addressof(m_response_info);
+						return true;
+					}
+
+					switch (m_auth.handle_401(m_response_info))
+					{
+					case http_client_auth::kSuccess:
+						break;
+					case http_client_auth::kBadPassword:
+                                                sends = 2;
+						break;
+					default:
+					case http_client_auth::kParseFailure:
+						LOG_ERROR("Bad server response for authentication");
+						return false;
+					}
+					req_buff.resize(initial_size); // rollback for new auth generation
+				}
+				LOG_ERROR("Client has incorrect username/password for server requiring authentication");
+				return false;
 			}
 			//---------------------------------------------------------------------------
-			inline bool invoke_post(const std::string& uri, const std::string& body,  const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
+			inline bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
-				return invoke(uri, "POST", body, ppresponse_info, additional_params);
+				return invoke(uri, "POST", body, timeout, ppresponse_info, additional_params);
 			}
 		private: 
 			//---------------------------------------------------------------------------
-			inline bool handle_reciev()
+			inline bool handle_reciev(std::chrono::milliseconds timeout)
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				bool keep_handling = true;
@@ -374,9 +431,9 @@ using namespace std;
 				{
 					if(need_more_data)
 					{
-						if(!m_net_client.recv(recv_buffer))
+						if(!m_net_client.recv(recv_buffer, timeout))
 						{
-							LOG_PRINT("Unexpected reciec fail", LOG_LEVEL_3);
+							MERROR("Unexpected recv fail");
 							m_state = reciev_machine_state_error;
             }
             if(!recv_buffer.size())
@@ -464,7 +521,7 @@ using namespace std;
 				CRITICAL_REGION_LOCAL(m_lock);
 				if(!recv_buff.size())
 				{
-					LOG_PRINT("Warning: Content-Len mode, but connection unexpectedly closed", LOG_LEVEL_3);
+					MERROR("Warning: Content-Len mode, but connection unexpectedly closed");
 					m_state = reciev_machine_state_done;
 					return true;
 				}
@@ -578,7 +635,7 @@ using namespace std;
         CRITICAL_REGION_LOCAL(m_lock);
 				if(!recv_buff.size())
 				{
-					LOG_PRINT("Warning: CHUNKED mode, but connection unexpectedly closed", LOG_LEVEL_3);
+					MERROR("Warning: CHUNKED mode, but connection unexpectedly closed");
 					m_state = reciev_machine_state_done;
 					return true;
 				}
@@ -665,7 +722,7 @@ using namespace std;
 			inline
 				bool parse_header(http_header_info& body_info, const std::string& m_cache_to_process)
 			{ 
-				LOG_FRAME("http_stream_filter::parse_cached_header(*)", LOG_LEVEL_4);
+				MTRACE("http_stream_filter::parse_cached_header(*)");
 				
 				STATIC_REGEXP_EXPR_1(rexp_mach_field, 
 					"\n?((Connection)|(Referer)|(Content-Length)|(Content-Type)|(Transfer-Encoding)|(Content-Encoding)|(Host)|(Cookie)|(User-Agent)"
@@ -708,7 +765,7 @@ using namespace std;
 					else if(result[i++].matched)//"User-Agent"
 						body_info.m_user_agent = result[field_val];
 					else if(result[i++].matched)//e.t.c (HAVE TO BE MATCHED!)
-					{;}
+						body_info.m_etc_fields.emplace_back(result[11], result[field_val]);
 					else
 					{CHECK_AND_ASSERT_MES(false, false, "http_stream_filter::parse_cached_header() not matched last entry in:"<<m_cache_to_process);}
 
@@ -833,7 +890,7 @@ using namespace std;
 				}else
 				{   //Apparently there are no signs of the form of transfer, will receive data until the connection is closed
 					m_state = reciev_machine_state_error;
-					LOG_PRINT("Undefinded transfer type, consider http_body_transfer_connection_close method. header: " << m_header_cache, LOG_LEVEL_2);
+					MERROR("Undefinded transfer type, consider http_body_transfer_connection_close method. header: " << m_header_cache);
 					return false;
 				} 
 				return false;
@@ -875,33 +932,6 @@ using namespace std;
 				return true;
 			}
 		};
-
-
-
-    /************************************************************************/
-    /*                                                                      */
-    /************************************************************************/
-  //inline 
-    template<class t_transport>
-    bool invoke_request(const std::string& url, t_transport& tr, unsigned int timeout, const http_response_info** ppresponse_info, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list())
-    {
-      http::url_content u_c;
-      bool res = parse_url(url, u_c);
-
-      if(!tr.is_connected() && !u_c.host.empty())
-      {
-        CHECK_AND_ASSERT_MES(res, false, "failed to parse url: " << url);
-
-        if(!u_c.port)
-          u_c.port = 80;//default for http
-
-        res = tr.connect(u_c.host, static_cast<int>(u_c.port), timeout);
-        CHECK_AND_ASSERT_MES(res, false, "failed to connect " << u_c.host << ":" << u_c.port);
-      }
-
-      return tr.invoke(u_c.uri, method, body, ppresponse_info, additional_params);
-    }
-
 	}
 }
 }

contrib/epee/include/net/http_client_abstract_invoke.h

@@ -1,98 +0,0 @@
-
-// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-// * Neither the name of the Andrey N. Sabelnikov nor the
-// names of its contributors may be used to endorse or promote products
-// derived from this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
-// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-
-#pragma once
-#include "storages/serializeble_struct_helper.h"
-
-namespace epee
-{
-  namespace net_utils
-  {
-    namespace http
-    {
-      template<class TArg, class TResult, class TTransport>
-      bool invoke_http_json_remote_command(const std::string& url, TArg& out_struct, TResult& result_struct, TTransport& transport, unsigned int timeout = 5000, const std::string& method = "GET")
-      {
-        std::string req_param;
-        StorageNamed::InMemStorageSpace::json::store_t_to_json(out_struct, req_param);
-
-        const http_response_info* pri = NULL;
-        if(!invoke_request(url, transport, timeout, &pri, method, req_param))
-        {
-          LOG_PRINT_L1("Failed to invoke http request to  " << url);
-          return false;
-        }
-
-        if(!pri->m_response_code)
-        {
-          LOG_PRINT_L1("Failed to invoke http request to  " << url << ", internal error (null response ptr)");
-          return false;
-        }
-
-        if(pri->m_response_code != 200)
-        {
-          LOG_PRINT_L1("Failed to invoke http request to  " << url << ", wrong response code: " << pri->m_response_code);
-          return false;
-        }
-
-        return StorageNamed::InMemStorageSpace::json::load_t_from_json(result_struct, pri->m_body);
-      }
-
-
-
-      template<class TArg, class TResult, class TTransport>
-      bool invoke_http_bin_remote_command(const std::string& url, TArg& out_struct, TResult& result_struct, TTransport& transport, unsigned int timeout = 5000, const std::string& method = "GET")
-      {
-        std::string req_param;
-        epee::StorageNamed::save_struct_as_storage_to_buff(out_struct, req_param); 
-
-        const http_response_info* pri = NULL;
-        if(!invoke_request(url, transport, timeout, &pri, method, req_param))
-        {
-          LOG_PRINT_L1("Failed to invoke http request to  " << url);
-          return false;
-        }
-
-        if(!pri->m_response_code)
-        {
-          LOG_PRINT_L1("Failed to invoke http request to  " << url << ", internal error (null response ptr)");
-          return false;
-        }
-
-        if(pri->m_response_code != 200)
-        {
-          LOG_PRINT_L1("Failed to invoke http request to  " << url << ", wrong response code: " << pri->m_response_code);
-          return false;
-        }
-
-        return epee::StorageNamed::load_struct_from_storage_buff(result_struct, pri->m_body); 
-      }
-
-
-    }
-  }
-}

contrib/epee/include/net/http_client_base.h

@@ -26,6 +26,9 @@
 
 #pragma once 
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 namespace epee
 {
   namespace net_utils

contrib/epee/include/net/http_client_via_api_helper.h

@@ -32,6 +32,9 @@
 #include <atlutil.h>
 #pragma comment(lib, "Wininet.lib")
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 namespace epee
 {
 namespace net_utils

contrib/epee/include/net/http_protocol_handler.h

@@ -30,11 +30,16 @@
 #ifndef _HTTP_SERVER_H_
 #define _HTTP_SERVER_H_
 
+#include <boost/optional/optional.hpp>
 #include <string>
 #include "net_utils_base.h"
 #include "to_nonconst_iterator.h"
+#include "http_auth.h"
 #include "http_base.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 namespace epee
 {
 namespace net_utils
@@ -49,7 +54,7 @@ namespace net_utils
 		struct http_server_config
 		{
 			std::string m_folder;
-			std::string m_required_user_agent;
+			boost::optional<login> m_user;
 			critical_section m_lock;
 		};
 
@@ -169,11 +174,20 @@ namespace net_utils
 			http_custom_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context)
 				: simple_http_connection_handler<t_connection_context>(psnd_hndlr, config),
 					m_config(config),
-					m_conn_context(conn_context)
+					m_conn_context(conn_context),
+					m_auth(m_config.m_user ? http_server_auth{*m_config.m_user} : http_server_auth{})
 			{}
 			inline bool handle_request(const http_request_info& query_info, http_response_info& response)
 			{
 				CHECK_AND_ASSERT_MES(m_config.m_phandler, false, "m_config.m_phandler is NULL!!!!");
+
+				const auto auth_response = m_auth.get_response(query_info);
+				if (auth_response)
+				{
+					response = std::move(*auth_response);
+					return true;
+				}
+
 				//fill with default values
 				response.m_mime_tipe = "text/plain";
 				response.m_response_code = 200;
@@ -202,6 +216,7 @@ namespace net_utils
 			//simple_http_connection_handler::config_type m_stub_config;
 			config_type& m_config;
 			t_connection_context& m_conn_context;
+			http_server_auth m_auth;
 		};
 	}
 }

contrib/epee/include/net/http_protocol_handler.inl

@@ -33,6 +33,9 @@
 #include "file_io_utils.h"
 #include "net_parse_helpers.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 #define HTTP_MAX_URI_LEN		 9000 
 #define HTTP_MAX_HEADER_LEN		 100000
 
@@ -133,7 +136,7 @@ namespace net_utils
 			std::string boundary;
 			if(!match_boundary(content_type, boundary))
 			{
-				LOG_PRINT("Failed to match boundary in content type: " << content_type, LOG_LEVEL_0);
+				MERROR("Failed to match boundary in content type: " << content_type);
 				return false;
 			}
 			
@@ -155,7 +158,7 @@ namespace net_utils
 					pos = body.find(boundary, std::distance(body.begin(), it_begin));
 					if(std::string::npos == pos)
 					{
-						LOG_PRINT("Error: Filed to match closing multipart tag", LOG_LEVEL_0);
+						MERROR("Error: Filed to match closing multipart tag");
 						it_end = body.end();
 					}else
 					{
@@ -177,7 +180,7 @@ namespace net_utils
 				out_values.push_back(multipart_entry());
 				if(!handle_part_of_multipart(it_begin, it_end, out_values.back()))
 				{
-					LOG_PRINT("Failed to handle_part_of_multipart", LOG_LEVEL_0);
+					MERROR("Failed to handle_part_of_multipart");
 					return false;
 				}
 
@@ -331,8 +334,6 @@ namespace net_utils
   template<class t_connection_context>
 	bool simple_http_connection_handler<t_connection_context>::handle_invoke_query_line()
 	{ 
-		LOG_FRAME("simple_http_connection_handler<t_connection_context>::handle_recognize_protocol_out(*)", LOG_LEVEL_3);
-
 		STATIC_REGEXP_EXPR_1(rexp_match_command_line, "^(((OPTIONS)|(GET)|(HEAD)|(POST)|(PUT)|(DELETE)|(TRACE)) (\\S+) HTTP/(\\d+).(\\d+))\r?\n", boost::regex::icase | boost::regex::normal);
 		//											    123         4     5      6      7     8        9        10          11     12    
 		//size_t match_len = 0;
@@ -379,8 +380,6 @@ namespace net_utils
 	{ 
 		//LOG_PRINT_L4("HTTP HEAD:\r\n" << m_cache.substr(0, pos));
 
-		LOG_FRAME("simple_http_connection_handler<t_connection_context>::analize_cached_request_header_and_invoke_state(*)", LOG_LEVEL_3);
-
 		m_query_info.m_full_request_buf_size = pos;
     m_query_info.m_request_head.assign(m_cache.begin(), m_cache.begin()+pos); 
 
@@ -391,13 +390,6 @@ namespace net_utils
 			return false;
 		}
 
-                if (!m_config.m_required_user_agent.empty() && m_query_info.m_header_info.m_user_agent != m_config.m_required_user_agent)
-                {
-			LOG_ERROR("simple_http_connection_handler<t_connection_context>::analize_cached_request_header_and_invoke_state(): unexpected user agent: " << m_query_info.m_header_info.m_user_agent);
-			m_state = http_state_error;
-			return false;
-                }
-
 		m_cache.erase(0, pos);
 
 		std::string req_command_str = m_query_info.m_full_request_str;
@@ -479,8 +471,6 @@ namespace net_utils
   template<class t_connection_context>
 	bool simple_http_connection_handler<t_connection_context>::parse_cached_header(http_header_info& body_info, const std::string& m_cache_to_process, size_t pos)
 	{ 
-		LOG_FRAME("http_stream_filter::parse_cached_header(*)", LOG_LEVEL_3);
-
 		STATIC_REGEXP_EXPR_1(rexp_mach_field, 
 			"\n?((Connection)|(Referer)|(Content-Length)|(Content-Type)|(Transfer-Encoding)|(Content-Encoding)|(Host)|(Cookie)|(User-Agent)"
 			//  12            3         4                5              6                   7                  8      9        10
@@ -576,7 +566,7 @@ namespace net_utils
 		m_config.m_lock.unlock();
 		if(!file_io_utils::load_file_to_string(destination_file_path.c_str(), response.m_body))
 		{
-			LOG_PRINT("URI \""<< query_info.m_full_request_str.substr(0, query_info.m_full_request_str.size()-2) << "\" [" << destination_file_path << "] Not Found (404 )" , LOG_LEVEL_1);
+			MWARNING("URI \""<< query_info.m_full_request_str.substr(0, query_info.m_full_request_str.size()-2) << "\" [" << destination_file_path << "] Not Found (404 )");
 			response.m_body = get_not_found_response_body(query_info.m_URI);
 			response.m_response_code = 404;
 			response.m_response_comment = "Not found";
@@ -584,7 +574,7 @@ namespace net_utils
 			return true;
 		}
 
-		LOG_PRINT(" -->> " << query_info.m_full_request_str << "\r\n<<--OK" , LOG_LEVEL_3);
+		MDEBUG(" -->> " << query_info.m_full_request_str << "\r\n<<--OK");
 		response.m_response_code = 200;
 		response.m_response_comment = "OK";
 		response.m_mime_tipe = get_file_mime_tipe(uri_to_path);

contrib/epee/include/net/http_server_cp.h

@@ -32,6 +32,10 @@
 
 #include "abstract_tcp_server_cp.h"
 #include "http_server.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 namespace epee
 {
 namespace net_utils

contrib/epee/include/net/http_server_cp2.h

@@ -32,6 +32,10 @@
 
 #include "abstract_tcp_server2.h"
 #include "http_protocol_handler.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 namespace epee
 {
 namespace net_utils

contrib/epee/include/net/http_server_handlers_map2.h

@@ -31,6 +31,9 @@
 #include "storages/portable_storage.h"
 #include "storages/portable_storage_template_helper.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 
 #define CHAIN_HTTP_TO_MAP2(context_type) bool handle_http_request(const epee::net_utils::http::http_request_info& query_info, \
               epee::net_utils::http::http_response_info& response, \
@@ -77,7 +80,7 @@
       uint64_t ticks3 = epee::misc_utils::get_tick_count(); \
       response_info.m_mime_tipe = "application/json"; \
       response_info.m_header_info.m_content_type = " application/json"; \
-      LOG_PRINT( s_pattern << " processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms", LOG_LEVEL_2); \
+      MDEBUG( s_pattern << " processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms"); \
     }
 
 #define MAP_URI_AUTO_JON2(s_pattern, callback_f, command_type) MAP_URI_AUTO_JON2_IF(s_pattern, callback_f, command_type, true)
@@ -104,7 +107,7 @@
       uint64_t ticks3 = epee::misc_utils::get_tick_count(); \
       response_info.m_mime_tipe = " application/octet-stream"; \
       response_info.m_header_info.m_content_type = " application/octet-stream"; \
-      LOG_PRINT( s_pattern << "() processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms", LOG_LEVEL_2); \
+      MDEBUG( s_pattern << "() processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms"); \
     }
 
 #define CHAIN_URI_MAP2(callback) else {callback(query_info, response_info, m_conn_context);handled = true;}
@@ -166,7 +169,7 @@
   uint64_t ticks3 = epee::misc_utils::get_tick_count(); \
   response_info.m_mime_tipe = "application/json"; \
   response_info.m_header_info.m_content_type = " application/json"; \
-  LOG_PRINT( query_info.m_URI << "[" << method_name << "] processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms", LOG_LEVEL_2);
+  MDEBUG( query_info.m_URI << "[" << method_name << "] processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms");
 
 #define MAP_JON_RPC_WE_IF(method_name, callback_f, command_type, cond) \
     else if((callback_name == method_name) && (cond)) \

contrib/epee/include/net/http_server_impl_base.h

@@ -36,6 +36,9 @@
 #include "net/http_server_cp2.h"
 #include "net/http_server_handlers_map2.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
 namespace epee
 {
 
@@ -52,7 +55,8 @@ namespace epee
         : m_net_server(external_io_service)
     {}
 
-    bool init(const std::string& bind_port = "0", const std::string& bind_ip = "0.0.0.0", const std::string &user_agent = "")
+    bool init(const std::string& bind_port = "0", const std::string& bind_ip = "0.0.0.0",
+      boost::optional<net_utils::http::login> user = boost::none)
     {
 
       //set self as callback handler
@@ -61,10 +65,9 @@ namespace epee
       //here set folder for hosting reqests
       m_net_server.get_config_object().m_folder = "";
 
-      // workaround till we get auth/encryption
-      m_net_server.get_config_object().m_required_user_agent = user_agent;
+      m_net_server.get_config_object().m_user = std::move(user);
 
-      LOG_PRINT_L0("Binding on " << bind_ip << ":" << bind_port);
+      MGINFO("Binding on " << bind_ip << ":" << bind_port);
       bool res = m_net_server.init_server(bind_port, bind_ip);
       if(!res)
       {
@@ -77,15 +80,14 @@ namespace epee
     bool run(size_t threads_count, bool wait = true)
     {
       //go to loop
-      LOG_PRINT("Run net_service loop( " << threads_count << " threads)...", LOG_LEVEL_0);
-      _fact_c("net/RPClog", "Run net_service loop( " << threads_count << " threads)...");
+      MINFO("Run net_service loop( " << threads_count << " threads)...");
       if(!m_net_server.run_server(threads_count, wait))
       {
         LOG_ERROR("Failed to run net tcp server!");
       }
 
       if(wait)
-        LOG_PRINT("net_service loop stopped.", LOG_LEVEL_0);
+        MINFO("net_service loop stopped.");
       return true;
     }
 

contrib/epee/include/net/levin_client.inl

@@ -30,6 +30,10 @@
 //------------------------------------------------------------------------------
 //------------------------------------------------------------------------------
 #include "string_tools.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 namespace epee
 {
 namespace levin

contrib/epee/include/net/levin_client_async.h

@@ -31,6 +31,9 @@
 #include "net_helper.h"
 #include "levin_base.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 
 namespace epee
 {

contrib/epee/include/net/levin_helper.h

@@ -31,6 +31,9 @@
 #include "levin_base.h"
 #include "serializeble_struct_helper.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 namespace epee
 {
 namespace levin

contrib/epee/include/net/levin_protocol_handler.h

@@ -32,6 +32,9 @@
 #include <boost/uuid/uuid_generators.hpp>
 #include "levin_base.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 namespace epee
 {
 namespace levin
@@ -85,7 +88,7 @@ namespace levin
 	{
 		if(!m_config.m_pcommands_handler)
 		{
-			LOG_ERROR("Command handler not set!");
+			LOG_ERROR_CC(m_conn_context, "Command handler not set!");
 			return false;
 		}
 		m_cach_in_buffer.append((const char*)ptr, cb);
@@ -100,7 +103,7 @@ namespace levin
 				{
 					if(m_cach_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cach_in_buffer.data()) != LEVIN_SIGNATURE)
 					{
-						LOG_ERROR("Signature missmatch on accepted connection");
+						LOG_ERROR_CC(m_conn_context, "Signature missmatch on accepted connection");
 						return false;
 					}
 					is_continue = false;
@@ -110,7 +113,7 @@ namespace levin
 					bucket_head* phead = (bucket_head*)m_cach_in_buffer.data();
 					if(LEVIN_SIGNATURE != phead->m_signature)
 					{
-						LOG_ERROR("Signature missmatch on accepted connection");
+						LOG_ERROR_CC(m_conn_context, "Signature missmatch on accepted connection");
 						return false;
 					}
 					m_current_head = *phead;
@@ -154,7 +157,7 @@ namespace levin
 				m_state = conn_state_reading_head;
 				break;
 			default:
-				LOG_ERROR("Undefined state in levin_server_impl::connection_handler, m_state=" << m_state);
+				LOG_ERROR_CC(m_conn_context, "Undefined state in levin_server_impl::connection_handler, m_state=" << m_state);
 				return false;
 			}
 		}

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -38,6 +38,8 @@
 #include <random>
 #include <chrono>
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
 
 namespace epee
 {
@@ -151,7 +153,7 @@ public:
         {
           if(ec == boost::asio::error::operation_aborted)
             return;
-          LOG_PRINT_CC(con.get_context_ref(), "Timeout on invoke operation happened, command: " << command, LOG_LEVEL_2);
+          MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command);
           std::string fake;
           cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref());
           con.close();
@@ -244,15 +246,15 @@ public:
     }
     CHECK_AND_ASSERT_MES_NO_RET(0 == boost::interprocess::ipcdetail::atomic_read32(&m_wait_count), "Failed to wait for operation completion. m_wait_count = " << m_wait_count);
 
-    LOG_PRINT_CC(m_connection_context, "~async_protocol_handler()", LOG_LEVEL_4);
+    MTRACE(m_connection_context << "~async_protocol_handler()");
   }
 
   bool start_outer_call()
   {
-    LOG_PRINT_CC_L4(m_connection_context, "[levin_protocol] -->> start_outer_call");
+    MTRACE(m_connection_context << "[levin_protocol] -->> start_outer_call");
     if(!m_pservice_endpoint->add_ref())
     {
-      LOG_PRINT_CC_RED(m_connection_context, "[levin_protocol] -->> start_outer_call failed", LOG_LEVEL_4);
+      MERROR(m_connection_context << "[levin_protocol] -->> start_outer_call failed");
       return false;
     }
     boost::interprocess::ipcdetail::atomic_inc32(&m_wait_count);
@@ -260,7 +262,7 @@ public:
   }
   bool finish_outer_call()
   {
-    LOG_PRINT_CC_L4(m_connection_context, "[levin_protocol] <<-- finish_outer_call");
+    MTRACE(m_connection_context << "[levin_protocol] <<-- finish_outer_call");
     boost::interprocess::ipcdetail::atomic_dec32(&m_wait_count);
     m_pservice_endpoint->release();
     return true;
@@ -316,13 +318,13 @@ public:
 
     if(!m_config.m_pcommands_handler)
     {
-      LOG_ERROR_CC(m_connection_context, "Commands handler not set!");
+      MERROR(m_connection_context << "Commands handler not set!");
       return false;
     }
 
     if(m_cache_in_buffer.size() +  cb > m_config.m_max_packet_size)
     {
-      LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size 
+      MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size
                           << ", packet received " << m_cache_in_buffer.size() +  cb 
                           << ", connection will be closed.");
       return false;
@@ -353,7 +355,7 @@ public:
 
           bool is_response = (m_oponent_protocol_ver == LEVIN_PROTOCOL_VER_1 && m_current_head.m_flags&LEVIN_PACKET_RESPONSE);
 
-          LOG_PRINT_CC_L4(m_connection_context, "LEVIN_PACKET_RECIEVED. [len=" << m_current_head.m_cb 
+          MDEBUG(m_connection_context << "LEVIN_PACKET_RECIEVED. [len=" << m_current_head.m_cb
             << ", flags" << m_current_head.m_flags 
             << ", r?=" << m_current_head.m_have_to_return_data 
             <<", cmd = " << m_current_head.m_command 
@@ -381,7 +383,7 @@ public:
               //use sync call scenario
               if(!boost::interprocess::ipcdetail::atomic_read32(&m_wait_count) && !boost::interprocess::ipcdetail::atomic_read32(&m_close_called))
               {
-                LOG_ERROR_CC(m_connection_context, "no active invoke when response came, wtf?");
+                MERROR(m_connection_context << "no active invoke when response came, wtf?");
                 return false;
               }else
               {
@@ -413,7 +415,7 @@ public:
               if(!m_pservice_endpoint->do_send(send_buff.data(), send_buff.size()))
                 return false;
               CRITICAL_REGION_END();
-              LOG_PRINT_CC_L4(m_connection_context, "LEVIN_PACKET_SENT. [len=" << m_current_head.m_cb 
+              MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << m_current_head.m_cb
                 << ", flags" << m_current_head.m_flags 
                 << ", r?=" << m_current_head.m_have_to_return_data 
                 <<", cmd = " << m_current_head.m_command 
@@ -431,7 +433,7 @@ public:
           {
             if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.data()) != LEVIN_SIGNATURE)
             {
-              LOG_ERROR_CC(m_connection_context, "Signature mismatch, connection will be closed");
+              MWARNING(m_connection_context << "Signature mismatch, connection will be closed");
               return false;
             }
             is_continue = false;
@@ -585,7 +587,7 @@ public:
     }
     CRITICAL_REGION_END();
 
-    LOG_PRINT_CC_L4(m_connection_context, "LEVIN_PACKET_SENT. [len=" << head.m_cb 
+    MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
                             << ", f=" << head.m_flags 
                             << ", r?=" << head.m_have_to_return_data 
                             << ", cmd = " << head.m_command 
@@ -597,7 +599,7 @@ public:
     {
       if(misc_utils::get_tick_count() - ticks_start > m_config.m_invoke_timeout)
       {
-        LOG_PRINT_CC_L2(m_connection_context, "invoke timeout (" << m_config.m_invoke_timeout << "), closing connection ");
+        MWARNING(m_connection_context << "invoke timeout (" << m_config.m_invoke_timeout << "), closing connection ");
         close();
         return LEVIN_ERROR_CONNECTION_TIMEDOUT;
       }
@@ -650,7 +652,7 @@ public:
       return -1;
     }
     CRITICAL_REGION_END();
-    LOG_PRINT_CC_L4(m_connection_context, "LEVIN_PACKET_SENT. [len=" << head.m_cb << 
+    LOG_DEBUG_CC(m_connection_context, "LEVIN_PACKET_SENT. [len=" << head.m_cb <<
       ", f=" << head.m_flags << 
       ", r?=" << head.m_have_to_return_data <<
       ", cmd = " << head.m_command << 

contrib/epee/include/net/net_helper.h

@@ -37,6 +37,7 @@
 #include <ostream>
 #include <string>
 #include <boost/asio.hpp>
+#include <boost/asio/steady_timer.hpp>
 #include <boost/preprocessor/selection/min.hpp>
 #include <boost/lambda/bind.hpp>
 #include <boost/lambda/lambda.hpp>
@@ -46,6 +47,9 @@
 //#include "profile_tools.h"
 #include "../string_tools.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 #ifndef MAKE_IP
 #define MAKE_IP( a1, a2, a3, a4 )	(a1|(a2<<8)|(a3<<16)|(a4<<24))
 #endif
@@ -95,7 +99,7 @@ namespace net_utils
 			// No deadline is required until the first socket operation is started. We
 			// set the deadline to positive infinity so that the actor takes no action
 			// until a specific deadline is set.
-			m_deadline.expires_at(boost::posix_time::pos_infin);
+			m_deadline.expires_at(std::chrono::steady_clock::time_point::max());
 
 			// Start the persistent actor that checks for deadline expiry.
 			check_deadline();
@@ -108,26 +112,16 @@ namespace net_utils
 			shutdown();
 		}
 
-		inline void set_recv_timeout(int reciev_timeout)
-		{
-			m_reciev_timeout = reciev_timeout;
-		}
-
     inline
-      bool connect(const std::string& addr, int port, unsigned int connect_timeout, unsigned int reciev_timeout, const std::string& bind_ip = "0.0.0.0")
+      bool connect(const std::string& addr, int port, std::chrono::milliseconds timeout, const std::string& bind_ip = "0.0.0.0")
     {
-      return connect(addr, std::to_string(port), connect_timeout, reciev_timeout, bind_ip);
+      return connect(addr, std::to_string(port), timeout, bind_ip);
     }
 
     inline
-			bool connect(const std::string& addr, const std::string& port, unsigned int connect_timeout, unsigned int reciev_timeout, const std::string& bind_ip = "0.0.0.0")
+			bool connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout, const std::string& bind_ip = "0.0.0.0")
 		{
-			m_connect_timeout = connect_timeout;
-			m_reciev_timeout = reciev_timeout;
       m_connected = false;
-			if(!m_reciev_timeout)
-				m_reciev_timeout = m_connect_timeout;
-
 			try
 			{
 				m_socket.close();
@@ -161,7 +155,7 @@ namespace net_utils
 				}
 
 				
-				m_deadline.expires_from_now(boost::posix_time::milliseconds(m_connect_timeout));
+				m_deadline.expires_from_now(timeout);
 
 
 				boost::system::error_code ec = boost::asio::error::would_block;
@@ -176,23 +170,23 @@ namespace net_utils
 				if (!ec && m_socket.is_open())
 				{
 					m_connected = true;
-					m_deadline.expires_at(boost::posix_time::pos_infin);
+					m_deadline.expires_at(std::chrono::steady_clock::time_point::max());
 					return true;
 				}else
 				{
-					LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3);
+					MWARNING("Some problems at connect, message: " << ec.message());
 					return false;
 				}
 
 			}
 			catch(const boost::system::system_error& er)
 			{
-				LOG_PRINT("Some problems at connect, message: " << er.what(), LOG_LEVEL_4);
+				MDEBUG("Some problems at connect, message: " << er.what());
 				return false;
 			}
 			catch(...)
 			{
-				LOG_PRINT("Some fatal problems.", LOG_LEVEL_4);
+				MDEBUG("Some fatal problems.");
 				return false;
 			}
 
@@ -228,12 +222,12 @@ namespace net_utils
 
 
 		inline 
-		bool send(const std::string& buff)
+		bool send(const std::string& buff, std::chrono::milliseconds timeout)
 		{
 
 			try
 			{
-				m_deadline.expires_from_now(boost::posix_time::milliseconds(m_reciev_timeout));
+				m_deadline.expires_from_now(timeout);
 
 				// Set up the variable that receives the result of the asynchronous
 				// operation. The error code is set to would_block to signal that the
@@ -261,7 +255,7 @@ namespace net_utils
 					return false;
 				}else
 				{
-					m_deadline.expires_at(boost::posix_time::pos_infin);
+					m_deadline.expires_at(std::chrono::steady_clock::time_point::max());
 				}
 			}
 
@@ -319,7 +313,7 @@ namespace net_utils
 					return false;
 				}else
 				{
-					m_deadline.expires_at(boost::posix_time::pos_infin);
+					m_deadline.expires_at(std::chrono::steady_clock::time_point::max());
 				}
 			}
 
@@ -347,7 +341,7 @@ namespace net_utils
 		}
 
 		inline 
-		bool recv(std::string& buff)
+		bool recv(std::string& buff, std::chrono::milliseconds timeout)
 		{
 
 			try
@@ -355,7 +349,7 @@ namespace net_utils
 				// Set a deadline for the asynchronous operation. Since this function uses
 				// a composed operation (async_read_until), the deadline applies to the
 				// entire operation, rather than individual reads from the socket.
-				m_deadline.expires_from_now(boost::posix_time::milliseconds(m_reciev_timeout));
+				m_deadline.expires_from_now(timeout);
 
 				// Set up the variable that receives the result of the asynchronous
 				// operation. The error code is set to would_block to signal that the
@@ -387,21 +381,21 @@ namespace net_utils
 
 				if (ec)
 				{
-                    LOG_PRINT_L4("READ ENDS: Connection err_code " << ec.value());
+                    MTRACE("READ ENDS: Connection err_code " << ec.value());
                     if(ec == boost::asio::error::eof)
                     {
-                      LOG_PRINT_L4("Connection err_code eof.");
+                      MTRACE("Connection err_code eof.");
                       //connection closed there, empty
                       return true;
                     }
 
-					LOG_PRINT_L3("Problems at read: " << ec.message());
+					MDEBUG("Problems at read: " << ec.message());
                     m_connected = false;
 					return false;
 				}else
 				{
-                    LOG_PRINT_L4("READ ENDS: Success. bytes_tr: " << bytes_transfered);
-					m_deadline.expires_at(boost::posix_time::pos_infin);
+                    MTRACE("READ ENDS: Success. bytes_tr: " << bytes_transfered);
+					m_deadline.expires_at(std::chrono::steady_clock::time_point::max());
 				}
 
 				/*if(!bytes_transfered)
@@ -429,7 +423,7 @@ namespace net_utils
 
 		}
 
-		inline bool recv_n(std::string& buff, int64_t sz)
+		inline bool recv_n(std::string& buff, int64_t sz, std::chrono::milliseconds timeout)
 		{
 
 			try
@@ -437,7 +431,7 @@ namespace net_utils
 				// Set a deadline for the asynchronous operation. Since this function uses
 				// a composed operation (async_read_until), the deadline applies to the
 				// entire operation, rather than individual reads from the socket.
-				m_deadline.expires_from_now(boost::posix_time::milliseconds(m_reciev_timeout));
+				m_deadline.expires_from_now(timeout);
 
 				// Set up the variable that receives the result of the asynchronous
 				// operation. The error code is set to would_block to signal that the
@@ -474,7 +468,7 @@ namespace net_utils
 					return false;
 				}else
 				{
-					m_deadline.expires_at(boost::posix_time::pos_infin);
+					m_deadline.expires_at(std::chrono::steady_clock::time_point::max());
 				}
 
 				if(bytes_transfered != buff.size())
@@ -536,7 +530,7 @@ namespace net_utils
 			// Check whether the deadline has passed. We compare the deadline against
 			// the current time since a new asynchronous operation may have moved the
 			// deadline before this actor had a chance to run.
-			if (m_deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now())
+			if (m_deadline.expires_at() <= std::chrono::steady_clock::now())
 			{
 				// The deadline has passed. The socket is closed so that any outstanding
 				// asynchronous operations are cancelled. This allows the blocked
@@ -547,7 +541,7 @@ namespace net_utils
 
 				// There is no longer an active deadline. The expiry is set to positive
 				// infinity so that the actor takes no action until a new deadline is set.
-				m_deadline.expires_at(boost::posix_time::pos_infin);
+				m_deadline.expires_at(std::chrono::steady_clock::time_point::max());
 			}
 
 			// Put the actor back to sleep.
@@ -559,11 +553,9 @@ namespace net_utils
 	protected:
 		boost::asio::io_service m_io_service;
 		boost::asio::ip::tcp::socket m_socket;
-		int m_connect_timeout;
-		int m_reciev_timeout;
 		bool m_initialized;
 		bool m_connected;
-		boost::asio::deadline_timer m_deadline;
+		boost::asio::steady_timer m_deadline;
 		volatile uint32_t m_shutdowned;
 	};
 

contrib/epee/include/net/net_parse_helpers.h

@@ -31,6 +31,8 @@
 #include "http_base.h"
 #include "reg_exp_definer.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
 
 namespace epee
 {

contrib/epee/include/net/net_utils_base.h

@@ -31,6 +31,10 @@
 
 #include <boost/uuid/uuid.hpp>
 #include "string_tools.h"
+#include "misc_log_ex.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
 
 #ifndef MAKE_IP
 #define MAKE_IP( a1, a2, a3, a4 )	(a1|(a2<<8)|(a3<<16)|(a4<<24))
@@ -142,20 +146,24 @@ namespace net_utils
     return ss.str();
   }
 
-#define LOG_PRINT_CC(ct, message, log_level) LOG_PRINT("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message, log_level)
-#define LOG_PRINT_CC_GREEN(ct, message, log_level) LOG_PRINT_GREEN("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message, log_level)
-#define LOG_PRINT_CC_RED(ct, message, log_level) LOG_PRINT_RED("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message, log_level)
-#define LOG_PRINT_CC_BLUE(ct, message, log_level) LOG_PRINT_BLUE("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message, log_level)
-#define LOG_PRINT_CC_YELLOW(ct, message, log_level) LOG_PRINT_YELLOW("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message, log_level)
-#define LOG_PRINT_CC_CYAN(ct, message, log_level) LOG_PRINT_CYAN("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message, log_level)
-#define LOG_PRINT_CC_MAGENTA(ct, message, log_level) LOG_PRINT_MAGENTA("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message, log_level)
-#define LOG_ERROR_CC(ct, message) LOG_PRINT_RED("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message, LOG_LEVEL_2)
+inline MAKE_LOGGABLE(connection_context_base, ct, os)
+{
+  os << "[" << epee::net_utils::print_connection_context_short(ct) << "] ";
+  return os;
+}
 
-#define LOG_PRINT_CC_L0(ct, message) LOG_PRINT_L0("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message)
-#define LOG_PRINT_CC_L1(ct, message) LOG_PRINT_L1("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message)
-#define LOG_PRINT_CC_L2(ct, message) LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message)
-#define LOG_PRINT_CC_L3(ct, message) LOG_PRINT_L3("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message)
-#define LOG_PRINT_CC_L4(ct, message) LOG_PRINT_L4("[" << epee::net_utils::print_connection_context_short(ct) << "]" << message)
+#define LOG_ERROR_CC(ct, message) MERROR(ct << message)
+#define LOG_WARNING_CC(ct, message) MWARNING(ct << message)
+#define LOG_INFO_CC(ct, message) MINFO(ct << message)
+#define LOG_DEBUG_CC(ct, message) MDEBUG(ct << message)
+#define LOG_TRACE_CC(ct, message) MTRACE(ct << message)
+#define LOG_CC(level, ct, message) MLOG(level, ct << message)
+
+#define LOG_PRINT_CC_L0(ct, message) LOG_PRINT_L0(ct << message)
+#define LOG_PRINT_CC_L1(ct, message) LOG_PRINT_L1(ct << message)
+#define LOG_PRINT_CC_L2(ct, message) LOG_PRINT_L2(ct << message)
+#define LOG_PRINT_CC_L3(ct, message) LOG_PRINT_L3(ct << message)
+#define LOG_PRINT_CC_L4(ct, message) LOG_PRINT_L4(ct << message)
 
 #define LOG_PRINT_CCONTEXT_L0(message) LOG_PRINT_CC_L0(context, message)
 #define LOG_PRINT_CCONTEXT_L1(message) LOG_PRINT_CC_L1(context, message)
@@ -163,13 +171,6 @@ namespace net_utils
 #define LOG_PRINT_CCONTEXT_L3(message) LOG_PRINT_CC_L3(context, message)
 #define LOG_ERROR_CCONTEXT(message)    LOG_ERROR_CC(context, message)
  
-#define LOG_PRINT_CCONTEXT_GREEN(message, log_level) LOG_PRINT_CC_GREEN(context, message, log_level)
-#define LOG_PRINT_CCONTEXT_RED(message, log_level) LOG_PRINT_CC_RED(context, message, log_level)
-#define LOG_PRINT_CCONTEXT_BLUE(message, log_level) LOG_PRINT_CC_BLUE(context, message, log_level) 
-#define LOG_PRINT_CCONTEXT_YELLOW(message, log_level) LOG_PRINT_CC_YELLOW(context, message, log_level) 
-#define LOG_PRINT_CCONTEXT_CYAN(message, log_level) LOG_PRINT_CC_CYAN(context, message, log_level) 
-#define LOG_PRINT_CCONTEXT_MAGENTA(message, log_level) LOG_PRINT_CC_MAGENTA(context, message, log_level) 
-
 #define CHECK_AND_ASSERT_MES_CC(condition, return_val, err_message) CHECK_AND_ASSERT_MES(condition, return_val, "[" << epee::net_utils::print_connection_context_short(context) << "]" << err_message)
 
 }

contrib/epee/include/profile_tools.h

@@ -52,8 +52,8 @@ namespace epee
 #endif
 
 #define START_WAY_POINTS() uint64_t _____way_point_time = epee::misc_utils::get_tick_count();
-#define WAY_POINT(name) {uint64_t delta = epee::misc_utils::get_tick_count()-_____way_point_time; LOG_PRINT("Way point " << name << ": " << delta, LOG_LEVEL_2);_____way_point_time = misc_utils::get_tick_count();}
-#define WAY_POINT2(name, avrg_obj) {uint64_t delta = epee::misc_utils::get_tick_count()-_____way_point_time; avrg_obj.push(delta); LOG_PRINT("Way point " << name << ": " << delta, LOG_LEVEL_2);_____way_point_time = misc_utils::get_tick_count();}
+#define WAY_POINT(name) {uint64_t delta = epee::misc_utils::get_tick_count()-_____way_point_time; MDEBUG("Way point " << name << ": " << delta);_____way_point_time = misc_utils::get_tick_count();}
+#define WAY_POINT2(name, avrg_obj) {uint64_t delta = epee::misc_utils::get_tick_count()-_____way_point_time; avrg_obj.push(delta); MDEBUG("Way point " << name << ": " << delta);_____way_point_time = misc_utils::get_tick_count();}
 
 
 #define TIME_MEASURE_START(var_name)    uint64_t var_name = epee::misc_utils::get_tick_count();
@@ -67,7 +67,7 @@ namespace profile_tools
 		{}
 		~local_call_account()
 		{
-			LOG_PRINT2("profile_details.log", "PROFILE "<<m_pname<<":av_time:\t" << (m_count_of_call ? (m_summary_time_used/m_count_of_call):0) <<" sum_time:\t"<<m_summary_time_used<<" call_count:\t" << m_count_of_call, LOG_LEVEL_0);
+			MINFO("PROFILE "<<m_pname<<":av_time:\t" << (m_count_of_call ? (m_summary_time_used/m_count_of_call):0) <<" sum_time:\t"<<m_summary_time_used<<" call_count:\t" << m_count_of_call);
 		}
 
 		size_t m_count_of_call;

contrib/epee/include/storages/activity_notifier.h

@@ -1,132 +0,0 @@
-// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-// * Neither the name of the Andrey N. Sabelnikov nor the
-// names of its contributors may be used to endorse or promote products
-// derived from this software without specific prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
-// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-
-
-
-#pragma once
-
-#include "inmemtoxml.h"
-
-//#include "levin/levin_server.h"
-
-namespace epee
-{
-
-class activity_printer_base
-{
-public:
-  activity_printer_base(){}
-  virtual ~activity_printer_base(){}
-};
-
-template<class A>
-class notify_activity_printer: public activity_printer_base
-{
-public:
-  notify_activity_printer(int level, A& arg, bool is_notify_mode = true):m_ref_arg(arg), m_level(level), m_is_notify_mode(is_notify_mode)
-  {
-    m_command_name = typeid(m_ref_arg).name();
-    m_command_name.erase(0, 7);
-    m_command_name.erase(m_command_name.size()-10,  m_command_name.size()-1); 
-    if(level == log_space::get_set_log_detalisation_level())
-    {
-      LOG_PRINT(m_command_name, level);
-    }
-    else if(level+1 == log_space::get_set_log_detalisation_level())
-    {
-      LOG_PRINT(" -->>" << m_command_name, level);
-    }
-    else if(level+2 == log_space::get_set_log_detalisation_level())
-    {
-      LOG_PRINT(" -->>" << m_command_name << "\n" << StorageNamed::xml::get_t_as_xml(m_ref_arg), level);
-    }
-  }
-
-  virtual ~notify_activity_printer()
-  {
-    if(m_is_notify_mode)
-    {
-      if(m_level+1 == log_space::get_set_log_detalisation_level())
-      {
-        LOG_PRINT(" <<--" << m_command_name, m_level);
-      }
-    }
-  }
-protected:
-  std::string m_command_name;
-  A& m_ref_arg;
-  int m_level;
-  bool m_is_notify_mode;
-};
-
-template<class A, class R>
-class command_activity_printer: public notify_activity_printer<A>
-{
-public:
-  command_activity_printer(int level, A& arg, R& rsp):notify_activity_printer(level, arg, false), m_ref_rsp(rsp)
-  {
-  }
-
-  virtual ~command_activity_printer()
-  {
-    if(m_level+1 == log_space::get_set_log_detalisation_level())
-    {
-      LOG_PRINT(" <<--" << m_command_name, m_level);
-    }
-    else if(m_level+2 == log_space::get_set_log_detalisation_level())
-    {
-      LOG_PRINT(" <<--" << m_command_name << "\n" << StorageNamed::trace_as_xml(m_ref_rsp), m_level);
-    }
-  }
-private:
-  R& m_ref_rsp;
-};
-
-template<class A, class R>
-activity_printer_base* create_activity_printer(int level, A& arg, R& rsp)
-{
-  return new command_activity_printer<A, R>(level, arg, rsp);
-}
-
-template<class A>
-activity_printer_base* create_activity_printer(int level, A& arg)
-{
-  return new notify_activity_printer<A>(level, arg);
-}
-
-}
-
-#define PRINT_COMMAND_ACTIVITY(level) boost::shared_ptr<activity_printer_base> local_activity_printer(create_activity_printer(level, in_struct, out_struct));
-#define PRINT_NOTIFY_ACTIVITY(level) boost::shared_ptr<activity_printer_base> local_activity_printer(create_activity_printer(level, in_struct));
-
-#define PRINT_ACTIVITY(level) \
-{std::string some_str = typeid(in_struct).name(); \
-  some_str.erase(0, 7); \
-  some_str.erase(some_str.size()-10,  some_str.size()-1); \
-  LOG_PRINT(some_str, level);}
-
-}
-

contrib/epee/include/storages/http_abstract_invoke.h

@@ -26,6 +26,9 @@
 // 
 
 #pragma once
+#include <boost/utility/string_ref.hpp>
+#include <chrono>
+#include <string>
 #include "portable_storage_template_helper.h"
 #include "net/http_base.h"
 #include "net/http_server_handlers_map2.h"
@@ -35,28 +38,28 @@ namespace epee
   namespace net_utils
   {
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_json_remote_command2(const std::string& url, t_request& out_struct, t_response& result_struct, t_transport& transport, unsigned int timeout = 5000, const std::string& method = "GET")
+    bool invoke_http_json(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(5), const boost::string_ref method = "GET")
     {
       std::string req_param;
       if(!serialization::store_t_to_json(out_struct, req_param))
         return false;
 
       const http::http_response_info* pri = NULL;
-      if(!invoke_request(url, transport, timeout, &pri, method, req_param))
+      if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri)))
       {
-        LOG_PRINT_L1("Failed to invoke http request to  " << url);
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri);
         return false;
       }
 
-      if(!pri->m_response_code)
+      if(!pri)
       {
-        LOG_PRINT_L1("Failed to invoke http request to  " << url << ", internal error (null response ptr)");
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri << ", internal error (null response ptr)");
         return false;
       }
 
       if(pri->m_response_code != 200)
       {
-        LOG_PRINT_L1("Failed to invoke http request to  " << url << ", wrong response code: " << pri->m_response_code);
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri << ", wrong response code: " << pri->m_response_code);
         return false;
       }
 
@@ -66,28 +69,28 @@ namespace epee
 
 
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_bin_remote_command2(const std::string& url, t_request& out_struct, t_response& result_struct, t_transport& transport, unsigned int timeout = 5000, const std::string& method = "GET")
+    bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(5), const boost::string_ref method = "GET")
     {
       std::string req_param;
       if(!serialization::store_t_to_binary(out_struct, req_param))
         return false;
 
       const http::http_response_info* pri = NULL;
-      if(!invoke_request(url, transport, timeout, &pri, method, req_param))
+      if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri)))
       {
-        LOG_PRINT_L1("Failed to invoke http request to  " << url);
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri);
         return false;
       }
 
-      if(!pri->m_response_code)
+      if(!pri)
       {
-        LOG_PRINT_L1("Failed to invoke http request to  " << url << ", internal error (null response ptr)");
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri << ", internal error (null response ptr)");
         return false;
       }
 
       if(pri->m_response_code != 200)
       {
-        LOG_PRINT_L1("Failed to invoke http request to  " << url << ", wrong response code: " << pri->m_response_code);
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri << ", wrong response code: " << pri->m_response_code);
         return false;
       }
 
@@ -95,15 +98,15 @@ namespace epee
     }
 
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_json_rpc(const std::string& url, const std::string& method_name, t_request& out_struct, t_response& result_struct, t_transport& transport, unsigned int timeout = 5000, const std::string& http_method = "GET", const std::string& req_id = "0")
+    bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(5), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
     {
       epee::json_rpc::request<t_request> req_t = AUTO_VAL_INIT(req_t);
       req_t.jsonrpc = "2.0";
       req_t.id = req_id;
-      req_t.method = method_name;
+      req_t.method = std::move(method_name);
       req_t.params = out_struct;
       epee::json_rpc::response<t_response, epee::json_rpc::error> resp_t = AUTO_VAL_INIT(resp_t);
-      if(!epee::net_utils::invoke_http_json_remote_command2(url, req_t, resp_t, transport, timeout, http_method))
+      if(!epee::net_utils::invoke_http_json(uri, req_t, resp_t, transport, timeout, http_method))
       {
         return false;
       }
@@ -117,9 +120,9 @@ namespace epee
     }
 
     template<class t_command, class t_transport>
-    bool invoke_http_json_rpc(const std::string& url, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, unsigned int timeout = 5000, const std::string& http_method = "GET", const std::string& req_id = "0")
+    bool invoke_http_json_rpc(const boost::string_ref uri, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(5), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
     {
-      return invoke_http_json_rpc(url, t_command::methodname(), out_struct, result_struct, transport, timeout, http_method, req_id);
+      return invoke_http_json_rpc(uri, t_command::methodname(), out_struct, result_struct, transport, timeout, http_method, req_id);
     }
 
   }

contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -30,6 +30,9 @@
 #include <boost/utility/value_init.hpp>
 #include "net/levin_base.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
 namespace epee
 {
   namespace net_utils
@@ -48,7 +51,7 @@ namespace epee
       int res = transport.invoke(command, buff_to_send, buff_to_recv);
       if( res <=0 )
       {
-        LOG_PRINT_RED("Failed to invoke command " << command << " return code " << res, LOG_LEVEL_1);
+        MERROR("Failed to invoke command " << command << " return code " << res);
         return false;
       }
       serialization::portable_storage stg_ret;
@@ -154,7 +157,7 @@ namespace epee
       int res = transport.notify(command, buff_to_send, conn_id);
       if(res <=0 )
       {
-        LOG_PRINT_RED_L0("Failed to notify command " << command << " return code " << res);
+        MERROR("Failed to notify command " << command << " return code " << res);
         return false;
       }
       return true;

contrib/epee/include/storages/portable_storage_from_json.h

@@ -365,12 +365,12 @@ namespace epee
         }
         catch(const std::exception& ex)
         {
-          LOG_PRINT_RED_L0("Failed to parse json, what: " << ex.what());
+          MERROR("Failed to parse json, what: " << ex.what());
           return false;
         }
         catch(...)
         {
-          LOG_PRINT_RED_L0("Failed to parse json");
+          MERROR("Failed to parse json");
           return false;
         }
       }

contrib/epee/src/CMakeLists.txt

@@ -0,0 +1,47 @@
+# Copyright (c) 2014-2017, The Monero Project
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+# 
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+# 
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+add_library(epee STATIC http_auth.cpp mlog.cpp)
+
+# Build and install libepee if we're building for GUI
+if (BUILD_GUI_DEPS)
+    if(IOS)
+        set(lib_folder lib-${ARCH})
+    else()
+        set(lib_folder lib)
+    endif()
+    install(TARGETS epee
+        ARCHIVE DESTINATION ${lib_folder})
+endif()
+
+target_link_libraries(epee
+  PUBLIC
+    crypto
+    ${Boost_FILESYSTEM_LIBRARY}
+  PRIVATE
+    ${EXTRA_LIBRARIES})

contrib/epee/src/http_auth.cpp

@@ -0,0 +1,796 @@
+// Copyright (c) 2014-2017, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "net/http_auth.h"
+
+#include <array>
+#include <boost/algorithm/string/find_iterator.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/fusion/adapted/std_tuple.hpp>
+#include <boost/fusion/algorithm/iteration/for_each.hpp>
+#include <boost/fusion/algorithm/iteration/iter_fold.hpp>
+#include <boost/fusion/algorithm/query/any.hpp>
+#include <boost/fusion/iterator/distance.hpp>
+#include <boost/fusion/iterator/value_of.hpp>
+#include <boost/fusion/sequence/intrinsic/begin.hpp>
+#include <boost/fusion/sequence/intrinsic/size.hpp>
+#include <boost/range/algorithm/copy.hpp>
+#include <boost/range/algorithm/find_if.hpp>
+#include <boost/range/iterator_range_core.hpp>
+#include <boost/range/join.hpp>
+#include <boost/spirit/include/karma_generate.hpp>
+#include <boost/spirit/include/karma_uint.hpp>
+#include <boost/spirit/include/qi_alternative.hpp>
+#include <boost/spirit/include/qi_and_predicate.hpp>
+#include <boost/spirit/include/qi_char.hpp>
+#include <boost/spirit/include/qi_char_class.hpp>
+#include <boost/spirit/include/qi_difference.hpp>
+#include <boost/spirit/include/qi_kleene.hpp>
+#include <boost/spirit/include/qi_parse.hpp>
+#include <boost/spirit/include/qi_plus.hpp>
+#include <boost/spirit/include/qi_no_case.hpp>
+#include <boost/spirit/include/qi_not_predicate.hpp>
+#include <boost/spirit/include/qi_raw.hpp>
+#include <boost/spirit/include/qi_rule.hpp>
+#include <boost/spirit/include/qi_sequence.hpp>
+#include <boost/spirit/include/qi_string.hpp>
+#include <boost/spirit/include/qi_symbols.hpp>
+#include <boost/spirit/include/qi_uint.hpp>
+#include <cassert>
+#include <iterator>
+#include <limits>
+#include <tuple>
+#include <type_traits>
+
+#include "crypto/crypto.h"
+#include "md5_l.h"
+#include "string_coding.h"
+
+/* This file uses the `u8` prefix and specifies all chars by ASCII numeric
+value. This is for maximum portability - C++ does not actually specify ASCII
+as the encoding type for unprefixed string literals, etc. Although rare, the
+effort required to support rare compiler encoding types is low.
+
+Also be careful of qi::ascii character classes (`qi::asci::alpha`, etc.) -
+non-ASCII characters will cause undefined behavior in the table lookup until
+boost 1.60. The expression `&qi::ascii::char_` will fail on non-ASCII
+characters without "consuming" the input character. */
+
+namespace
+{
+  namespace http = epee::net_utils::http;
+
+  // string_ref is only constexpr if length is given
+  template<std::size_t N>
+  constexpr boost::string_ref ceref(const char (&arg)[N])
+  {
+    return boost::string_ref(arg, N - 1);
+  }
+
+  constexpr const auto client_auth_field = ceref(u8"Authorization");
+  constexpr const auto server_auth_field = ceref(u8"WWW-authenticate");
+  constexpr const auto auth_realm = ceref(u8"monero-rpc");
+  constexpr const char comma = 44;
+  constexpr const char equal_sign = 61;
+  constexpr const char quote = 34;
+  constexpr const char zero = 48;
+  constexpr const auto sess_algo = ceref(u8"-sess");
+
+  constexpr const unsigned client_reserve_size = 512; //!< std::string::reserve size for clients
+
+  //// Digest Algorithms
+
+  template<std::size_t N>
+  std::array<char, N * 2> to_hex(const std::array<std::uint8_t, N>& digest) noexcept
+  {
+    static constexpr const char alphabet[] = u8"0123456789abcdef";
+    static_assert(sizeof(alphabet) == 17, "bad alphabet size");
+
+    // TODO upgrade (improve performance) of to hex in epee string tools
+    std::array<char, N * 2> out{{}};
+    auto current = out.begin();
+    for (const std::uint8_t byte : digest)
+    {
+      *current = alphabet[byte >> 4];
+      ++current;
+      *current = alphabet[byte & 0x0F];
+      ++current;
+    }
+    return out;
+  }
+
+  struct md5_
+  {
+    static constexpr const boost::string_ref name = ceref(u8"MD5");
+
+    struct update
+    {
+      template<typename T>
+      void operator()(const T& arg) const
+      {
+        const boost::iterator_range<const char*> data(boost::as_literal(arg));
+        md5::MD5Update(
+          std::addressof(ctx),
+          reinterpret_cast<const std::uint8_t*>(data.begin()),
+          data.size()
+        );
+      }
+      void operator()(const std::string& arg) const
+      {
+        (*this)(boost::string_ref(arg));
+      }
+
+      md5::MD5_CTX& ctx;
+    };
+
+    template<typename... T>
+    std::array<char, 32> operator()(const T&... args) const
+    {      
+      md5::MD5_CTX ctx{};
+      md5::MD5Init(std::addressof(ctx));
+      boost::fusion::for_each(std::tie(args...), update{ctx});
+
+      std::array<std::uint8_t, 16> digest{{}};
+      md5::MD5Final(digest.data(), std::addressof(ctx));
+      return to_hex(digest);
+    }
+  };
+  constexpr const boost::string_ref md5_::name;
+
+  //! Digest Algorithms available for HTTP Digest Auth. Sort better algos to the left
+  constexpr const std::tuple<md5_> digest_algorithms{};
+
+  //// Various String Utilities
+
+  struct ascii_tolower_
+  {
+    template<typename Char>
+    constexpr Char operator()(Char value) const noexcept
+    {
+      static_assert(std::is_integral<Char>::value, "only integral types supported");
+      return (65 <= value && value <= 90) ? (value + 32) : value;
+    }
+  };
+  constexpr const ascii_tolower_ ascii_tolower{};
+
+  struct ascii_iequal_
+  {
+    template<typename Char>
+    constexpr bool operator()(Char left, Char right) const noexcept
+    {
+      return ascii_tolower(left) == ascii_tolower(right);
+    }
+  };
+  constexpr const ascii_iequal_ ascii_iequal{};
+
+  struct http_list_separator_
+  {
+    template<typename Char>
+    bool operator()(Char value) const noexcept
+    {
+      static_assert(std::is_integral<Char>::value, "only integral types supported");
+      return boost::spirit::char_encoding::ascii::isascii_(value) &&
+        (value == comma || boost::spirit::char_encoding::ascii::isspace(value));
+    }
+  };
+  constexpr const http_list_separator_ http_list_separator{};
+
+  std::string to_string(boost::iterator_range<const char*> source)
+  {
+    return {source.begin(), source.size()};
+  }
+
+  template<typename T>
+  void add_first_field(std::string& str, const char* const name, const T& value)
+  {
+    str.append(name);
+    str.push_back(equal_sign);
+    boost::copy(value, std::back_inserter(str));
+  }
+
+  template<typename T>
+  void add_field(std::string& str, const char* const name, const T& value)
+  {
+    str.push_back(comma);
+    add_first_field(str, name, value);
+  }
+
+  template<typename T>
+  using quoted_result = boost::joined_range<
+    const boost::joined_range<const boost::string_ref, const T>, const boost::string_ref
+  >;
+
+  template<typename T>
+  quoted_result<T> quoted(const T& arg)
+  {
+    return boost::range::join(boost::range::join(ceref(u8"\""), arg), ceref(u8"\""));
+  }
+
+  //// Digest Authentication
+
+  template<typename Digest>
+  typename std::result_of<Digest()>::type generate_a1(
+    Digest digest, const http::login& creds, const boost::string_ref realm)
+  {
+    return digest(creds.username, u8":", realm, u8":", creds.password);
+  }
+
+  template<typename Digest>
+  typename std::result_of<Digest()>::type generate_a1(
+    Digest digest, const http::http_client_auth::session& user)
+  {
+    return generate_a1(std::move(digest), user.credentials, user.server.realm);
+  }
+
+  template<typename T>
+  void init_client_value(std::string& str,
+    const boost::string_ref algorithm, const http::http_client_auth::session& user,
+    const boost::string_ref uri, const T& response)
+  {
+    str.append(u8"Digest ");
+    add_first_field(str, u8"algorithm", algorithm);
+    add_field(str, u8"nonce", quoted(user.server.nonce));
+    add_field(str, u8"realm", quoted(user.server.realm));
+    add_field(str, u8"response", quoted(response));
+    add_field(str, u8"uri", quoted(uri));
+    add_field(str, u8"username", quoted(user.credentials.username));
+    if (!user.server.opaque.empty())
+      add_field(str, u8"opaque", quoted(user.server.opaque));
+  }
+
+  //! Implements superseded algorithm specified in RFC 2069
+  template<typename Digest>
+  struct old_algorithm
+  {
+    explicit old_algorithm(Digest digest_) : digest(std::move(digest_)) {}
+
+    std::string operator()(const http::http_client_auth::session& user,
+      const boost::string_ref method, const boost::string_ref uri) const
+    {
+      const auto response = digest(
+        generate_a1(digest, user), u8":", user.server.nonce, u8":", digest(method, u8":", uri)
+      );
+      std::string out{};
+      out.reserve(client_reserve_size);
+      init_client_value(out, Digest::name, user, uri, response);
+      return out;
+    }
+  private:
+    Digest digest;
+  };
+
+  //! Implements the `qop=auth` algorithm specified in RFC 2617
+  template<typename Digest>
+  struct auth_algorithm
+  {
+    explicit auth_algorithm(Digest digest_) : digest(std::move(digest_)) {}
+
+    std::string operator()(const http::http_client_auth::session& user,
+      const boost::string_ref method, const boost::string_ref uri) const
+    {
+      namespace karma = boost::spirit::karma;
+      using counter_type = decltype(user.counter);
+      static_assert(
+        std::numeric_limits<counter_type>::radix == 2, "unexpected radix for counter"
+      );
+      static_assert(
+        std::numeric_limits<counter_type>::digits <= 32,
+        "number of digits will cause underflow on padding below"
+      );
+
+      std::string out{};
+      out.reserve(client_reserve_size);
+
+      karma::generate(std::back_inserter(out), karma::hex(user.counter));
+      out.insert(out.begin(), 8 - out.size(), zero); // zero left pad
+      if (out.size() != 8)
+        return {};
+
+      std::array<char, 8> nc{{}};
+      boost::copy(out, nc.data());
+      const auto response = digest(
+        generate_a1(digest, user), u8":", user.server.nonce, u8":", nc, u8"::auth:", digest(method, u8":", uri)
+      );
+      out.clear();
+      init_client_value(out, Digest::name, user, uri, response);
+      add_field(out, u8"qop", ceref(u8"auth"));
+      add_field(out, u8"nc", nc);
+      return out;
+    }
+
+  private:
+    Digest digest;
+  };
+
+  //! Processes client "Authorization" and server "WWW-authenticate" HTTP fields
+  struct auth_message
+  {
+    using iterator = const char*;
+    enum status{ kFail = 0, kStale, kPass };
+
+    //! \return Status of the `response` field from the client
+    static status verify(const boost::string_ref method, const boost::string_ref request,
+      const http::http_server_auth::session& user)
+    {
+      const auto parsed = parse(request);
+      if (parsed &&
+          boost::equals(parsed->username, user.credentials.username) &&
+          boost::fusion::any(digest_algorithms, has_valid_response{*parsed, user, method}))
+      {
+        if (boost::equals(parsed->nonce, user.nonce))
+        {
+          // RFC 2069 format does not verify nc value - allow just once
+          if (user.counter == 1 || (!parsed->qop.empty() && parsed->counter() == user.counter))
+          {
+            return kPass;
+          }
+        }
+        return kStale;
+      }
+      return kFail;
+    }
+
+    //! \return Information needed to generate client authentication `response`s.
+    static http::http_client_auth::session::keys extract(
+      const http::http_response_info& response, const bool is_first)
+    {
+      using field = std::pair<std::string, std::string>;
+
+      server_parameters best{};
+
+      const std::list<field>& fields = response.m_header_info.m_etc_fields;
+      auto current = fields.begin();
+      const auto end = fields.end();
+      while (true)
+      {
+        current = std::find_if(current, end, [] (const field& value) {
+          return boost::equals(server_auth_field, value.first, ascii_iequal);
+        });
+        if (current == end)
+          break;
+
+        const auto parsed = parse(current->second);
+        if (parsed)
+        {
+          server_parameters local_best = parsed->algorithm.empty() ?
+            server_parameters{*parsed, boost::fusion::find<md5_>(digest_algorithms)} :
+            boost::fusion::iter_fold(digest_algorithms, server_parameters{}, matches_algorithm{*parsed});
+
+          if (local_best.index < best.index)
+            best = std::move(local_best);
+        }
+        ++current;
+      }
+      if (is_first || boost::equals(best.stale, ceref(u8"true"), ascii_iequal))
+        return best.take();
+      return {}; // authentication failed with bad user/pass
+    }
+
+  private:
+    explicit auth_message()
+      : algorithm()
+      , cnonce()
+      , nc()
+      , nonce()
+      , qop()
+      , realm()
+      , response()
+      , stale()
+      , uri()
+      , username() {
+    }
+
+    static boost::optional<auth_message> parse(const boost::string_ref request)
+    {
+      struct parser
+      {
+        using field_parser = std::function<bool(const parser&, iterator&, iterator, auth_message&)>;
+
+        explicit parser() : field_table(), skip_whitespace(), header(), quoted_string(), token(), fields() {
+          using namespace std::placeholders;
+          namespace qi = boost::spirit::qi;
+
+          struct parse_nc
+          {
+            bool operator()(const parser&, iterator& current, const iterator end, auth_message& result) const
+            {
+              return qi::parse(
+                current, end,
+                (qi::raw[qi::uint_parser<std::uint32_t, 16, 8, 8>{}]),
+                result.nc
+              );
+            }
+          };
+          struct parse_token
+          {
+            bool operator()(const parser& parse, iterator& current, const iterator end,
+              boost::iterator_range<iterator>& result) const
+            {
+              return qi::parse(current, end, parse.token, result);
+            }
+          };
+          struct parse_string
+          {
+            bool operator()(const parser& parse, iterator& current, const iterator end,
+              boost::iterator_range<iterator>& result) const
+            {
+              return qi::parse(current, end, parse.quoted_string, result);
+            }
+            bool operator()(const parser& parse, iterator& current, const iterator end) const
+            {
+              return qi::parse(current, end, parse.quoted_string);
+            }
+          };
+          struct parse_response
+          {
+            bool operator()(const parser&, iterator& current, const iterator end, auth_message& result) const
+            {
+              using byte = qi::uint_parser<std::uint8_t, 16, 2, 2>;
+              return qi::parse(
+                current, end,
+                (qi::lit(quote) >> qi::raw[+(byte{})] >> qi::lit(quote)),
+                result.response
+              );
+            }
+          };
+
+          field_table.add
+            (u8"algorithm", std::bind(parse_token{}, _1, _2, _3, std::bind(&auth_message::algorithm, _4)))
+            (u8"cnonce", std::bind(parse_string{}, _1, _2, _3, std::bind(&auth_message::cnonce, _4)))
+            (u8"domain", std::bind(parse_string{}, _1, _2, _3)) // ignore field
+            (u8"nc", parse_nc{})
+            (u8"nonce", std::bind(parse_string{}, _1, _2, _3, std::bind(&auth_message::nonce, _4)))
+            (u8"opaque", std::bind(parse_string{}, _1, _2, _3, std::bind(&auth_message::opaque, _4)))
+            (u8"qop", std::bind(parse_token{}, _1, _2, _3, std::bind(&auth_message::qop, _4)))
+            (u8"realm", std::bind(parse_string{}, _1, _2, _3, std::bind(&auth_message::realm, _4)))
+            (u8"response", parse_response{})
+            (u8"stale", std::bind(parse_token{}, _1, _2, _3, std::bind(&auth_message::stale, _4)))
+            (u8"uri", std::bind(parse_string{}, _1, _2, _3, std::bind(&auth_message::uri, _4)))
+            (u8"username", std::bind(parse_string{}, _1, _2, _3, std::bind(&auth_message::username, _4)));
+
+          skip_whitespace = *(&qi::ascii::char_ >> qi::ascii::space);
+          header = skip_whitespace >> qi::ascii::no_case[u8"digest"] >> skip_whitespace;
+          quoted_string = (qi::lit(quote) >> qi::raw[+(u8"\\\"" | (qi::ascii::char_ - quote))] >> qi::lit(quote));
+          token =
+            (!qi::lit(quote) >> qi::raw[+(&qi::ascii::char_ >> (qi::ascii::graph - qi::ascii::char_(u8"()<>@,;:\\\"/[]?={}")))]) |
+            quoted_string;
+          fields = field_table >> skip_whitespace >> equal_sign >> skip_whitespace;
+        }
+
+        boost::optional<auth_message> operator()(const boost::string_ref request) const
+        { 
+          namespace qi = boost::spirit::qi;
+           
+          iterator current = request.begin();
+          const iterator end = request.end();
+
+          if (!qi::parse(current, end, header))
+          {
+            return boost::none;
+          }
+
+          auth_message info{};
+          field_parser null_parser{};
+          std::reference_wrapper<const field_parser> field = null_parser;
+
+          do // require at least one field; require field after `,` character
+          {
+            if (!qi::parse(current, end, fields, field) || !field(*this, current, end, info))
+            {
+              return boost::none;
+            }
+            qi::parse(current, end, skip_whitespace);
+          } while (qi::parse(current, end, qi::char_(comma) >> skip_whitespace));
+          return boost::make_optional(current == end, info);
+        }
+
+      private:
+        boost::spirit::qi::symbols<
+          char, field_parser, boost::spirit::qi::tst<char, field_parser>, ascii_tolower_
+        > field_table;
+        boost::spirit::qi::rule<iterator> skip_whitespace;
+        boost::spirit::qi::rule<iterator> header;
+        boost::spirit::qi::rule<iterator, boost::iterator_range<iterator>()> quoted_string;
+        boost::spirit::qi::rule<iterator, boost::iterator_range<iterator>()> token;
+        boost::spirit::qi::rule<iterator, std::reference_wrapper<const field_parser>()> fields;
+      }; // parser
+
+      static const parser parse_;
+      return parse_(request);
+    }
+
+    struct has_valid_response
+    {
+      template<typename Digest, typename Result>
+      Result generate_old_response(Digest digest, const Result& key, const Result& auth) const
+      {
+        return digest(key, u8":", request.nonce, u8":", auth);
+      }
+
+      template<typename Digest, typename Result>
+      Result generate_new_response(Digest digest, const Result& key, const Result& auth) const
+      {
+        return digest(
+          key, u8":", request.nonce, u8":", request.nc, u8":", request.cnonce, u8":", request.qop, u8":", auth
+        );
+      }
+
+      template<typename Result>
+      bool check(const Result& result) const
+      {
+        return boost::equals(request.response, result, ascii_iequal);
+      }
+
+      template<typename Digest>
+      bool operator()(const Digest& digest) const
+      {
+        if (boost::starts_with(request.algorithm, Digest::name, ascii_iequal) ||
+            (request.algorithm.empty() && std::is_same<md5_, Digest>::value))
+        {
+          auto key = generate_a1(digest, user.credentials, auth_realm);
+          if (boost::ends_with(request.algorithm, sess_algo, ascii_iequal))
+          {
+            key = digest(key, u8":", request.nonce, u8":", request.cnonce);
+          }
+
+          auto auth = digest(method, u8":", request.uri);
+          if (request.qop.empty())
+          {
+            return check(generate_old_response(std::move(digest), std::move(key), std::move(auth)));
+          }
+          else if (boost::equals(ceref(u8"auth"), request.qop, ascii_iequal))
+          {
+            return check(generate_new_response(std::move(digest), std::move(key), std::move(auth)));
+          }
+        }
+        return false;
+      }
+
+      const auth_message& request;
+      const http::http_server_auth::session& user;
+      const boost::string_ref method;
+    };
+
+    boost::optional<std::uint32_t> counter() const
+    {
+      namespace qi = boost::spirit::qi;
+      using hex = qi::uint_parser<std::uint32_t, 16>;
+      std::uint32_t value = 0;
+      const bool converted = qi::parse(nc.begin(), nc.end(), hex{}, value);
+      return boost::make_optional(converted, value);
+    }
+
+    struct server_parameters
+    {
+      server_parameters()
+        : nonce(), opaque(), realm(), stale(), value_generator()
+        , index(boost::fusion::size(digest_algorithms))
+      {}
+
+      template<typename DigestIter>
+      explicit server_parameters(const auth_message& request, const DigestIter& digest)
+        : nonce(request.nonce)
+        , opaque(request.opaque)
+        , stale(request.stale)
+        , realm(request.realm)
+        , value_generator()
+        , index(boost::fusion::distance(boost::fusion::begin(digest_algorithms), digest))
+      {
+        using digest_type = typename boost::fusion::result_of::value_of<DigestIter>::type;
+
+        // debug check internal state of the auth_message class
+        assert(
+          (std::is_same<digest_type, md5_>::value) ||
+          boost::equals((*digest).name, request.algorithm, ascii_iequal)
+        );
+        if (request.qop.empty())
+          value_generator = old_algorithm<digest_type>{*digest};
+        else
+        {
+          for (auto elem = boost::make_split_iterator(request.qop, boost::token_finder(http_list_separator));
+               !elem.eof();
+               ++elem)
+          {
+            if (boost::equals(ceref(u8"auth"), *elem, ascii_iequal))
+            {
+              value_generator = auth_algorithm<digest_type>{*digest};
+              break;
+            }
+          }
+          if (!value_generator) // no supported qop mode
+            index = boost::fusion::size(digest_algorithms);
+        }
+      }
+
+      http::http_client_auth::session::keys take()
+      {
+        return {to_string(nonce), to_string(opaque), to_string(realm), std::move(value_generator)};
+      }
+
+      boost::iterator_range<iterator> nonce;
+      boost::iterator_range<iterator> opaque;
+      boost::iterator_range<iterator> realm;
+      boost::iterator_range<iterator> stale;
+      http::http_client_auth::session::keys::algorithm value_generator;
+      unsigned index;
+    };
+
+    struct matches_algorithm
+    {
+      template<typename DigestIter>
+      server_parameters operator()(server_parameters current, const DigestIter& digest) const
+      {
+        if (!current.value_generator)
+        {
+          if (boost::equals(response.algorithm, (*digest).name, ascii_iequal))
+          {
+            current = server_parameters{response, digest};
+          }
+        }
+        return current;
+      }
+      const auth_message& response;
+    };
+
+
+    boost::iterator_range<iterator> algorithm;
+    boost::iterator_range<iterator> cnonce;
+    boost::iterator_range<iterator> nc;
+    boost::iterator_range<iterator> nonce;
+    boost::iterator_range<iterator> opaque;
+    boost::iterator_range<iterator> qop;
+    boost::iterator_range<iterator> realm;
+    boost::iterator_range<iterator> response;
+    boost::iterator_range<iterator> stale;
+    boost::iterator_range<iterator> uri;
+    boost::iterator_range<iterator> username; 
+  }; // auth_message
+
+  struct add_challenge
+  {
+    template<typename Digest>
+    void operator()(const Digest& digest) const
+    {
+      static constexpr const auto fvalue = ceref(u8"Digest qop=\"auth\"");
+
+      for (unsigned i = 0; i < 2; ++i)
+      {
+        std::string out(fvalue);
+
+        const auto algorithm = boost::range::join(
+          Digest::name, (i == 0 ? boost::string_ref{} : sess_algo)
+        );
+        add_field(out, u8"algorithm", algorithm);
+        add_field(out, u8"realm", quoted(auth_realm));
+        add_field(out, u8"nonce", quoted(nonce));
+        add_field(out, u8"stale", is_stale ? ceref("true") : ceref("false"));
+        
+        fields.push_back(std::make_pair(std::string(server_auth_field), std::move(out)));
+      }
+    }
+
+    const boost::string_ref nonce;
+    std::list<std::pair<std::string, std::string>>& fields;
+    const bool is_stale;
+  };
+
+  http::http_response_info create_digest_response(const boost::string_ref nonce, const bool is_stale)
+  {
+    epee::net_utils::http::http_response_info rc{};
+    rc.m_response_code = 401;
+    rc.m_response_comment = u8"Unauthorized";
+    rc.m_mime_tipe = u8"text/html";
+    rc.m_body = 
+      u8"<html><head><title>Unauthorized Access</title></head><body><h1>401 Unauthorized</h1></body></html>";
+
+    boost::fusion::for_each(
+      digest_algorithms, add_challenge{nonce, rc.m_additional_fields, is_stale}
+    );
+    
+    return rc;
+  }
+}
+
+namespace epee
+{
+  namespace net_utils
+  {
+    namespace http
+    {
+      http_server_auth::http_server_auth(login credentials)
+        : user(session{std::move(credentials)}) {
+      }
+
+      boost::optional<http_response_info> http_server_auth::do_get_response(const http_request_info& request)
+      {
+        assert(user);
+        using field = std::pair<std::string, std::string>;
+
+        const std::list<field>& fields = request.m_header_info.m_etc_fields;
+        const auto auth = boost::find_if(fields, [] (const field& value) {
+          return boost::equals(client_auth_field, value.first, ascii_iequal);
+        });
+
+        bool is_stale = false;
+        if (auth != fields.end())
+        {
+          ++(user->counter);
+          switch (auth_message::verify(request.m_http_method_str, auth->second, *user))
+          {
+          case auth_message::kPass:
+            return boost::none;
+
+          case auth_message::kStale:
+            is_stale = true;
+            break;
+
+          default:
+          case auth_message::kFail:
+            break;
+          }
+        }
+        user->counter = 0;
+        {
+          std::array<std::uint8_t, 16> rand_128bit{{}};
+          crypto::rand(rand_128bit.size(), rand_128bit.data());
+          user->nonce = string_encoding::base64_encode(rand_128bit.data(), rand_128bit.size());
+        }
+        return create_digest_response(user->nonce, is_stale);
+      }
+
+      http_client_auth::http_client_auth(login credentials)
+        : user(session{std::move(credentials)}) {
+      }
+
+      http_client_auth::status http_client_auth::do_handle_401(const http_response_info& response)
+      {
+        assert(user);
+        const bool first_auth = (user->counter == 0);
+        user->server = auth_message::extract(response, first_auth);
+        if (user->server.generator)
+        {
+          user->counter = 0;
+          return kSuccess;
+        }
+        return first_auth ? kParseFailure : kBadPassword;
+      }
+
+      boost::optional<std::pair<std::string, std::string>> http_client_auth::do_get_auth_field(
+        const boost::string_ref method, const boost::string_ref uri)
+      {
+        assert(user);
+        if (user->server.generator)
+        {
+          ++(user->counter);
+          return std::make_pair(std::string(client_auth_field), user->server.generator(*user, method, uri));
+        }
+        return boost::none;
+      }
+    }
+  }
+}
+

contrib/epee/src/mlog.cpp

@@ -0,0 +1,338 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+
+#ifndef _MLOG_H_
+#define _MLOG_H_
+
+#include <atomic>
+#include "misc_log_ex.h"
+
+INITIALIZE_EASYLOGGINGPP
+
+#define MLOG_BASE_FORMAT "%datetime{%Y-%M-%d %H:%m:%s.%g}\t%thread\t%level\t%logger\t%loc\t%msg"
+
+using namespace epee;
+
+static std::string generate_log_filename(const char *base)
+{
+  std::string filename(base);
+  char tmp[200];
+  struct tm tm;
+  time_t now = time(NULL);
+  if
+#ifdef WIN32
+  (!gmtime_s(&tm, &now))
+#else
+  (!gmtime_r(&now, &tm))
+#endif
+    strcpy(tmp, "unknown");
+  else
+    strftime(tmp, sizeof(tmp), "%Y-%m-%d-%H-%M-%S", &tm);
+  filename += "-";
+  filename += tmp;
+  return filename;
+}
+
+std::string mlog_get_default_log_path(const char *default_filename)
+{
+  std::string process_name = epee::string_tools::get_current_module_name();
+  std::string default_log_folder = epee::string_tools::get_current_module_folder();
+  std::string default_log_file = process_name;
+  std::string::size_type a = default_log_file.rfind('.');
+  if ( a != std::string::npos )
+    default_log_file.erase( a, default_log_file.size());
+  if ( ! default_log_file.empty() )
+    default_log_file += ".log";
+  else
+    default_log_file = default_filename;
+
+  return (boost::filesystem::path(default_log_folder) / boost::filesystem::path(default_log_file)).string();
+}
+
+static void mlog_set_common_prefix()
+{
+  static const char * const expected_filename = "contrib/epee/src/mlog.cpp";
+  const char *path = __FILE__, *expected_ptr = strstr(path, expected_filename);
+  if (!expected_ptr)
+    return;
+  el::Loggers::setFilenameCommonPrefix(std::string(path, expected_ptr - path));
+}
+
+static const char *get_default_categories(int level)
+{
+  const char *categories = "";
+  switch (level)
+  {
+    case 0:
+      categories = "*:WARNING,net:FATAL,net.p2p:FATAL,global:INFO,verify:FATAL,stacktrace:INFO";
+      break;
+    case 1:
+      categories = "*:WARNING,global:INFO,stacktrace:INFO";
+      break;
+    case 2:
+      categories = "*:DEBUG";
+      break;
+    case 3:
+      categories = "*:TRACE";
+      break;
+    case 4:
+      categories = "*:TRACE";
+      break;
+    default:
+      break;
+  }
+  return categories;
+}
+
+void mlog_configure(const std::string &filename_base, bool console)
+{
+  el::Configurations c;
+  c.setGlobally(el::ConfigurationType::Filename, filename_base);
+  c.setGlobally(el::ConfigurationType::ToFile, "true");
+  const char *log_format = getenv("MONERO_LOG_FORMAT");
+  if (!log_format)
+    log_format = MLOG_BASE_FORMAT;
+  c.setGlobally(el::ConfigurationType::Format, log_format);
+  c.setGlobally(el::ConfigurationType::ToStandardOutput, console ? "true" : "false");
+  c.setGlobally(el::ConfigurationType::MaxLogFileSize, "104850000"); // 100 MB - 7600 bytes
+  el::Loggers::setDefaultConfigurations(c, true);
+
+  el::Loggers::addFlag(el::LoggingFlag::HierarchicalLogging);
+  el::Loggers::addFlag(el::LoggingFlag::CreateLoggerAutomatically);
+  el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
+  el::Loggers::addFlag(el::LoggingFlag::ColoredTerminalOutput);
+  el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
+  el::Helpers::installPreRollOutCallback([&filename_base](const char *name, size_t){
+    std::string rname = generate_log_filename(filename_base.c_str());
+    rename(name, rname.c_str());
+  });
+  mlog_set_common_prefix();
+  const char *monero_log = getenv("MONERO_LOGS");
+  if (!monero_log)
+  {
+    monero_log = get_default_categories(0);
+  }
+  mlog_set_categories(monero_log);
+}
+
+void mlog_set_categories(const char *categories)
+{
+  el::Loggers::setCategories(categories);
+  MGINFO("New log categories: " << categories);
+}
+
+// maps epee style log level to new logging system
+void mlog_set_log_level(int level)
+{
+  const char *categories = get_default_categories(level);
+  el::Loggers::setCategories(categories);
+  MGINFO("New log categories: " << categories);
+}
+
+void mlog_set_log(const char *log)
+{
+  long level;
+  char *ptr = NULL;
+
+  level = strtoll(log, &ptr, 10);
+  if (ptr && *ptr)
+  {
+    // we can have a default level, eg, 2,foo:ERROR
+    if (*ptr == ',') {
+      std::string new_categories = std::string(get_default_categories(level)) + ptr;
+      mlog_set_categories(new_categories.c_str());
+    }
+    else {
+      mlog_set_categories(log);
+    }
+  }
+  else if (level >= 0 && level <= 4)
+  {
+    mlog_set_log_level(level);
+  }
+  else
+  {
+    MERROR("Invalid numerical log level: " << log);
+  }
+}
+
+namespace epee
+{
+
+bool is_stdout_a_tty()
+{
+  static std::atomic<bool> initialized(false);
+  static std::atomic<bool> is_a_tty(false);
+
+  if (!initialized.load(std::memory_order_acquire))
+  {
+#if defined(WIN32)
+    is_a_tty.store(0 != _isatty(_fileno(stdout)), std::memory_order_relaxed);
+#else
+    is_a_tty.store(0 != isatty(fileno(stdout)), std::memory_order_relaxed);
+#endif
+    initialized.store(true, std::memory_order_release);
+  }
+
+  return is_a_tty.load(std::memory_order_relaxed);
+}
+
+void set_console_color(int color, bool bright)
+{
+  if (!is_stdout_a_tty())
+    return;
+
+  switch(color)
+  {
+  case console_color_default:
+    {
+#ifdef WIN32
+      HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+      SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE| (bright ? FOREGROUND_INTENSITY:0));
+#else
+      if(bright)
+        std::cout << "\033[1;37m";
+      else
+        std::cout << "\033[0m";
+#endif
+    }
+    break;
+  case console_color_white:
+    {
+#ifdef WIN32
+      HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+      SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0));
+#else
+      if(bright)
+        std::cout << "\033[1;37m";
+      else
+        std::cout << "\033[0;37m";
+#endif
+    }
+    break;
+  case console_color_red:
+    {
+#ifdef WIN32
+      HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+      SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0));
+#else
+      if(bright)
+        std::cout << "\033[1;31m";
+      else
+        std::cout << "\033[0;31m";
+#endif
+    }
+    break;
+  case console_color_green:
+    {
+#ifdef WIN32
+      HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+      SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0));
+#else
+      if(bright)
+        std::cout << "\033[1;32m";
+      else
+        std::cout << "\033[0;32m";
+#endif
+    }
+    break;
+
+  case console_color_blue:
+    {
+#ifdef WIN32
+      HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+      SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY);//(bright ? FOREGROUND_INTENSITY:0));
+#else
+      if(bright)
+        std::cout << "\033[1;34m";
+      else
+        std::cout << "\033[0;34m";
+#endif
+    }
+    break;
+
+  case console_color_cyan:
+    {
+#ifdef WIN32
+      HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+      SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0));
+#else
+      if(bright)
+        std::cout << "\033[1;36m";
+      else
+        std::cout << "\033[0;36m";
+#endif
+    }
+    break;
+
+  case console_color_magenta:
+    {
+#ifdef WIN32
+      HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+      SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0));
+#else
+      if(bright)
+        std::cout << "\033[1;35m";
+      else
+        std::cout << "\033[0;35m";
+#endif
+    }
+    break;
+
+  case console_color_yellow:
+    {
+#ifdef WIN32
+      HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+      SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0));
+#else
+      if(bright)
+        std::cout << "\033[1;33m";
+      else
+        std::cout << "\033[0;33m";
+#endif
+    }
+    break;
+
+  }
+}
+
+void reset_console_color() {
+  if (!is_stdout_a_tty())
+    return;
+
+#ifdef WIN32
+  HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+  SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE);
+#else
+  std::cout << "\033[0m";
+  std::cout.flush();
+#endif
+}
+
+}
+
+#endif //_MLOG_H_

contrib/otshell_utils/CMakeLists.txt

@@ -1,14 +0,0 @@
-cmake_minimum_required (VERSION 2.6)
-project (otshell CXX)
-
-# Add executable
-
-file(GLOB otshell_utils_sources # All files in directory:
-	"*.h"
-	"*.hpp"
-	"*.cpp"
-)
-
-add_library (otshell_utils ${otshell_utils_sources})
-set_target_properties (otshell_utils PROPERTIES OUTPUT_NAME "otshell_utils")
-#target_link_libraries (upnpc-static ${LDLIBS}) # to add used libs

contrib/otshell_utils/LICENCE.txt

@@ -1,21 +0,0 @@
-
-This are some files also from OpenTransactions / otshell project,
-developed thanks to the awesome OpenTransaction project, organization and developers :)
-
-Parts of code here was also developed thanks to the excellent Monero project,
-thanks to Monero project, organization and developers :)
-
-[Some] files/code here (in external/otshell_utils) are under licence defined in
-src/doc/LICENCE-otshell.txt ;
-Others are from monero, with licence in src/doc/LICENCE-monero.txt ;
-
-For me (rfree) the licence seem compatbile so no problem, personally (as author of many parts of the code,
-possibly not all) I do not worry who uses it how; I'am not a lawyer.
-
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Please share :-) This licence can be used e.g. for parts of code that are usable in both open-source FOSS project
-Monero and Open Transactions, to share and develop both faster.
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-
-

contrib/otshell_utils/ccolor.cpp

@@ -1,116 +0,0 @@
-#include "ccolor.hpp"
-#include <cstdarg>
-
-// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal
-// from http://wiznet.gr/src/ccolor.zip
-// edited by rfree - as part of https://github.com/rfree/Open-Transactions/
-
-using namespace std;
-
-#ifdef _MSC_VER
-
-#define snprintf c99_snprintf
-
-inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) {
-	int count = -1;
-	if (size != 0)
-		 count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
-	if (count == -1)
-		 count = _vscprintf(format, ap);
-	return count;
-}
-
-inline int c99_snprintf(char* str, size_t size, const char* format, ...) {
-	int count;
-	va_list ap;
-	va_start(ap, format);
-	count = c99_vsnprintf(str, size, format, ap);
-	va_end(ap);
-	return count;
-}
-#endif // _MSC_VER
-
-#define CC_CONSOLE_COLOR_DEFAULT "\033[0m"
-#define CC_FORECOLOR(C) "\033[" #C "m"
-#define CC_BACKCOLOR(C) "\033[" #C "m"
-#define CC_ATTR(A) "\033[" #A "m"
-
-namespace zkr
-{
-	enum Color
-	{
-		Black,
-		Red,
-		Green,
-		Yellow,
-		Blue,
-		Magenta,
-		Cyan,
-		White,
-		Default = 9
-	};
-
-	enum Attributes
-	{
-		Reset,
-		Bright,
-		Dim,
-		Underline,
-		Blink,
-		Reverse,
-		Hidden
-	};
-
-	char * cc::color(int attr, int fg, int bg)
-	{
-				static const int size = 20;
-        static char command[size];
-
-        /* Command is the control command to the terminal */
-        snprintf(command, size, "%c[%d;%d;%dm", 0x1B, attr, fg + 30, bg + 40);
-        return command;
-	}
-
-
-	const char *cc::console = CC_CONSOLE_COLOR_DEFAULT;
-	const char *cc::underline = CC_ATTR(4);
-	const char *cc::bold = CC_ATTR(1);
-
-	const char *cc::fore::black = CC_FORECOLOR(30);
-	const char *cc::fore::blue = CC_FORECOLOR(34);
-	const char *cc::fore::red = CC_FORECOLOR(31);
-	const char *cc::fore::magenta = CC_FORECOLOR(35);
-	const char *cc::fore::green = CC_FORECOLOR(92);
-	const char *cc::fore::cyan = CC_FORECOLOR(36);
-	const char *cc::fore::yellow = CC_FORECOLOR(33);
-	const char *cc::fore::white = CC_FORECOLOR(37);
-	const char *cc::fore::console = CC_FORECOLOR(39);
-
-	const char *cc::fore::lightblack = CC_FORECOLOR(90);
-	const char *cc::fore::lightblue = CC_FORECOLOR(94);
-	const char *cc::fore::lightred = CC_FORECOLOR(91);
-	const char *cc::fore::lightmagenta = CC_FORECOLOR(95);
-	const char *cc::fore::lightgreen = CC_FORECOLOR(92);
-	const char *cc::fore::lightcyan = CC_FORECOLOR(96);
-	const char *cc::fore::lightyellow = CC_FORECOLOR(93);
-	const char *cc::fore::lightwhite = CC_FORECOLOR(97);
-
-	const char *cc::back::black = CC_BACKCOLOR(40);
-	const char *cc::back::blue = CC_BACKCOLOR(44);
-	const char *cc::back::red = CC_BACKCOLOR(41);
-	const char *cc::back::magenta = CC_BACKCOLOR(45);
-	const char *cc::back::green = CC_BACKCOLOR(42);
-	const char *cc::back::cyan = CC_BACKCOLOR(46);
-	const char *cc::back::yellow = CC_BACKCOLOR(43);
-	const char *cc::back::white = CC_BACKCOLOR(47);
-	const char *cc::back::console = CC_BACKCOLOR(49);
-
-	const char *cc::back::lightblack = CC_BACKCOLOR(100);
-	const char *cc::back::lightblue = CC_BACKCOLOR(104);
-	const char *cc::back::lightred = CC_BACKCOLOR(101);
-	const char *cc::back::lightmagenta = CC_BACKCOLOR(105);
-	const char *cc::back::lightgreen = CC_BACKCOLOR(102);
-	const char *cc::back::lightcyan = CC_BACKCOLOR(106);
-	const char *cc::back::lightyellow = CC_BACKCOLOR(103);
-	const char *cc::back::lightwhite = CC_BACKCOLOR(107);
-}

contrib/otshell_utils/ccolor.hpp

@@ -1,73 +0,0 @@
-// ccolor.hpp
-
-// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal
-// from http://wiznet.gr/src/ccolor.zip
-// edited by rfree - as part of https://github.com/rfree/Open-Transactions/
-
-#ifndef INCLUDE_OT_ccolor
-#define INCLUDE_OT_ccolor
-
-#include <iostream>
-#include <stdio.h>
-
-namespace zkr
-{
-	class cc
-	{
-	public:
-
-		class fore
-		{
-		public:
-			static const char *black;
-			static const char *blue;
-			static const char *red;
-			static const char *magenta;
-			static const char *green;
-			static const char *cyan;
-			static const char *yellow;
-			static const char *white;
-			static const char *console;
-
-			static const char *lightblack;
-			static const char *lightblue;
-			static const char *lightred;
-			static const char *lightmagenta;
-			static const char *lightgreen;
-			static const char *lightcyan;
-			static const char *lightyellow;
-			static const char *lightwhite;
-		};
-
-		class back
-		{
-		public:
-			static const char *black;
-			static const char *blue;
-			static const char *red;
-			static const char *magenta;
-			static const char *green;
-			static const char *cyan;
-			static const char *yellow;
-			static const char *white;
-			static const char *console;
-
-			static const char *lightblack;
-			static const char *lightblue;
-			static const char *lightred;
-			static const char *lightmagenta;
-			static const char *lightgreen;
-			static const char *lightcyan;
-			static const char *lightyellow;
-			static const char *lightwhite;
-		};
-
-		static char *color(int attr, int fg, int bg);
-		static const char *console;
-		static const char *underline;
-		static const char *bold;
-	};
-}
-
-#endif
-

contrib/otshell_utils/lib_common1.hpp

@@ -1,53 +0,0 @@
-/* See other files here for the LICENCE that applies here. */
-
-
-#ifndef INCLUDE_OT_NEWCLI_COMMON1
-#define INCLUDE_OT_NEWCLI_COMMON1
-
-#include <string>
-#include <cstring>
-#include <vector>
-#include <map>
-#include <list>
-#include <algorithm>
-#include <iostream>
-#include <fstream>
-#include <sstream>
-#include <set>
-#include <iterator>
-#include <stdexcept>
-
-#include <functional>
-#include <memory>
-#include <atomic>
-#include <boost/thread.hpp>
-#include <boost/thread/mutex.hpp>
-#include <boost/thread/recursive_mutex.hpp>
-#include <boost/thread/lock_guard.hpp>
-
-
-// list of thigs from libraries that we pull into namespace nOT::nNewcli
-// we might still need to copy/paste it in few places to make IDEs pick it up correctly
-#define INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 \
-	using std::string; \
-	using std::vector; \
-	using std::vector; \
-	using std::list; \
-	using std::set; \
-	using std::map; \
-	using std::ostream; \
-	using std::istream; \
-	using std::cin; \
-	using std::cerr; \
-	using std::cout; \
-	using std::cerr; \
-	using std::endl; \
-	using std::function; \
-	using std::unique_ptr; \
-	using std::shared_ptr; \
-	using std::weak_ptr; \
-	using std::enable_shared_from_this; \
-	using boost::lock_guard; \
-
-#endif
-

contrib/otshell_utils/runoptions.cpp

@@ -1,69 +0,0 @@
-/* See other files here for the LICENCE that applies here. */
-/* See header file .hpp for info */
-
-#include "runoptions.hpp"
-
-#include "lib_common1.hpp"
-
-namespace nOT {
-
-INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces
-
-// (no debug - this is the default)
-// +nodebug (no debug)
-// +debug ...... --asdf
-// +debug +debugcerr .... --asfs
-// +debug +debugfile .... --asfs
-
-cRunOptions::cRunOptions()
-	: mRunMode(eRunModeCurrent), mDebug(false), mDebugSendToFile(false), mDebugSendToCerr(false)
-	,mDoRunDebugshow(false)
-{ }
-
-vector<string> cRunOptions::ExecuteRunoptionsAndRemoveThem(const vector<string> & args) {
-	vector<string> arg_clear; // will store only the arguments that are not removed
-
-	for (auto arg : args) {
-		bool thisIsRunoption=false;
-
-		if (arg.size()>0) {
-			if (arg.at(0) == '+') thisIsRunoption=true;
-		}
-
-		if (thisIsRunoption) Exec(arg); // ***
-		if (! thisIsRunoption) arg_clear.push_back(arg);
-	}
-
-	Normalize();
-
-	return arg_clear;
-}
-
-void cRunOptions::Exec(const string & runoption) { // eg: Exec("+debug");
-	if (runoption == "+nodebug") { mDebug=false; }
-	else if (runoption == "+debug") { mDebug=true; }
-	else if (runoption == "+debugcerr") { mDebug=true;  mDebugSendToCerr=true; }
-	else if (runoption == "+debugfile") { mDebug=true;  mDebugSendToFile=true; }
-	else if (runoption == "+demo") { mRunMode=eRunModeDemo; }
-	else if (runoption == "+normal") { mRunMode=eRunModeNormal; }
-	else if (runoption == "+current") { mRunMode=eRunModeCurrent; }
-	else if (runoption == "+debugshow") { mDebug=true;  mDebugSendToCerr=true;  mDoRunDebugshow=true; }
-	else {
-		cerr << "Unknown runoption in Exec: '" << runoption << "'" << endl;
-		throw std::runtime_error("Unknown runoption");
-	}
-	// cerr<<"debug="<<mDebug<<endl;
-}
-
-void cRunOptions::Normalize() {
-	if (mDebug) {
-		if (!(  mDebugSendToFile || mDebugSendToCerr  ))  mDebugSendToCerr=true; // if debug is on then send to something, e.g. to cerr
-	}
-}
-
-
-cRunOptions gRunOptions; // (extern)
-
-} // namespace OT
-
-

contrib/otshell_utils/runoptions.hpp

@@ -1,58 +0,0 @@
-/* See other files here for the LICENCE that applies here. */
-/*
-Template for new files, replace word "template" and later delete this line here.
-*/
-
-#ifndef INCLUDE_OT_NEWCLI_runoptions_hpp
-#define INCLUDE_OT_NEWCLI_runoptions_hpp
-
-#include "lib_common1.hpp"
-
-namespace nOT {
-
-INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces
-
-/** Global options to run this program main() Eg used for developer's special options like +setdemo +setdebug.
-This is NOT for all the other options that are parsed and executed by program. */
-class cRunOptions {
-	public:
-		enum tRunMode { ///< Type of run mode - is this normal, or demonstration etc.
-			eRunModeCurrent=1, ///< currently developed version
-			eRunModeDemo, ///< best currently available Demo of something nice
-			eRunModeNormal, ///< do the normal things that the program should do
-		};
-
-	private:
-		tRunMode mRunMode; ///< selected run mode
-
-		bool mDebug; // turn debug on, Eg: +debug without it probably nothing will be written to debug (maybe just error etc)
-		bool mDebugSendToFile; // send to file, Eg: for +debugfile ; also turns on debug
-		bool mDebugSendToCerr; // send to cerr, Eg: for +debugcerr ; also turns on debug
-		// if debug is set but not any other DebugSend* then we will default to sending to debugcerr
-
-		bool mDoRunDebugshow;
-
-	public:
-		tRunMode getTRunMode() const { return mRunMode; }
-		bool getDebug() const { return mDebug; }
-		bool getDebugSendToFile() const { return mDebugSendToFile; }
-		bool getDebugSendToCerr() const { return mDebugSendToCerr; }
-		bool getDoRunDebugshow() const { return mDoRunDebugshow; }
-
-		cRunOptions();
-
-		vector<string> ExecuteRunoptionsAndRemoveThem(const vector<string> & args);
-		void Exec(const string & runoption); // eg: Exec("+debug");
-
-		void Normalize();
-};
-
-extern cRunOptions gRunOptions;
-
-
-} // namespace nOT
-
-
-
-#endif
-

contrib/otshell_utils/utils.cpp

@@ -1,806 +0,0 @@
-/// @file
-/// @author rfree (current maintainer in monero.cc project)
-/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug
-
-/* See other files here for the LICENCE that applies here. */
-/* See header file .hpp for info */
-
-#include <algorithm>
-#include <functional>
-#include <cctype>
-#include <locale>
-#include <fstream>
-#include <iostream>
-#include <iomanip>
-#include <chrono>
-
-#include "utils.hpp"
-
-#include "ccolor.hpp"
-
-#include "lib_common1.hpp"
-
-#include "runoptions.hpp"
-
-#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined (WIN64)
-	#define OS_TYPE_WINDOWS
-#elif defined(__unix__) || defined(__posix) || defined(__linux) || defined(__darwin) || defined(__APPLE__) || defined(__clang__)
-	#define OS_TYPE_POSIX
-#else
-	#warning "Compiler/OS platform is not recognized. Just assuming it will work as POSIX then"
-	#define OS_TYPE_POSIX
-#endif
-
-#if defined(OS_TYPE_WINDOWS)
-	#include <windows.h>
-	#include <process.h>
-#elif defined(OS_TYPE_POSIX)
-	#include <sys/types.h>
-	#include <sys/stat.h>
-	#include <unistd.h>
-#else
-	#error "Compiler/OS platform detection failed - not supported"
-#endif
-
-
-namespace nOT {
-namespace nUtils {
-
-INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces
-
-// ====================================================================
-
-// Numerical values of the debug levels - see hpp
-const int _debug_level_nr_dbg3=20;
-const int _debug_level_nr_dbg2=30;
-const int _debug_level_nr_dbg1=40;
-const int _debug_level_nr_info=50;
-const int _debug_level_nr_note=60;
-const int _debug_level_nr_fact=75;
-const int _debug_level_nr_mark=80;
-const int _debug_level_nr_warn=90;
-const int _debug_level_nr_erro=100;
-
-// ====================================================================
-
-myexception::myexception(const char * what)
-	: std::runtime_error(what)
-{ }
-
-myexception::myexception(const std::string &what)
-	: std::runtime_error(what)
-{ }
-
-void myexception::Report() const {
-	_erro("Error: " << what());
-}
-
-//myexception::~myexception() { }
-
-// ====================================================================
-
-// text trimming
-// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
-std::string & ltrim(std::string &s) {
-	s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<int, int>(std::isspace))));
-	return s;
-}
-
-std::string & rtrim(std::string &s) {
-	s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
-  return s;
-}
-
-std::string & trim(std::string &s) {
-	return ltrim(rtrim(s));
-}
-
-std::string get_current_time() {
-  std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
-  time_t time_now = std::chrono::system_clock::to_time_t(now);
-  std::chrono::high_resolution_clock::duration duration = now.time_since_epoch();
-  int64_t micro = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
-
-  // std::localtime() - This function may not be thread-safe.
-	#ifdef OS_TYPE_WINDOWS
-		struct tm * tm_pointer = std::localtime( &time_now ); // thread-safe on mingw-w64 (thread local variable) and on MSVC btw
-		// http://stackoverflow.com/questions/18551409/localtime-r-support-on-mingw
-		// tm_pointer points to thread-local data, memory is owned/managed by the system/library
-	#else
-		// linux, freebsd, have this
-		struct tm tm_object; // automatic storage duration http://en.cppreference.com/w/cpp/language/storage_duration
-		struct tm * tm_pointer = & tm_object; // just point to our data
-		auto x = localtime_r( &time_now , tm_pointer    ); // modifies our own (this thread) data in tm_object, this is safe http://linux.die.net/man/3/localtime_r
-		if (x != tm_pointer) return "(internal error in get_current_time)"; // redundant check in case of broken implementation of localtime_r
-	#endif
-	// tm_pointer now points to proper time data, and that memory is automatically managed
-	if (!tm_pointer) return "(internal error in get_current_time - NULL)"; // redundant check in case of broken implementation of used library methods
-
-	std::stringstream stream;
-		stream << std::setfill('0')
-		<< std::setw(2) << tm_pointer->tm_year+1900
-		<< '-' << std::setw(2) << tm_pointer->tm_mon+1
-		<< '-' << std::setw(2) << tm_pointer->tm_mday
-		<< ' ' << std::setw(2) << tm_pointer->tm_hour
-		<< ':' << std::setw(2) << tm_pointer->tm_min
-		<< ':' << std::setw(2) << tm_pointer->tm_sec
-		<< '.' << std::setw(6) << (micro%1000000); // 6 because microseconds
-  return stream.str();
-}
-
-cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data)
-
-boost::recursive_mutex gLoggerGuard; // extern
-std::atomic<int> gLoggerGuardDepth; // extern
-
-std::atomic<int> & gLoggerGuardDepth_Get() {
-	// TODO std::once would be nicer here
-
-	static bool once=0;
-
-	if (!once) { // initialize it once
-		once=1;
-		gLoggerGuardDepth=0;	
-	}
-
-	return gLoggerGuardDepth; // global, atomic counter
-}
-
-
-// ====================================================================
-
-namespace nDetail {
-
-const char* DbgShortenCodeFileName(const char *s) {
-	const char *p = s;
-	const char *a = s;
-
-	bool inc=1;
-	while (*p) {
-		++p;
-		if (inc && ('\0' != * p)) { a=p; inc=false; } // point to the current character (if valid) becasue previous one was slash
-		if ((*p)=='/') { a=p; inc=true; } // point at current slash (but set inc to try to point to next character)
-	}
-	return a;
-}
-
-}
-
-// a workaround for MSVC compiler; e.g. see https://bugs.webkit.org/show_bug.cgi?format=multiple&id=125795
-#ifndef _MSC_VER
-template<typename T, typename ...Args>
-std::unique_ptr<T> make_unique( Args&& ...args )
-{
-	return std::unique_ptr<T>( new T( std::forward<Args>(args)... ) );
-}
-#else
-	using std::make_unique;
-#endif
-// ====================================================================
-
-char cFilesystemUtils::GetDirSeparatorSys() {
-	// TODO nicer os detection?
-	#if defined(OS_TYPE_POSIX)
-		return '/';
-	#elif defined(OS_TYPE_WINDOWS)
-		return '\\';
-	#else
-		#error "Do not know how to compile this for your platform."
-	#endif
-}
-
-char cFilesystemUtils::GetDirSeparatorInter() {
-	return '/';
-}
-
-string cFilesystemUtils::FileInternalToSystem(const std::string &name) {
-	string ret;
-	ret.resize(name.size());
-	std::replace_copy(name.begin(), name.end(), ret.begin(), 
-		GetDirSeparatorInter() , GetDirSeparatorSys());
-	return ret;
-}
-
-string cFilesystemUtils::FileSystemToInternal(const std::string &name) {
-	string ret;
-	ret.reserve(name.size());
-	std::replace_copy(name.begin(), name.end(), ret.begin(), 
-		GetDirSeparatorSys() , GetDirSeparatorInter());
-	return ret;
-}
-
-bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) {
-	const bool dbg=false;
-	//struct stat st;
-	const char dirchS = cFilesystemUtils::GetDirSeparatorSys();
-	const char dirchI = cFilesystemUtils::GetDirSeparatorInter();
-	std::istringstream iss(dir);
-	string partI; // current par is in internal format (though it should not matter since it doesn't contain any slashes). eg "bar"
-	string sofarS=""; // sofarS - the so far created dir part is in SYSTEM format. eg "foo/bar"
-	if (dir.size()<1) return false; // illegal name
-	// dir[0] is valid from here
-	if  ( only_below  &&  ((dir[0]==dirchS) || (dir[0]==dirchI))) return false; // no jumping to top (on any os)
-
-	while (getline(iss,partI,dirchI)) { // get new component eg "bar" into part
-		if (dbg) cout << '['<<partI<<']' << endl;
-		sofarS += partI;
-		if (partI.size()<1) return false; // bad format?
-		if ((only_below) && (partI=="..")) return false; // trying to go up
-
-		if (dbg) cout << "test ["<<sofarS<<"]"<<endl;
-		// TODO nicer os detection?
-		#if defined(OS_TYPE_POSIX)
-			struct stat st;
-			bool exists = stat(sofarS.c_str() ,&st) == 0; // *
-			if (exists) {
-				if (! S_ISDIR(st.st_mode)) {
-					// std::cerr << "This exists, but as a file: [" << sofar << "]" << (size_t)st.st_ino << endl;
-					return false; // exists but is a file nor dir
-				}
-			}
-		#elif defined(OS_TYPE_WINDOWS)
-			DWORD dwAttrib = GetFileAttributesA(sofarS.c_str());
-			bool exists = (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY));
-		#else
-			#error "Do not know how to compile this for your platform."
-		#endif
-
-		if (!exists) {
-			if (dbg) cout << "mkdir ["<<sofarS<<"]"<<endl;
-			#if defined(OS_TYPE_POSIX)
-				bool ok = 0==  mkdir(sofarS.c_str(), 0700); // ***
-			#elif defined(OS_TYPE_WINDOWS)
-				bool ok = (bool) CreateDirectoryA(sofarS.c_str(), NULL); // TODO use -W() after conversion to unicode UTF16
-			#else
-				#error "Do not know how to compile this for your platform."
-			#endif
-			if (!ok) return false;
-		}
-		sofarS += dirchS;
-	}
-	return true;
-}
-// ====================================================================
-
-namespace nDetail {
-
-struct channel_use_info { ///< feedback information about using (e.g. opening) given debug channel - used internally by logging system 
-/// TODO not yet used in code
-/// e.g. used to write into channel net/in/all that given message was a first logged message from never-before-logged thread or PID etc
-	bool m_was_interesting; ///< anything interesting happened when using the channel?
-	std::vector<std::string> m_extra_msg; ///< any additional messages about this channel use
-};	
-
-cDebugScopeGuard::cDebugScopeGuard() : mLevel(-1) {
-}
-	
-cDebugScopeGuard::~cDebugScopeGuard() {
-	if (mLevel != -1) {
-		gCurrentLogger.write_stream(mLevel,mChan) << mMsg << " ... end" << gCurrentLogger.endline() << std::flush;
-	}
-}
-
-void cDebugScopeGuard::Assign(const string &chan, const int level, const string &msg) {
-	mChan=chan;
-	mLevel=level;
-	mMsg=msg;
-}
-
-} // namespace nDetail
-
-// ====================================================================
-
-cLogger::cLogger() :
-mStream(NULL),
-mStreamBrokenDebug(NULL),
-mIsBroken(true), // before constructor finishes
-mLevel(_debug_level_nr_warn),
-mThread2Number_Biggest(0), // the CURRENT biggest value (no thread yet in map)
-mPid2Number_Biggest(0)
-{
-	mStream = & std::cout;
-	mStreamBrokenDebug = & std::cerr; // the backup stream
-	*mStreamBrokenDebug << "Creating the logger system" << endl;
-	mIsBroken=false; // ok, constr. succeeded, so string is not broken now
-
-	// this is here, because it could be using logging itself to log creation of first thread/PID etc
-	Thread2Number( boost::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1
-	Pid2Number( getpid() ); // add this proces ID as first one
-}
-
-cLogger::~cLogger() {
-	for (auto pair : mChannels) {
-		std::ofstream *ptr = pair.second;
-		delete ptr;
-		pair.second=NULL;
-	}
-}
-
-void cLogger::SetStreamBroken() {
-	SetStreamBroken("(no additional details about this problem)");
-}
-
-void cLogger::SetStreamBroken(const std::string &msg) {
-	_dbg_dbg("Stream is broken (msg: " << msg << ")");
-	if (!mIsBroken) { // if not already marked as broken
-		_dbg_dbg("(It was not broken before)");
-		std::cerr << OT_CODE_STAMP << "WARNING: due to a problem in the debug/logging system itself ("<<msg<<") - we are switching back to fallback stream (e.g. cerr)" << std::endl;
-		if (mStreamBrokenDebug == nullptr) {
-			std::cerr << OT_CODE_STAMP << " ERROR: in addition, while reporting this problem, mStreamBrokenDebug stream is NULL: " << mStreamBrokenDebug << std::endl;
-		} else {
-			(*mStreamBrokenDebug) << OT_CODE_STAMP << "WARNING: due to debug stream problem ("<<msg<<") - switching back to fallback stream (e.g. cerr)" << std::endl;
-		}
-		mIsBroken = true;
-	}
-}
-
-std::ostream & cLogger::write_stream(int level) {
-	return write_stream(level,"");
-}
-
-std::ostream & cLogger::write_stream(int level, const std::string & channel ) {
-	_dbg_dbg("level="<<level<<" channel="<<channel);
-	if (level >= mLevel) {
-		if (mStream) { // TODO now disabling mStream also disables writting to any channel
-			_dbg_dbg("Selecting output...");
-			ostream & output = SelectOutput(level,channel);
-			_dbg_dbg("Selecting output... done, output=" << (void*)(&output));
-			#if defined(OS_TYPE_WINDOWS)
-			output << windows_stream(level);
-			#endif
-			output << icon(level) << ' ';
-			boost::thread::id this_id = boost::this_thread::get_id();
-			output << "{" << Thread2Number(this_id) << "}";
-			auto nicePid = Pid2Number(getpid());
-			if (nicePid>0) output << " {p" << nicePid << "}";
-			output << ' ';
-			return output; // <--- return
-		} else _dbg_dbg("Not writting: No mStream");
-	} else _dbg_dbg("Not writting: Too low level level="<<level<<" not >= mLevel="<<mLevel);
-	return g_nullstream;
-}
-
-std::string cLogger::GetLogBaseDir() const {
-	return "log";
-}
-
-void cLogger::OpenNewChannel(const std::string & channel) noexcept {
-	try {
-		_dbg_dbg("Openning channel for channel="<<channel);
-		OpenNewChannel_(channel);
-	}
-	catch (const std::exception &except) {
-		SetStreamBroken(OT_CODE_STAMP + " Got exception when opening debug channel: " + ToStr(except.what()));
-	}
-	catch (...) {
-		SetStreamBroken(OT_CODE_STAMP + " Got not-standard exception when opening debug channel.");
-	}
-}
-
-void cLogger::OpenNewChannel_(const std::string & channel) { // channel=="net/sleep"
-	_dbg_dbg("Openning channel for channel="<<channel);
-	size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparatorInter());
-
-	string fname_system; // the full file name in system format
-
-	if (last_split==string::npos) { // The channel name has no directory, eg channel=="test"
-		string dir = GetLogBaseDir();
-		string basefile = channel + ".log";
-		string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile;
-		fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <-
-	}
-	else { // there is a directory eg channel=="net/sleep"
-		// net/sleep 
-		//    ^----- last_split	
-		string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparatorInter() + channel.substr(0, last_split);
-		string basefile = channel.substr(last_split+1) + ".log";
-		string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile;
-		fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <-
-		bool dirok = cFilesystemUtils::CreateDirTree(dir);
-		if (!dirok) { string err = "In logger failed to open directory (" + dir +") for channel (" + channel +")"; throw std::runtime_error(err); }
-	}
-
-	_dbg_dbg("Openning fname_system="<<fname_system);
-	std::ofstream * thefile = new std::ofstream( fname_system.c_str() ); // file system
-	*thefile << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl;
-//    	cerr << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl;
-  	_dbg_dbg( "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" );
-	mChannels.insert( std::pair<string,std::ofstream*>(channel , thefile ) ); // <- created the channel mapping
-}
-
-std::ostream & cLogger::SelectOutput(int level, const std::string & channel) noexcept {
-	try {
-		if (mIsBroken) { 
-			_dbg_dbg("The stream is broken mIsBroken="<<mIsBroken<<" so will return backup stream");
-			return *mStreamBrokenDebug;
-		}
-		if (channel=="") { 
-			_dbg_dbg("No channel given (channel="<<channel<<") so will return main stream");
-			return *mStream;
-		}
-
-		auto obj = mChannels.find(channel);
-		if (obj == mChannels.end()) { // not found - need to make new channel
-			_dbg_dbg("No stream openened for channel="<<channel<<" so will create it now");
-			OpenNewChannel(channel); // <- create channel
-			obj = mChannels.find(channel); // find again
-			if (obj == mChannels.end()) { // still not found! something is wrong
-				SetStreamBroken( OT_CODE_STAMP + " WARNING: can not get stream for channel="+ToStr(channel)+" level="+ToStr(channel) );
-				return *mStreamBrokenDebug;
-			}
-		}
-		auto the_stream_ptr = obj->second;
-		_dbg_dbg("Found the stream file for channel="<<channel<<" as the_stream_ptr="<<the_stream_ptr);
-		ASRT(the_stream_ptr);
-		return *the_stream_ptr; // <--- RETURN
-	} 
-	catch (std::exception &except) { 
-		SetStreamBroken( OT_CODE_STAMP + " Got exception: " + ToStr(except.what()) );
-		_dbg_dbg("Exception! Returning broken stream");
-		return *mStreamBrokenDebug;
-	}
-	catch (...) {
-		SetStreamBroken( OT_CODE_STAMP + " Got not-standard exception.");
-		_dbg_dbg("Exception! Returning broken stream");
-		return *mStreamBrokenDebug;
-	}
-
-	// dead code
-}
-
-void cLogger::setOutStreamFile(const string &fname) { // switch to using this file
-	_mark("WILL SWITCH DEBUG NOW to file: " << fname);
-	mOutfile =  make_unique<std::ofstream>(fname);
-	mStream = & (*mOutfile);
-	_mark("Started new debug, to file: " << fname);
-}
-
-void cLogger::setOutStreamFromGlobalOptions() {
-	if ( gRunOptions.getDebug() ) {
-		if ( gRunOptions.getDebugSendToFile() ) {
-			mOutfile =  make_unique<std::ofstream> ("debuglog.txt");
-			mStream = & (*mOutfile);
-		}
-		else if ( gRunOptions.getDebugSendToCerr() ) {
-			mStream = & std::cerr;
-		}
-		else {
-			mStream = & g_nullstream;
-		}
-	}
-	else {
-		mStream = & g_nullstream;
-	}
-}
-
-void cLogger::setDebugLevel(int level) {
-	bool note_before = (mLevel > level); // report the level change before or after the change? (on higher level)
-	if (note_before) _note("Setting debug level to "<<level);
-	mLevel = level;
-	if (!note_before) _note("Setting debug level to "<<level);
-}
-
-std::string cLogger::icon(int level) const {
-	// TODO replan to avoid needles converting back and forth char*, string etc
-
-	using namespace zkr;
-    #if defined(OS_TYPE_POSIX)
-	if (level >= 100) return cc::back::lightred     + ToStr(cc::fore::lightyellow) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ;
-	if (level >=  90) return cc::back::lightyellow  + ToStr(cc::fore::black) + ToStr("Warn  ") + ToStr(cc::fore::red)+ " " ;
-	if (level >=  80) return cc::back::lightmagenta + ToStr(cc::fore::black) + ToStr("MARK  "); //+ zkr::cc::console + ToStr(cc::fore::lightmagenta)+ " ";
-    if (level >=  75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT  ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " ";
-	if (level >=  70) return cc::fore::green    + ToStr("Note  ");
-	if (level >=  50) return cc::fore::cyan   + ToStr("info  ");
-	if (level >=  40) return cc::fore::lightwhite    + ToStr("dbg   ");
-	if (level >=  30) return cc::fore::lightblue   + ToStr("dbg   ");
-	if (level >=  20) return cc::fore::blue    + ToStr("dbg   ");
-
-    #elif defined(OS_TYPE_WINDOWS)
-    if (level >= 100) return ToStr("ERROR ");
-    if (level >=  90) return ToStr("Warn  ");
-    if (level >=  80) return ToStr("MARK  ");
-    if (level >=  75) return ToStr("FACT  ");
-    if (level >=  70) return ToStr("Note  ");
-    if (level >=  50) return ToStr("info  ");
-    if (level >=  40) return ToStr("dbg   ");
-    if (level >=  30) return ToStr("dbg   ");
-    if (level >=  20) return ToStr("dbg   ");
-    #endif
-
-	return "  ";
-}
-
-std::string cLogger::endline() const {
-	#if defined(OS_TYPE_POSIX)
-	return ToStr("") + zkr::cc::console + ToStr("\n"); // TODO replan to avoid needles converting back and forth char*, string etc
-    #elif defined(OS_TYPE_WINDOWS)
-    return ToStr("\n");
-    #endif
-}
-
-int cLogger::Thread2Number(const boost::thread::id id) {
-	auto found = mThread2Number.find( id );
-	if (found == mThread2Number.end()) { // new one
-		mThread2Number_Biggest++;
-		mThread2Number[id] = mThread2Number_Biggest;
-		_info_c("dbg/main", "This is a new thread (used in debug), thread id="<<id); // can cause some recursion
-		return mThread2Number_Biggest;
-	} else {
-		return mThread2Number[id];
-	}
-}
-
-int cLogger::Pid2Number(const t_anypid id) {
-	auto found = mPid2Number.find( id );
-	if (found == mPid2Number.end()) { // new one
-		mPid2Number_Biggest++;
-		mPid2Number[id] = mPid2Number_Biggest;
-		_info_c("dbg/main", "This is a new process (used in debug), process pid="<<id); // can cause some recursion
-		return mPid2Number_Biggest;
-	} else {
-		return mPid2Number[id];
-	}
-}
-
-// ====================================================================
-// object gCurrentLogger is defined later - in global namespace below
-
-
-// ====================================================================
-// vector debug
-
-void DisplayStringEndl(std::ostream & out, const std::string text) {
-	out << text;
-	out << std::endl;
-}
-
-std::string SpaceFromEscape(const std::string &s) {
-	std::ostringstream  newStr;
-		for(size_t i = 0; i < s.length();i++) {
-			if(s[i] == '\\' && s[i+1] ==32)
-				newStr<<"";
-			else
-				newStr<<s[i];
-			}
-	return newStr.str();
-}
-
-std::string EscapeFromSpace(const std::string &s) {
-	std::ostringstream  newStr;
-	for(size_t i = 0; i < s.length();i++) {
-		if(s[i] == 32)
-			newStr << "\\" << " ";
-		else
-			newStr << s[i];
-	}
-	return newStr.str();
-}
-
-
-std::string EscapeString(const std::string &s) {
-	std::ostringstream  newStr;
-		for(size_t i = 0; i < s.length();i++) {
-			if(s[i] >=32 && s[i] <= 126)
-				newStr<<s[i];
-			else
-				newStr<<"\\"<< (int) s[i];
-			}
-
-	return newStr.str();
-}
-
-
-bool CheckIfBegins(const std::string & beggining, const std::string & all) {
-	if (all.compare(0, beggining.length(), beggining) == 0) {
-		return 1;
-	}
-	else {
-		return 0;
-	}
-}
-
-bool CheckIfEnds (std::string const & ending, std::string const & all){
-	if (all.length() >= ending.length()) {
-		return (0 == all.compare (all.length() - ending.length(), ending.length(), ending));
-	} else {
-		return false;
-	}
-}
-
-
-vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib) {
-	vector<string> ret;
-	for ( auto rec : possib) { // check of possibilities
-		if (CheckIfBegins(sofar,rec)) {
-			rec = EscapeFromSpace(rec);
-			ret.push_back(rec); // this record matches
-		}
-	}
-	return ret;
-}
-
-char GetLastChar(const std::string & str) { // TODO unicode?
-	auto s = str.length();
-	if (s==0) throw std::runtime_error("Getting last character of empty string (" + ToStr(s) + ")" + OT_CODE_STAMP);
-	return str.at( s - 1);
-}
-
-std::string GetLastCharIf(const std::string & str) { // TODO unicode?
-	auto s = str.length();
-	if (s==0) return ""; // empty string signalizes ther is nothing to be returned
-	return std::string( 1 , str.at( s - 1) );
-}
-
-// ====================================================================
-
-// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no.
-// Use it like this: ASRT( x>y );  with the semicolon at end, a clever trick forces this syntax :)
-
-void Assert(bool result, const std::string &stamp, const std::string &condition) {
-	if (!result) {
-		_erro("Assert failed at "+stamp+": ASSERT( " << condition << ")");
-		throw std::runtime_error("Assert failed at "+stamp+": ASSERT( " + condition + ")");
-	}
-}
-
-// ====================================================================
-// advanced string
-
-const std::string GetMultiline(string endLine) {
-	std::string result(""); // Taken from OT_CLI_ReadUntilEOF
-	while (true) {
-		std::string input_line("");
-		if (std::getline(std::cin, input_line, '\n'))
-		{
-			input_line += "\n";
-				if (input_line[0] == '~')
-					break;
-			result += input_line;
-		}
-		if (std::cin.eof() )
-		{
-			std::cin.clear();
-				break;
-		}
-		if (std::cin.fail() )
-		{
-			std::cin.clear();
-				break;
-		}
-		if (std::cin.bad())
-		{
-			std::cin.clear();
-				break;
-		}
-	}
-	return result;
-}
-
-vector<string> SplitString(const string & str){
-		std::istringstream iss(str);
-		vector<string> vec { std::istream_iterator<string>{iss}, std::istream_iterator<string>{} };
-		return vec;
-}
-
-bool checkPrefix(const string & str, char prefix) {
-	if (str.at(0) == prefix)
-		return true;
-	return false;
-}
-
-// ====================================================================
-// operation on files
-
-
-#ifdef __unix
-
-void cEnvUtils::GetTmpTextFile() {
-	// TODO make this name configurable (depending on project)
-	char filename[] = "/tmp/otshellutils_text.XXXXXX";
-	fd = mkstemp(filename);
-	if (fd == -1) {
-		_erro("Can't create the file: " << filename);
-		return;
-	}
-	mFilename = filename;
-}
-
-void cEnvUtils::CloseFile() {
-	close(fd);
-	unlink( mFilename.c_str() );
-}
-
-void  cEnvUtils::OpenEditor() {
-	char* editor = std::getenv("OT_EDITOR"); //TODO Read editor from configuration file
-	if (editor == NULL)
-		editor = std::getenv("VISUAL");
-	if (editor == NULL)
-		editor = std::getenv("EDITOR");
-
-	string command;
-	if (editor != NULL)
-		command = ToStr(editor) + " " + mFilename;
-	else
-		command = "/usr/bin/editor " + mFilename;
-	_dbg3("Opening editor with command: " << command);
-	if ( system( command.c_str() ) == -1 )
-		_erro("Cannot execute system command: " << command);
-}
-
-const string cEnvUtils::ReadFromTmpFile() {
-	std::ifstream ifs(mFilename);
-	string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
-	return msg;
-}
-
-const string cEnvUtils::Compose() {
-	GetTmpTextFile();
-	OpenEditor();
-	string input = ReadFromTmpFile();
-	CloseFile();
-	return input;
-}
-
-#endif
-
-const string cEnvUtils::ReadFromFile(const string path) {
-	std::ifstream ifs(path);
-	string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>());
-	return msg;
-}
-
-void hintingToTxt(std::fstream & file, string command, vector<string> &commands) {
-	if(file.good()) {
-		file<<command<<"~"<<endl;
-		for (auto a: commands) {
-			file <<a<< " ";
-			file.flush();
-		}
-		file<<endl;
-	}
-}
-
-string stringToColor(const string &hash) {
-  // Generete vector with all possible light colors
-  vector <string> lightColors;
-  using namespace zkr;
-  lightColors.push_back(cc::fore::lightblue);
-  lightColors.push_back(cc::fore::lightred);
-  lightColors.push_back(cc::fore::lightmagenta);
-  lightColors.push_back(cc::fore::lightgreen);
-  lightColors.push_back(cc::fore::lightcyan);
-  lightColors.push_back(cc::fore::lightyellow);
-  lightColors.push_back(cc::fore::lightwhite);
-
-  int sum=0;
-
-  for (auto ch : hash) sum+=ch;
-  auto color = sum%(lightColors.size()-1);
-
-  return lightColors.at( color );
-}
-
-
-// ====================================================================
-// algorthms
-
-
-} // namespace nUtil
-
-
-} // namespace OT
-
-// global namespace
-
-const extern int _dbg_ignore = 0; // see description in .hpp
-
-std::string GetObjectName() {
-	//static std::string * name=nullptr;
-	//if (!name) name = new std::string("(global)");
-	return "";
-}
-
-// ====================================================================
-
-nOT::nUtils::cLogger gCurrentLogger;
-

contrib/otshell_utils/utils.hpp

@@ -1,532 +0,0 @@
-/// @file
-/// @author rfree (current maintainer in monero.cc project)
-/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug
-
-/* See other files here for the LICENCE that applies here. */
-
-#include "ccolor.hpp"
-#ifndef INCLUDE_OT_NEWCLI_UTILS
-#define INCLUDE_OT_NEWCLI_UTILS
-
-#include "lib_common1.hpp"
-#ifdef __unix
-	#include <unistd.h>
-#endif
-
-#if defined(_WIN32)
-    #include"windows_stream.h"
-#endif
-
-#ifndef CFG_WITH_TERMCOLORS
-	//#error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)."
-#endif
-
-///Macros related to automatic deduction of class name etc; 
-#define MAKE_CLASS_NAME(NAME) private: static std::string GetObjectName() { return #NAME; }
-#define MAKE_STRUCT_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } public:
-
-// define this to debug the debug system itself:
-// #define opt_debug_debug
-
-#ifdef opt_debug_debug
-	#define _dbg_dbg(X) do { std::cerr<<"_dbg_dbg: " << OT_CODE_STAMP << " {thread=" << boost::this_thread::get_id()<<"} " \
-	<< " {pid="<<getpid()<<"} " <<  ": " << X << std::endl; } while(0)
-#else
-	#define _dbg_dbg(X) do { } while(0)
-#endif
-
-namespace nOT {
-
-namespace nUtils {
-
-/// @brief general based for my runtime errors
-class myexception : public std::runtime_error {
-	public:
-		myexception(const char * what);
-		myexception(const std::string &what);
-        //virtual ~myexception();
-		virtual void Report() const;
-};
-
-/// @macro Use this macro INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 as a shortcut for various using std::string etc.
-INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces
-
-// ======================================================================================
-/// text trimming functions (they do mutate the passes string); they trim based on std::isspace. also return it's reference again
-/// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
-std::string & trim(std::string &s); ///< trim text http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring
-std::string & ltrim(std::string &s); ///< left trim
-std::string & rtrim(std::string &s); ///< right trim
-
-// ======================================================================================
-
-std::string get_current_time();
-
-// string conversions
-template <class T>
-std::string ToStr(const T & obj) {
-	std::ostringstream oss;
-	oss << obj;
-	return oss.str();
-}
-
-struct cNullstream : std::ostream {
-	cNullstream() : std::ios(0), std::ostream(0) {}
-};
-extern cNullstream g_nullstream; // a stream that does nothing (eats/discards data)
-// ========== debug ==========
-// _dbg_ignore is moved to global namespace (on purpose)
-
-// TODO make _dbg_ignore thread-safe everywhere
-
-extern boost::recursive_mutex gLoggerGuard; // the mutex guarding logging/debugging code e.g. protecting streams, files, etc
-
-std::atomic<int> & gLoggerGuardDepth_Get(); // getter for the global singleton of counter (it guarantees initializing it to 0). This counter shows the current recursion (re-entrant) level of debug macros.
-
-// TODO more debug of the debug system:
-// detect lock() error e.g. recursive limit
-// detect stream e.g. operator<< error
-
-#define _debug_level(LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \
-	_dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" VAR: " << VAR ); \
-	auto level=LEVEL; short int part=0; \
-	try { \
-		boost::lock_guard<boost::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \
-		part=1; \
-		try { \
-			++nOT::nUtils::gLoggerGuardDepth_Get(); \
-/* int counter = nOT::nUtils::gLoggerGuardDepth_Get();  if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="<<counter<<gCurrentLogger.endline(); */ \
-			gCurrentLogger.write_stream(LEVEL,"") << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \
-			part=9; \
-		} catch(...) { \
-			gCurrentLogger.write_stream(std::max(level,90),"") << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << "(ERROR IN DEBUG)" << gCurrentLogger.endline(); \
-			--nOT::nUtils::gLoggerGuardDepth_Get(); throw ; \
-		} \
-		--nOT::nUtils::gLoggerGuardDepth_Get(); \
-	} catch(...) { if (part<8) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." <<gCurrentLogger.endline();   throw ; } \
-	} } while(0)
-
-// info for code below: oss object is normal stack variable, using it does not need lock protection
-#define _debug_level_c(CHANNEL,LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \
-	_dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" CHANNEL="<<CHANNEL<<" VAR: " << VAR ); \
-	auto level=LEVEL; short int part=0; \
-	try { \
-		boost::lock_guard<boost::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \
-		part=1; \
-		try { \
-			++nOT::nUtils::gLoggerGuardDepth_Get(); \
-			std::ostringstream oss; \
-			oss << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \
-			std::string as_string = oss.str(); \
-			_dbg_dbg("START will write to log LEVEL="<<LEVEL<<" to CHANNEL="<<CHANNEL<<" as_string="<<as_string); \
-/* int counter = nOT::nUtils::gLoggerGuardDepth_Get();  if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="<<counter<<gCurrentLogger.endline(); */ \
-			gCurrentLogger.write_stream(LEVEL,""     ) << as_string << gCurrentLogger.endline() << std::flush; \
-			gCurrentLogger.write_stream(LEVEL,CHANNEL) << as_string << gCurrentLogger.endline() << std::flush; \
-			_dbg_dbg("DONE will write to log LEVEL="<<LEVEL<<" to CHANNEL="<<CHANNEL<<" as_string="<<as_string); \
-			part=9; \
-		} catch(...) { \
-			gCurrentLogger.write_stream(std::max(level,90),CHANNEL) << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << "(ERROR IN DEBUG)" << gCurrentLogger.endline(); \
-			--nOT::nUtils::gLoggerGuardDepth_Get(); throw ; \
-		} \
-		--nOT::nUtils::gLoggerGuardDepth_Get(); \
-	} catch(...) { if (part<8) gCurrentLogger.write_stream(100,CHANNEL)<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." <<gCurrentLogger.endline();   throw ; } \
-	} } while(0)
-
-// Numerical values of the debug levels - are defined here as const ints. Full name (with namespace) given for clarity.
-extern const int _debug_level_nr_dbg3;
-extern const int _debug_level_nr_dbg2;
-extern const int _debug_level_nr_dbg1;
-extern const int _debug_level_nr_info;
-extern const int _debug_level_nr_note;
-extern const int _debug_level_nr_fact;
-extern const int _debug_level_nr_mark;
-extern const int _debug_level_nr_warn;
-extern const int _debug_level_nr_erro;
-
-#define _dbg3(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg3,VAR) // details - most detailed
-#define _dbg2(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg2,VAR) // details - a bit more important
-#define _dbg1(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg1,VAR) // details - more important
-#define _info(VAR) _debug_level( nOT::nUtils::_debug_level_nr_info,VAR) // information
-#define _note(VAR) _debug_level( nOT::nUtils::_debug_level_nr_note,VAR) // more interesting information
-#define _fact(VAR) _debug_level( nOT::nUtils::_debug_level_nr_fact,VAR) // interesting events that could be interesting even for user, for logical/business things
-#define _mark(VAR) _debug_level( nOT::nUtils::_debug_level_nr_mark,VAR) // marked actions
-#define _warn(VAR) _debug_level( nOT::nUtils::_debug_level_nr_warn,VAR) // some problems
-#define _erro(VAR) _debug_level( nOT::nUtils::_debug_level_nr_erro,VAR) // errors
-
-#define _dbg3_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg3, VAR) // details - most detailed
-#define _dbg2_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg2, VAR) // details - a bit more important
-#define _dbg1_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg1, VAR) // details - more important
-#define _info_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_info, VAR) // information
-#define _note_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_note, VAR) // more interesting information
-#define _fact_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_fact, VAR) // interesting events that could be interesting even for user, for logical/business things
-#define _mark_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_mark, VAR) // marked actions
-#define _warn_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_warn, VAR) // some problems
-#define _erro_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_erro, VAR) // errors
-
-// lock // because of VAR
-#define _scope_debug_level_c(CHANNEL,LEVEL,VAR) \
-	std::ostringstream debug_detail_oss; \
-	nOT::nUtils::gLoggerGuard.lock(); \
-	debug_detail_oss << OT_CODE_STAMP << ' ' << VAR ; \
-	nOT::nUtils::nDetail::cDebugScopeGuard debugScopeGuard; \
-	if (_dbg_ignore<LEVEL) debugScopeGuard.Assign(CHANNEL,LEVEL, debug_detail_oss.str()); \
-	if (_dbg_ignore<LEVEL) _debug_level_c(CHANNEL,LEVEL,debug_detail_oss.str() + " ... begin"); \
-	nOT::nUtils::gLoggerGuard.unlock();
-#define _scope_debug_level(LEVEL,VAR) _scope_debug_level_c("",LEVEL,VAR)
-
-#define _scope_dbg1(VAR) _scope_debug_level( _debug_level_nr_dbg3, VAR)
-#define _scope_dbg2(VAR) _scope_debug_level( _debug_level_nr_dbg2, VAR)
-#define _scope_dbg3(VAR) _scope_debug_level( _debug_level_nr_dbg1, VAR) 
-#define _scope_info(VAR) _scope_debug_level( _debug_level_nr_info, VAR) 
-#define _scope_note(VAR) _scope_debug_level( _debug_level_nr_note, VAR) 
-#define _scope_fact(VAR) _scope_debug_level( _debug_level_nr_fact, VAR) 
-#define _scope_mark(VAR) _scope_debug_level( _debug_level_nr_mark, VAR) 
-#define _scope_warn(VAR) _scope_debug_level( _debug_level_nr_warn, VAR) 
-#define _scope_erro(VAR) _scope_debug_level( _debug_level_nr_erro, VAR) 
-
-/***
-@brief do not use this namespace directly, it is implementation detail.
-*/
-namespace nDetail { 
-
-/***
-@brief a Debug scope-guard, to log a debug message when current scope is left. Do NOT use this directly,
-only use it via the macros like _scope_dbg1 etc.
-*/
-class cDebugScopeGuard {
-	protected:
-		string mMsg;
-		int mLevel;
-		string mChan;
-	public:
-		cDebugScopeGuard();
-		~cDebugScopeGuard();
-		void Assign(const string &chan, const int level, const string &msg);
-};
-
-const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to some part of the string that was given, skipping directory names, for log/debug
-
-} // namespace nDetail
-
-// ========== logger ==========
-
-namespace nDetail {
-	struct channel_use_info;
-} // namespace nDetail
-
-/*** 
-@brief Class to write debug into. Used it by calling the debug macros _dbg1(...) _info(...) _erro(...) etc, NOT directly!
-@author rfree (maintainer) 
-@thread this class is NOT thread safe and must used only by one thread at once (use it via ot_debug_macros like _info macro they do proper locking)
-*/
-class cLogger {
-	public:
-		cLogger();
-		~cLogger();
-		std::ostream & write_stream(int level); ///< starts a new message on given level (e.g. writes out the icon/tag) and returns stream to output to
-		std::ostream & write_stream(int level, const std::string & channel); ///< the same but with name of the debug channel
-
-		void setOutStreamFromGlobalOptions(); // set debug level, file etc - according to global Options
-		void setOutStreamFile(const std::string &fname); // switch to using this file
-		void setDebugLevel(int level); // change the debug level e.g. to mute debug from now
-
-		std::string icon(int level) const; ///< returns "icon" for given debug level. It is text, might include color controll characters
-		std::string endline() const; ///< returns string to be written at end of message
-
-	protected:
-		typedef long int t_anypid; // a portable representation of PID. long int should cover all platforms
-
-		void SetStreamBroken(); ///< call in case of internal error in logger (e.g. can not open a file)
-		void SetStreamBroken(const std::string &msg); ///< same but with error message
-
-		unique_ptr<std::ofstream> mOutfile;
-		std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream
-		std::ostream * mStreamBrokenDebug; ///< pointing only! this is a pointer to some stream that should be used when normal debugging is broken eg std::cerr
-		bool mIsBroken; ///< is the debugging system broken (this should be set when internal problems occur and should cause fallback to std::cerr)
-
-		std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class
-
-		int mLevel; ///< current debug level
-
-		std::ostream & SelectOutput(int level, const std::string & channel) noexcept; ///< returns a proper stream for this level and channel (always usable string)
-		void OpenNewChannel(const std::string & channel) noexcept; ///< tries to prepare this channel. does NOT guarantee to created mChannels[] entry!
-		void OpenNewChannel_(const std::string & channel);  ///< internal function, will throw in case of problems
-		std::string GetLogBaseDir() const;
-
-		std::map< boost::thread::id , int > mThread2Number; ///<  change long thread IDs into a short nice number to show
-		int mThread2Number_Biggest; ///<  current biggest value held there (biggest key) - works as growing-only counter basically
-		int Thread2Number(const boost::thread::id id); ///<  convert the system's thread id into a nice short our id; make one if new thread
-
-		std::map< t_anypid , int > mPid2Number; ///<  change long proces PID into a short nice number to show
-		int mPid2Number_Biggest; ///<  current biggest value held there (biggest key) - works as growing-only counter basically
-		int Pid2Number(const t_anypid id); ///<  convert the system's PID id into a nice short our id; make one if new thread
-};
-
-
-
-// ====================================================================
-// vector debug
-
-template <class T>
-std::string vectorToStr(const T & v) {
-	std::ostringstream oss;
-	for(auto rec: v) {
-		oss << rec <<",";
-		}
-	return oss.str();
-}
-
-template <class T>
-void DisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
-	std::copy( v.begin(), v.end(), std::ostream_iterator<T>(out, delim.c_str()) );
-}
-
-template <class T>
-void EndlDisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
-	out << std::endl;
-	DisplayVector(out,v,delim);
-}
-
-template <class T>
-void DisplayVectorEndl(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") {
-	DisplayVector(out,v,delim);
-	out << std::endl;
-}
-
-template <class T>
-void DbgDisplayVector(const std::vector<T> &v, const std::string &delim=" ") {
-	std::cerr << "[";
-	std::copy( v.begin(), v.end(), std::ostream_iterator<T>(std::cerr, delim.c_str()) );
-	std::cerr << "]";
-}
-
-string stringToColor(const string &hash);
-template <class T, class T2>
-void DisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") {
-	auto *no_color = zkr::cc::fore::console;
-	for(auto var : m) {
-		out << stringToColor(var.first) << var.first << delim << var.second << no_color << endl;
-	}
-
-}
-
-template <class T, class T2>
-void EndlDisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") {
-	out << endl;
-	for(auto var : m) {
-		out << var.first << delim << var.second << endl;
-	}
-}
-
-template <class T, class T2>
-void DbgDisplayMap(const std::map<T, T2> &m, const std::string &delim=" ") {
-	for(auto var : m) {
-		std::cerr << var.first << delim << var.second << endl;
-	}
-}
-
-
-template <class T>
-void DbgDisplayVectorEndl(const std::vector<T> &v, const std::string &delim=" ") {
-	DbgDisplayVector(v,delim);
-	std::cerr << std::endl;
-}
-
-void DisplayStringEndl(std::ostream & out, const std::string text);
-
-bool CheckIfBegins(const std::string & beggining, const std::string & all);
-bool CheckIfEnds (std::string const & ending, std::string const & all);
-std::string SpaceFromEscape(const std::string &s);
-std::string EscapeFromSpace(const std::string &s);
-vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib);
-char GetLastChar(const std::string & str);
-std::string GetLastCharIf(const std::string & str); // TODO unicode?
-std::string EscapeString(const std::string &s);
-
-
-template <class T>
-std::string DbgVector(const std::vector<T> &v, const std::string &delim="|") {
-	std::ostringstream oss;
-	oss << "[";
-	bool first=true;
-	for(auto vElement : v) { if (!first) oss<<delim; first=false; oss <<vElement ; }
-	oss << "]";
-	//std::copy( v.begin(), v.end(), std::ostream_iterator<T>(oss, delim.c_str()) );
-	return oss.str();
-}
-
-template <class T>
-std::ostream & operator<<(std::ostream & os, const map< T, vector<T> > & obj){
-	os << "[";
-	for(auto const & elem : obj) {
-		os << " [" << elem.first << "=" << DbgVector(elem.second) << "] ";
-	}
-	os << "]";
-  return os;
-}
-
-template <class T, class T2>
-std::string DbgMap(const map<T, T2> & map) {
-	std::ostringstream oss;
-		oss << map;
-	return oss.str();
-}
-
-// ====================================================================
-// assert
-
-// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no.
-// Use it like this: ASRT( x>y );  with the semicolon at end, a clever trick forces this syntax :)
-#define ASRT(x) do { if (!(x)) nOT::nUtils::Assert(false, OT_CODE_STAMP, #x); } while(0)
-
-void Assert(bool result, const std::string &stamp, const std::string &condition);
-
-// ====================================================================
-// advanced string
-
-const std::string GetMultiline(string endLine = "~");
-vector<string> SplitString(const string & str);
-
-bool checkPrefix(const string & str, char prefix = '^');
-
-// ====================================================================
-// nUse utils
-
-enum class eSubjectType {Account, Asset, User, Server, Unknown};
-
-string SubjectType2String(const eSubjectType & type);
-eSubjectType String2SubjectType(const string & type);
-
-// ====================================================================
-// operation on files
-
-/// @brief tools related to filesystem
-/// @author rfree (maintainer)
-class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later)
-	public:
-		static bool CreateDirTree(const std::string & dir, bool only_below=false);
-		static char GetDirSeparatorSys(); /// < eg '/' or '\'
-		static char GetDirSeparatorInter(); /// < internal is '/'
-		static string FileInternalToSystem(const std::string &name); ///< converts from internal file name string to system file name string
-		static string FileSystemToInternal(const std::string &name); ///< converts from system file name string to internal file name string
-};
-
-
-/// @brief utils to e.g. edit a file from console
-/// @author rfree (maintainer)
-class cEnvUtils {
-	int fd;
-	string mFilename;
-
-	void GetTmpTextFile();
-	void CloseFile();
-	void OpenEditor();
-	const string ReadFromTmpFile();
-public:
-	const string Compose();
-	const string ReadFromFile(const string path);
-};
-void hintingToTxt(std::fstream & file, string command, vector<string> &commands);
-void generateQuestions (std::fstream & file, string command);
-void generateAnswers (std::fstream & file, string command, vector<string> &completions);
-
-// ====================================================================
-
-namespace nOper { // nOT::nUtils::nOper
-// cool shortcut operators, like vector + vecotr operator working same as string (appending)
-// isolated to namespace because it's unorthodox ide to implement this
-
-using namespace std;
-
-// TODO use && and move?
-template <class T>
-vector<T> operator+(const vector<T> &a, const vector<T> &b) {
-	vector<T> ret = a;
-	ret.insert( ret.end() , b.begin(), b.end() );
-	return ret;
-}
-
-template <class T>
-vector<T> operator+(const T &a, const vector<T> &b) {
-	vector<T> ret(1,a);
-	ret.insert( ret.end() , b.begin(), b.end() );
-	return ret;
-}
-
-template <class T>
-vector<T> operator+(const vector<T> &a, const T &b) {
-	vector<T> b_vector(1,a);
-	return a + b_vector;
-}
-
-template <class T>
-vector<T>& operator+=(vector<T> &a, const vector<T> &b) {
-	a.insert( a.end() , b.begin(), b.end() );
-	return a;
-}
-
-// map
-template <class TK,class TV>
-map<TK,TV> operator+(const map<TK,TV> &a, const map<TK,TV> &b) {
-	map<TK,TV> ret = a;
-	for (const auto & elem : b) {
-		ret.insert(elem);
-	}
-	return ret;
-}
-
-
-} // nOT::nUtils::nOper
-
-// ====================================================================
-
-// ====================================================================
-
-// Algorithms
-
-// ====================================================================
-// ====================================================================
-
-
-/**
-@brief Special type that on creation will be initialized to have value INIT given as template argument. 
-Might be usefull e.g. to express in the declaration of class what will be the default value of member variable
-See also http://www.boost.org/doc/libs/1_56_0/libs/utility/value_init.htm
-Probably not needed when using boost in your project.
-*/
-template <class T, T INIT>
-class value_init {
-	private:
-		T data;
-	public:
-		value_init();
-
-		T& operator=(const T& v) { data=v; return *this; }
-		operator T const &() const { return data; }
-		operator T&() { return data;	}
-};
-
-template <class T, T INIT>
-value_init<T, INIT>::value_init() :	data(INIT) { }
-
-} // namespace nUtils
-
-} // namespace nOT
-
-
-// global namespace
-extern nOT::nUtils::cLogger gCurrentLogger; ///< The current main logger. Usually do not use it directly, instead use macros like _dbg1 etc
-
-std::string GetObjectName(); ///< Method to return name of current object; To use in debug; Can be shadowed in your classes. (Might be not used currently)
-
-const extern int _dbg_ignore; ///< the global _dbg_ignore, but local code (blocks, classes etc) you could shadow it in your code blocks, 
-// to override debug compile-time setting for given block/class, e.g. to disable debug in one of your methods or increase it there.
-// Or to make it runtime by providing a class normal member and editing it in runtime
-
-#define OT_CODE_STAMP ( nOT::nUtils::ToStr("[") + nOT::nUtils::nDetail::DbgShortenCodeFileName(__FILE__) + nOT::nUtils::ToStr("+") + nOT::nUtils::ToStr(__LINE__) + nOT::nUtils::ToStr(" ") + (GetObjectName()) + nOT::nUtils::ToStr("::") + nOT::nUtils::ToStr(__FUNCTION__) + nOT::nUtils::ToStr("]"))
-
-
-
-
-#endif
-

contrib/otshell_utils/windows_stream.cpp

@@ -1,64 +0,0 @@
-#if defined(_WIN32)
-#include "windows_stream.h"
-#include <windows.h>
-
-windows_stream::windows_stream(unsigned int pLevel)
-    :
-      mLevel(pLevel)
-{
-}
-
-std::ostream& operator << (std::ostream &stream, windows_stream const& object)
-{
-    HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
-
-    if (object.mLevel >= 100)
-    {
-        SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_RED | BACKGROUND_INTENSITY);
-        return stream;
-    }
-    if (object.mLevel >= 90)
-    {
-        SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_GREEN | BACKGROUND_RED | BACKGROUND_INTENSITY);
-        return stream;
-    }
-    if (object.mLevel >=  80)
-    {
-        SetConsoleTextAttribute(h_stdout, BACKGROUND_BLUE | BACKGROUND_RED | BACKGROUND_INTENSITY);
-        return stream;
-    }
-    if (object.mLevel >=  75)
-    {
-        SetConsoleTextAttribute(h_stdout, BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_INTENSITY);
-        return stream;
-    }
-    if (object.mLevel >=  70)
-    {
-        SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_INTENSITY);
-        return stream;
-    }
-    if (object.mLevel >=  50)
-    {
-        SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY);
-        return stream;
-    }
-    if (object.mLevel >=  40)
-    {
-        SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_RED | FOREGROUND_INTENSITY);
-        return stream;
-    }
-    if (object.mLevel >=  30)
-    {
-        SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY);
-        return stream;
-    }
-    if (object.mLevel >=  20)
-    {
-        SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE);
-        return stream;
-    }
-
-    return stream;
-}
-
-#endif

contrib/otshell_utils/windows_stream.h

@@ -1,20 +0,0 @@
-#ifndef WINDOWS_STREAM_H
-#define WINDOWS_STREAM_H
-
-#if defined(_WIN32)
-
-#include <string>
-#include <iostream>
-
-class windows_stream
-{
-public:
-    windows_stream(unsigned int pLevel);
-    friend std::ostream& operator<<(std::ostream &stream, windows_stream const& object);
-private:
-    unsigned int mLevel = 0;
-};
-
-#endif // _WIN32
-
-#endif // WINDOWS_STREAM_H

external/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 # 
 # All rights reserved.
 # 

external/boost/archive/portable_binary_archive.hpp

@@ -0,0 +1,55 @@
+#ifndef PORTABLE_BINARY_ARCHIVE_HPP
+#define PORTABLE_BINARY_ARCHIVE_HPP
+
+// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . 
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// MS compatible compilers support #pragma once
+#if defined(_MSC_VER)
+# pragma once
+#endif
+
+#include <boost/config.hpp>
+#include <boost/cstdint.hpp>
+#include <boost/static_assert.hpp>
+#include <boost/archive/archive_exception.hpp>
+
+#include <climits>
+#if CHAR_BIT != 8
+#error This code assumes an eight-bit byte.
+#endif
+
+#include <boost/archive/basic_archive.hpp>
+#include <boost/detail/endian.hpp>
+
+#include <boost/archive/impl/archive_serializer_map.ipp>
+
+namespace boost { namespace archive {
+
+enum portable_binary_archive_flags {
+    endian_big        = 0x4000,
+    endian_little     = 0x8000
+};
+
+//#if ( endian_big <= boost::archive::flags_last )
+//#error archive flags conflict
+//#endif
+
+inline void
+reverse_bytes(signed char size, char *address){
+    if (size <= 0)
+        throw archive_exception(archive_exception::other_exception);
+    char * first = address;
+    char * last = first + size - 1;
+    for(;first < last;++first, --last){
+        char x = *last;
+        *last = *first;
+        *first = x;
+    }
+}
+
+} }
+
+#endif // PORTABLE_BINARY_ARCHIVE_HPP

external/boost/archive/portable_binary_iarchive.hpp

@@ -0,0 +1,373 @@
+#ifndef PORTABLE_BINARY_IARCHIVE_HPP
+#define PORTABLE_BINARY_IARCHIVE_HPP
+
+// MS compatible compilers support #pragma once
+#if defined(_MSC_VER)
+# pragma once
+#endif
+
+#if defined(_MSC_VER)
+#pragma warning( push )
+#pragma warning( disable : 4244 )
+#endif
+
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// portable_binary_iarchive.hpp
+
+// (C) Copyright 2002-7 Robert Ramey - http://www.rrsd.com . 
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  See http://www.boost.org for updates, documentation, and revision history.
+
+#include <istream>
+#include <boost/version.hpp>
+#include <boost/serialization/string.hpp>
+#include <boost/serialization/item_version_type.hpp>
+#include <boost/archive/archive_exception.hpp>
+#include <boost/archive/basic_binary_iprimitive.hpp>
+#include <boost/archive/detail/common_iarchive.hpp>
+#include <boost/archive/detail/register_archive.hpp>
+
+#include <boost/archive/portable_binary_archive.hpp>
+#include <boost/archive/impl/basic_binary_iprimitive.ipp>
+
+namespace boost { namespace archive {
+
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// exception to be thrown if integer read from archive doesn't fit
+// variable being loaded
+class portable_binary_iarchive_exception : 
+    public boost::archive::archive_exception
+{
+public:
+    enum exception_code {
+        incompatible_integer_size
+    } m_exception_code ;
+    portable_binary_iarchive_exception(exception_code c = incompatible_integer_size ) :
+        boost::archive::archive_exception(boost::archive::archive_exception::other_exception),
+        m_exception_code(c)
+    {}
+    virtual const char *what( ) const throw( )
+    {
+        const char *msg = "programmer error";
+        switch(m_exception_code){
+        case incompatible_integer_size:
+            msg = "integer cannot be represented";
+            break;
+        default:
+            msg = boost::archive::archive_exception::what();
+            assert(false);
+            break;
+        }
+        return msg;
+    }
+};
+
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// "Portable" input binary archive.  It addresses integer size and endienness so 
+// that binary archives can be passed across systems. Note:floating point types
+// not addressed here
+class portable_binary_iarchive :
+    public boost::archive::basic_binary_iprimitive<
+        portable_binary_iarchive,
+        std::istream::char_type, 
+        std::istream::traits_type
+    >,
+    public boost::archive::detail::common_iarchive<
+        portable_binary_iarchive
+    >
+    {
+    typedef boost::archive::basic_binary_iprimitive<
+        portable_binary_iarchive,
+        std::istream::char_type, 
+        std::istream::traits_type
+    > primitive_base_t;
+    typedef boost::archive::detail::common_iarchive<
+        portable_binary_iarchive
+    > archive_base_t;
+#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+public:
+#else
+    friend archive_base_t;
+    friend primitive_base_t; // since with override load below
+    friend class boost::archive::detail::interface_iarchive<
+        portable_binary_iarchive
+    >;
+    friend class boost::archive::load_access;
+protected:
+#endif
+    unsigned int m_flags;
+    void load_impl(boost::intmax_t & l, char maxsize);
+
+    // default fall through for any types not specified here
+    template<class T>
+    void load(T & t){
+        boost::intmax_t l;
+        load_impl(l, sizeof(T));
+        // use cast to avoid compile time warning
+        //t = static_cast< T >(l);
+        t = T(l);
+    }
+    void load(boost::serialization::item_version_type & t){
+        boost::intmax_t l;
+        load_impl(l, sizeof(boost::serialization::item_version_type));
+        // use cast to avoid compile time warning
+        t = boost::serialization::item_version_type(l);
+    }
+    void load(boost::archive::version_type & t){
+        boost::intmax_t l;
+        load_impl(l, sizeof(boost::archive::version_type));
+        // use cast to avoid compile time warning
+        t = boost::archive::version_type(l);
+    }
+    void load(boost::archive::class_id_type & t){
+        boost::intmax_t l;
+        load_impl(l, sizeof(boost::archive::class_id_type));
+        // use cast to avoid compile time warning
+        t = boost::archive::class_id_type(static_cast<int>(l));
+    }
+    void load(std::string & t){
+        this->primitive_base_t::load(t);
+    }
+    #ifndef BOOST_NO_STD_WSTRING
+    void load(std::wstring & t){
+        this->primitive_base_t::load(t);
+    }
+    #endif
+    void load(float & t){
+        this->primitive_base_t::load(t);
+        // floats not supported
+        //BOOST_STATIC_ASSERT(false);
+    }
+    void load(double & t){
+        this->primitive_base_t::load(t);
+        // doubles not supported
+        //BOOST_STATIC_ASSERT(false);
+    }
+    void load(char & t){
+        this->primitive_base_t::load(t);
+    }
+    void load(unsigned char & t){
+        this->primitive_base_t::load(t);
+    }
+    typedef boost::archive::detail::common_iarchive<portable_binary_iarchive> 
+        detail_common_iarchive;
+#if BOOST_VERSION > 105800
+    template<class T>
+    void load_override(T & t){
+        this->detail_common_iarchive::load_override(t);
+    }
+    void load_override(boost::archive::class_name_type & t);
+    // binary files don't include the optional information 
+    void load_override(boost::archive::class_id_optional_type &){}
+#else
+    template<class T>
+    void load_override(T & t, int){
+        this->detail_common_iarchive::load_override(t, 0);
+    }
+    void load_override(boost::archive::class_name_type & t, int);
+    // binary files don't include the optional information 
+    void load_override(boost::archive::class_id_optional_type &, int){}
+#endif
+
+    void init(unsigned int flags);
+public:
+    portable_binary_iarchive(std::istream & is, unsigned flags = 0) :
+        primitive_base_t(
+            * is.rdbuf(), 
+            0 != (flags & boost::archive::no_codecvt)
+        ),
+        archive_base_t(flags),
+        m_flags(0)
+    {
+        init(flags);
+    }
+
+    portable_binary_iarchive(
+        std::basic_streambuf<
+            std::istream::char_type, 
+            std::istream::traits_type
+        > & bsb, 
+        unsigned int flags
+    ) :
+        primitive_base_t(
+            bsb, 
+            0 != (flags & boost::archive::no_codecvt)
+        ),
+        archive_base_t(flags),
+        m_flags(0)
+    {
+        init(flags);
+    }
+};
+
+} }
+
+// required by export in boost version > 1.34
+#ifdef BOOST_SERIALIZATION_REGISTER_ARCHIVE
+    BOOST_SERIALIZATION_REGISTER_ARCHIVE(portable_binary_iarchive)
+#endif
+
+// required by export in boost <= 1.34
+#define BOOST_ARCHIVE_CUSTOM_IARCHIVE_TYPES portable_binary_iarchive
+
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// portable_binary_iarchive.cpp
+
+// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . 
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  See http://www.boost.org for updates, documentation, and revision history.
+
+#include <istream>
+#include <string>
+
+#include <boost/detail/endian.hpp>
+#include <boost/serialization/throw_exception.hpp>
+#include <boost/archive/archive_exception.hpp>
+
+namespace boost { namespace archive {
+
+inline void 
+portable_binary_iarchive::load_impl(boost::intmax_t & l, char maxsize){
+    signed char size;
+    l = 0;
+    this->primitive_base_t::load(size);
+
+    if(0 == size){
+        return;
+    }
+
+    bool negative = (size < 0);
+    if(negative)
+        size = -size;
+
+    if(size > maxsize)
+        boost::serialization::throw_exception(
+            portable_binary_iarchive_exception()
+        );
+
+    char * cptr = reinterpret_cast<char *>(& l);
+    #ifdef BOOST_BIG_ENDIAN
+        cptr += (sizeof(boost::intmax_t) - size);
+    #endif
+    this->primitive_base_t::load_binary(cptr, size);
+
+    #ifdef BOOST_BIG_ENDIAN
+        if(m_flags & endian_little)
+    #else
+        if(m_flags & endian_big)
+    #endif
+    reverse_bytes(size, cptr);
+    
+    if(negative)
+        l = -l;
+}
+
+#if BOOST_VERSION > 105800
+inline void
+portable_binary_iarchive::load_override(
+    boost::archive::class_name_type & t
+){
+    std::string cn;
+    cn.reserve(BOOST_SERIALIZATION_MAX_KEY_SIZE);
+    load_override(cn);
+    if(cn.size() > (BOOST_SERIALIZATION_MAX_KEY_SIZE - 1))
+        boost::serialization::throw_exception(
+            boost::archive::archive_exception(
+                boost::archive::archive_exception::invalid_class_name)
+            );
+    std::memcpy(t, cn.data(), cn.size());
+    // borland tweak
+    t.t[cn.size()] = '\0';
+}
+#else
+inline void
+portable_binary_iarchive::load_override(
+    boost::archive::class_name_type & t, int
+){
+    std::string cn;
+    cn.reserve(BOOST_SERIALIZATION_MAX_KEY_SIZE);
+    load_override(cn, 0);
+    if(cn.size() > (BOOST_SERIALIZATION_MAX_KEY_SIZE - 1))
+        boost::serialization::throw_exception(
+            boost::archive::archive_exception(
+                boost::archive::archive_exception::invalid_class_name)
+            );
+    std::memcpy(t, cn.data(), cn.size());
+    // borland tweak
+    t.t[cn.size()] = '\0';
+}
+#endif
+
+inline void 
+portable_binary_iarchive::init(unsigned int flags){
+    if(0 == (flags & boost::archive::no_header)){
+        // read signature in an archive version independent manner
+        std::string file_signature;
+        * this >> file_signature;
+        if(file_signature != boost::archive::BOOST_ARCHIVE_SIGNATURE())
+            boost::serialization::throw_exception(
+                boost::archive::archive_exception(
+                    boost::archive::archive_exception::invalid_signature
+                )
+            );
+        // make sure the version of the reading archive library can
+        // support the format of the archive being read
+        boost::archive::library_version_type input_library_version;
+        * this >> input_library_version;
+
+        // ignore archive version checking
+        /*
+        // extra little .t is to get around borland quirk
+        if(boost::archive::BOOST_ARCHIVE_VERSION() < input_library_version)
+            boost::serialization::throw_exception(
+                boost::archive::archive_exception(
+                    boost::archive::archive_exception::unsupported_version
+                )
+            );
+        */
+        
+        #if BOOST_WORKAROUND(__MWERKS__, BOOST_TESTED_AT(0x3205))
+        this->set_library_version(input_library_version);
+        //#else
+        //#if ! BOOST_WORKAROUND(BOOST_MSVC, <= 1200)
+        //detail::
+        //#endif
+        boost::archive::detail::basic_iarchive::set_library_version(
+            input_library_version
+        );
+        #endif
+    }
+    unsigned char x;
+    load(x);
+    m_flags = x << CHAR_BIT;
+}
+
+} }
+
+namespace boost {
+namespace archive {
+
+namespace detail {
+    template class archive_serializer_map<portable_binary_iarchive>;
+}
+
+// template class basic_binary_iprimitive<
+//    portable_binary_iarchive,
+//    std::istream::char_type, 
+//    std::istream::traits_type
+//> ;
+
+} // namespace archive
+} // namespace boost
+
+#if defined(_MSC_VER)
+#pragma warning( pop )
+#endif
+
+#endif // PORTABLE_BINARY_IARCHIVE_HPP

external/boost/archive/portable_binary_oarchive.hpp

@@ -0,0 +1,315 @@
+#ifndef PORTABLE_BINARY_OARCHIVE_HPP
+#define PORTABLE_BINARY_OARCHIVE_HPP
+
+// MS compatible compilers support #pragma once
+#if defined(_MSC_VER)
+# pragma once
+#endif
+
+#if defined(_MSC_VER)
+#pragma warning( push )
+#pragma warning( disable : 4244 )
+#endif
+
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// portable_binary_oarchive.hpp
+
+// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . 
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  See http://www.boost.org for updates, documentation, and revision history.
+
+#include <ostream>
+#include <boost/version.hpp>
+#include <boost/serialization/string.hpp>
+#include <boost/archive/archive_exception.hpp>
+#include <boost/archive/basic_binary_oprimitive.hpp>
+#include <boost/archive/detail/common_oarchive.hpp>
+#include <boost/archive/detail/register_archive.hpp>
+
+#include <boost/archive/portable_binary_archive.hpp>
+#include <boost/archive/impl/basic_binary_oprimitive.ipp>
+
+namespace boost { namespace archive {
+
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// exception to be thrown if integer read from archive doesn't fit
+// variable being loaded
+class portable_binary_oarchive_exception : 
+    public boost::archive::archive_exception
+{
+public:
+    typedef enum {
+        invalid_flags 
+    } exception_code;
+    portable_binary_oarchive_exception(exception_code c = invalid_flags )
+    {}
+    virtual const char *what( ) const throw( )
+    {
+        const char *msg = "programmer error";
+        switch(code){
+        case invalid_flags:
+            msg = "cannot be both big and little endian";
+        default:
+            boost::archive::archive_exception::what();
+        }
+        return msg;
+    }
+};
+
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// "Portable" output binary archive.  This is a variation of the native binary 
+// archive. it addresses integer size and endienness so that binary archives can
+// be passed across systems. Note:floating point types not addressed here
+
+class portable_binary_oarchive :
+    public boost::archive::basic_binary_oprimitive<
+        portable_binary_oarchive,
+        std::ostream::char_type, 
+        std::ostream::traits_type
+    >,
+    public boost::archive::detail::common_oarchive<
+        portable_binary_oarchive
+    >
+{
+    typedef boost::archive::basic_binary_oprimitive<
+        portable_binary_oarchive,
+        std::ostream::char_type, 
+        std::ostream::traits_type
+    > primitive_base_t;
+    typedef boost::archive::detail::common_oarchive<
+        portable_binary_oarchive
+    > archive_base_t;
+#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+public:
+#else
+    friend archive_base_t;
+    friend primitive_base_t; // since with override save below
+    friend class boost::archive::detail::interface_oarchive<
+        portable_binary_oarchive
+    >;
+    friend class boost::archive::save_access;
+protected:
+#endif
+    unsigned int m_flags;
+    void save_impl(const boost::intmax_t l, const char maxsize);
+    // add base class to the places considered when matching
+    // save function to a specific set of arguments.  Note, this didn't
+    // work on my MSVC 7.0 system so we use the sure-fire method below
+    // using archive_base_t::save;
+
+    // default fall through for any types not specified here
+    template<class T>
+    void save(const T & t){
+        save_impl(t, sizeof(T));
+    }
+    void save(const std::string & t){
+        this->primitive_base_t::save(t);
+    }
+    #ifndef BOOST_NO_STD_WSTRING
+    void save(const std::wstring & t){
+        this->primitive_base_t::save(t);
+    }
+    #endif
+    void save(const float & t){
+        this->primitive_base_t::save(t);
+        // floats not supported
+        //BOOST_STATIC_ASSERT(false);
+    }
+    void save(const double & t){
+        this->primitive_base_t::save(t);
+        // doubles not supported
+        //BOOST_STATIC_ASSERT(false);
+    }
+    void save(const char & t){
+        this->primitive_base_t::save(t);
+    }
+    void save(const unsigned char & t){
+        this->primitive_base_t::save(t);
+    }
+
+    // default processing - kick back to base class.  Note the
+    // extra stuff to get it passed borland compilers
+    typedef boost::archive::detail::common_oarchive<portable_binary_oarchive> 
+        detail_common_oarchive;
+#if BOOST_VERSION > 105800
+    template<class T>
+    void save_override(T & t){
+        this->detail_common_oarchive::save_override(t);
+    }
+    // explicitly convert to char * to avoid compile ambiguities
+    void save_override(const boost::archive::class_name_type & t){
+        const std::string s(t);
+        * this << s;
+    }
+    // binary files don't include the optional information 
+    void save_override(
+        const boost::archive::class_id_optional_type & /* t */
+    ){}
+#else
+    template<class T>
+    void save_override(T & t, int){
+        this->detail_common_oarchive::save_override(t, 0);
+    }
+    // explicitly convert to char * to avoid compile ambiguities
+    void save_override(const boost::archive::class_name_type & t, int){
+        const std::string s(t);
+        * this << s;
+    }
+    // binary files don't include the optional information 
+    void save_override(
+        const boost::archive::class_id_optional_type & /* t */, int
+    ){}
+#endif
+
+    void init(unsigned int flags);
+public:
+    portable_binary_oarchive(std::ostream & os, unsigned flags = 0) :
+        primitive_base_t(
+            * os.rdbuf(),
+            0 != (flags & boost::archive::no_codecvt)
+        ),
+        archive_base_t(flags),
+        m_flags(flags & (endian_big | endian_little))
+    {
+        init(flags);
+    }
+
+    portable_binary_oarchive(
+        std::basic_streambuf<
+            std::ostream::char_type, 
+            std::ostream::traits_type
+        > & bsb, 
+        unsigned int flags
+    ) :
+        primitive_base_t(
+            bsb, 
+            0 != (flags & boost::archive::no_codecvt)
+        ),
+        archive_base_t(flags),
+        m_flags(0)
+    {
+        init(flags);
+    }
+};
+
+} }
+
+// required by export in boost version > 1.34
+#ifdef BOOST_SERIALIZATION_REGISTER_ARCHIVE
+    BOOST_SERIALIZATION_REGISTER_ARCHIVE(portable_binary_oarchive)
+#endif
+
+// required by export in boost <= 1.34
+#define BOOST_ARCHIVE_CUSTOM_OARCHIVE_TYPES portable_binary_oarchive
+
+/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8
+// portable_binary_oarchive.cpp
+
+// (C) Copyright 2002-7 Robert Ramey - http://www.rrsd.com . 
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+//  See http://www.boost.org for updates, documentation, and revision history.
+
+#include <ostream>
+#include <boost/detail/endian.hpp>
+
+namespace boost { namespace archive {
+
+inline void 
+portable_binary_oarchive::save_impl(
+    const boost::intmax_t l,
+    const char maxsize
+){
+    signed char size = 0;
+
+    if(l == 0){
+        this->primitive_base_t::save(size);
+        return;
+    }
+
+    boost::intmax_t ll;
+    bool negative = (l < 0);
+    if(negative)
+        ll = -l;
+    else
+        ll = l;
+
+    do{
+        ll >>= CHAR_BIT;
+        ++size;
+    }while(ll != 0);
+
+    this->primitive_base_t::save(
+        static_cast<signed char>(negative ? -size : size)
+    );
+
+    if(negative)
+        ll = -l;
+    else
+        ll = l;
+    char * cptr = reinterpret_cast<char *>(& ll);
+    #ifdef BOOST_BIG_ENDIAN
+        cptr += (sizeof(boost::intmax_t) - size);
+        if(m_flags & endian_little)
+            reverse_bytes(size, cptr);
+    #else
+        if(m_flags & endian_big)
+            reverse_bytes(size, cptr);
+    #endif
+    this->primitive_base_t::save_binary(cptr, size);
+}
+
+inline void 
+portable_binary_oarchive::init(unsigned int flags) {
+    if(m_flags == (endian_big | endian_little)){
+        boost::serialization::throw_exception(
+            portable_binary_oarchive_exception()
+        );
+    }
+    if(0 == (flags & boost::archive::no_header)){
+        // write signature in an archive version independent manner
+        const std::string file_signature(
+            boost::archive::BOOST_ARCHIVE_SIGNATURE()
+        );
+        * this << file_signature;
+        // ignore archive version checking
+        const boost::archive::library_version_type v{};
+        /*
+        // write library version
+        const boost::archive::library_version_type v(
+            boost::archive::BOOST_ARCHIVE_VERSION()
+        );
+        */
+        * this << v;
+    }
+    save(static_cast<unsigned char>(m_flags >> CHAR_BIT));
+}
+
+} }
+
+namespace boost {
+namespace archive {
+
+namespace detail {
+    template class archive_serializer_map<portable_binary_oarchive>;
+}
+
+// template class basic_binary_oprimitive<
+//     portable_binary_oarchive,
+//     std::ostream::char_type, 
+//     std::ostream::traits_type
+// > ;
+
+} // namespace archive
+} // namespace boost
+
+#if defined(_MSC_VER)
+#pragma warning( pop )
+#endif
+
+#endif // PORTABLE_BINARY_OARCHIVE_HPP

external/db_drivers/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 # 
 # All rights reserved.
 # 

external/db_drivers/liblmdb/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 # 
 # All rights reserved.
 # 
@@ -30,6 +30,11 @@ if(FREEBSD)
   add_definitions(-DMDB_DSYNC=O_SYNC)
 endif()
 
+if(ANDROID)
+  add_definitions("-DANDROID=1")
+endif()
+
+
 set (lmdb_sources
 mdb.c
 midl.c)

external/db_drivers/liblmdb/mdb.c

@@ -809,6 +809,16 @@ typedef struct MDB_txbody {
 	uint32_t	mtb_magic;
 		/** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */
 	uint32_t	mtb_format;
+		/**	The ID of the last transaction committed to the database.
+		 *	This is recorded here only for convenience; the value can always
+		 *	be determined by reading the main database meta pages.
+		 */
+	volatile txnid_t		mtb_txnid;
+		/** The number of slots that have been used in the reader table.
+		 *	This always records the maximum count, it is not decremented
+		 *	when readers release their slots.
+		 */
+	volatile unsigned	mtb_numreaders;
 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 	char	mtb_rmname[MNAME_LEN];
 #elif defined(MDB_USE_SYSV_SEM)
@@ -820,16 +830,6 @@ typedef struct MDB_txbody {
 		 */
 	mdb_mutex_t	mtb_rmutex;
 #endif
-		/**	The ID of the last transaction committed to the database.
-		 *	This is recorded here only for convenience; the value can always
-		 *	be determined by reading the main database meta pages.
-		 */
-	volatile txnid_t		mtb_txnid;
-		/** The number of slots that have been used in the reader table.
-		 *	This always records the maximum count, it is not decremented
-		 *	when readers release their slots.
-		 */
-	volatile unsigned	mtb_numreaders;
 } MDB_txbody;
 
 	/** The actual reader table definition. */
@@ -1953,13 +1953,15 @@ static void
 mdb_cursor_unref(MDB_cursor *mc)
 {
 	int i;
-	if (!mc->mc_snum || !mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0]))
-		return;
-	for (i=0; i<mc->mc_snum; i++)
-		mdb_page_unref(mc->mc_txn, mc->mc_pg[i]);
-	if (mc->mc_ovpg) {
-		mdb_page_unref(mc->mc_txn, mc->mc_ovpg);
-		mc->mc_ovpg = 0;
+	if (mc->mc_txn->mt_rpages[0].mid) {
+		if (!mc->mc_snum || !mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0]))
+			return;
+		for (i=0; i<mc->mc_snum; i++)
+			mdb_page_unref(mc->mc_txn, mc->mc_pg[i]);
+		if (mc->mc_ovpg) {
+			mdb_page_unref(mc->mc_txn, mc->mc_ovpg);
+			mc->mc_ovpg = 0;
+		}
 	}
 	mc->mc_snum = mc->mc_top = 0;
 	mc->mc_pg[0] = NULL;

external/miniupnpc/CMakeLists.txt

@@ -32,10 +32,10 @@ endif (NO_GETADDRINFO)
 if (NOT WIN32)
   add_definitions (-DMINIUPNPC_SET_SOCKET_TIMEOUT)
   add_definitions (-D_BSD_SOURCE -D_DEFAULT_SOURCE)
-  if (NOT CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+  if (NOT CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" AND NOT CMAKE_SYSTEM_NAME STREQUAL "DragonFly")
     # add_definitions (-D_POSIX_C_SOURCE=200112L)
     add_definitions (-D_XOPEN_SOURCE=600)
-  endif (NOT CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+  endif (NOT CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" AND NOT CMAKE_SYSTEM_NAME STREQUAL "DragonFly")
 else (NOT WIN32)
   add_definitions (-D_WIN32_WINNT=0x0501) # XP or higher for getnameinfo and friends
 endif (NOT WIN32)

external/miniupnpc/testdesc/new_LiveBox_desc.xml

@@ -87,4 +87,4 @@
       </device>
     </deviceList>
   </device>
-</root>
+</root>

external/unbound/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 # 
 # All rights reserved.
 # 
@@ -30,22 +30,6 @@ cmake_minimum_required(VERSION 2.8.7)
 
 project(unbound C)
 
-if (APPLE)
-  if (NOT OpenSSL_DIR)
-      EXECUTE_PROCESS(COMMAND brew --prefix openssl
-        OUTPUT_VARIABLE OPENSSL_ROOT_DIR
-        OUTPUT_STRIP_TRAILING_WHITESPACE)
-    message(STATUS "Using OpenSSL found at ${OPENSSL_ROOT_DIR}")
-  endif()
-endif()
-
-find_package(OpenSSL REQUIRED)
-if(STATIC)
-  if(UNIX)
-    set(OPENSSL_LIBRARIES "${OPENSSL_LIBRARIES};${CMAKE_DL_LIBS}")
-  endif()
-endif()
-
 find_package(Threads)
 
 include(configure_checks.cmake)

external/unbound/compat/getentropy_linux.c

@@ -30,7 +30,13 @@
 #ifdef HAVE_SYS_SYSCTL_H
 #include <sys/sysctl.h>
 #endif
+#ifdef __ANDROID__
+#include <sys/vfs.h> 
+#define statvfs statfs 
+#define fstatvfs fstatfs 
+#else
 #include <sys/statvfs.h>
+#endif
 #include <sys/socket.h>
 #include <sys/mount.h>
 #include <sys/mman.h>

include/INode.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 // 
 // All rights reserved.
 // 

include/IWallet.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 // 
 // All rights reserved.
 // 

snapcraft.yaml

@@ -1,5 +1,5 @@
 name: monero
-version: 0  # TODO: change this to release version in CI builds
+version: 0.10.1 # Current stable version
 summary: "Monero: the secure, private, untraceable cryptocurrency https://getmonero.org"
 description: |
     Monero is a private, secure, untraceable, decentralised digital currency.
@@ -9,37 +9,28 @@ grade: devel
 confinement: strict
 
 apps:
-    d:
+    monerod:
         daemon: forking
-        command: daemon.bash
+        command: |
+          monerod --detach --data-dir ${SNAP_DATA}
         plugs:
             - network
             - network-bind
-
-    log:
-        command: log.bash
-
-    monero:
-        command: wallet.bash
+    monero-wallet-rpc:
+        command: |
+            monero-wallet-rpc --log-file ${SNAP_USER_DATA}
         plugs:
+            - home
+            - network
+            - network-bind
+    monero-wallet-cli:
+        command: |
+            monero-wallet-cli --log-file ${SNAP_USER_DATA}
+        plugs:
+            - home
             - network
 
 parts:
-    wrapper:
-        plugin: dump
-        source: .
-        stage-packages:
-            - rlwrap
-        organize:
-            contrib/snap/daemon.bash: daemon.bash
-            contrib/snap/log.bash: log.bash
-            contrib/snap/wallet.bash: wallet.bash
-        snap:
-            - daemon.bash
-            - log.bash
-            - wallet.bash
-            - usr/bin/rlwrap
- 
     cmake-build:
         plugin: cmake
         configflags:
@@ -51,7 +42,6 @@ parts:
         source: .
         build-packages:
             - gcc
-            - cmake
             - pkg-config
             - libunbound-dev
             - libevent-dev
@@ -68,6 +58,8 @@ parts:
             - libminiupnpc10
             - libunbound2
             - libunwind8
-        snap:
+        prime:
             - bin
-            - usr
+            - usr/lib/
+            - -usr/lib/gcc
+            - -usr/share

src/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 #
 # All rights reserved.
 #
@@ -99,6 +99,7 @@ endfunction ()
 add_subdirectory(common)
 add_subdirectory(crypto)
 add_subdirectory(ringct)
+add_subdirectory(cryptonote_basic)
 add_subdirectory(cryptonote_core)
 add_subdirectory(blockchain_db)
 add_subdirectory(mnemonics)

src/blockchain_db/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 #
 # All rights reserved.
 #
@@ -63,6 +63,7 @@ target_link_libraries(blockchain_db
   PUBLIC
     common
     crypto
+    ringct
     ${LMDB_LIBRARY}
     ${BDB_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}

src/blockchain_db/berkeleydb/db_bdb.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are
@@ -31,7 +31,7 @@
 #include <memory>  // std::unique_ptr
 #include <cstring>  // memcpy
 
-#include "cryptonote_core/cryptonote_format_utils.h"
+#include "cryptonote_basic/cryptonote_format_utils.h"
 #include "crypto/crypto.h"
 #include "profile_tools.h"
 
@@ -1813,9 +1813,10 @@ bool BlockchainBDB::has_key_image(const crypto::key_image& img) const
 // Ostensibly BerkeleyDB has batch transaction support built-in,
 // so the following few functions will be NOP.
 
-void BlockchainBDB::batch_start(uint64_t batch_num_blocks)
+bool BlockchainBDB::batch_start(uint64_t batch_num_blocks)
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);
+    return false;
 }
 
 void BlockchainBDB::batch_commit()

src/blockchain_db/berkeleydb/db_bdb.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are
@@ -324,7 +324,7 @@ public:
                             );
 
   virtual void set_batch_transactions(bool batch_transactions);
-  virtual void batch_start(uint64_t batch_num_blocks=0);
+  virtual bool batch_start(uint64_t batch_num_blocks=0);
   virtual void batch_commit();
   virtual void batch_stop();
   virtual void batch_abort();

src/blockchain_db/blockchain_db.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 // 
 // All rights reserved.
 // 
@@ -29,9 +29,12 @@
 #include <boost/range/adaptor/reversed.hpp>
 
 #include "blockchain_db.h"
-#include "cryptonote_core/cryptonote_format_utils.h"
+#include "cryptonote_basic/cryptonote_format_utils.h"
 #include "profile_tools.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "blockchain.db"
+
 using epee::string_tools::pod_to_hex;
 
 namespace cryptonote
@@ -125,11 +128,7 @@ uint64_t BlockchainDB::add_block( const block& blk
   TIME_MEASURE_FINISH(time1);
   time_blk_hash += time1;
 
-  // call out to subclass implementation to add the block & metadata
-  time1 = epee::misc_utils::get_tick_count();
-  add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash);
-  TIME_MEASURE_FINISH(time1);
-  time_add_block1 += time1;
+  uint64_t prev_height = height();
 
   // call out to add the transactions
 
@@ -146,9 +145,12 @@ uint64_t BlockchainDB::add_block( const block& blk
   TIME_MEASURE_FINISH(time1);
   time_add_transaction += time1;
 
-  // DB's new height based on this added block is only incremented after this
-  // function returns, so height() here returns the new previous height.
-  uint64_t prev_height = height();
+  // call out to subclass implementation to add the block & metadata
+  time1 = epee::misc_utils::get_tick_count();
+  add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash);
+  TIME_MEASURE_FINISH(time1);
+  time_add_block1 += time1;
+
   m_hardfork->add(blk, prev_height);
 
   block_txn_stop();
@@ -198,6 +200,45 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
   remove_transaction_data(tx_hash, tx);
 }
 
+block BlockchainDB::get_block_from_height(const uint64_t& height) const
+{
+  blobdata bd = get_block_blob_from_height(height);
+  block b;
+  if (!parse_and_validate_block_from_blob(bd, b))
+    throw new DB_ERROR("Failed to parse block from blob retrieved from the db");
+
+  return b;
+}
+
+block BlockchainDB::get_block(const crypto::hash& h) const
+{
+  blobdata bd = get_block_blob(h);
+  block b;
+  if (!parse_and_validate_block_from_blob(bd, b))
+    throw new DB_ERROR("Failed to parse block from blob retrieved from the db");
+
+  return b;
+}
+
+bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) const
+{
+  blobdata bd;
+  if (!get_tx_blob(h, bd))
+    return false;
+  if (!parse_and_validate_tx_from_blob(bd, tx))
+    throw new DB_ERROR("Failed to parse transaction from blob retrieved from the db");
+
+  return true;
+}
+
+transaction BlockchainDB::get_tx(const crypto::hash& h) const
+{
+  transaction tx;
+  if (!get_tx(h, tx))
+    throw new TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str());
+  return tx;
+}
+
 void BlockchainDB::reset_stats()
 {
   num_calls = 0;

src/blockchain_db/blockchain_db.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -34,9 +34,10 @@
 #include <string>
 #include <exception>
 #include "crypto/hash.h"
-#include "cryptonote_core/cryptonote_basic.h"
-#include "cryptonote_core/difficulty.h"
-#include "cryptonote_core/hardfork.h"
+#include "cryptonote_protocol/blobdatatype.h"
+#include "cryptonote_basic/cryptonote_basic.h"
+#include "cryptonote_basic/difficulty.h"
+#include "cryptonote_basic/hardfork.h"
 
 /** \file
  * Cryptonote Blockchain Database Interface
@@ -655,16 +656,17 @@ public:
    * been called.  In either case, it should end the batch and write to its
    * backing store.
    *
-   * If a batch is already in-progress, this function should throw a DB_ERROR.
-   * This exception may change in the future if it is deemed necessary to
-   * have a more granular exception type for this scenario.
+   * If a batch is already in-progress, this function must return false.
+   * If a batch was started by this call, it must return true.
    *
    * If any of this cannot be done, the subclass should throw the corresponding
    * subclass of DB_EXCEPTION
    *
    * @param batch_num_blocks number of blocks to batch together
+   *
+   * @return true if we started the batch, false if already started
    */
-  virtual void batch_start(uint64_t batch_num_blocks=0) = 0;
+  virtual bool batch_start(uint64_t batch_num_blocks=0) = 0;
 
   /**
    * @brief ends a batch transaction
@@ -753,7 +755,20 @@ public:
    *
    * @return the block requested
    */
-  virtual block get_block(const crypto::hash& h) const = 0;
+  virtual cryptonote::blobdata get_block_blob(const crypto::hash& h) const = 0;
+
+  /**
+   * @brief fetches the block with the given hash
+   *
+   * Returns the requested block.
+   *
+   * If the block does not exist, the subclass should throw BLOCK_DNE
+   *
+   * @param h the hash to look for
+   *
+   * @return the block requested
+   */
+  block get_block(const crypto::hash& h) const;
 
   /**
    * @brief gets the height of the block with a given hash
@@ -783,7 +798,7 @@ public:
   virtual block_header get_block_header(const crypto::hash& h) const = 0;
 
   /**
-   * @brief fetch a block by height
+   * @brief fetch a block blob by height
    *
    * The subclass should return the block at the given height.
    *
@@ -792,9 +807,21 @@ public:
    *
    * @param height the height to look for
    *
+   * @return the block blob
+   */
+  virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const = 0;
+
+  /**
+   * @brief fetch a block by height
+   *
+   * If the block does not exist, that is to say if the blockchain is not
+   * that high, then the subclass should throw BLOCK_DNE
+   *
+   * @param height the height to look for
+   *
    * @return the block
    */
-  virtual block get_block_from_height(const uint64_t& height) const = 0;
+  block get_block_from_height(const uint64_t& height) const;
 
   /**
    * @brief fetch a block's timestamp
@@ -1008,16 +1035,38 @@ public:
   /**
    * @brief fetches the transaction with the given hash
    *
-   * The subclass should return the transaction stored which has the given
-   * hash.
-   *
    * If the transaction does not exist, the subclass should throw TX_DNE.
    *
    * @param h the hash to look for
    *
    * @return the transaction with the given hash
    */
-  virtual transaction get_tx(const crypto::hash& h) const = 0;
+  transaction get_tx(const crypto::hash& h) const;
+
+  /**
+   * @brief fetches the transaction with the given hash
+   *
+   * If the transaction does not exist, the subclass should return false.
+   *
+   * @param h the hash to look for
+   *
+   * @return true iff the transaction was found
+   */
+  bool get_tx(const crypto::hash& h, transaction &tx) const;
+
+  /**
+   * @brief fetches the transaction blob with the given hash
+   *
+   * The subclass should return the transaction stored which has the given
+   * hash.
+   *
+   * If the transaction does not exist, the subclass should return false.
+   *
+   * @param h the hash to look for
+   *
+   * @return true iff the transaction was found
+   */
+  virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0;
 
   /**
    * @brief fetches the total number of transactions ever
@@ -1169,7 +1218,7 @@ public:
    * @param offsets a list of amount-specific output indices
    * @param outputs return-by-reference a list of outputs' metadata
    */
-  virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) = 0;
+  virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) = 0;
   
   /*
    * FIXME: Need to check with git blame and ask what this does to

src/blockchain_db/db_types.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -34,7 +34,6 @@ namespace cryptonote
 
   const std::unordered_set<std::string> blockchain_db_types = 
   { "lmdb"
-  , "berkeley"
   };
 
 } // namespace cryptonote

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are
@@ -34,10 +34,14 @@
 #include <cstring>  // memcpy
 #include <random>
 
-#include "cryptonote_core/cryptonote_format_utils.h"
+#include "cryptonote_basic/cryptonote_format_utils.h"
 #include "crypto/crypto.h"
 #include "profile_tools.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "blockchain.db.lmdb"
+
+
 #if defined(__i386) || defined(__x86_64)
 #define MISALIGNED_OK	1
 #endif
@@ -373,7 +377,50 @@ void mdb_txn_safe::allow_new_txns()
   creation_gate.clear();
 }
 
+void lmdb_resized(MDB_env *env)
+{
+  mdb_txn_safe::prevent_new_txns();
 
+  MGINFO("LMDB map resize detected.");
+
+  MDB_envinfo mei;
+
+  mdb_env_info(env, &mei);
+  uint64_t old = mei.me_mapsize;
+
+  mdb_txn_safe::wait_no_active_txns();
+
+  int result = mdb_env_set_mapsize(env, 0);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to set new mapsize: ", result).c_str()));
+
+  mdb_env_info(env, &mei);
+  uint64_t new_mapsize = mei.me_mapsize;
+
+  MGINFO("LMDB Mapsize increased." << "  Old: " << old / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
+
+  mdb_txn_safe::allow_new_txns();
+}
+
+inline int lmdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
+{
+  int res = mdb_txn_begin(env, parent, flags, txn);
+  if (res == MDB_MAP_RESIZED) {
+    lmdb_resized(env);
+    res = mdb_txn_begin(env, parent, flags, txn);
+  }
+  return res;
+}
+
+inline int lmdb_txn_renew(MDB_txn *txn)
+{
+  int res = mdb_txn_renew(txn);
+  if (res == MDB_MAP_RESIZED) {
+    lmdb_resized(mdb_txn_env(txn));
+    res = mdb_txn_renew(txn);
+  }
+  return res;
+}
 
 void BlockchainLMDB::do_resize(uint64_t increase_size)
 {
@@ -388,14 +435,14 @@ void BlockchainLMDB::do_resize(uint64_t increase_size)
     boost::filesystem::space_info si = boost::filesystem::space(path);
     if(si.available < add_size)
     {
-      LOG_PRINT_RED_L0("!! WARNING: Insufficient free space to extend database !!: " << si.available / 1LL << 20L);
+      MERROR("!! WARNING: Insufficient free space to extend database !!: " << si.available / 1LL << 20L);
       return;
     }
   }
   catch(...)
   {
     // print something but proceed.
-    LOG_PRINT_YELLOW("Unable to query free disk space.", LOG_LEVEL_0);
+    MWARNING("Unable to query free disk space.");
   }
 
   MDB_envinfo mei;
@@ -437,7 +484,7 @@ void BlockchainLMDB::do_resize(uint64_t increase_size)
   if (result)
     throw0(DB_ERROR(lmdb_error("Failed to set new mapsize: ", result).c_str()));
 
-  LOG_PRINT_GREEN("LMDB Mapsize increased." << "  Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB", LOG_LEVEL_0);
+  MGINFO("LMDB Mapsize increased." << "  Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
 
   mdb_txn_safe::allow_new_txns();
 }
@@ -521,7 +568,7 @@ void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
   // size-based check
   if (need_resize(threshold_size))
   {
-    LOG_PRINT_L0("[batch] DB resize needed");
+    MGINFO("[batch] DB resize needed");
     do_resize(increase_size);
   }
 }
@@ -543,6 +590,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
   uint64_t min_block_size = 4 * 1024;
 
   uint64_t block_stop = 0;
+  uint64_t m_height = height();
   if (m_height > 1)
     block_stop = m_height - 1;
   uint64_t block_start = 0;
@@ -556,7 +604,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
   {
     LOG_PRINT_L1("No existing blocks to check for average block size");
   }
-  else if (m_cum_count)
+  else if (m_cum_count >= num_prev_blocks)
   {
     avg_block_size = m_cum_size / m_cum_count;
     LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size);
@@ -565,6 +613,9 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
   }
   else
   {
+    MDB_txn *rtxn;
+    mdb_txn_cursors *rcurs;
+    block_rtxn_start(&rtxn, &rcurs);
     for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
     {
       uint32_t block_size = get_block_size(block_num);
@@ -573,6 +624,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
       // some blocks were to be skipped for being outliers.
       ++num_blocks_used;
     }
+    block_rtxn_stop();
     avg_block_size = total_block_size / num_blocks_used;
     LOG_PRINT_L1("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
   }
@@ -593,6 +645,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
   mdb_txn_cursors *m_cursors = &m_wcursors;
+  uint64_t m_height = height();
 
   CURSOR(block_heights)
   blk_height bh = {blk_hash, m_height};
@@ -622,6 +675,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   CURSOR(blocks)
   CURSOR(block_info)
 
+  // this call to mdb_cursor_put will change height()
   MDB_val_copy<blobdata> blob(block_to_blob(blk));
   result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND);
   if (result)
@@ -654,6 +708,7 @@ void BlockchainLMDB::remove_block()
 
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
+  uint64_t m_height = height();
 
   if (m_height == 0)
     throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain"));
@@ -691,9 +746,10 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
   mdb_txn_cursors *m_cursors = &m_wcursors;
+  uint64_t m_height = height();
 
   int result;
-  uint64_t tx_id = m_num_txs;
+  uint64_t tx_id = num_txs();
 
   CURSOR(txs)
   CURSOR(tx_indices)
@@ -726,7 +782,6 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
   if (result)
     throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str()));
 
-  m_num_txs++;
   return tx_id;
 }
 
@@ -774,8 +829,6 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
   // Don't delete the tx_indices entry until the end, after we're done with val_tx_id
   if (mdb_cursor_del(m_cur_tx_indices, 0))
       throw1(DB_ERROR("Failed to add removal of tx index to db transaction"));
-
-  m_num_txs--;
 }
 
 uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
@@ -787,6 +840,8 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
   mdb_txn_cursors *m_cursors = &m_wcursors;
+  uint64_t m_height = height();
+  uint64_t m_num_outputs = num_outputs();
 
   int result = 0;
 
@@ -839,7 +894,6 @@ uint64_t BlockchainLMDB::add_output(const crypto::hash& tx_hash,
   if ((result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &data, MDB_APPENDDUP)))
       throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str()));
 
-  m_num_outputs++;
   return ok.amount_index;
 }
 
@@ -924,8 +978,6 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in
   result = mdb_cursor_del(m_cur_output_amounts, 0);
   if (result)
     throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str()));
-
-  m_num_outputs--;
 }
 
 void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
@@ -990,7 +1042,7 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const
 
 void BlockchainLMDB::check_open() const
 {
-  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+//  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   if (!m_open)
     throw0(DB_ERROR("DB operation attempted on a not-open DB instance"));
 }
@@ -1018,7 +1070,6 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
   m_write_txn = nullptr;
   m_write_batch_txn = nullptr;
   m_batch_active = false;
-  m_height = 0;
   m_cum_size = 0;
   m_cum_count = 0;
 
@@ -1048,10 +1099,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 
   // check for existing LMDB files in base directory
   boost::filesystem::path old_files = direc.parent_path();
-  if (boost::filesystem::exists(old_files / "data.mdb") || boost::filesystem::exists(old_files / "lock.mdb"))
+  if (boost::filesystem::exists(old_files / CRYPTONOTE_BLOCKCHAINDATA_FILENAME)
+      || boost::filesystem::exists(old_files / CRYPTONOTE_BLOCKCHAINDATA_LOCK_FILENAME))
   {
     LOG_PRINT_L0("Found existing LMDB files in " << old_files.string());
-    LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart");
+    LOG_PRINT_L0("Move " << CRYPTONOTE_BLOCKCHAINDATA_FILENAME << " and/or " << CRYPTONOTE_BLOCKCHAINDATA_LOCK_FILENAME << " to " << filename << ", or delete them, and then restart");
     throw DB_ERROR("Database could not be opened");
   }
 
@@ -1143,17 +1195,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
   if ((result = mdb_stat(txn, m_blocks, &db_stats)))
     throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
   LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries);
-  m_height = db_stats.ms_entries;
-
-  // get and keep current number of txs
-  if ((result = mdb_stat(txn, m_txs, &db_stats)))
-    throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str()));
-  m_num_txs = db_stats.ms_entries;
-
-  // get and keep current number of outputs
-  if ((result = mdb_stat(txn, m_output_txs, &db_stats)))
-    throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str()));
-  m_num_outputs = db_stats.ms_entries;
+  uint64_t m_height = db_stats.ms_entries;
 
   bool compatible = true;
 
@@ -1164,7 +1206,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
   {
     if (*(const uint32_t*)v.mv_data > VERSION)
     {
-      LOG_PRINT_RED_L0("Existing lmdb database was made by a later version. We don't know how it will change yet.");
+      MWARNING("Existing lmdb database was made by a later version. We don't know how it will change yet.");
       compatible = false;
     }
 #if VERSION > 0
@@ -1194,8 +1236,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
     txn.abort();
     mdb_env_close(m_env);
     m_open = false;
-    LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version.");
-    LOG_PRINT_RED_L0("Please delete the existing database and resync.");
+    MFATAL("Existing lmdb database is incompatible with this version.");
+    MFATAL("Please delete the existing database and resync.");
     return;
   }
 
@@ -1212,7 +1254,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
         txn.abort();
         mdb_env_close(m_env);
         m_open = false;
-        LOG_PRINT_RED_L0("Failed to write version to database.");
+        MERROR("Failed to write version to database.");
         return;
       }
     }
@@ -1260,7 +1302,7 @@ void BlockchainLMDB::reset()
   check_open();
 
   mdb_txn_safe txn;
-  if (auto result = mdb_txn_begin(m_env, NULL, 0, txn))
+  if (auto result = lmdb_txn_begin(m_env, NULL, 0, txn))
     throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
 
   if (auto result = mdb_drop(txn, m_blocks, 0))
@@ -1294,8 +1336,6 @@ void BlockchainLMDB::reset()
     throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str()));
 
   txn.commit();
-  m_height = 0;
-  m_num_outputs = 0;
   m_cum_size = 0;
   m_cum_count = 0;
 }
@@ -1306,9 +1346,9 @@ std::vector<std::string> BlockchainLMDB::get_filenames() const
   std::vector<std::string> filenames;
 
   boost::filesystem::path datafile(m_folder);
-  datafile /= "data.mdb";
+  datafile /= CRYPTONOTE_BLOCKCHAINDATA_FILENAME;
   boost::filesystem::path lockfile(m_folder);
-  lockfile /= "lock.mdb";
+  lockfile /= CRYPTONOTE_BLOCKCHAINDATA_LOCK_FILENAME;
 
   filenames.push_back(datafile.string());
   filenames.push_back(lockfile.string());
@@ -1345,7 +1385,7 @@ void BlockchainLMDB::unlock()
     txn_ptr = m_write_txn; \
   else \
   { \
-    if (auto mdb_res = mdb_txn_begin(m_env, NULL, flags, auto_txn)) \
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \
       throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \
   } \
 
@@ -1379,7 +1419,7 @@ void BlockchainLMDB::unlock()
     txn_ptr = m_write_txn; \
   else \
   { \
-    if (auto mdb_res = mdb_txn_begin(m_env, NULL, flags, auto_txn)) \
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, flags, auto_txn)) \
       throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \
   } \
 
@@ -1420,12 +1460,12 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h, uint64_t *height) const
   return ret;
 }
 
-block BlockchainLMDB::get_block(const crypto::hash& h) const
+cryptonote::blobdata BlockchainLMDB::get_block_blob(const crypto::hash& h) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  return get_block_from_height(get_block_height(h));
+  return get_block_blob_from_height(get_block_height(h));
 }
 
 uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const
@@ -1458,7 +1498,7 @@ block_header BlockchainLMDB::get_block_header(const crypto::hash& h) const
   return get_block(h);
 }
 
-block BlockchainLMDB::get_block_from_height(const uint64_t& height) const
+cryptonote::blobdata BlockchainLMDB::get_block_blob_from_height(const uint64_t& height) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -1479,13 +1519,9 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const
   blobdata bd;
   bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size);
 
-  block b;
-  if (!parse_and_validate_block_from_blob(bd, b))
-    throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
-
   TXN_POSTFIX_RDONLY();
 
-  return b;
+  return bd;
 }
 
 uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
@@ -1515,6 +1551,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
+  uint64_t m_height = height();
 
   // if no blocks, return 0
   if (m_height == 0)
@@ -1666,6 +1703,7 @@ crypto::hash BlockchainLMDB::top_block_hash() const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
+  uint64_t m_height = height();
   if (m_height != 0)
   {
     return get_block_hash_from_height(m_height - 1);
@@ -1678,6 +1716,7 @@ block BlockchainLMDB::get_top_block() const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
+  uint64_t m_height = height();
 
   if (m_height != 0)
   {
@@ -1692,8 +1731,42 @@ uint64_t BlockchainLMDB::height() const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
+  TXN_PREFIX_RDONLY();
+  int result;
 
-  return m_height;
+  // get current height
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_blocks, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
+  return db_stats.ms_entries;
+}
+
+uint64_t BlockchainLMDB::num_txs() const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  TXN_PREFIX_RDONLY();
+  int result;
+
+  // get current height
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_txs, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str()));
+  return db_stats.ms_entries;
+}
+
+uint64_t BlockchainLMDB::num_outputs() const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  TXN_PREFIX_RDONLY();
+  int result;
+
+  // get current height
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_output_txs, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str()));
+  return db_stats.ms_entries;
 }
 
 bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
@@ -1791,7 +1864,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const
   return ret;
 }
 
-transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
+bool BlockchainLMDB::get_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -1810,20 +1883,15 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
     get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET);
   }
   if (get_result == MDB_NOTFOUND)
-    throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
+    return false;
   else if (get_result)
     throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
 
-  blobdata bd;
   bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size);
 
-  transaction tx;
-  if (!parse_and_validate_tx_from_blob(bd, tx))
-    throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
-
   TXN_POSTFIX_RDONLY();
 
-  return tx;
+  return true;
 }
 
 uint64_t BlockchainLMDB::get_tx_count() const
@@ -2234,15 +2302,15 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c
 }
 
 // batch_num_blocks: (optional) Used to check if resize needed before batch transaction starts.
-void BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
+bool BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   if (! m_batch_transactions)
     throw0(DB_ERROR("batch transactions not enabled"));
   if (m_batch_active)
-    throw0(DB_ERROR("batch transaction already in progress"));
+    return false;
   if (m_write_batch_txn != nullptr)
-    throw0(DB_ERROR("batch transaction already in progress"));
+    return false;
   if (m_write_txn)
     throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
   check_open();
@@ -2253,7 +2321,7 @@ void BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
   m_write_batch_txn = new mdb_txn_safe();
 
   // NOTE: need to make sure it's destroyed properly when done
-  if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
+  if (auto mdb_res = lmdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
   {
     delete m_write_batch_txn;
     m_write_batch_txn = nullptr;
@@ -2268,6 +2336,7 @@ void BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
   memset(&m_wcursors, 0, sizeof(m_wcursors));
 
   LOG_PRINT_L3("batch transaction: begin");
+  return true;
 }
 
 void BlockchainLMDB::batch_commit()
@@ -2276,9 +2345,12 @@ void BlockchainLMDB::batch_commit()
   if (! m_batch_transactions)
     throw0(DB_ERROR("batch transactions not enabled"));
   if (! m_batch_active)
-    throw0(DB_ERROR("batch transaction not in progress"));
+    throw1(DB_ERROR("batch transaction not in progress"));
   if (m_write_batch_txn == nullptr)
-    throw0(DB_ERROR("batch transaction not in progress"));
+    throw1(DB_ERROR("batch transaction not in progress"));
+  if (m_writer != boost::this_thread::get_id())
+    throw1(DB_ERROR("batch transaction owned by other thread"));
+
   check_open();
 
   LOG_PRINT_L3("batch transaction: committing...");
@@ -2300,9 +2372,11 @@ void BlockchainLMDB::batch_stop()
   if (! m_batch_transactions)
     throw0(DB_ERROR("batch transactions not enabled"));
   if (! m_batch_active)
-    throw0(DB_ERROR("batch transaction not in progress"));
+    throw1(DB_ERROR("batch transaction not in progress"));
   if (m_write_batch_txn == nullptr)
-    throw0(DB_ERROR("batch transaction not in progress"));
+    throw1(DB_ERROR("batch transaction not in progress"));
+  if (m_writer != boost::this_thread::get_id())
+    throw1(DB_ERROR("batch transaction owned by other thread"));
   check_open();
   LOG_PRINT_L3("batch transaction: committing...");
   TIME_MEASURE_START(time1);
@@ -2324,7 +2398,11 @@ void BlockchainLMDB::batch_abort()
   if (! m_batch_transactions)
     throw0(DB_ERROR("batch transactions not enabled"));
   if (! m_batch_active)
-    throw0(DB_ERROR("batch transaction not in progress"));
+    throw1(DB_ERROR("batch transaction not in progress"));
+  if (m_write_batch_txn == nullptr)
+    throw1(DB_ERROR("batch transaction not in progress"));
+  if (m_writer != boost::this_thread::get_id())
+    throw1(DB_ERROR("batch transaction owned by other thread"));
   check_open();
   // for destruction of batch transaction
   m_write_txn = nullptr;
@@ -2362,12 +2440,12 @@ bool BlockchainLMDB::block_rtxn_start(MDB_txn **mtxn, mdb_txn_cursors **mcur) co
     m_tinfo.reset(new mdb_threadinfo);
     memset(&m_tinfo->m_ti_rcursors, 0, sizeof(m_tinfo->m_ti_rcursors));
     memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
-    if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
       throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str()));
     ret = true;
   } else if (!m_tinfo->m_ti_rflags.m_rf_txn)
   {
-    if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn))
+    if (auto mdb_res = lmdb_txn_renew(m_tinfo->m_ti_rtxn))
       throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str()));
     ret = true;
   }
@@ -2400,12 +2478,12 @@ void BlockchainLMDB::block_txn_start(bool readonly)
       m_tinfo.reset(new mdb_threadinfo);
       memset(&m_tinfo->m_ti_rcursors, 0, sizeof(m_tinfo->m_ti_rcursors));
       memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
-      if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
+      if (auto mdb_res = lmdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn))
         throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str()));
       didit = true;
     } else if (!m_tinfo->m_ti_rflags.m_rf_txn)
     {
-      if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn))
+      if (auto mdb_res = lmdb_txn_renew(m_tinfo->m_ti_rtxn))
         throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str()));
       didit = true;
     }
@@ -2431,7 +2509,7 @@ void BlockchainLMDB::block_txn_start(bool readonly)
   {
     m_writer = boost::this_thread::get_id();
     m_write_txn = new mdb_txn_safe();
-    if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, *m_write_txn))
+    if (auto mdb_res = lmdb_txn_begin(m_env, NULL, 0, *m_write_txn))
     {
       delete m_write_txn;
       m_write_txn = nullptr;
@@ -2497,6 +2575,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
+  uint64_t m_height = height();
 
   if (m_height % 1000 == 0)
   {
@@ -2508,8 +2587,6 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
     }
   }
 
-  uint64_t num_txs = m_num_txs;
-  uint64_t num_outputs = m_num_outputs;
   try
   {
     BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
@@ -2520,8 +2597,6 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
   }
   catch (...)
   {
-    m_num_txs = num_txs;
-    m_num_outputs = num_outputs;
     block_txn_abort();
     throw;
   }
@@ -2536,8 +2611,6 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
 
   block_txn_start(false);
 
-  uint64_t num_txs = m_num_txs;
-  uint64_t num_outputs = m_num_outputs;
   try
   {
     BlockchainDB::pop_block(blk, txs);
@@ -2545,13 +2618,9 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
   }
   catch (...)
   {
-    m_num_txs = num_txs;
-    m_num_outputs = num_outputs;
 	block_txn_abort();
     throw;
   }
-
-  --m_height;
 }
 
 void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
@@ -2582,7 +2651,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6
   TXN_POSTFIX_RDONLY();
 }
 
-void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs)
+void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   TIME_MEASURE_START(db3);
@@ -2600,7 +2669,14 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui
 
     auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH);
     if (get_result == MDB_NOTFOUND)
+    {
+      if (allow_partial)
+      {
+        MDEBUG("Partial result: " << outputs.size() << "/" << offsets.size());
+        break;
+      }
       throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) + ", index " + boost::lexical_cast<std::string>(index) + ", count " + boost::lexical_cast<std::string>(get_num_outputs(amount)) + "), but key does not exist").c_str()));
+    }
     else if (get_result)
       throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str()));
 
@@ -2837,31 +2913,36 @@ void BlockchainLMDB::fixup()
     ptr = (char *)k.mv_data; \
     ptr[sizeof(name)-2] = 's'
 
-#define LOGIF(y)    if (y <= epee::log_space::log_singletone::get_log_detalisation_level())
+#define LOGIF(y)    if (ELPP->vRegistry()->allowed(y, MONERO_DEFAULT_LOG_CATEGORY))
 
 void BlockchainLMDB::migrate_0_1()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-  uint64_t i, z;
+  uint64_t i, z, m_height;
   int result;
   mdb_txn_safe txn(false);
   MDB_val k, v;
   char *ptr;
 
-  LOG_PRINT_YELLOW("Migrating blockchain from DB version 0 to 1 - this may take a while:", LOG_LEVEL_0);
-  LOG_PRINT_L0("updating blocks, hf_versions, outputs, txs, and spent_keys tables...");
-
-  LOG_PRINT_L0("Total number of blocks: " << m_height);
-  LOG_PRINT_L1("block migration will update block_heights, block_info, and hf_versions...");
+  MLOG_YELLOW(el::Level::Info, "Migrating blockchain from DB version 0 to 1 - this may take a while:");
+  MINFO("updating blocks, hf_versions, outputs, txs, and spent_keys tables...");
 
   do {
-    LOG_PRINT_L1("migrating block_heights:");
-    MDB_dbi o_heights;
-
-    unsigned int flags;
     result = mdb_txn_begin(m_env, NULL, 0, txn);
     if (result)
       throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+
+    MDB_stat db_stats;
+    if ((result = mdb_stat(txn, m_blocks, &db_stats)))
+      throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
+    m_height = db_stats.ms_entries;
+    MINFO("Total number of blocks: " << m_height);
+    MINFO("block migration will update block_heights, block_info, and hf_versions...");
+
+    MINFO("migrating block_heights:");
+    MDB_dbi o_heights;
+
+    unsigned int flags;
     result = mdb_dbi_flags(txn, m_block_heights, &flags);
     if (result)
       throw0(DB_ERROR(lmdb_error("Failed to retrieve block_heights flags: ", result).c_str()));
@@ -2893,7 +2974,7 @@ void BlockchainLMDB::migrate_0_1()
     while(1) {
       if (!(i % 2000)) {
         if (i) {
-          LOGIF(1) {
+          LOGIF(el::Level::Info) {
             std::cout << i << " / " << z << "  \r" << std::flush;
           }
           txn.commit();
@@ -2984,7 +3065,7 @@ void BlockchainLMDB::migrate_0_1()
       MDB_val k, v;
       if (!(i % 2000)) {
         if (i) {
-          LOGIF(1) {
+          LOGIF(el::Level::Info) {
             std::cout << i << " / " << z << "  \r" << std::flush;
           }
           txn.commit();
@@ -3116,7 +3197,7 @@ void BlockchainLMDB::migrate_0_1()
     while(1) {
       if (!(i % 2000)) {
         if (i) {
-          LOGIF(1) {
+          LOGIF(el::Level::Info) {
             std::cout << i << " / " << z << "  \r" << std::flush;
           }
           txn.commit();
@@ -3217,8 +3298,6 @@ void BlockchainLMDB::migrate_0_1()
     lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
     mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
     txn.commit();
-    m_num_txs = 0;
-    m_num_outputs = 0;
   } while(0);
 
   do {
@@ -3262,7 +3341,7 @@ void BlockchainLMDB::migrate_0_1()
     while(1) {
       if (!(i % 1000)) {
         if (i) {
-          LOGIF(1) {
+          LOGIF(el::Level::Info) {
             std::cout << i << " / " << z << "  \r" << std::flush;
           }
           MDB_val_set(pk, "txblk");
@@ -3289,12 +3368,9 @@ void BlockchainLMDB::migrate_0_1()
           throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str()));
         if (!i) {
           MDB_stat ms;
-          mdb_stat(txn, m_output_txs, &ms);
-          m_num_outputs = ms.ms_entries;
           mdb_stat(txn, m_txs, &ms);
-          m_num_txs = i = ms.ms_entries;
+          i = ms.ms_entries;
           if (i) {
-            m_num_txs = i;
             MDB_val_set(pk, "txblk");
             result = mdb_cursor_get(c_props, &pk, &k, MDB_SET);
             if (result)

src/blockchain_db/lmdb/db_lmdb.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are
@@ -170,13 +170,13 @@ public:
 
   virtual bool block_exists(const crypto::hash& h, uint64_t *height = NULL) const;
 
-  virtual block get_block(const crypto::hash& h) const;
-
   virtual uint64_t get_block_height(const crypto::hash& h) const;
 
   virtual block_header get_block_header(const crypto::hash& h) const;
 
-  virtual block get_block_from_height(const uint64_t& height) const;
+  virtual cryptonote::blobdata get_block_blob(const crypto::hash& h) const;
+
+  virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const;
 
   virtual uint64_t get_block_timestamp(const uint64_t& height) const;
 
@@ -207,7 +207,7 @@ public:
 
   virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const;
 
-  virtual transaction get_tx(const crypto::hash& h) const;
+  virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
 
   virtual uint64_t get_tx_count() const;
 
@@ -219,7 +219,7 @@ public:
 
   virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index);
   virtual output_data_t get_output_key(const uint64_t& global_index) const;
-  virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs);
+  virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false);
 
   virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const;
   virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
@@ -245,7 +245,7 @@ public:
                             );
 
   virtual void set_batch_transactions(bool batch_transactions);
-  virtual void batch_start(uint64_t batch_num_blocks=0);
+  virtual bool batch_start(uint64_t batch_num_blocks=0);
   virtual void batch_commit();
   virtual void batch_stop();
   virtual void batch_abort();
@@ -310,6 +310,9 @@ private:
 
   virtual void remove_spent_key(const crypto::key_image& k_image);
 
+  uint64_t num_txs() const;
+  uint64_t num_outputs() const;
+
   // Hard fork
   virtual void set_hard_fork_version(uint64_t height, uint8_t version);
   virtual uint8_t get_hard_fork_version(uint64_t height) const;
@@ -367,11 +370,8 @@ private:
 
   MDB_dbi m_properties;
 
-  uint64_t m_height;
-  uint64_t m_num_txs;
-  uint64_t m_num_outputs;
   mutable uint64_t m_cum_size;	// used in batch size estimation
-  mutable int m_cum_count;
+  mutable unsigned int m_cum_count;
   std::string m_folder;
   mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
   mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB

src/blockchain_utilities/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2016, The Monero Project
+# Copyright (c) 2014-2017, The Monero Project
 #
 # All rights reserved.
 #
@@ -72,6 +72,7 @@ target_link_libraries(blockchain_import
     cryptonote_core
     blockchain_db
     p2p
+    epee
     ${Boost_FILESYSTEM_LIBRARY}
     ${Boost_SYSTEM_LIBRARY}
     ${Boost_THREAD_LIBRARY}
@@ -98,6 +99,7 @@ target_link_libraries(blockchain_export
     cryptonote_core
     blockchain_db
     p2p
+    epee
     ${Boost_FILESYSTEM_LIBRARY}
     ${Boost_SYSTEM_LIBRARY}
     ${Boost_THREAD_LIBRARY}
@@ -118,6 +120,7 @@ target_link_libraries(cn_deserialize
   LINK_PRIVATE
     cryptonote_core
 	p2p
+    epee
     ${CMAKE_THREAD_LIBS_INIT})
 
 add_dependencies(cn_deserialize

src/blockchain_utilities/README.md

@@ -1,6 +1,6 @@
 # Monero Blockchain Utilities
 
-Copyright (c) 2014-2016, The Monero Project
+Copyright (c) 2014-2017, The Monero Project
 
 ## Introduction
 

src/blockchain_utilities/blockchain_export.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -38,8 +38,11 @@
 #include "blockchain_db/db_types.h"
 #include "version.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
+
 namespace po = boost::program_options;
-using namespace epee; // log_space
+using namespace epee;
 
 std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim)
 {
@@ -54,6 +57,8 @@ std::string join_set_strings(const std::unordered_set<std::string>& db_types_all
 
 int main(int argc, char* argv[])
 {
+  epee::string_tools::set_module_name_and_folder(argv[0]);
+
   std::string default_db_type = "lmdb";
 
   std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types;
@@ -122,10 +127,9 @@ int main(int argc, char* argv[])
   log_level    = command_line::get_arg(vm, arg_log_level);
   block_stop = command_line::get_arg(vm, arg_block_stop);
 
-  log_space::get_set_log_detalisation_level(true, log_level);
-  log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
+  mlog_configure(mlog_get_default_log_path("monero-blockchain-export.log"), true);
+  mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
   LOG_PRINT_L0("Starting...");
-  LOG_PRINT_L0("Setting log level = " << log_level);
 
   bool opt_testnet = command_line::get_arg(vm, arg_testnet_on);
   bool opt_blocks_dat = command_line::get_arg(vm, arg_blocks_dat);

src/blockchain_utilities/blockchain_import.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -34,7 +34,7 @@
 #include <boost/filesystem.hpp>
 #include "bootstrap_file.h"
 #include "bootstrap_serialization.h"
-#include "cryptonote_core/cryptonote_format_utils.h"
+#include "cryptonote_basic/cryptonote_format_utils.h"
 #include "serialization/binary_utils.h" // dump_binary(), parse_binary()
 #include "serialization/json_utils.h" // dump_json()
 #include "include_base_utils.h"
@@ -44,6 +44,9 @@
 
 #include "fake_core.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
+
 namespace
 {
 // CONFIG
@@ -132,7 +135,7 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int&
 #if !defined(BERKELEY_DB)
   if (db_type == "berkeley")
   {
-    LOG_ERROR("BerkeleyDB support disabled.");
+    MFATAL("BerkeleyDB support disabled.");
     return false;
   }
 #endif
@@ -163,7 +166,7 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int&
 	continue;
       if (db_type == "lmdb")
       {
-	LOG_PRINT_L1("LMDB flag: " << it);
+	MINFO("LMDB flag: " << it);
 	if (it == "nosync")
 	  db_flags |= MDB_NOSYNC;
 	else if (it == "nometasync")
@@ -211,7 +214,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
     if (simple_core.support_batch)
       use_batch = true;
     else
-      LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
+      MWARNING("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
   }
 
   if (use_batch)
@@ -260,14 +263,14 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
   boost::system::error_code ec;
   if (!boost::filesystem::exists(fs_import_file_path, ec))
   {
-    LOG_PRINT_L0("bootstrap file not found: " << fs_import_file_path);
+    MFATAL("bootstrap file not found: " << fs_import_file_path);
     return false;
   }
 
   BootstrapFile bootstrap;
   // BootstrapFile bootstrap(import_file_path);
   uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path);
-  LOG_PRINT_L0("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height)  total blocks: " << total_source_blocks);
+  MINFO("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height)  total blocks: " << total_source_blocks);
 
   std::cout << ENDL;
   std::cout << "Preparing to read blocks..." << ENDL;
@@ -280,7 +283,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
   uint64_t num_imported = 0;
   if (import_file.fail())
   {
-    LOG_PRINT_L0("import_file.open() fail");
+    MFATAL("import_file.open() fail");
     return false;
   }
 
@@ -309,7 +312,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
 
   // These are what we'll try to use, and they don't have to be a determination
   // from source and destination blockchains, but those are the defaults.
-  LOG_PRINT_L0("start block: " << start_height << "  stop block: " <<
+  MINFO("start block: " << start_height << "  stop block: " <<
       block_stop);
 
   bool use_batch = false;
@@ -318,13 +321,13 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
     if (simple_core.support_batch)
       use_batch = true;
     else
-      LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
+      MWARNING("WARNING: batch transactions enabled but unsupported or unnecessary for this database type - ignoring");
   }
 
   if (use_batch)
     simple_core.batch_start(db_batch_size);
 
-  LOG_PRINT_L0("Reading blockchain from bootstrap file...");
+  MINFO("Reading blockchain from bootstrap file...");
   std::cout << ENDL;
 
   // Within the loop, we skip to start_height before we start adding.
@@ -338,7 +341,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
     // TODO: bootstrap.read_chunk();
     if (! import_file) {
       std::cout << refresh_string;
-      LOG_PRINT_L0("End of file reached");
+      MINFO("End of file reached");
       quit = 1;
       break;
     }
@@ -349,29 +352,29 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
     {
       throw std::runtime_error("Error in deserialization of chunk size");
     }
-    LOG_PRINT_L3("chunk_size: " << chunk_size);
+    MDEBUG("chunk_size: " << chunk_size);
 
     if (chunk_size > BUFFER_SIZE)
     {
-      LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
+      MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
       throw std::runtime_error("Aborting: chunk size exceeds buffer size");
     }
     if (chunk_size > 100000)
     {
-      LOG_PRINT_L0("NOTE: chunk_size " << chunk_size << " > 100000");
+      MINFO("NOTE: chunk_size " << chunk_size << " > 100000");
     }
     else if (chunk_size == 0) {
-      LOG_PRINT_L0("ERROR: chunk_size == 0");
+      MFATAL("ERROR: chunk_size == 0");
       return 2;
     }
     import_file.read(buffer_block, chunk_size);
     if (! import_file) {
-      LOG_PRINT_L0("ERROR: unexpected end of file: bytes read before error: "
+      MFATAL("ERROR: unexpected end of file: bytes read before error: "
           << import_file.gcount() << " of chunk_size " << chunk_size);
       return 2;
     }
     bytes_read += chunk_size;
-    LOG_PRINT_L3("Total bytes read: " << bytes_read);
+    MDEBUG("Total bytes read: " << bytes_read);
 
     if (h + NUM_BLOCKS_PER_CHUNK < start_height + 1)
     {
@@ -384,7 +387,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
         << " / " << block_stop
         << std::flush;
       std::cout << ENDL << ENDL;
-      LOG_PRINT_L0("Specified block number reached - stopping.  block: " << h-1 << "  total blocks: " << h);
+      MINFO("Specified block number reached - stopping.  block: " << h-1 << "  total blocks: " << h);
       quit = 1;
       break;
     }
@@ -405,14 +408,14 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
         if ((h-1) % display_interval == 0)
         {
           std::cout << refresh_string;
-          LOG_PRINT_L0("loading block number " << h-1);
+          MDEBUG("loading block number " << h-1);
         }
         else
         {
-          LOG_PRINT_L3("loading block number " << h-1);
+          MDEBUG("loading block number " << h-1);
         }
         b = bp.block;
-        LOG_PRINT_L2("block prev_id: " << b.prev_id << ENDL);
+        MDEBUG("block prev_id: " << b.prev_id << ENDL);
 
         if ((h-1) % progress_interval == 0)
         {
@@ -427,12 +430,12 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
         archived_txs = bp.txs;
 
         // std::cout << refresh_string;
-        // LOG_PRINT_L1("txs: " << archived_txs.size());
+        // MDEBUG("txs: " << archived_txs.size());
 
         // if archived_txs is invalid
         // {
         //   std::cout << refresh_string;
-        //   LOG_PRINT_RED_L0("exception while de-archiving txs, height=" << h);
+        //   MFATAL("exception while de-archiving txs, height=" << h);
         //   quit = 1;
         //   break;
         // }
@@ -445,20 +448,20 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
           ++tx_num;
           // if tx is invalid
           // {
-          //   LOG_PRINT_RED_L0("exception while indexing tx from txs, height=" << h <<", tx_num=" << tx_num);
+          //   MFATAL("exception while indexing tx from txs, height=" << h <<", tx_num=" << tx_num);
           //   quit = 1;
           //   break;
           // }
 
           // std::cout << refresh_string;
-          // LOG_PRINT_L1("tx hash: " << get_transaction_hash(tx));
+          // MDEBUG("tx hash: " << get_transaction_hash(tx));
 
           // crypto::hash hsh = null_hash;
           // size_t blob_size = 0;
           // NOTE: all tx hashes except for coinbase tx are available in the block data
           // get_transaction_hash(tx, hsh, blob_size);
-          // LOG_PRINT_L0("tx " << tx_num << "  " << hsh << " : " << ENDL);
-          // LOG_PRINT_L0(obj_to_json_str(tx) << ENDL);
+          // MDEBUG("tx " << tx_num << "  " << hsh << " : " << ENDL);
+          // MDEBUG(obj_to_json_str(tx) << ENDL);
 
           // add blocks with verification.
           // for Blockchain and blockchain_storage add_new_block().
@@ -472,10 +475,10 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
             uint8_t version = simple_core.m_storage.get_current_hard_fork_version();
             tx_verification_context tvc = AUTO_VAL_INIT(tvc);
             bool r = true;
-            r = simple_core.m_pool.add_tx(tx, tvc, true, true, version);
+            r = simple_core.m_pool.add_tx(tx, tvc, true, true, false, version);
             if (!r)
             {
-              LOG_PRINT_RED_L0("failed to add transaction to transaction pool, height=" << h <<", tx_num=" << tx_num);
+              MFATAL("failed to add transaction to transaction pool, height=" << h <<", tx_num=" << tx_num);
               quit = 1;
               break;
             }
@@ -499,8 +502,8 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
 
           if (bvc.m_verifivation_failed)
           {
-            LOG_PRINT_L0("Failed to add block to blockchain, verification failed, height = " << h);
-            LOG_PRINT_L0("skipping rest of file");
+            MFATAL("Failed to add block to blockchain, verification failed, height = " << h);
+            MFATAL("skipping rest of file");
             // ok to commit previously batched data because it failed only in
             // verification of potential new block with nothing added to batch
             // yet
@@ -509,8 +512,8 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
           }
           if (! bvc.m_added_to_main_chain)
           {
-            LOG_PRINT_L0("Failed to add block to blockchain, height = " << h);
-            LOG_PRINT_L0("skipping rest of file");
+            MFATAL("Failed to add block to blockchain, height = " << h);
+            MFATAL("skipping rest of file");
             // make sure we don't commit partial block data
             quit = 2;
             break;
@@ -527,9 +530,9 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
           coins_generated = bp.coins_generated;
 
           // std::cout << refresh_string;
-          // LOG_PRINT_L2("block_size: " << block_size);
-          // LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty);
-          // LOG_PRINT_L2("coins_generated: " << coins_generated);
+          // MDEBUG("block_size: " << block_size);
+          // MDEBUG("cumulative_difficulty: " << cumulative_difficulty);
+          // MDEBUG("coins_generated: " << coins_generated);
 
           try
           {
@@ -538,7 +541,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
           catch (const std::exception& e)
           {
             std::cout << refresh_string;
-            LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what());
+            MFATAL("Error adding block to blockchain: " << e.what());
             quit = 2; // make sure we don't commit partial block data
             break;
           }
@@ -563,7 +566,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
     catch (const std::exception& e)
     {
       std::cout << refresh_string;
-      LOG_PRINT_RED_L0("exception while reading from file, height=" << h << ": " << e.what());
+      MFATAL("exception while reading from file, height=" << h << ": " << e.what());
       return 2;
     }
   } // while
@@ -582,10 +585,10 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
       simple_core.batch_stop();
     }
     simple_core.m_storage.get_db().show_stats();
-    LOG_PRINT_L0("Number of blocks imported: " << num_imported);
+    MINFO("Number of blocks imported: " << num_imported);
     if (h > 0)
       // TODO: if there was an error, the last added block is probably at zero-based height h-2
-      LOG_PRINT_L0("Finished at block: " << h-1 << "  total blocks: " << h);
+      MINFO("Finished at block: " << h-1 << "  total blocks: " << h);
   }
   std::cout << ENDL;
   return 0;
@@ -593,6 +596,8 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
 
 int main(int argc, char* argv[])
 {
+  epee::string_tools::set_module_name_and_folder(argv[0]);
+
   std::string default_db_type = "lmdb";
   std::string default_db_engine_compiled = "blockchain_db";
 
@@ -602,7 +607,7 @@ int main(int argc, char* argv[])
   std::string available_dbs = join_set_strings(db_types_all, ", ");
   available_dbs = "available: " + available_dbs;
 
-  uint32_t log_level = LOG_LEVEL_0;
+  uint32_t log_level = 0;
   uint64_t num_blocks = 0;
   uint64_t block_stop = 0;
   std::string m_config_folder;
@@ -719,10 +724,9 @@ int main(int argc, char* argv[])
   m_config_folder = command_line::get_arg(vm, data_dir_arg);
   db_arg_str = command_line::get_arg(vm, arg_database);
 
-  log_space::get_set_log_detalisation_level(true, log_level);
-  log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
-  LOG_PRINT_L0("Starting...");
-  LOG_PRINT_L0("Setting log level = " << log_level);
+  mlog_configure(mlog_get_default_log_path("monero-blockchain-import.log"), true);
+  mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+  MINFO("Starting...");
 
   boost::filesystem::path fs_import_file_path;
 
@@ -767,23 +771,23 @@ int main(int argc, char* argv[])
     db_engine_compiled = "memory";
   }
 
-  LOG_PRINT_L0("database: " << db_type);
-  LOG_PRINT_L0("database flags: " << db_flags);
-  LOG_PRINT_L0("verify:  " << std::boolalpha << opt_verify << std::noboolalpha);
+  MINFO("database: " << db_type);
+  MINFO("database flags: " << db_flags);
+  MINFO("verify:  " << std::boolalpha << opt_verify << std::noboolalpha);
   if (opt_batch)
   {
-    LOG_PRINT_L0("batch:   " << std::boolalpha << opt_batch << std::noboolalpha
+    MINFO("batch:   " << std::boolalpha << opt_batch << std::noboolalpha
         << "  batch size: " << db_batch_size);
   }
   else
   {
-    LOG_PRINT_L0("batch:   " << std::boolalpha << opt_batch << std::noboolalpha);
+    MINFO("batch:   " << std::boolalpha << opt_batch << std::noboolalpha);
   }
-  LOG_PRINT_L0("resume:  " << std::boolalpha << opt_resume  << std::noboolalpha);
-  LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha);
+  MINFO("resume:  " << std::boolalpha << opt_resume  << std::noboolalpha);
+  MINFO("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha);
 
-  LOG_PRINT_L0("bootstrap file path: " << import_file_path);
-  LOG_PRINT_L0("database path:       " << m_config_folder);
+  MINFO("bootstrap file path: " << import_file_path);
+  MINFO("database path:       " << m_config_folder);
 
   try
   {
@@ -813,15 +817,15 @@ int main(int argc, char* argv[])
   if (! vm["pop-blocks"].defaulted())
   {
     num_blocks = command_line::get_arg(vm, arg_pop_blocks);
-    LOG_PRINT_L0("height: " << simple_core.m_storage.get_current_blockchain_height());
+    MINFO("height: " << simple_core.m_storage.get_current_blockchain_height());
     pop_blocks(simple_core, num_blocks);
-    LOG_PRINT_L0("height: " << simple_core.m_storage.get_current_blockchain_height());
+    MINFO("height: " << simple_core.m_storage.get_current_blockchain_height());
     return 0;
   }
 
   if (! vm["drop-hard-fork"].defaulted())
   {
-    LOG_PRINT_L0("Dropping hard fork tables...");
+    MINFO("Dropping hard fork tables...");
     simple_core.m_storage.get_db().drop_hard_fork_info();
     return 0;
   }

src/blockchain_utilities/blockchain_utilities.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blocksdat_file.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -28,6 +28,8 @@
 
 #include "blocksdat_file.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
 
 namespace po = boost::program_options;
 
@@ -50,7 +52,7 @@ bool BlocksdatFile::open_writer(const boost::filesystem::path& file_path, uint64
     {
       if (!boost::filesystem::is_directory(dir_path))
       {
-        LOG_PRINT_RED_L0("export directory path is a file: " << dir_path);
+        MFATAL("export directory path is a file: " << dir_path);
         return false;
       }
     }
@@ -58,7 +60,7 @@ bool BlocksdatFile::open_writer(const boost::filesystem::path& file_path, uint64
     {
       if (!boost::filesystem::create_directory(dir_path))
       {
-        LOG_PRINT_RED_L0("Failed to create directory " << dir_path);
+        MFATAL("Failed to create directory " << dir_path);
         return false;
       }
     }
@@ -66,7 +68,7 @@ bool BlocksdatFile::open_writer(const boost::filesystem::path& file_path, uint64
 
   m_raw_data_file = new std::ofstream();
 
-  LOG_PRINT_L0("creating file");
+  MINFO("creating file");
 
   m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
   if (m_raw_data_file->fail())
@@ -123,21 +125,21 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
 
   uint64_t block_start = 0;
   uint64_t block_stop = 0;
-  LOG_PRINT_L0("source blockchain height: " <<  m_blockchain_storage->get_current_blockchain_height()-1);
+  MINFO("source blockchain height: " <<  m_blockchain_storage->get_current_blockchain_height()-1);
   if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
   {
-    LOG_PRINT_L0("Using requested block height: " << requested_block_stop);
+    MINFO("Using requested block height: " << requested_block_stop);
     block_stop = requested_block_stop;
   }
   else
   {
     block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
-    LOG_PRINT_L0("Using block height of source blockchain: " << block_stop);
+    MINFO("Using block height of source blockchain: " << block_stop);
   }
-  LOG_PRINT_L0("Storing blocks raw data...");
+  MINFO("Storing blocks raw data...");
   if (!BlocksdatFile::open_writer(output_file, block_stop))
   {
-    LOG_PRINT_RED_L0("failed to open raw file for write");
+    MFATAL("failed to open raw file for write");
     return false;
   }
   for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
@@ -157,7 +159,7 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   std::cout << refresh_string;
   std::cout << "block " << m_cur_height-1 << "/" << block_stop << ENDL;
 
-  LOG_PRINT_L0("Number of blocks exported: " << num_blocks_written);
+  MINFO("Number of blocks exported: " << num_blocks_written);
 
   return BlocksdatFile::close();
 }

src/blockchain_utilities/blocksdat_file.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -34,8 +34,8 @@
 
 #include <boost/iostreams/filtering_streambuf.hpp>
 
-#include "cryptonote_core/cryptonote_basic.h"
-#include "cryptonote_core/cryptonote_boost_serialization.h"
+#include "cryptonote_basic/cryptonote_basic.h"
+#include "cryptonote_basic/cryptonote_boost_serialization.h"
 #include "cryptonote_core/blockchain.h"
 #include "blockchain_db/blockchain_db.h"
 #include "blockchain_db/lmdb/db_lmdb.h"

src/blockchain_utilities/bootstrap_file.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -32,6 +32,8 @@
 
 #include "bootstrap_file.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
 
 namespace po = boost::program_options;
 
@@ -59,7 +61,7 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
     {
       if (!boost::filesystem::is_directory(dir_path))
       {
-        LOG_PRINT_RED_L0("export directory path is a file: " << dir_path);
+        MFATAL("export directory path is a file: " << dir_path);
         return false;
       }
     }
@@ -67,7 +69,7 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
     {
       if (!boost::filesystem::create_directory(dir_path))
       {
-        LOG_PRINT_RED_L0("Failed to create directory " << dir_path);
+        MFATAL("Failed to create directory " << dir_path);
         return false;
       }
     }
@@ -80,14 +82,14 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
 
   if (! boost::filesystem::exists(file_path))
   {
-    LOG_PRINT_L0("creating file");
+    MDEBUG("creating file");
     do_initialize_file = true;
     num_blocks = 0;
   }
   else
   {
     num_blocks = count_blocks(file_path.string());
-    LOG_PRINT_L0("appending to existing file with height: " << num_blocks-1 << "  total blocks: " << num_blocks);
+    MDEBUG("appending to existing file with height: " << num_blocks-1 << "  total blocks: " << num_blocks);
   }
   m_height = num_blocks;
 
@@ -138,7 +140,7 @@ bool BootstrapFile::initialize_file()
   uint32_t bd_size = 0;
 
   blobdata bd = t_serializable_object_to_blob(bfi);
-  LOG_PRINT_L1("bootstrap::file_info size: " << bd.size());
+  MDEBUG("bootstrap::file_info size: " << bd.size());
   bd_size = bd.size();
 
   if (! ::serialization::dump_binary(bd_size, blob))
@@ -149,7 +151,7 @@ bool BootstrapFile::initialize_file()
   *output_stream_header << bd;
 
   bd = t_serializable_object_to_blob(bbi);
-  LOG_PRINT_L1("bootstrap::blocks_info size: " << bd.size());
+  MDEBUG("bootstrap::blocks_info size: " << bd.size());
   bd_size = bd.size();
 
   if (! ::serialization::dump_binary(bd_size, blob))
@@ -172,10 +174,10 @@ void BootstrapFile::flush_chunk()
   m_output_stream->flush();
 
   uint32_t chunk_size = m_buffer.size();
-  // LOG_PRINT_L0("chunk_size " << chunk_size);
+  // MTRACE("chunk_size " << chunk_size);
   if (chunk_size > BUFFER_SIZE)
   {
-    LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
+    MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
   }
 
   std::string blob;
@@ -196,14 +198,14 @@ void BootstrapFile::flush_chunk()
   long num_chars_written = pos_after - pos_before;
   if (static_cast<unsigned long>(num_chars_written) != chunk_size)
   {
-    LOG_PRINT_RED_L0("Error writing chunk:  height: " << m_cur_height << "  chunk_size: " << chunk_size << "  num chars written: " << num_chars_written);
+    MFATAL("Error writing chunk:  height: " << m_cur_height << "  chunk_size: " << chunk_size << "  num chars written: " << num_chars_written);
     throw std::runtime_error("Error writing chunk");
   }
 
   m_buffer.clear();
   delete m_output_stream;
   m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
-  LOG_PRINT_L1("flushed chunk:  chunk_size: " << chunk_size);
+  MDEBUG("flushed chunk:  chunk_size: " << chunk_size);
 }
 
 void BootstrapFile::write_block(block& block)
@@ -267,10 +269,10 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   m_blockchain_storage = _blockchain_storage;
   m_tx_pool = _tx_pool;
   uint64_t progress_interval = 100;
-  LOG_PRINT_L0("Storing blocks raw data...");
+  MINFO("Storing blocks raw data...");
   if (!BootstrapFile::open_writer(output_file))
   {
-    LOG_PRINT_RED_L0("failed to open raw file for write");
+    MFATAL("failed to open raw file for write");
     return false;
   }
   block b;
@@ -280,16 +282,16 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   // height.
   uint64_t block_start = m_height;
   uint64_t block_stop = 0;
-  LOG_PRINT_L0("source blockchain height: " <<  m_blockchain_storage->get_current_blockchain_height()-1);
+  MINFO("source blockchain height: " <<  m_blockchain_storage->get_current_blockchain_height()-1);
   if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
   {
-    LOG_PRINT_L0("Using requested block height: " << requested_block_stop);
+    MINFO("Using requested block height: " << requested_block_stop);
     block_stop = requested_block_stop;
   }
   else
   {
     block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
-    LOG_PRINT_L0("Using block height of source blockchain: " << block_stop);
+    MINFO("Using block height of source blockchain: " << block_stop);
   }
   for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
   {
@@ -315,9 +317,9 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   std::cout << refresh_string;
   std::cout << "block " << m_cur_height-1 << "/" << block_stop << ENDL;
 
-  LOG_PRINT_L0("Number of blocks exported: " << num_blocks_written);
+  MINFO("Number of blocks exported: " << num_blocks_written);
   if (num_blocks_written > 0)
-    LOG_PRINT_L0("Largest chunk: " << m_max_chunk << " bytes");
+    MINFO("Largest chunk: " << m_max_chunk << " bytes");
 
   return BootstrapFile::close();
 }
@@ -338,11 +340,11 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file)
 
   if (file_magic != blockchain_raw_magic)
   {
-    LOG_PRINT_RED_L0("bootstrap file not recognized");
+    MFATAL("bootstrap file not recognized");
     throw std::runtime_error("Aborting");
   }
   else
-    LOG_PRINT_L0("bootstrap file recognized");
+    MINFO("bootstrap file recognized");
 
   uint32_t buflen_file_info;
 
@@ -352,7 +354,7 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file)
     throw std::runtime_error("Error reading expected number of bytes");
   if (! ::serialization::parse_binary(str1, buflen_file_info))
     throw std::runtime_error("Error in deserialization of buflen_file_info");
-  LOG_PRINT_L1("bootstrap::file_info size: " << buflen_file_info);
+  MINFO("bootstrap::file_info size: " << buflen_file_info);
 
   if (buflen_file_info > sizeof(buf1))
     throw std::runtime_error("Error: bootstrap::file_info size exceeds buffer size");
@@ -363,9 +365,9 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file)
   bootstrap::file_info bfi;
   if (! ::serialization::parse_binary(str1, bfi))
     throw std::runtime_error("Error in deserialization of bootstrap::file_info");
-  LOG_PRINT_L0("bootstrap file v" << unsigned(bfi.major_version) << "." << unsigned(bfi.minor_version));
-  LOG_PRINT_L0("bootstrap magic size: " << sizeof(file_magic));
-  LOG_PRINT_L0("bootstrap header size: " << bfi.header_size);
+  MINFO("bootstrap file v" << unsigned(bfi.major_version) << "." << unsigned(bfi.minor_version));
+  MINFO("bootstrap magic size: " << sizeof(file_magic));
+  MINFO("bootstrap header size: " << bfi.header_size);
 
   uint64_t full_header_size = sizeof(file_magic) + bfi.header_size;
   import_file.seekg(full_header_size);
@@ -379,7 +381,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
   boost::system::error_code ec;
   if (!boost::filesystem::exists(raw_file_path, ec))
   {
-    LOG_PRINT_L0("bootstrap file not found: " << raw_file_path);
+    MFATAL("bootstrap file not found: " << raw_file_path);
     throw std::runtime_error("Aborting");
   }
   std::ifstream import_file;
@@ -388,14 +390,14 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
   uint64_t h = 0;
   if (import_file.fail())
   {
-    LOG_PRINT_L0("import_file.open() fail");
+    MFATAL("import_file.open() fail");
     throw std::runtime_error("Aborting");
   }
 
   uint64_t full_header_size; // 4 byte magic + length of header structures
   full_header_size = seek_to_first_chunk(import_file);
 
-  LOG_PRINT_L0("Scanning blockchain from bootstrap file...");
+  MINFO("Scanning blockchain from bootstrap file...");
   block b;
   bool quit = false;
   uint64_t bytes_read = 0;
@@ -409,7 +411,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
     import_file.read(buf1, sizeof(chunk_size));
     if (!import_file) {
       std::cout << refresh_string;
-      LOG_PRINT_L1("End of file reached");
+      MDEBUG("End of file reached");
       quit = true;
       break;
     }
@@ -425,38 +427,38 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
     str1.assign(buf1, sizeof(chunk_size));
     if (! ::serialization::parse_binary(str1, chunk_size))
       throw std::runtime_error("Error in deserialization of chunk_size");
-    LOG_PRINT_L3("chunk_size: " << chunk_size);
+    MDEBUG("chunk_size: " << chunk_size);
 
     if (chunk_size > BUFFER_SIZE)
     {
       std::cout << refresh_string;
-      LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE
+      MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE
           << "  height: " << h-1);
       throw std::runtime_error("Aborting: chunk size exceeds buffer size");
     }
     if (chunk_size > 100000)
     {
       std::cout << refresh_string;
-      LOG_PRINT_L0("NOTE: chunk_size " << chunk_size << " > 100000" << "  height: "
+      MDEBUG("NOTE: chunk_size " << chunk_size << " > 100000" << "  height: "
           << h-1);
     }
     else if (chunk_size <= 0) {
       std::cout << refresh_string;
-      LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << "  height: " << h-1);
+      MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << "  height: " << h-1);
       throw std::runtime_error("Aborting");
     }
     // skip to next expected block size value
     import_file.seekg(chunk_size, std::ios_base::cur);
     if (! import_file) {
       std::cout << refresh_string;
-      LOG_PRINT_L0("ERROR: unexpected end of file: bytes read before error: "
+      MFATAL("ERROR: unexpected end of file: bytes read before error: "
           << import_file.gcount() << " of chunk_size " << chunk_size);
       throw std::runtime_error("Aborting");
     }
     bytes_read += chunk_size;
 
     // std::cout << refresh_string;
-    LOG_PRINT_L3("Number bytes scanned: " << bytes_read);
+    MDEBUG("Number bytes scanned: " << bytes_read);
   }
 
   import_file.close();

src/blockchain_utilities/bootstrap_file.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -34,7 +34,7 @@
 
 #include <boost/iostreams/filtering_streambuf.hpp>
 
-#include "cryptonote_core/cryptonote_basic.h"
+#include "cryptonote_basic/cryptonote_basic.h"
 #include "cryptonote_core/blockchain.h"
 
 #include <algorithm>

src/blockchain_utilities/bootstrap_serialization.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -28,8 +28,8 @@
 
 #pragma once
 
-#include "cryptonote_core/cryptonote_boost_serialization.h"
-#include "cryptonote_core/difficulty.h"
+#include "cryptonote_basic/cryptonote_boost_serialization.h"
+#include "cryptonote_basic/difficulty.h"
 
 
 namespace cryptonote

src/blockchain_utilities/cn_deserialize.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -26,15 +26,18 @@
 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include "cryptonote_core/cryptonote_basic.h"
-#include "cryptonote_core/tx_extra.h"
+#include "cryptonote_basic/cryptonote_basic.h"
+#include "cryptonote_basic/tx_extra.h"
 #include "cryptonote_core/blockchain.h"
 #include "blockchain_utilities.h"
 #include "common/command_line.h"
 #include "version.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
+
 namespace po = boost::program_options;
-using namespace epee; // log_space
+using namespace epee;
 
 using namespace cryptonote;
 
@@ -87,8 +90,7 @@ int main(int argc, char* argv[])
     return 1;
   }
 
-  log_space::get_set_log_detalisation_level(true, log_level);
-  log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
+  mlog_configure("", true);
 
   std::string m_config_folder;
 

src/blockchain_utilities/fake_core.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2016, The Monero Project
+// Copyright (c) 2014-2017, The Monero Project
 //
 // All rights reserved.
 //
@@ -119,9 +119,9 @@ struct fake_core_db
     return m_storage.get_db().add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
   }
 
-  void batch_start(uint64_t batch_num_blocks = 0)
+  bool batch_start(uint64_t batch_num_blocks = 0)
   {
-    m_storage.get_db().batch_start(batch_num_blocks);
+    return m_storage.get_db().batch_start(batch_num_blocks);
   }
 
   void batch_stop()