Charon/blockchain
1
0
Fork 0
forked from lthn/blockchain
Code Pull requests Activity

merge from master

Browse source
This commit is contained in:
cryptozoidberg 2022-04-25 14:54:30 +02:00
commit 072b022514
No known key found for this signature in database
GPG key ID: 22DEB97A54C6FDEC
86 changed files with 42043 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.gitmodules vendored
View file

@ -1,9 +1,6 @@
[submodule "contrib/miniupnp"]
path = contrib/miniupnp
url = https://github.com/miniupnp/miniupnp
[submodule "contrib/db/libmdbx"]
path = contrib/db/libmdbx
url = https://github.com/leo-yuriev/libmdbx.git
[submodule "src/gui/qt-daemon/layout"]
path = src/gui/qt-daemon/layout
url = https://github.com/hyle-team/zano_ui.git
@ -11,4 +8,4 @@
[submodule "contrib/tor-connect"]
path = contrib/tor-connect
url = https://github.com/hyle-team/tor-connect.git
branch = main
branch = main

1
contrib/db/libmdbx

@ -1 +0,0 @@
Subproject commit b7ed67543fefb0878dba1c70dea2a81201041314

20
contrib/db/libmdbx/.circleci/config.yml Normal file
View file

@ -0,0 +1,20 @@
version: 2
jobs:
build:
docker:
- image: circleci/buildpack-deps:artful
environment:
- TESTDB: /tmp/test.db
- TESTLOG: /tmp/test.log
steps:
- checkout
- run: make all
- run: ulimit -c unlimited && make check
- run:
command: |
mkdir -p /tmp/artifacts
mv -t /tmp/artifacts $TESTLOG $TESTDB core.*
when: on_fail
- store_artifacts:
path: /tmp/artifacts
destination: test-artifacts

3
contrib/db/libmdbx/.clang-format Normal file
View file

@ -0,0 +1,3 @@
BasedOnStyle: LLVM
Standard: Cpp11
ReflowComments: true

35
contrib/db/libmdbx/.gitignore vendored Normal file
View file

@ -0,0 +1,35 @@
*.[ao]
*.bak
*.exe
*.gcda
*.gcno
*.gcov
*.lo
*.orig
*.rej
*.so
*[~#]
.idea
.le.ini
.vs/
cmake-build-*
@*
core
mdbx_example
libmdbx.creator.user
mdbx_chk
mdbx_copy
mdbx_dump
mdbx_load
mdbx_stat
mdbx_test
test.log
test/tmp.db
test/tmp.db-lck
tmp.db
tmp.db-lck
valgrind.*
src/elements/version.c
src/elements/config.h
dist/
*.tar*

61
contrib/db/libmdbx/.travis.yml Normal file
View file

@ -0,0 +1,61 @@
language: c cpp
matrix:
include:
- os: linux
dist: precise
env: CC=cc CXX=c++
- os: linux
dist: trusty
compiler: clang
env: CC=clang CXX=clang++
- os: linux
dist: xenial
compiler: gcc
env: CC=gcc CXX=g++
- os: linux
dist: bionic
compiler: clang
env: CC=clang CXX=clang++
- os: osx
osx_image: xcode11
env: CC=cc CXX=c++
- os: osx
osx_image: xcode9.4
env: CC=cc CXX=c++
script: >
if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then
git fetch --unshallow --tags --prune &&
git submodule foreach --recursive git fetch --unshallow --tags --prune &&
(if which clang-format-6.0 > /dev/null && make reformat && [[ -n $(git diff) ]];
then
echo "You must run 'make reformat' before submitting a pull request";
echo "";
git diff;
exit -1;
fi) &&
make --keep-going all && MALLOC_CHECK_=7 MALLOC_PERTURB_=42 make --keep-going check
else
[ ! -s cov-int/scm_log.txt ] || cat cov-int/scm_log.txt;
fi && sleep 3
env:
global:
- secure: "M+W+heGGyRQJoBq2W0uqWVrpL4KBXmL0MFL7FSs7f9vmAaDyEgziUXeZRj3GOKzW4kTef3LpIeiu9SmvqSMoQivGGiomZShqPVl045o/OUgRCAT7Al1RLzEZ0efSHpIPf0PZ6byEf6GR2ML76OfuL6JxTVdnz8iVyO2sgLE1HbX1VeB+wgd/jfMeOBhCCXskfK6MLyZihfMYsiYZYSaV98ZDhDLSlzuuRIgzb0bMi8aL6AErs0WLW0NelRBeHkKPYfAUc85pdQHscgrJw6Rh/zT6+8BQ/q5f4IgWhiu4xoRg3Ngl7SNoedRQh93ADM3UG2iGl6HDFpVORaXcFWKAtuYY+kHQ0HB84BRYpQmeBuXNpltsfxQ3d1Q3u0RlE45zRvmr2+X1mFnkcNUAWISLPbsOUlriDQM8irGwRpho77/uYnRC00bJsHW//s6+uPf9zrAw1nI4f0y3PAWukGF/xs6HAI3FZPsuSSnx18Tj3Opgbc9Spop+V3hkhdiJoPGpNKTkFX4ZRXfkPgoRVJmtp4PpbpH0Ps/mCriKjMEfGGi0HcVCi0pEGLXiecdqJ5KPg5+22zNycEujQBJcNTKd9shN+R3glrbmhAxTEzGdGwxXXJ2ybwJ2PWJLMYZ7g98nLyX+uQPaA3BlsbYJHNeS5283/9pJsd9DzfHKsN2nFSc="
before_install:
- echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-
- ${CC} --version
- ${CXX} --version
addons:
coverity_scan:
project:
name: "ReOpen/libmdbx"
version: 0.1
description: "Build submitted via Travis CI"
notification_email: leo@yuriev.ru
build_command_prepend: "git fetch --unshallow --tags --prune && make dist"
build_command: "make MDBX_OPTIONS=-DMDBX_DEBUG=2 -C dist all"
branch_pattern: coverity_scan

32
contrib/db/libmdbx/AUTHORS Normal file
View file

@ -0,0 +1,32 @@
Contributors
============
Alexey Naumov <alexey.naumov@gmail.com>
Chris Mikkelson <cmikk@qwest.net>
Claude Brisson <claude.brisson@gmail.com>
David Barbour <dmbarbour@gmail.com>
David Wilson <dw@botanicus.net>
dreamsxin <dreamsxin@126.com>
Hallvard Furuseth <hallvard@openldap.org>, <h.b.furuseth@usit.uio.no>
Heiko Becker <heirecka@exherbo.org>
Howard Chu <hyc@openldap.org>, <hyc@symas.com>
Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
James Rouzier <rouzier@gmail.com>
Jean-Christophe DUBOIS <jcd@tribudubois.net>
John Hewson <john@jahewson.com>
Klaus Malorny <klaus.malorny@knipp.de>
Kurt Zeilenga <kurt.zeilenga@isode.com>
Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.com>
Lorenz Bauer <lmb@cloudflare.com>
Luke Yeager <lyeager@nvidia.com>
Martin Hedenfalk <martin@bzero.se>
Ondrej Kuznik <ondrej.kuznik@acision.com>
Orivej Desh <orivej@gmx.fr>
Oskari Timperi <oskari.timperi@iki.fi>
Pavel Medvedev <pmedvedev@gmail.com>
Philipp Storz <philipp.storz@bareos.com>
Quanah Gibson-Mount <quanah@openldap.org>
Salvador Ortiz <sog@msg.com.mx>
Sebastien Launay <sebastien@slaunay.fr>
Vladimir Romanov <vromanov@gmail.com>
Zano Foundation <crypto.sowle@gmail.com>

192
contrib/db/libmdbx/CMakeLists.dist-minimal Normal file
View file

@ -0,0 +1,192 @@
##
## This is the minimal template for CMakeList.txt which could be used
## to build libmdbx from the "amalgamated form" of libmdbx's source code.
##
## The amalgamated form is intended to embedding libmdbx in other projects
## in cases when using as git-submodule is not acceptable or inconveniently.
##
## The amalgamated form could be generated from full git repository
## on Linux just by `make dist`.
##
##
## Copyright 2019 Leonid Yuriev <leo@yuriev.ru>
## and other libmdbx authors: please see AUTHORS file.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
##
##
## libmdbx = { Revised and extended descendant of Symas LMDB. }
## Please see README.md at https://github.com/leo-yuriev/libmdbx
##
## Libmdbx is superior to LMDB in terms of features and reliability,
## not inferior in performance. libmdbx works on Linux, FreeBSD, MacOS X
## and other systems compliant with POSIX.1-2008, but also support Windows
## as a complementary platform.
##
## The next version is under active non-public development and will be
## released as MithrilDB and libmithrildb for libraries & packages.
## Admittedly mythical Mithril is resembling silver but being stronger and
## lighter than steel. Therefore MithrilDB is rightly relevant name.
##
## MithrilDB will be radically different from libmdbx by the new database
## format and API based on C++17, as well as the Apache 2.0 License.
## The goal of this revolution is to provide a clearer and robust API,
## add more features and new valuable properties of database.
##
## The Future will (be) Positive. Всё будет хорошо.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
if(NOT CMAKE_VERSION VERSION_LESS 3.9)
cmake_policy(SET CMP0069 NEW)
endif()
if(NOT CMAKE_VERSION VERSION_LESS 3.12)
cmake_policy(SET CMP0075 NEW)
endif()
if(NOT CMAKE_VERSION VERSION_LESS 3.13)
cmake_policy(SET CMP0077 NEW)
endif()
if(DEFINED PROJECT_NAME)
set(SUBPROJECT ON)
set(NOT_SUBPROJECT OFF)
else()
set(SUBPROJECT OFF)
set(NOT_SUBPROJECT ON)
project(libmdbx C CXX)
endif()
find_package(Threads REQUIRED)
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING
"Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif()
list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11)
if(NOT HAS_C11 LESS 0)
set(MDBX_C_STANDARD 11)
else()
set(MDBX_C_STANDARD 99)
endif()
message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx")
# not supported by this (minimal) script
add_definitions(-DMDBX_AVOID_CRT=0)
# provide build timestamp
string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC)
add_definitions(-DMDBX_BUILD_TIMESTAMP="${MDBX_BUILD_TIMESTAMP}")
# provide compiler info
execute_process(COMMAND sh -c "${CMAKE_C_COMPILER} --version | head -1"
OUTPUT_VARIABLE MDBX_BUILD_COMPILER
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
RESULT_VARIABLE rc)
if(rc OR NOT MDBX_BUILD_COMPILER)
string(STRIP "${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION}" MDBX_BUILD_COMPILER)
endif()
add_definitions(-DMDBX_BUILD_COMPILER="${MDBX_BUILD_COMPILER}")
# provide cpu/arch-system pair
if(CMAKE_C_COMPILER_TARGET)
set(MDBX_BUILD_TARGET "${CMAKE_C_COMPILER_TARGET}")
elseif(CMAKE_C_PLATFORM_ID AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME)
string(STRIP "${CMAKE_C_PLATFORM_ID}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_LIBRARY_ARCHITECTURE)
string(STRIP "${CMAKE_LIBRARY_ARCHITECTURE}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_GENERATOR_PLATFORM AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME)
string(STRIP "${CMAKE_GENERATOR_PLATFORM}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_SYSTEM_ARCH)
string(STRIP "${CMAKE_SYSTEM_ARCH}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
else()
string(STRIP "${CMAKE_SYSTEM_PROCESSOR}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
endif()
add_definitions(-DMDBX_BUILD_TARGET="${MDBX_BUILD_TARGET}")
# provide build target-config
if(CMAKE_CONFIGURATION_TYPES)
add_definitions(-DMDBX_BUILD_CONFIG="$<CONFIG>")
else()
add_definitions(-DMDBX_BUILD_CONFIG="${CMAKE_BUILD_TYPE}")
endif()
# provide build cflags
set(MDBX_BUILD_FLAGS "")
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS})
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES})
if(CMAKE_CONFIGURATION_TYPES)
add_definitions(-DMDBX_BUILD_FLAGS_CONFIG="$<$<CONFIG:Debug>:${CMAKE_C_FLAGS_DEBUG} ${CMAKE_C_DEFINES_DEBUG}>$<$<CONFIG:Release>:${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_DEFINES_RELEASE}>$<$<CONFIG:RelWithDebInfo>:${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_DEFINES_RELWITHDEBINFO}>$<$<CONFIG:MinSizeRel>:${CMAKE_C_FLAGS_MINSIZEREL} ${CMAKE_C_DEFINES_MINSIZEREL}>")
else()
string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPERCASE)
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}})
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES_${CMAKE_BUILD_TYPE_UPPERCASE}})
endif()
list(REMOVE_DUPLICATES MDBX_BUILD_FLAGS)
string(REPLACE ";" " " MDBX_BUILD_FLAGS "${MDBX_BUILD_FLAGS}")
add_definitions(-DMDBX_BUILD_FLAGS="${MDBX_BUILD_FLAGS}")
# shared library
if(NOT DEFINED MDBX_BUILD_SHARED_LIBRARY)
if(DEFINED BUILD_SHARED_LIBS)
option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS})
else()
option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ON)
endif()
endif()
if(MDBX_BUILD_SHARED_LIBRARY)
add_library(mdbx SHARED mdbx.c mdbx.h)
set_target_properties(mdbx PROPERTIES
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON
PUBLIC_HEADER mdbx.h)
target_compile_definitions(mdbx PRIVATE LIBMDBX_EXPORTS INTERFACE LIBMDBX_IMPORTS)
if(DEFINED INTERPROCEDURAL_OPTIMIZATION)
set_target_properties(mdbx PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
endif()
target_link_libraries(mdbx PRIVATE ${CMAKE_THREAD_LIBS_INIT})
if(WIN32)
target_link_libraries(mdbx PRIVATE ntdll.lib)
endif()
endif()
# static library used for tools, to avoid rpath/dll-path troubles
add_library(mdbx-static STATIC EXCLUDE_FROM_ALL mdbx.c mdbx.h)
set_target_properties(mdbx-static PROPERTIES
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON
PUBLIC_HEADER mdbx.h)
target_link_libraries(mdbx-static INTERFACE ${CMAKE_THREAD_LIBS_INIT})
if(DEFINED INTERPROCEDURAL_OPTIMIZATION)
set_target_properties(mdbx-static PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
endif()
if(WIN32)
target_link_libraries(mdbx-static INTERFACE ntdll.lib)
endif()
# mdbx-tools
foreach(TOOL mdbx_chk mdbx_copy mdbx_stat mdbx_dump mdbx_load)
add_executable(${TOOL} ${TOOL}.c)
set_target_properties(${TOOL} PROPERTIES
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON)
if(DEFINED INTERPROCEDURAL_OPTIMIZATION)
set_target_properties(${TOOL} PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
endif()
target_link_libraries(${TOOL} mdbx-static)
endforeach()
cmake_policy(POP)

342
contrib/db/libmdbx/CMakeLists.txt Normal file
View file

@ -0,0 +1,342 @@
##
## Copyright 2019 Leonid Yuriev <leo@yuriev.ru>
## and other libmdbx authors: please see AUTHORS file.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
##
##
## libmdbx = { Revised and extended descendant of Symas LMDB. }
## Please see README.md at https://github.com/leo-yuriev/libmdbx
##
## Libmdbx is superior to LMDB in terms of features and reliability,
## not inferior in performance. libmdbx works on Linux, FreeBSD, MacOS X
## and other systems compliant with POSIX.1-2008, but also support Windows
## as a complementary platform.
##
## The next version is under active non-public development and will be
## released as MithrilDB and libmithrildb for libraries & packages.
## Admittedly mythical Mithril is resembling silver but being stronger and
## lighter than steel. Therefore MithrilDB is rightly relevant name.
##
## MithrilDB will be radically different from libmdbx by the new database
## format and API based on C++17, as well as the Apache 2.0 License.
## The goal of this revolution is to provide a clearer and robust API,
## add more features and new valuable properties of database.
##
## The Future will (be) Positive. Всё будет хорошо.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
if(NOT CMAKE_VERSION VERSION_LESS 3.13)
cmake_policy(SET CMP0077 NEW)
endif()
if(NOT CMAKE_VERSION VERSION_LESS 3.12)
cmake_policy(SET CMP0075 NEW)
endif()
if(NOT CMAKE_VERSION VERSION_LESS 3.9)
cmake_policy(SET CMP0069 NEW)
include(CheckIPOSupported)
check_ipo_supported(RESULT CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE)
else()
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE FALSE)
endif()
if(DEFINED PROJECT_NAME)
set(SUBPROJECT ON)
set(NOT_SUBPROJECT OFF)
if(NOT DEFINED BUILD_TESTING)
set(BUILD_TESTING OFF)
endif()
else()
set(SUBPROJECT OFF)
set(NOT_SUBPROJECT ON)
project(libmdbx C CXX)
if(NOT DEFINED BUILD_TESTING)
set(BUILD_TESTING ON)
endif()
endif()
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING
"Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif()
macro(add_mdbx_option NAME DESCRIPTION DEFAULT)
list(APPEND MDBX_BUILD_OPTIONS ${NAME})
if(NOT ${DEFAULT} STREQUAL "AUTO")
option(${NAME} "${DESCRIPTION}" ${DEFAULT})
endif()
endmacro()
# only for compatibility testing
# set(CMAKE_CXX_STANDARD 14)
if(NOT "$ENV{TEAMCITY_PROCESS_FLOW_ID}" STREQUAL "")
set(CI TEAMCITY)
message(STATUS "TeamCity CI")
elseif(NOT "$ENV{TRAVIS}" STREQUAL "")
set(CI TRAVIS)
message(STATUS "Travis CI")
elseif(NOT "$ENV{CIRCLECI}" STREQUAL "")
set(CI CIRCLE)
message(STATUS "Circle CI")
elseif(NOT "$ENV{APPVEYOR}" STREQUAL "")
set(CI APPVEYOR)
message(STATUS "AppVeyor CI")
elseif(NOT "$ENV{CI}" STREQUAL "")
set(CI "$ENV{CI}")
message(STATUS "Other CI (${CI})")
else()
message(STATUS "Assume No any CI environment")
unset(CI)
endif()
# output all mdbx-related targets in single directory
if(NOT DEFINED MDBX_OUTPUT_DIR)
set(MDBX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
endif()
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR})
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR})
set(CMAKE_PDB_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR})
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR})
include(CheckLibraryExists)
include(CheckIncludeFiles)
include(CheckCCompilerFlag)
include(CheckSymbolExists)
include(CheckCSourceRuns)
include(CheckCXXSourceRuns)
include(CheckCSourceCompiles)
include(CheckCXXSourceCompiles)
include(TestBigEndian)
include(CheckFunctionExists)
include(FindPackageMessage)
include(CheckStructHasMember)
include(CMakeDependentOption)
include(GNUInstallDirs)
if(CMAKE_C_COMPILER_ID STREQUAL "MSVC" AND MSVC_VERSION LESS 1900)
message(SEND_ERROR "MSVC compiler ${MSVC_VERSION} is too old for building MDBX."
" At least 'Microsoft Visual Studio 2015' is required.")
endif()
# Set default build type to Release. This is to ease a User's life.
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE STRING
"Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif()
string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPERCASE)
include(cmake/utils.cmake)
include(cmake/compiler.cmake)
include(cmake/profile.cmake)
find_program(ECHO echo)
find_program(CAT cat)
find_program(GIT git)
find_program(LD ld)
# CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H)
# CHECK_INCLUDE_FILES(sys/uio.h HAVE_SYS_UIO_H)
# CHECK_INCLUDE_FILES(sys/stat.h HAVE_SYS_STAT_H)
CHECK_FUNCTION_EXISTS(pow NOT_NEED_LIBM)
if(NOT_NEED_LIBM)
set(LIB_MATH "")
else()
set(CMAKE_REQUIRED_LIBRARIES m)
CHECK_FUNCTION_EXISTS(pow HAVE_LIBM)
if(HAVE_LIBM)
set(LIB_MATH m)
else()
message(FATAL_ERROR "No libm found for math support")
endif()
endif()
find_package(Threads REQUIRED)
if(SUBPROJECT)
if(NOT DEFINED BUILD_SHARED_LIBS)
option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" OFF)
endif()
if(NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE)
option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independed (PIC)" ON)
endif()
else()
option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" ON)
option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independed (PIC)" ON)
if (CC_HAS_ARCH_NATIVE)
option(BUILD_FOR_NATIVE_CPU "Generate code for the compiling machine CPU" OFF)
endif()
if(CMAKE_CONFIGURATION_TYPES OR NOT CMAKE_BUILD_TYPE_UPPERCASE STREQUAL "DEBUG")
set(INTERPROCEDURAL_OPTIMIZATION_DEFAULT ON)
else()
set(INTERPROCEDURAL_OPTIMIZATION_DEFAULT OFF)
endif()
if(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE
OR GCC_LTO_AVAILABLE OR MSVC_LTO_AVAILABLE OR CLANG_LTO_AVAILABLE)
option(INTERPROCEDURAL_OPTIMIZATION "Enable interprocedural/LTO optimization" ${INTERPROCEDURAL_OPTIMIZATION_DEFAULT})
endif()
if(INTERPROCEDURAL_OPTIMIZATION)
if(GCC_LTO_AVAILABLE)
set(LTO_ENABLED TRUE)
set(CMAKE_AR ${CMAKE_GCC_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE)
set(CMAKE_NM ${CMAKE_GCC_NM} CACHE PATH "Path to nm program with LTO-plugin" FORCE)
set(CMAKE_RANLIB ${CMAKE_GCC_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE)
message(STATUS "MDBX indulge Link-Time Optimization by GCC")
elseif(CLANG_LTO_AVAILABLE)
set(LTO_ENABLED TRUE)
set(CMAKE_AR ${CMAKE_CLANG_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE)
set(CMAKE_NM ${CMAKE_CLANG_NM} CACHE PATH "Path to nm program with LTO-plugin" FORCE)
set(CMAKE_RANLIB ${CMAKE_CLANG_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE)
message(STATUS "MDBX indulge Link-Time Optimization by CLANG")
elseif(MSVC_LTO_AVAILABLE)
set(LTO_ENABLED TRUE)
message(STATUS "MDBX indulge Link-Time Optimization by MSVC")
elseif(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE)
message(STATUS "MDBX indulge Interprocedural Optimization by CMake")
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
set(LTO_ENABLED TRUE)
else()
message(WARNING "Unable to engage interprocedural/LTO optimization.")
endif()
else()
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION FALSE)
set(LTO_ENABLED FALSE)
endif()
find_program(VALGRIND valgrind)
if(VALGRIND)
# LY: cmake is ugly and nasty.
# - therefore memcheck-options should be defined before including ctest;
# - otherwise ctest may ignore it.
set(MEMORYCHECK_SUPPRESSIONS_FILE
"${PROJECT_SOURCE_DIR}/test/valgrind_suppress.txt"
CACHE FILEPATH "Suppressions file for Valgrind" FORCE)
set(MEMORYCHECK_COMMAND_OPTIONS
"--trace-children=yes --leak-check=full --track-origins=yes --error-exitcode=42 --error-markers=@ --errors-for-leak-kinds=definite --fair-sched=yes --suppressions=${MEMORYCHECK_SUPPRESSIONS_FILE}"
CACHE STRING "Valgrind options" FORCE)
set(VALGRIND_COMMAND_OPTIONS "${MEMORYCHECK_COMMAND_OPTIONS}" CACHE STRING "Valgrind options" FORCE)
endif()
#
# Enable 'make tags' target.
find_program(CTAGS ctags)
if(CTAGS)
add_custom_target(tags COMMAND ${CTAGS} -R -f tags
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
add_custom_target(ctags DEPENDS tags)
endif(CTAGS)
#
# Enable 'make reformat' target.
find_program(CLANG_FORMAT
NAMES clang-format-6.0 clang-format-5.0 clang-format-4.0
clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format)
if(CLANG_FORMAT AND UNIX)
add_custom_target(reformat
VERBATIM
COMMAND
git ls-files |
grep -E \\.\(c|cxx|cc|cpp|h|hxx|hpp\)\(\\.in\)?\$ |
xargs ${CLANG_FORMAT} -i --style=file
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
endif()
if(NOT "${PROJECT_BINARY_DIR}" STREQUAL "${PROJECT_SOURCE_DIR}")
add_custom_target(distclean)
add_custom_command(TARGET distclean
COMMAND ${CMAKE_COMMAND} -E remove_directory "${PROJECT_BINARY_DIR}"
COMMENT "Removing the build directory and its content")
elseif(IS_DIRECTORY .git AND GIT)
add_custom_target(distclean)
add_custom_command(TARGET distclean
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${GIT} submodule foreach --recursive git clean -f -X -d
COMMAND ${GIT} clean -f -X -d
COMMENT "Removing all build files from the source directory")
endif()
setup_compile_flags()
endif(SUBPROJECT)
list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11)
if(NOT HAS_C11 LESS 0)
set(MDBX_C_STANDARD 11)
else()
set(MDBX_C_STANDARD 99)
endif()
message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx")
##############################################################################
##############################################################################
#
# #### ##### ##### # #### # # ####
# # # # # # # # # ## # #
# # # # # # # # # # # # ####
# # # ##### # # # # # # # #
# # # # # # # # # ## # #
# #### # # # #### # # ####
#
set(MDBX_BUILD_OPTIONS ENABLE_ASAN MDBX_USE_VALGRIND ENABLE_GPROF ENABLE_GCOV)
add_mdbx_option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS})
add_mdbx_option(MDBX_ALLOY_BUILD "Build MDBX library as single object file" ON)
add_mdbx_option(MDBX_TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON)
add_mdbx_option(MDBX_TXN_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO)
mark_as_advanced(MDBX_TXN_CHECKPID)
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
add_mdbx_option(MDBX_DISABLE_GNU_SOURCE "Don't use nonstandard GNU/Linux extension functions" OFF)
mark_as_advanced(MDBX_DISABLE_GNU_SOURCE)
endif()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
add_mdbx_option(MDBX_OSX_SPEED_INSTEADOF_DURABILITY "Disable use fcntl(F_FULLFSYNC) in favor of speed" OFF)
mark_as_advanced(MDBX_OSX_SPEED_INSTEADOF_DURABILITY)
endif()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
add_mdbx_option(MDBX_AVOID_CRT "Avoid dependence from MSVC CRT" ${NOT_SUBPROJECT})
if(NOT MDBX_BUILD_SHARED_LIBRARY)
add_mdbx_option(MDBX_CONFIG_MANUAL_TLS_CALLBACK
"Provide mdbx_dll_handler() for manual initialization" OFF)
mark_as_advanced(MDBX_CONFIG_MANUAL_TLS_CALLBACK)
endif()
else()
add_mdbx_option(MDBX_USE_ROBUST "Use POSIX.1-2008 robust mutexes" AUTO)
mark_as_advanced(MDBX_USE_ROBUST)
add_mdbx_option(MDBX_USE_OFDLOCKS "Use Open file description locks (aka OFD locks, non-POSIX)" AUTO)
mark_as_advanced(MDBX_USE_OFDLOCKS)
endif()
option(MDBX_ENABLE_TESTS "Build MDBX tests." ${BUILD_TESTING})
################################################################################
################################################################################
add_subdirectory(src)
if(MDBX_ENABLE_TESTS)
add_subdirectory(test)
endif()
set(PACKAGE "libmdbx")
set(CPACK_PACKAGE_VERSION_MAJOR ${MDBX_VERSION_MAJOR})
set(CPACK_PACKAGE_VERSION_MINOR ${MDBX_VERSION_MINOR})
set(CPACK_PACKAGE_VERSION_PATCH ${MDBX_VERSION_RELEASE})
set(CPACK_PACKAGE_VERSION_COMMIT ${MDBX_VERSION_REVISION})
set(PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${CPACK_PACKAGE_VERSION_COMMIT}")
message(STATUS "libmdbx package version is ${PACKAGE_VERSION}")
cmake_policy(POP)

22
contrib/db/libmdbx/COPYRIGHT Normal file
View file

@ -0,0 +1,22 @@
Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
Copyright 2011-2015 Howard Chu, Symas Corp.
Copyright 2015,2016 Peter-Service R&D LLC.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted only as authorized by the OpenLDAP
Public License.
A copy of this license is available in the file LICENSE in the
top-level directory of the distribution or, alternatively, at
<http://www.OpenLDAP.org/license.html>.
OpenLDAP is a registered trademark of the OpenLDAP Foundation.
Individual files and/or contributed packages may be copyright by
other parties and/or subject to additional restrictions.
This work also contains materials derived from public sources.
Additional information about OpenLDAP can be obtained at
<http://www.openldap.org/>.

362
contrib/db/libmdbx/GNUmakefile Normal file
View file

@ -0,0 +1,362 @@
# This makefile is for GNU Make, and nowadays provided
# just for compatibility and preservation of traditions.
# Please use CMake in case of any difficulties or problems.
#
# Preprocessor macros (for MDBX_OPTIONS) of interest...
# Note that the defaults should already be correct for most platforms;
# you should not need to change any of these. Read their descriptions
# in README and source code if you do. There may be other macros of interest.
SHELL := /bin/bash
# install sandbox
SANDBOX ?=
# install prefixes (inside sandbox)
prefix ?= /usr/local
mandir ?= $(prefix)/man
# lib/bin suffix for multiarch/biarch, e.g. '.x86_64'
suffix ?=
CC ?= gcc
LD ?= ld
MDBX_OPTIONS ?= -DNDEBUG=1
CFLAGS ?= -Os -g3 -Wall -Werror -Wextra -Wpedantic -ffunction-sections -fPIC -fvisibility=hidden -std=gnu11 -pthread -Wno-tautological-compare
# LY: '--no-as-needed,-lrt' for ability to built with modern glibc, but then run with the old
LDFLAGS ?= $(shell $(LD) --help 2>/dev/null | grep -q -- --gc-sections && echo '-Wl,--gc-sections,-z,relro,-O1')$(shell $(LD) --help 2>/dev/null | grep -q -- -dead_strip && echo '-Wl,-dead_strip')
EXE_LDFLAGS ?= -pthread
################################################################################
UNAME := $(shell uname -s 2>/dev/null || echo Unknown)
define uname2sosuffix
case "$(UNAME)" in
Darwin*|Mach*) echo dylib;;
CYGWIN*|MINGW*|MSYS*|Windows*) echo dll;;
*) echo so;;
esac
endef
SO_SUFFIX := $(shell $(uname2sosuffix))
HEADERS := mdbx.h
LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX)
TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk
MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 mdbx_chk.1
.PHONY: mdbx all install clean test dist check
all: $(LIBRARIES) $(TOOLS)
mdbx: libmdbx.a libmdbx.$(SO_SUFFIX)
tools: $(TOOLS)
strip: all
strip libmdbx.$(SO_SUFFIX) $(TOOLS)
clean:
rm -rf $(TOOLS) mdbx_test @* *.[ao] *.[ls]o *~ tmp.db/* \
*.gcov *.log *.err src/*.o test/*.o mdbx_example dist \
config.h src/elements/config.h src/elements/version.c *.tar*
libmdbx.a: mdbx-static.o
$(AR) rs $@ $?
libmdbx.$(SO_SUFFIX): mdbx-dylib.o
$(CC) $(CFLAGS) $^ -pthread -shared $(LDFLAGS) -o $@
#> dist-cutoff-begin
ifeq ($(wildcard mdbx.c),mdbx.c)
#< dist-cutoff-end
################################################################################
# Amalgamated source code, i.e. distributed after `make dists`
MAN_SRCDIR := man1/
config.h: mdbx.c $(lastword $(MAKEFILE_LIST))
(echo '#define MDBX_BUILD_TIMESTAMP "$(shell date +%Y-%m-%dT%H:%M:%S%z)"' \
&& echo '#define MDBX_BUILD_FLAGS "$(CFLAGS) $(LDFLAGS)"' \
&& echo '#define MDBX_BUILD_COMPILER "$(shell set -o pipefail; $(CC) --version | head -1 || echo 'Please use GCC or CLANG compatible compiler')"' \
&& echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || echo 'Please use GCC or CLANG compatible compiler')"' \
) > $@
mdbx-dylib.o: config.h mdbx.c $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c mdbx.c -o $@
mdbx-static.o: config.h mdbx.c $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c mdbx.c -o $@
mdbx_%: mdbx_%.c libmdbx.a
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) -o $@
#> dist-cutoff-begin
else
################################################################################
# Plain (non-amalgamated) sources with test
define uname2osal
case "$(UNAME)" in
CYGWIN*|MINGW*|MSYS*|Windows*) echo windows;;
*) echo unix;;
esac
endef
define uname2titer
case "$(UNAME)" in
Darwin*|Mach*) echo 2;;
*) echo 12;;
esac
endef
DIST_EXTRA := LICENSE README.md CMakeLists.txt GNUmakefile $(addprefix man1/, $(MANPAGES))
DIST_SRC := mdbx.h mdbx.c $(addsuffix .c, $(TOOLS))
TEST_DB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.db
TEST_LOG ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.log
TEST_OSAL := $(shell $(uname2osal))
TEST_ITER := $(shell $(uname2titer))
TEST_SRC := test/osal-$(TEST_OSAL).cc $(filter-out $(wildcard test/osal-*.cc), $(wildcard test/*.cc))
TEST_INC := $(wildcard test/*.h)
TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC))
CXX ?= g++
CXXSTD ?= $(shell $(CXX) -std=c++27 -c test/test.cc -o /dev/null 2>/dev/null && echo -std=c++17 || echo -std=c++11)
CXXFLAGS := $(CXXSTD) $(filter-out -std=gnu11,$(CFLAGS))
MAN_SRCDIR := src/man1/
ALLOY_DEPS := $(wildcard src/elements/*)
MDBX_VERSION_GIT = ${shell set -o pipefail; git describe --tags | sed -n 's|^v*\([0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\)\(.*\)|\1|p' || echo 'Please fetch tags and/or install latest git version'}
MDBX_GIT_TIMESTAMP = $(shell git show --no-patch --format=%cI HEAD || echo 'Please install latest get version')
MDBX_GIT_DESCRIBE = $(shell git describe --tags --long --dirty=-dirty || echo 'Please fetch tags and/or install latest git version')
MDBX_VERSION_SUFFIX = $(shell set -o pipefail; echo -n '$(MDBX_GIT_DESCRIBE)' | tr -c -s '[a-zA-Z0-9]' _)
MDBX_BUILD_SOURCERY = $(shell set -o pipefail; $(MAKE) -s src/elements/version.c && (openssl dgst -r -sha256 src/elements/version.c || sha256sum src/elements/version.c || shasum -a 256 src/elements/version.c) 2>/dev/null | cut -d ' ' -f 1 || echo 'Please install openssl or sha256sum or shasum')_$(MDBX_VERSION_SUFFIX)
test check: all mdbx_example mdbx_test
rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \
(./mdbx_test --progress --console=no --repeat=$(TEST_ITER) --pathname=$(TEST_DB) --dont-cleanup-after basic && \
./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=1 --pathname=$(TEST_DB) --dont-cleanup-after basic) \
| tee -a $(TEST_LOG) | tail -n 42) \
&& ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy
mdbx_example: mdbx.h example/example-mdbx.c libmdbx.$(SO_SUFFIX)
$(CC) $(CFLAGS) -I. example/example-mdbx.c ./libmdbx.$(SO_SUFFIX) -o $@
check-singleprocess: all mdbx_test
rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \
(./mdbx_test --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after --hill && \
./mdbx_test --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \
./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after --nested) \
| tee -a $(TEST_LOG) | tail -n 42) \
&& ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy
check-fault: all mdbx_test
rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --inject-writefault=42 --dump-config --dont-cleanup-after basic | tee -a $(TEST_LOG) | tail -n 42) \
; ./mdbx_chk -vvnw $(TEST_DB) && ([ ! -e $(TEST_DB)-copy ] || ./mdbx_chk -vvn $(TEST_DB)-copy)
VALGRIND=valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt
memcheck check-valgrind: all mdbx_test
@echo "$(MDBX_OPTIONS)" | grep -q MDBX_USE_VALGRIND || echo "WARNING: Please build libmdbx with -DMDBX_USE_VALGRIND to avoid false-positives from Valgrind !!!" >&2
rm -f valgrind-*.log $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \
($(VALGRIND) ./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=4 --pathname=$(TEST_DB) --dont-cleanup-after basic && \
$(VALGRIND) ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \
$(VALGRIND) ./mdbx_test --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-after basic) \
| tee -a $(TEST_LOG) | tail -n 42) \
&& $(VALGRIND) ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy
define test-rule
$(patsubst %.cc,%.o,$(1)): $(1) $(TEST_INC) mdbx.h $(lastword $(MAKEFILE_LIST))
$(CXX) $(CXXFLAGS) $(MDBX_OPTIONS) -c $(1) -o $$@
endef
$(foreach file,$(TEST_SRC),$(eval $(call test-rule,$(file))))
mdbx_%: src/tools/mdbx_%.c libmdbx.a
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) -o $@
mdbx_test: $(TEST_OBJ) libmdbx.$(SO_SUFFIX)
$(CXX) $(CXXFLAGS) $(TEST_OBJ) -Wl,-rpath . -L . -l mdbx $(EXE_LDFLAGS) -o $@
git_DIR := $(shell if [ -d .git ]; then echo .git; elif [ -s .git -a -f .git ]; then grep '^gitdir: ' .git | cut -d ':' -f 2; else echo "Please use libmdbx as a git-submodule or the amalgamated source code" >&2 && echo git_directory; fi)
src/elements/version.c: src/elements/version.c.in $(lastword $(MAKEFILE_LIST)) $(git_DIR)/HEAD $(git_DIR)/index $(git_DIR)/refs/tags
sed \
-e "s|@MDBX_GIT_TIMESTAMP@|$(MDBX_GIT_TIMESTAMP)|" \
-e "s|@MDBX_GIT_TREE@|$(shell git show --no-patch --format=%T HEAD || echo 'Please install latest get version')|" \
-e "s|@MDBX_GIT_COMMIT@|$(shell git show --no-patch --format=%H HEAD || echo 'Please install latest get version')|" \
-e "s|@MDBX_GIT_DESCRIBE@|$(MDBX_GIT_DESCRIBE)|" \
-e "s|\$${MDBX_VERSION_MAJOR}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 1)|" \
-e "s|\$${MDBX_VERSION_MINOR}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 2)|" \
-e "s|\$${MDBX_VERSION_RELEASE}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 3)|" \
-e "s|\$${MDBX_VERSION_REVISION}|$(shell git rev-list --count --no-merges HEAD || echo 'Please fetch tags and/or install latest git version')|" \
src/elements/version.c.in > $@
src/elements/config.h: src/elements/version.c $(lastword $(MAKEFILE_LIST))
(echo '#define MDBX_BUILD_TIMESTAMP "$(shell date +%Y-%m-%dT%H:%M:%S%z)"' \
&& echo '#define MDBX_BUILD_FLAGS "$(CFLAGS) $(LDFLAGS)"' \
&& echo '#define MDBX_BUILD_COMPILER "$(shell set -o pipefail; $(CC) --version | head -1 || echo 'Please use GCC or CLANG compatible compiler')"' \
&& echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || echo 'Please use GCC or CLANG compatible compiler')"' \
&& echo '#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)' \
) > $@
mdbx-dylib.o: src/elements/config.h src/elements/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c src/alloy.c -o $@
mdbx-static.o: src/elements/config.h src/elements/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
$(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c src/alloy.c -o $@
.PHONY: dist
dist: libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz $(lastword $(MAKEFILE_LIST))
libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz: $(addprefix dist/, $(DIST_SRC) $(DIST_EXTRA)) $(addprefix dist/man1/,$(MANPAGES))
tar -c -a -f $@ --owner=0 --group=0 -C dist $(DIST_SRC) $(DIST_EXTRA) \
&& rm dist/@tmp-shared_internals.inc
dist/mdbx.h: mdbx.h src/elements/version.c $(lastword $(MAKEFILE_LIST))
mkdir -p dist && cp $< $@
dist/GNUmakefile: GNUmakefile
mkdir -p dist && sed -e '/^#> dist-cutoff-begin/,/^#< dist-cutoff-end/d' $< > $@
dist/@tmp-shared_internals.inc: src/elements/version.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST))
mkdir -p dist && sed \
-e 's|#pragma once|#define MDBX_ALLOY 1\n#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)|' \
-e 's|#include "../../mdbx.h"|@INCLUDE "mdbx.h"|' \
-e '/#include "defs.h"/r src/elements/defs.h' \
-e '/#include "osal.h"/r src/elements/osal.h' \
src/elements/internals.h > $@
dist/mdbx.c: dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST))
mkdir -p dist && (cat dist/@tmp-shared_internals.inc \
&& cat src/elements/core.c src/elements/osal.c src/elements/version.c \
&& echo '#if defined(_WIN32) || defined(_WIN64)' \
&& cat src/elements/lck-windows.c && echo '#else /* LCK-implementation */' \
&& cat src/elements/lck-posix.c && echo '#endif /* LCK-implementation */' \
) | grep -v -e '#include "' -e '#pragma once' | sed 's|@INCLUDE|#include|' > $@
define dist-tool-rule
dist/$(1).c: src/tools/$(1).c src/tools/wingetopt.h src/tools/wingetopt.c \
dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST))
mkdir -p dist && sed \
-e '/#include "..\/elements\/internals.h"/r dist/@tmp-shared_internals.inc' \
-e '/#include "wingetopt.h"/r src/tools/wingetopt.c' \
src/tools/$(1).c \
| grep -v -e '#include "' -e '#pragma once' -e '#define MDBX_ALLOY' \
| sed 's|@INCLUDE|#include|' > $$@
endef
$(foreach file,$(TOOLS),$(eval $(call dist-tool-rule,$(file))))
dist/man1/mdbx_%.1: src/man1/mdbx_%.1
mkdir -p dist/man1/ && cp $< $@
dist/LICENSE: LICENSE
mkdir -p dist/man1/ && cp $< $@
dist/README.md: README.md
mkdir -p dist/man1/ && cp $< $@
dist/CMakeLists.txt: CMakeLists.dist-minimal
mkdir -p dist/man1/ && cp $< $@
endif
################################################################################
# Cross-compilation simple test
CROSS_LIST = mips-linux-gnu-gcc \
powerpc64-linux-gnu-gcc powerpc-linux-gnu-gcc \
arm-linux-gnueabihf-gcc aarch64-linux-gnu-gcc \
sh4-linux-gnu-gcc mips64-linux-gnuabi64-gcc
# hppa-linux-gnu-gcc - don't supported by current qemu release
# s390x-linux-gnu-gcc - qemu troubles (hang/abort)
# sparc64-linux-gnu-gcc - qemu troubles (fcntl for F_SETLK/F_GETLK)
# alpha-linux-gnu-gcc - qemu (or gcc) troubles (coredump)
CROSS_LIST_NOQEMU = hppa-linux-gnu-gcc s390x-linux-gnu-gcc \
sparc64-linux-gnu-gcc alpha-linux-gnu-gcc
cross-gcc:
@echo "CORRESPONDING CROSS-COMPILERs ARE REQUIRED."
@echo "FOR INSTANCE: apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu"
@for CC in $(CROSS_LIST_NOQEMU) $(CROSS_LIST); do \
echo "===================== $$CC"; \
$(MAKE) clean && CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static $(MAKE) all || exit $$?; \
done
# Unfortunately qemu don't provide robust support for futexes.
# Therefore it is impossible to run full multi-process tests.
cross-qemu:
@echo "CORRESPONDING CROSS-COMPILERs AND QEMUs ARE REQUIRED."
@echo "FOR INSTANCE: "
@echo " 1) apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu"
@echo " 2) apt install binfmt-support qemu-user-static qemu-user qemu-system-arm qemu-system-mips qemu-system-misc qemu-system-ppc qemu-system-sparc"
@for CC in $(CROSS_LIST); do \
echo "===================== $$CC + qemu"; \
$(MAKE) clean && \
CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static MDBX_OPTIONS="-DMDBX_SAFE4QEMU $(MDBX_OPTIONS)" \
$(MAKE) check-singleprocess || exit $$?; \
done
#< dist-cutoff-end
install: $(LIBRARIES) $(TOOLS) $(HEADERS)
mkdir -p $(SANDBOX)$(prefix)/bin$(suffix) \
&& cp -t $(SANDBOX)$(prefix)/bin$(suffix) $(TOOLS) && \
mkdir -p $(SANDBOX)$(prefix)/lib$(suffix) \
&& cp -t $(SANDBOX)$(prefix)/lib$(suffix) $(LIBRARIES) && \
mkdir -p $(SANDBOX)$(prefix)/include \
&& cp -t $(SANDBOX)$(prefix)/include $(HEADERS) && \
mkdir -p $(SANDBOX)$(mandir)/man1 \
&& cp -t $(SANDBOX)$(mandir)/man1 $(addprefix $(MAN_SRCDIR), $(MANPAGES))
################################################################################
# Benchmarking by ioarena
IOARENA ?= $(shell \
(test -x ../ioarena/@BUILD/src/ioarena && echo ../ioarena/@BUILD/src/ioarena) || \
(test -x ../../@BUILD/src/ioarena && echo ../../@BUILD/src/ioarena) || \
(test -x ../../src/ioarena && echo ../../src/ioarena) || which ioarena)
NN ?= 25000000
ifneq ($(wildcard $(IOARENA)),)
.PHONY: bench clean-bench re-bench
clean-bench:
rm -rf bench-*.txt _ioarena/*
re-bench: clean-bench bench
define bench-rule
bench-$(1)_$(2).txt: $(3) $(IOARENA) $(lastword $(MAKEFILE_LIST))
LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \
$(IOARENA) -D $(1) -B crud -m nosync -n $(2) \
| tee $$@ | grep throughput && \
LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \
$(IOARENA) -D $(1) -B get,iterate -m sync -r 4 -n $(2) \
| tee -a $$@ | grep throughput \
|| mv -f $$@ $$@.error
endef
$(eval $(call bench-rule,mdbx,$(NN),libmdbx.$(SO_SUFFIX)))
$(eval $(call bench-rule,sophia,$(NN)))
$(eval $(call bench-rule,leveldb,$(NN)))
$(eval $(call bench-rule,rocksdb,$(NN)))
$(eval $(call bench-rule,wiredtiger,$(NN)))
$(eval $(call bench-rule,forestdb,$(NN)))
$(eval $(call bench-rule,lmdb,$(NN)))
$(eval $(call bench-rule,nessdb,$(NN)))
$(eval $(call bench-rule,sqlite3,$(NN)))
$(eval $(call bench-rule,ejdb,$(NN)))
$(eval $(call bench-rule,vedisdb,$(NN)))
$(eval $(call bench-rule,dummy,$(NN)))
$(eval $(call bench-rule,debug,10))
bench: bench-mdbx_$(NN).txt
.PHONY: bench-debug
bench-debug: bench-debug_10.txt
bench-quartet: bench-mdbx_$(NN).txt bench-lmdb_$(NN).txt bench-rocksdb_$(NN).txt bench-wiredtiger_$(NN).txt
endif

47
contrib/db/libmdbx/LICENSE Normal file
View file

@ -0,0 +1,47 @@
The OpenLDAP Public License
Version 2.8, 17 August 2003
Redistribution and use of this software and associated documentation
("Software"), with or without modification, are permitted provided
that the following conditions are met:
1. Redistributions in source form must retain copyright statements
and notices,
2. Redistributions in binary form must reproduce applicable copyright
statements and notices, this list of conditions, and the following
disclaimer in the documentation and/or other materials provided
with the distribution, and
3. Redistributions must contain a verbatim copy of this document.
The OpenLDAP Foundation may revise this license from time to time.
Each revision is distinguished by a version number. You may use
this Software under terms of this license revision or under the
terms of any subsequent revision of the license.
THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS
CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S)
OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
The names of the authors and copyright holders must not be used in
advertising or otherwise to promote the sale, use or other dealing
in this Software without specific, written prior permission. Title
to copyright in this Software shall at all times remain with copyright
holders.
OpenLDAP is a registered trademark of the OpenLDAP Foundation.
Copyright 1999-2003 The OpenLDAP Foundation, Redwood City,
California, USA. All Rights Reserved. Permission to copy and
distribute verbatim copies of this document is granted.

591
contrib/db/libmdbx/README.md Normal file
View file

@ -0,0 +1,591 @@
### The [repository now only mirrored on the Github](https://abf.io/erthink/libmdbx) due to illegal discriminatory restrictions for Russian Crimea and for sovereign crimeans.
<!-- Required extensions: pymdownx.betterem, pymdownx.tilde, pymdownx.emoji, pymdownx.tasklist, pymdownx.superfences -->
-----
libmdbx
======================================
_libmdbx_ is an extremely fast, compact, powerful, embedded
transactional [key-value
store](https://en.wikipedia.org/wiki/Key-value_database)
database, with permissive [OpenLDAP Public License](LICENSE).
_libmdbx_ has a specific set of properties and capabilities,
focused on creating unique lightweight solutions with
extraordinary performance.
The next version is under active non-public development and will be
released as **_MithrilDB_** and `libmithrildb` for libraries & packages.
Admittedly mythical [Mithril](https://en.wikipedia.org/wiki/Mithril) is
resembling silver but being stronger and lighter than steel. Therefore
_MithrilDB_ is rightly relevant name.
> _MithrilDB_ will be radically different from _libmdbx_ by the new
> database format and API based on C++17, as well as the [Apache 2.0
> License](https://www.apache.org/licenses/LICENSE-2.0). The goal of this
> revolution is to provide a clearer and robust API, add more features and
> new valuable properties of database.
*The Future will (be) [Positive](https://www.ptsecurity.com). Всё будет хорошо.*
[![Build Status](https://travis-ci.org/leo-yuriev/libmdbx.svg?branch=master)](https://travis-ci.org/leo-yuriev/libmdbx)
[![Build status](https://ci.appveyor.com/api/projects/status/ue94mlopn50dqiqg/branch/master?svg=true)](https://ci.appveyor.com/project/leo-yuriev/libmdbx/branch/master)
[![Coverity Scan Status](https://scan.coverity.com/projects/12915/badge.svg)](https://scan.coverity.com/projects/reopen-libmdbx)
## Table of Contents
- [Overview](#overview)
- [Comparison with other databases](#comparison-with-other-databases)
- [History & Acknowledgments](#history)
- [Description](#description)
- [Key features](#key-features)
- [Improvements over LMDB](#improvements-over-lmdb)
- [Gotchas](#gotchas)
- [Usage](#usage)
- [Building](#building)
- [Bindings](#bindings)
- [Performance comparison](#performance-comparison)
- [Integral performance](#integral-performance)
- [Read scalability](#read-scalability)
- [Sync-write mode](#sync-write-mode)
- [Lazy-write mode](#lazy-write-mode)
- [Async-write mode](#async-write-mode)
- [Cost comparison](#cost-comparison)
-----
## Overview
_libmdbx_ is revised and extended descendant of amazing [Lightning
Memory-Mapped
Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
_libmdbx_ inherits all features and characteristics from
[LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database),
but resolves some issues and adds several features.
- _libmdbx_ guarantee data integrity after crash unless this was explicitly
neglected in favour of write performance.
- _libmdbx_ allows multiple processes to read and update several key-value
tables concurrently, while being
[ACID](https://en.wikipedia.org/wiki/ACID)-compliant, with minimal
overhead and Olog(N) operation cost.
- _libmdbx_ enforce
[serializability](https://en.wikipedia.org/wiki/Serializability) for
writers by single
[mutex](https://en.wikipedia.org/wiki/Mutual_exclusion) and affords
[wait-free](https://en.wikipedia.org/wiki/Non-blocking_algorithm#Wait-freedom)
for parallel readers without atomic/interlocked operations, while
writing and reading transactions do not block each other.
- _libmdbx_ uses [B+Trees](https://en.wikipedia.org/wiki/B%2B_tree) and
[Memory-Mapping](https://en.wikipedia.org/wiki/Memory-mapped_file),
doesn't use [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging)
which might be a caveat for some workloads.
- _libmdbx_ implements a simplified variant of the [Berkeley
DB](https://en.wikipedia.org/wiki/Berkeley_DB) and/or
[dbm](https://en.wikipedia.org/wiki/DBM_(computing)) API.
- _libmdbx_ supports Linux, Windows, MacOS, FreeBSD and other systems
compliant with POSIX.1-2008.
### Comparison with other databases
For now please refer to [chapter of "BoltDB comparison with other
databases"](https://github.com/coreos/bbolt#comparison-with-other-databases)
which is also (mostly) applicable to _libmdbx_.
### History
At first the development was carried out within the
[ReOpenLDAP](https://github.com/leo-yuriev/ReOpenLDAP) project. About a
year later _libmdbx_ was separated into standalone project, which was
[presented at Highload++ 2015
conference](http://www.highload.ru/2015/abstracts/1831.html).
Since 2017 _libmdbx_ is used in [Fast Positive Tables](https://github.com/leo-yuriev/libfpta),
and development is funded by [Positive Technologies](https://www.ptsecurity.com).
### Acknowledgments
Howard Chu <hyc@openldap.org> is the author of LMDB, from which
originated the MDBX in 2015.
Martin Hedenfalk <martin@bzero.se> is the author of `btree.c` code, which
was used for begin development of LMDB.
-----
Description
===========
## Key features
1. Key-value pairs are stored in ordered map(s), keys are always sorted,
range lookups are supported.
2. Data is [memory-mapped](https://en.wikipedia.org/wiki/Memory-mapped_file)
into each worker DB process, and could be accessed zero-copy from transactions.
3. Transactions are
[ACID](https://en.wikipedia.org/wiki/ACID)-compliant, through to
[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
and [CoW](https://en.wikipedia.org/wiki/Copy-on-write). Writes are
strongly serialized and aren't blocked by reads, transactions can't
conflict with each other. Reads are guaranteed to get only commited data
([relaxing serializability](https://en.wikipedia.org/wiki/Serializability#Relaxing_serializability)).
4. Read transactions are
[non-blocking](https://en.wikipedia.org/wiki/Non-blocking_algorithm),
don't use [atomic operations](https://en.wikipedia.org/wiki/Linearizability#High-level_atomic_operations).
Readers don't block each other and aren't blocked by writers. Read
performance scales linearly with CPU core count.
> Nonetheless, "connect to DB" (starting the first read transaction in a thread) and
> "disconnect from DB" (closing DB or thread termination) requires a lock
> acquisition to register/unregister at the "readers table".
5. Keys with multiple values are stored efficiently without key
duplication, sorted by value, including integers (valuable for
secondary indexes).
6. Efficient operation on short fixed length keys,
including 32/64-bit integer types.
7. [WAF](https://en.wikipedia.org/wiki/Write_amplification) (Write
Amplification Factor) и RAF (Read Amplification Factor) are Olog(N).
8. No [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) and
transaction journal. In case of a crash no recovery needed. No need for
regular maintenance. Backups can be made on the fly on working DB
without freezing writers.
9. No additional memory management, all done by basic OS services.
## Improvements over LMDB
_libmdbx_ is superior to _legendary [LMDB](https://symas.com/lmdb/)_ in
terms of features and reliability, not inferior in performance. In
comparison to LMDB, _libmdbx_ make things "just work" perfectly and
out-of-the-box, not silently and catastrophically break down. The list
below is pruned down to the improvements most notable and obvious from
the user's point of view.
1. Automatic on-the-fly database size control by preset parameters, both
reduction and increment.
> _libmdbx_ manage the database size according to parameters specified
> by `mdbx_env_set_geometry()` function,
> ones include the growth step and the truncation threshold.
2. Automatic continuous zero-overhead database compactification.
> _libmdbx_ logically move as possible a freed pages
> at end of allocation area into unallocated space,
> and then release such space if a lot of.
3. LIFO policy for recycling a Garbage Collection items. On systems with a disk
write-back cache, this can significantly increase write performance, up to
several times in a best case scenario.
> LIFO means that for reuse pages will be taken which became unused the lastest.
> Therefore the loop of database pages circulation becomes as short as possible.
> In other words, the number of pages, that are overwritten in memory
> and on disk during a series of write transactions, will be as small as possible.
> Thus creates ideal conditions for the efficient operation of the disk write-back cache.
4. Fast estimation of range query result volume, i.e. how many items can
be found between a `KEY1` and a `KEY2`. This is prerequisite for build
and/or optimize query execution plans.
> _libmdbx_ performs a rough estimate based only on b-tree pages that
> are common for the both stacks of cursors that were set to corresponing
> keys.
5. `mdbx_chk` tool for database integrity check.
6. Guarantee of database integrity even in asynchronous unordered write-to-disk mode.
> _libmdbx_ propose additional trade-off by implementing append-like manner for updates
> in `NOSYNC` and `MAPASYNC` modes, that avoid database corruption after a system crash
> contrary to LMDB. Nevertheless, the `MDBX_UTTERLY_NOSYNC` mode available to match LMDB behaviour,
> and for a special use-cases.
7. Automated steady flush to disk upon volume of changes and/or by
timeout via cheap polling.
8. Sequence generation and three cheap persistent 64-bit markers with ACID.
9. Support for keys and values of zero length, including multi-values
(aka sorted duplicates).
10. The handler of lack-of-space condition with a callback,
that allow you to control and resolve such situations.
11. Support for opening a database in the exclusive mode, including on a network share.
12. Extended transaction info, including dirty and leftover space info
for a write transaction, reading lag and hold over space for read
transactions.
13. Extended whole-database info (aka environment) and reader enumeration.
14. Extended update or delete, _at once_ with getting previous value
and addressing the particular item from multi-value with the same key.
15. Support for explicitly updating the existing record, not insertion a new one.
16. All cursors are uniformly, can be reused and should be closed explicitly,
regardless ones were opened within write or read transaction.
17. Correct update of current record with `MDBX_CURRENT` flag when size
of key or data was changed, including sorted duplicated.
18. Opening database handles is spared from race conditions and
pre-opening is not needed.
19. Ability to determine whether the particular data is on a dirty page
or not, that allows to avoid copy-out before updates.
20. Ability to determine whether the cursor is pointed to a key-value
pair, to the first, to the last, or not set to anything.
21. Returning `MDBX_EMULTIVAL` error in case of ambiguous update or delete.
22. On **MacOS** the `fcntl(F_FULLFSYNC)` syscall is used _by
default_ to synchronize data with the disk, as this is [the only way to
guarantee data
durability](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html)
in case of power failure. Unfortunately, in scenarios with high write
intensity, the use of `F_FULLFSYNC` significant degrades performance
compared to LMDB, where the `fsync()` syscall is used. Therefore,
_libmdbx_ allows you to override this behavior by defining the
`MDBX_OSX_SPEED_INSTEADOF_DURABILITY=1` option while build the library.
23. On **Windows** the `LockFileEx()` syscall is used for locking, since
it allows place the database on network drives, and provides protection
against incompetent user actions (aka
[poka-yoke](https://en.wikipedia.org/wiki/Poka-yoke)). Therefore
_libmdbx_ may be a little lag in performance tests from LMDB where a
named mutexes are used.
## Gotchas
1. There cannot be more than one writer at a time.
> On the other hand, this allows serialize an updates and eliminate any
> possibility of conflicts, deadlocks or logical errors.
2. No [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) means
relatively big [WAF](https://en.wikipedia.org/wiki/Write_amplification)
(Write Amplification Factor). Because of this syncing data to disk might
be quite resource intensive and be main performance bottleneck during
intensive write workload.
> As compromise _libmdbx_ allows several modes of lazy and/or periodic
> syncing, including `MAPASYNC` mode, which modificate data in memory and
> asynchronously syncs data to disk, moment to sync is picked by OS.
>
> Although this should be used with care, synchronous transactions in a DB
> with transaction journal will require 2 IOPS minimum (probably 3-4 in
> practice) because of filesystem overhead, overhead depends on
> filesystem, not on record count or record size. In _libmdbx_ IOPS count
> will grow logarithmically depending on record count in DB (height of B+
> tree) and will require at least 2 IOPS per transaction too.
3. [CoW](https://en.wikipedia.org/wiki/Copy-on-write) for
[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
is done on memory page level with
[B+trees](https://ru.wikipedia.org/wiki/B-%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE).
Therefore altering data requires to copy about Olog(N) memory pages,
which uses [memory bandwidth](https://en.wikipedia.org/wiki/Memory_bandwidth) and is main
performance bottleneck in `MDBX_MAPASYNC` mode.
> This is unavoidable, but isn't that bad. Syncing data to disk requires
> much more similar operations which will be done by OS, therefore this is
> noticeable only if data sync to persistent storage is fully disabled.
> _libmdbx_ allows to safely save data to persistent storage with minimal
> performance overhead. If there is no need to save data to persistent
> storage then it's much more preferable to use `std::map`.
4. Massive altering of data during a parallel long read operation will
increase the process work set, may exhaust entire free database space and
result in subsequent write performance degradation.
> _libmdbx_ mostly solve this issue by lack-of-space callback and `MDBX_LIFORECLAIM` mode.
> See [`mdbx.h`](mdbx.h) with API description for details.
> The "next" version of libmdbx (MithrilDB) will completely solve this.
5. There are no built-in checksums or digests to verify database integrity.
> The "next" version of _libmdbx_ (MithrilDB) will solve this issue employing [Merkle Tree](https://en.wikipedia.org/wiki/Merkle_tree).
--------------------------------------------------------------------------------
Usage
=====
## Source code embedding
_libmdbx_ provides two official ways for integration in source code form:
1. Using the amalgamated source code.
> The amalgamated source code includes all files requires to build and
> use _libmdbx_, but not for testing _libmdbx_ itself.
2. Adding the complete original source code as a `git submodule`.
> This allows you to build as _libmdbx_ and testing tool.
> On the other hand, this way requires you to pull git tags, and use C++11 compiler for test tool.
**_Please, avoid using any other techniques._** Otherwise, at least
don't ask for support and don't name such chimeras `libmdbx`.
The amalgamated source code could be created from original clone of git
repository on Linux by executing `make dist`. As a result, the desired
set of files will be formed in the `dist` subdirectory.
## Building
Both amalgamated and original source code provides build through the use
[CMake](https://cmake.org/) or [GNU
Make](https://www.gnu.org/software/make/) with
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)). All build ways
are completely traditional and have minimal prerequirements like
`build-essential`, i.e. the non-obsolete C/C++ compiler and a
[SDK](https://en.wikipedia.org/wiki/Software_development_kit) for the
target platform. Obviously you need building tools itself, i.e. `git`,
`cmake` or GNU `make` with `bash`.
So just use CMake or GNU Make in your habitual manner and feel free to
fill an issue or make pull request in the case something will be
unexpected or broken down.
#### DSO/DLL unloading and destructors of Thread-Local-Storage objects
When building _libmdbx_ as a shared library or use static _libmdbx_ as a
part of another dynamic library, it is advisable to make sure that your
system ensures the correctness of the call destructors of
Thread-Local-Storage objects when unloading dynamic libraries.
If this is not the case, then unloading a dynamic-link library with
_libmdbx_ code inside, can result in either a resource leak or a crash
due to calling destructors from an already unloaded DSO/DLL object. The
problem can only manifest in a multithreaded application, which makes
the unloading of shared dynamic libraries with _libmdbx_ code inside,
after using _libmdbx_. It is known that TLS-destructors are properly
maintained in the following cases:
- On all modern versions of Windows (Windows 7 and later).
- On systems with the
[`__cxa_thread_atexit_impl()`](https://sourceware.org/glibc/wiki/Destructor%20support%20for%20thread_local%20variables)
function in the standard C library, including systems with GNU libc
version 2.18 and later.
- On systems with libpthread/ntpl from GNU libc with bug fixes
[#21031](https://sourceware.org/bugzilla/show_bug.cgi?id=21031) and
[#21032](https://sourceware.org/bugzilla/show_bug.cgi?id=21032), or
where there are no similar bugs in the pthreads implementation.
### Linux and other platforms with GNU Make
To build the library it is enough to execute `make all` in the directory
of source code, and `make check` for execute the basic tests.
If the `make` installed on the system is not GNU Make, there will be a
lot of errors from make when trying to build. In this case, perhaps you
should use `gmake` instead of `make`, or even `gnu-make`, etc.
### FreeBSD and related platforms
As a rule, in such systems, the default is to use Berkeley Make. And GNU
Make is called by the gmake command or may be missing. In addition,
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) may be absent.
You need to install the required components: GNU Make, bash, C and C++
compilers compatible with GCC or CLANG. After that, to build the
library, it is enough execute `gmake all` (or `make all`) in the
directory with source code, and `gmake check` (or `make check`) to run
the basic tests.
### Windows
For build _libmdbx_ on Windows the _original_ CMake and [Microsoft Visual
Studio](https://en.wikipedia.org/wiki/Microsoft_Visual_Studio) are
recommended.
Building by MinGW, MSYS or Cygwin is potentially possible. However,
these scripts are not tested and will probably require you to modify the
CMakeLists.txt or Makefile respectively.
It should be noted that in _libmdbx_ was efforts to resolve
runtime dependencies from CRT and other libraries Visual Studio.
For this is enough define the `MDBX_AVOID_CRT` during build.
An example of running a basic test script can be found in the
[CI-script](appveyor.yml) for [AppVeyor](https://www.appveyor.com/). To
run the [long stochastic test scenario](test/long_stochastic.sh),
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) is required, and
the such testing is recommended with place the test data on the
[RAM-disk](https://en.wikipedia.org/wiki/RAM_drive).
### MacOS
Current [native build tools](https://en.wikipedia.org/wiki/Xcode) for
MacOS include GNU Make, CLANG and an outdated version of bash.
Therefore, to build the library, it is enough to run `make all` in the
directory with source code, and run `make check` to execute the base
tests. If something goes wrong, it is recommended to install
[Homebrew](https://brew.sh/) and try again.
To run the [long stochastic test scenario](test/long_stochastic.sh), you
will need to install the current (not outdated) version of
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)). To do this, we
recommend that you install [Homebrew](https://brew.sh/) and then execute
`brew install bash`.
## Bindings
| Runtime | GitHub | Author |
| -------- | ------ | ------ |
| Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) |
| .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) |
--------------------------------------------------------------------------------
Performance comparison
======================
All benchmarks were done by [IOArena](https://github.com/pmwkaa/ioarena)
and multiple [scripts](https://github.com/pmwkaa/ioarena/tree/HL%2B%2B2015)
runs on Lenovo Carbon-2 laptop, i7-4600U 2.1 GHz, 8 Gb RAM,
SSD SAMSUNG MZNTD512HAGL-000L1 (DXT23L0Q) 512 Gb.
## Integral performance
Here showed sum of performance metrics in 3 benchmarks:
- Read/Search on 4 CPU cores machine;
- Transactions with [CRUD](https://en.wikipedia.org/wiki/CRUD)
operations in sync-write mode (fdatasync is called after each
transaction);
- Transactions with [CRUD](https://en.wikipedia.org/wiki/CRUD)
operations in lazy-write mode (moment to sync data to persistent storage
is decided by OS).
*Reasons why asynchronous mode isn't benchmarked here:*
1. It doesn't make sense as it has to be done with DB engines, oriented
for keeping data in memory e.g. [Tarantool](https://tarantool.io/),
[Redis](https://redis.io/)), etc.
2. Performance gap is too high to compare in any meaningful way.
![Comparison #1: Integral Performance](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-1.png)
--------------------------------------------------------------------------------
## Read Scalability
Summary performance with concurrent read/search queries in 1-2-4-8
threads on 4 CPU cores machine.
![Comparison #2: Read Scalability](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-2.png)
--------------------------------------------------------------------------------
## Sync-write mode
- Linear scale on left and dark rectangles mean arithmetic mean
transactions per second;
- Logarithmic scale on right is in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**10,000 transactions in sync-write mode**. In case of a crash all data
is consistent and state is right after last successful transaction.
[fdatasync](https://linux.die.net/man/2/fdatasync) syscall is used after
each write transaction in this mode.
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 10,000 small key-value records.
![Comparison #3: Sync-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-3.png)
--------------------------------------------------------------------------------
## Lazy-write mode
- Linear scale on left and dark rectangles mean arithmetic mean of
thousands transactions per second;
- Logarithmic scale on right in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**100,000 transactions in lazy-write mode**. In case of a crash all data
is consistent and state is right after one of last transactions, but
transactions after it will be lost. Other DB engines use
[WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) or transaction
journal for that, which in turn depends on order of operations in
journaled filesystem. _libmdbx_ doesn't use WAL and hands I/O operations
to filesystem and OS kernel (mmap).
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 100,000 small key-value
records.
![Comparison #4: Lazy-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-4.png)
--------------------------------------------------------------------------------
## Async-write mode
- Linear scale on left and dark rectangles mean arithmetic mean of
thousands transactions per second;
- Logarithmic scale on right in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**1,000,000 transactions in async-write mode**. In case of a crash all
data will be consistent and state will be right after one of last
transactions, but lost transaction count is much higher than in
lazy-write mode. All DB engines in this mode do as little writes as
possible on persistent storage. _libmdbx_ uses
[msync(MS_ASYNC)](https://linux.die.net/man/2/msync) in this mode.
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 10,000 small key-value records.
![Comparison #5: Async-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-5.png)
--------------------------------------------------------------------------------
## Cost comparison
Summary of used resources during lazy-write mode benchmarks:
- Read and write IOPS;
- Sum of user CPU time and sys CPU time;
- Used space on persistent storage after the test and closed DB, but not
waiting for the end of all internal housekeeping operations (LSM
compactification, etc).
_ForestDB_ is excluded because benchmark showed it's resource
consumption for each resource (CPU, IOPS) much higher than other engines
which prevents to meaningfully compare it with them.
All benchmark data is gathered by
[getrusage()](http://man7.org/linux/man-pages/man2/getrusage.2.html)
syscall and by scanning data directory.
![Comparison #6: Cost comparison](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-6.png)
--------------------------------------------------------------------------------
```
$ objdump -f -h -j .text libmdbx.so
libmdbx.so: file format elf64-x86-64
architecture: i386:x86-64, flags 0x00000150:
HAS_SYMS, DYNAMIC, D_PAGED
start address 0x0000000000003710
Sections:
Idx Name Size VMA LMA File off Algn
11 .text 00015eff 0000000000003710 0000000000003710 00003710 2**4
CONTENTS, ALLOC, LOAD, READONLY, CODE
```

99
contrib/db/libmdbx/appveyor.yml Normal file
View file

@ -0,0 +1,99 @@
version: 0.3.2.{build}
environment:
matrix:
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
TOOLSET: 142
MDBX_BUILD_SHARED_LIBRARY: OFF
MDBX_AVOID_CRT: OFF
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
TOOLSET: 142
MDBX_BUILD_SHARED_LIBRARY: ON
MDBX_AVOID_CRT: ON
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
TOOLSET: 142
MDBX_BUILD_SHARED_LIBRARY: OFF
MDBX_AVOID_CRT: ON
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
CMAKE_GENERATOR: Visual Studio 16 2019
TOOLSET: 142
MDBX_BUILD_SHARED_LIBRARY: ON
MDBX_AVOID_CRT: OFF
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
CMAKE_GENERATOR: Visual Studio 15 2017
TOOLSET: 141
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
CMAKE_GENERATOR: Visual Studio 14 2015
TOOLSET: 140
branches:
except:
- coverity_scan
configuration:
- Debug
- Release
platform:
- Win32
- x64
before_build:
- git clean -x -f -d
- git submodule sync
- git fetch --tags --prune
- git submodule update --init --recursive
- git submodule foreach --recursive git fetch --tags --prune
- cmake --version
build_script:
- ps: |
Write-Output "*******************************************************************************"
Write-Output "Configuration: $env:CONFIGURATION"
Write-Output "Platform: $env:PLATFORM"
Write-Output "Toolchain: $env:CMAKE_GENERATOR v$env:TOOLSET"
Write-Output "Options: MDBX_AVOID_CRT=$env:MDBX_AVOID_CRT MDBX_BUILD_SHARED_LIBRARY=$env:MDBX_BUILD_SHARED_LIBRARY"
Write-Output "*******************************************************************************"
md _build -Force | Out-Null
cd _build
$generator = $env:CMAKE_GENERATOR
if ($env:TOOLSET -lt 142) {
if ($env:PLATFORM -eq "x64") {
$generator = "$generator Win64"
}
& cmake -G "$generator" -D CMAKE_CONFIGURATION_TYPES="Debug;Release" -D MDBX_AVOID_CRT:BOOL=$env:MDBX_AVOID_CRT -D MDBX_BUILD_SHARED_LIBRARY:BOOL=$env:MDBX_BUILD_SHARED_LIBRARY ..
} else {
& cmake -G "$generator" -A $env:PLATFORM -D CMAKE_CONFIGURATION_TYPES="Debug;Release" -DMDBX_AVOID_CRT:BOOL=$env:MDBX_AVOID_CRT -D MDBX_BUILD_SHARED_LIBRARY:BOOL=$env:MDBX_BUILD_SHARED_LIBRARY ..
}
if ($LastExitCode -ne 0) {
throw "Exec: $ErrorMessage"
}
Write-Output "*******************************************************************************"
& cmake --build . --config $env:CONFIGURATION
if ($LastExitCode -ne 0) {
throw "Exec: $ErrorMessage"
}
Write-Output "*******************************************************************************"
test_script:
- ps: |
if (($env:PLATFORM -ne "ARM") -and ($env:PLATFORM -ne "ARM64")) {
& ./$env:CONFIGURATION/mdbx_test.exe --progress --console=no --pathname=test.db --dont-cleanup-after basic > test.log
Get-Content test.log | Select-Object -last 42
if ($LastExitCode -ne 0) {
throw "Exec: $ErrorMessage"
} else {
& ./$env:CONFIGURATION/mdbx_chk.exe -nvv test.db | Tee-Object -file chk.log | Select-Object -last 42
}
}
on_failure:
- ps: Push-AppveyorArtifact \projects\libmdbx\_build\test.log
- ps: Push-AppveyorArtifact \projects\libmdbx\_build\test.db
- ps: Push-AppveyorArtifact \projects\libmdbx\_build\chk.log

666
contrib/db/libmdbx/cmake/compiler.cmake Normal file
View file

@ -0,0 +1,666 @@
## Copyright (c) 2012-2019 Leonid Yuriev <leo@yuriev.ru>.
##
## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
if (CMAKE_VERSION MATCHES ".*MSVC.*")
message(FATAL_ERROR "CMake from MSVC kit is unfit! "
"Please use the original CMake from https://cmake.org/download/")
endif()
if (NOT (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED))
message(FATAL_ERROR "This module required C or C++ to be enabled")
endif()
include(CMakeDependentOption)
if(CMAKE_CXX_COMPILER_LOADED)
include(CheckCXXSourceRuns)
include(CheckCXXSourceCompiles)
include(CheckCXXCompilerFlag)
endif()
if(CMAKE_C_COMPILER_LOADED)
include(CheckCSourceRuns)
include(CheckCSourceCompiles)
include(CheckCCompilerFlag)
endif()
# Check if the same compile family is used for both C and CXX
if(CMAKE_C_COMPILER_LOADED AND CMAKE_CXX_COMPILER_LOADED AND
NOT (CMAKE_C_COMPILER_ID STREQUAL CMAKE_CXX_COMPILER_ID))
message(WARNING "CMAKE_C_COMPILER_ID (${CMAKE_C_COMPILER_ID}) is different "
"from CMAKE_CXX_COMPILER_ID (${CMAKE_CXX_COMPILER_ID}). "
"The final binary may be unusable.")
endif()
if(CMAKE_CXX_COMPILER_LOADED)
set(CMAKE_PRIMARY_LANG "CXX")
else()
set(CMAKE_PRIMARY_LANG "C")
endif()
macro(check_compiler_flag flag variable)
if(CMAKE_CXX_COMPILER_LOADED)
check_cxx_compiler_flag(${flag} ${variable})
else()
check_c_compiler_flag(${flag} ${variable})
endif()
endmacro(check_compiler_flag)
# We support building with Clang and gcc. First check
# what we're using for build.
if(CMAKE_C_COMPILER_LOADED AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
set(CMAKE_COMPILER_IS_CLANG ON)
set(CMAKE_COMPILER_IS_GNUCC OFF)
endif()
if(CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXx_COMPILER_ID STREQUAL "Clang")
set(CMAKE_COMPILER_IS_CLANG ON)
set(CMAKE_COMPILER_IS_GNUCXX OFF)
endif()
# Hard coding the compiler version is ugly from cmake POV, but
# at least gives user a friendly error message. The most critical
# demand for C++ compiler is support of C++11 lambdas, added
# only in version 4.5 https://gcc.gnu.org/projects/cxx0x.html
if(CMAKE_COMPILER_IS_GNUCC)
if(CMAKE_C_COMPILER_VERSION VERSION_LESS 4.5)
message(FATAL_ERROR "
Your GCC version is ${CMAKE_C_COMPILER_VERSION}, please update")
endif()
endif()
if(CMAKE_COMPILER_IS_GNUCXX)
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.5)
message(FATAL_ERROR "
Your G++ version is ${CMAKE_CXX_COMPILER_VERSION}, please update")
endif()
endif()
if(CMAKE_C_COMPILER_LOADED)
# Check for Elbrus lcc
execute_process(COMMAND ${CMAKE_C_COMPILER} --version
OUTPUT_VARIABLE tmp_lcc_probe_version
RESULT_VARIABLE tmp_lcc_probe_result ERROR_QUIET)
if(tmp_lcc_probe_result EQUAL 0)
string(FIND "${tmp_lcc_probe_version}" "lcc:" tmp_lcc_marker)
string(FIND "${tmp_lcc_probe_version}" ":e2k-" tmp_e2k_marker)
if(tmp_lcc_marker GREATER -1 AND tmp_e2k_marker GREATER tmp_lcc_marker)
execute_process(COMMAND ${CMAKE_C_COMPILER} -print-version
OUTPUT_VARIABLE CMAKE_C_COMPILER_VERSION
RESULT_VARIABLE tmp_lcc_probe_result)
set(CMAKE_COMPILER_IS_ELBRUSC ON)
set(CMAKE_C_COMPILER_ID "Elbrus")
else()
set(CMAKE_COMPILER_IS_ELBRUSC OFF)
endif()
unset(tmp_lcc_marker)
unset(tmp_e2k_marker)
endif()
unset(tmp_lcc_probe_version)
unset(tmp_lcc_probe_result)
endif()
if(CMAKE_CXX_COMPILER_LOADED)
# Check for Elbrus l++
execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version
OUTPUT_VARIABLE tmp_lxx_probe_version
RESULT_VARIABLE tmp_lxx_probe_result ERROR_QUIET)
if(tmp_lxx_probe_result EQUAL 0)
string(FIND "${tmp_lxx_probe_version}" "lcc:" tmp_lcc_marker)
string(FIND "${tmp_lxx_probe_version}" ":e2k-" tmp_e2k_marker)
if(tmp_lcc_marker GREATER -1 AND tmp_e2k_marker GREATER tmp_lcc_marker)
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -print-version
OUTPUT_VARIABLE CMAKE_CXX_COMPILER_VERSION
RESULT_VARIABLE tmp_lxx_probe_result)
set(CMAKE_COMPILER_IS_ELBRUSCXX ON)
set(CMAKE_CXX_COMPILER_ID "Elbrus")
else()
set(CMAKE_COMPILER_IS_ELBRUSCXX OFF)
endif()
unset(tmp_lcc_marker)
unset(tmp_e2k_marker)
endif()
unset(tmp_lxx_probe_version)
unset(tmp_lxx_probe_result)
endif()
if(CMAKE_CL_64)
set(MSVC64 1)
endif()
if(WIN32 AND CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG})
execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -dumpmachine
OUTPUT_VARIABLE __GCC_TARGET_MACHINE
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(__GCC_TARGET_MACHINE MATCHES "amd64|x86_64|AMD64")
set(MINGW64 1)
endif()
unset(__GCC_TARGET_MACHINE)
endif()
if(CMAKE_COMPILER_IS_ELBRUSC OR CMAKE_SYSTEM_PROCESSOR MATCHES "e2k.*|E2K.*|elbrus.*|ELBRUS.*")
set(E2K TRUE)
set(CMAKE_SYSTEM_ARCH "Elbrus")
elseif((MSVC64 OR MINGW64) AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(X86_64 TRUE)
set(CMAKE_SYSTEM_ARCH "x86_64")
elseif(MINGW OR (MSVC AND NOT CMAKE_CROSSCOMPILING))
set(X86_32 TRUE)
set(CMAKE_SYSTEM_ARCH "x86")
elseif(CMAKE_COMPILER_IS_ELBRUSC OR CMAKE_SYSTEM_PROCESSOR MATCHES "e2k.*|E2K.*|elbrus.*|ELBRUS.*")
set(E2K TRUE)
set(CMAKE_SYSTEM_ARCH "Elbrus")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(X86_64 TRUE)
set(CMAKE_SYSTEM_ARCH "x86_64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i686.*|i386.*|x86.*")
set(X86_32 TRUE)
set(CMAKE_SYSTEM_ARCH "x86")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|ARM64.*)" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(AARCH64 TRUE)
set(CMAKE_SYSTEM_ARCH "ARM64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm.*|ARM.*)")
set(ARM32 TRUE)
set(CMAKE_SYSTEM_ARCH "ARM")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64le.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(PPC64LE TRUE)
set(CMAKE_SYSTEM_ARCH "PPC64LE")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(PPC64 TRUE)
set(CMAKE_SYSTEM_ARCH "PPC64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc).*")
set(PPC32 TRUE)
set(CMAKE_SYSTEM_ARCH "PPC")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(mips|MIPS)64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(MIPS64 TRUE)
set(CMAKE_SYSTEM_ARCH "MIPS64")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(mips|MIPS).*")
set(MIPS32 TRUE)
set(CMAKE_SYSTEM_ARCH "MIPS")
endif()
if(MSVC)
check_compiler_flag("/WX" CC_HAS_WERROR)
else()
#
# GCC started to warn for unused result starting from 4.2, and
# this is when it introduced -Wno-unused-result
# GCC can also be built on top of llvm runtime (on mac).
check_compiler_flag("-Wno-unknown-pragmas" CC_HAS_WNO_UNKNOWN_PRAGMAS)
check_compiler_flag("-Wextra" CC_HAS_WEXTRA)
check_compiler_flag("-Werror" CC_HAS_WERROR)
check_compiler_flag("-fexceptions" CC_HAS_FEXCEPTIONS)
check_cxx_compiler_flag("-fcxx-exceptions" CC_HAS_FCXX_EXCEPTIONS)
check_compiler_flag("-funwind-tables" CC_HAS_FUNWIND_TABLES)
check_compiler_flag("-fno-omit-frame-pointer" CC_HAS_FNO_OMIT_FRAME_POINTER)
check_compiler_flag("-fno-common" CC_HAS_FNO_COMMON)
check_compiler_flag("-ggdb" CC_HAS_GGDB)
check_compiler_flag("-fvisibility=hidden" CC_HAS_VISIBILITY)
check_compiler_flag("-march=native" CC_HAS_ARCH_NATIVE)
check_compiler_flag("-Og" CC_HAS_DEBUG_FRENDLY_OPTIMIZATION)
check_compiler_flag("-Wall" CC_HAS_WALL)
check_compiler_flag("-Ominimal" CC_HAS_OMINIMAL)
check_compiler_flag("-ffunction-sections -fdata-sections" CC_HAS_SECTIONS)
check_compiler_flag("-ffast-math" CC_HAS_FASTMATH)
# Check for an omp support
set(CMAKE_REQUIRED_FLAGS "-fopenmp -Werror")
check_cxx_source_compiles("int main(void) {
#pragma omp parallel
return 0;
}" HAVE_OPENMP)
set(CMAKE_REQUIRED_FLAGS "")
endif()
# Check for LTO support by GCC
if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG})
unset(gcc_collect)
unset(gcc_lto_wrapper)
if(NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 4.7)
execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -v
OUTPUT_VARIABLE gcc_info_v ERROR_VARIABLE gcc_info_v)
string(REGEX MATCH "^(.+\nCOLLECT_GCC=)([^ \n]+)(\n.+)$" gcc_collect_valid ${gcc_info_v})
if(gcc_collect_valid)
string(REGEX REPLACE "^(.+\nCOLLECT_GCC=)([^ \n]+)(\n.+)$" "\\2" gcc_collect ${gcc_info_v})
endif()
string(REGEX MATCH "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" gcc_lto_wrapper_valid ${gcc_info_v})
if(gcc_lto_wrapper_valid)
string(REGEX REPLACE "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" "\\2" gcc_lto_wrapper ${gcc_info_v})
endif()
set(gcc_suffix "")
if(gcc_collect_valid AND gcc_collect)
string(REGEX MATCH "^(.*cc)(-.+)$" gcc_suffix_valid ${gcc_collect})
if(gcc_suffix_valid)
string(REGEX MATCH "^(.*cc)(-.+)$" "\\2" gcc_suffix ${gcc_collect})
endif()
endif()
get_filename_component(gcc_dir ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} DIRECTORY)
if(NOT CMAKE_GCC_AR)
find_program(CMAKE_GCC_AR NAMES gcc${gcc_suffix}-ar gcc-ar${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_GCC_NM)
find_program(CMAKE_GCC_NM NAMES gcc${gcc_suffix}-nm gcc-nm${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_GCC_RANLIB)
find_program(CMAKE_GCC_RANLIB NAMES gcc${gcc_suffix}-ranlib gcc-ranlib${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH)
endif()
unset(gcc_dir)
unset(gcc_suffix_valid)
unset(gcc_suffix)
unset(gcc_lto_wrapper_valid)
unset(gcc_collect_valid)
unset(gcc_collect)
unset(gcc_info_v)
endif()
if(CMAKE_GCC_AR AND CMAKE_GCC_NM AND CMAKE_GCC_RANLIB AND gcc_lto_wrapper)
message(STATUS "Found GCC's LTO toolset: ${gcc_lto_wrapper}, ${CMAKE_GCC_AR}, ${CMAKE_GCC_RANLIB}")
set(GCC_LTO_CFLAGS "-flto -fno-fat-lto-objects -fuse-linker-plugin")
set(GCC_LTO_AVAILABLE TRUE)
message(STATUS "Link-Time Optimization by GCC is available")
else()
set(GCC_LTO_AVAILABLE FALSE)
message(STATUS "Link-Time Optimization by GCC is NOT available")
endif()
unset(gcc_lto_wrapper)
endif()
# check for LTO by MSVC
if(MSVC)
if(NOT MSVC_VERSION LESS 1600)
set(MSVC_LTO_AVAILABLE TRUE)
message(STATUS "Link-Time Optimization by MSVC is available")
else()
set(MSVC_LTO_AVAILABLE FALSE)
message(STATUS "Link-Time Optimization by MSVC is NOT available")
endif()
endif()
# Check for LTO support by CLANG
if(CMAKE_COMPILER_IS_CLANG)
if(NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 3.5)
execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -print-search-dirs
OUTPUT_VARIABLE clang_search_dirs)
unset(clang_bindir)
unset(clang_libdir)
string(REGEX MATCH "^(.*programs: =)([^:]*:)*([^:]+/llvm[-.0-9]+/bin[^:]*)(:[^:]*)*(\n.+)$" clang_bindir_valid ${clang_search_dirs})
if(clang_bindir_valid)
string(REGEX REPLACE "^(.*programs: =)([^:]*:)*([^:]+/llvm[-.0-9]+/bin[^:]*)(:[^:]*)*(\n.+)$" "\\3" clang_bindir ${clang_search_dirs})
get_filename_component(clang_libdir "${clang_bindir}/../lib" REALPATH)
if(clang_libdir)
message(STATUS "Found CLANG/LLVM directories: ${clang_bindir}, ${clang_libdir}")
endif()
endif()
if(NOT (clang_bindir AND clang_libdir))
message(STATUS "Could NOT find CLANG/LLVM directories (bin and/or lib).")
endif()
if(NOT CMAKE_CLANG_LD AND clang_bindir)
find_program(CMAKE_CLANG_LD NAMES llvm-link link llvm-ld ld PATHS ${clang_bindir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_CLANG_AR AND clang_bindir)
find_program(CMAKE_CLANG_AR NAMES llvm-ar ar PATHS ${clang_bindir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_CLANG_NM AND clang_bindir)
find_program(CMAKE_CLANG_NM NAMES llvm-nm nm PATHS ${clang_bindir} NO_DEFAULT_PATH)
endif()
if(NOT CMAKE_CLANG_RANLIB AND clang_bindir)
find_program(CMAKE_CLANG_RANLIB NAMES llvm-ranlib ranlib PATHS ${clang_bindir} NO_DEFAULT_PATH)
endif()
set(clang_lto_plugin_name "LLVMgold${CMAKE_SHARED_LIBRARY_SUFFIX}")
if(NOT CMAKE_LD_GOLD AND clang_bindir)
find_program(CMAKE_LD_GOLD NAMES ld.gold PATHS)
endif()
if(NOT CLANG_LTO_PLUGIN AND clang_libdir)
find_file(CLANG_LTO_PLUGIN ${clang_lto_plugin_name} PATH ${clang_libdir} NO_DEFAULT_PATH)
endif()
if(CLANG_LTO_PLUGIN)
message(STATUS "Found CLANG/LLVM's plugin for LTO: ${CLANG_LTO_PLUGIN}")
else()
message(STATUS "Could NOT find CLANG/LLVM's plugin (${clang_lto_plugin_name}) for LTO.")
endif()
if(CMAKE_CLANG_LD AND CMAKE_CLANG_AR AND CMAKE_CLANG_NM AND CMAKE_CLANG_RANLIB)
message(STATUS "Found CLANG/LLVM's binutils for LTO: ${CMAKE_CLANG_AR}, ${CMAKE_CLANG_RANLIB}")
else()
message(STATUS "Could NOT find CLANG/LLVM's binutils (ar, ranlib, nm) for LTO.")
endif()
unset(clang_lto_plugin_name)
unset(clang_libdir)
unset(clang_bindir_valid)
unset(clang_bindir)
unset(clang_search_dirs)
endif()
if((CLANG_LTO_PLUGIN AND CMAKE_LD_GOLD) AND
(CMAKE_CLANG_LD AND CMAKE_CLANG_AR AND CMAKE_CLANG_NM AND CMAKE_CLANG_RANLIB))
set(CLANG_LTO_AVAILABLE TRUE)
message(STATUS "Link-Time Optimization by CLANG/LLVM is available")
elseif(CMAKE_TOOLCHAIN_FILE AND NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 7.0)
set(CLANG_LTO_AVAILABLE TRUE)
if (NOT CMAKE_CLANG_AR)
set(CMAKE_CLANG_AR ${CMAKE_AR})
endif()
if (NOT CMAKE_CLANG_NM)
set(CMAKE_CLANG_NM ${CMAKE_NM})
endif()
if (NOT CMAKE_CLANG_RANLIB)
set(CMAKE_CLANG_RANLIB ${CMAKE_RANLIB })
endif()
message(STATUS "Assume Link-Time Optimization by CLANG/LLVM is available via ${CMAKE_TOOLCHAIN_FILE}")
else()
set(CLANG_LTO_AVAILABLE FALSE)
message(STATUS "Link-Time Optimization by CLANG/LLVM is NOT available")
endif()
endif()
# Perform build type specific configuration.
option(ENABLE_BACKTRACE "Enable output of fiber backtrace information in 'show
fiber' administrative command. Only works on x86 architectures, if compiled
with gcc. If GNU binutils and binutils-dev libraries are installed, backtrace
is output with resolved function (symbol) names. Otherwise only frame
addresses are printed." OFF)
set(HAVE_BFD False)
if(ENABLE_BACKTRACE)
if(NOT (X86_32 OR X86_64) OR NOT CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG})
# We only know this option to work with gcc
message(FATAL_ERROR "ENABLE_BACKTRACE option is set but the system
is not x86 based (${CMAKE_SYSTEM_PROCESSOR}) or the compiler
is not GNU GCC (${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER}).")
endif()
# Use GNU bfd if present.
find_library(BFD_LIBRARY NAMES libbfd.a)
if(BFD_LIBRARY)
check_library_exists(${BFD_LIBRARY} bfd_init "" HAVE_BFD_LIB)
endif()
find_library(IBERTY_LIBRARY NAMES libiberty.a)
if(IBERTY_LIBRARY)
check_library_exists(${IBERTY_LIBRARY} cplus_demangle "" HAVE_IBERTY_LIB)
endif()
set(CMAKE_REQUIRED_DEFINITIONS -DPACKAGE=${PACKAGE} -DPACKAGE_VERSION=${PACKAGE_VERSION})
check_include_files(bfd.h HAVE_BFD_H)
set(CMAKE_REQUIRED_DEFINITIONS)
find_package(ZLIB)
if(HAVE_BFD_LIB AND HAVE_BFD_H AND HAVE_IBERTY_LIB AND ZLIB_FOUND)
set(HAVE_BFD ON)
set(BFD_LIBRARIES ${BFD_LIBRARY} ${IBERTY_LIBRARY} ${ZLIB_LIBRARIES})
find_package_message(BFD_LIBRARIES "Found libbfd and dependencies"
${BFD_LIBRARIES})
if(TARGET_OS_FREEBSD AND NOT TARGET_OS_DEBIAN_FREEBSD)
set(BFD_LIBRARIES ${BFD_LIBRARIES} iconv)
endif()
endif()
endif()
macro(setup_compile_flags)
# LY: save initial C/CXX flags
if(NOT INITIAL_CMAKE_FLAGS_SAVED)
if(MSVC)
string(REGEX REPLACE "^(.*)(/EHsc)( *)(.*)$" "\\1/EHs\\3\\4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
set(INITIAL_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_C_FLAGS ${CMAKE_C_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_STATIC_LINKER_FLAGS ${CMAKE_STATIC_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_MODULE_LINKER_FLAGS ${CMAKE_MODULE_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE)
set(INITIAL_CMAKE_FLAGS_SAVED TRUE CACHE INTERNAL "State of initial CMake's flags" FORCE)
endif()
# LY: reset C/CXX flags
set(CXX_FLAGS ${INITIAL_CMAKE_CXX_FLAGS})
set(C_FLAGS ${INITIAL_CMAKE_C_FLAGS})
set(EXE_LINKER_FLAGS ${INITIAL_CMAKE_EXE_LINKER_FLAGS})
set(SHARED_LINKER_FLAGS ${INITIAL_CMAKE_SHARED_LINKER_FLAGS})
set(STATIC_LINKER_FLAGS ${INITIAL_CMAKE_STATIC_LINKER_FLAGS})
set(MODULE_LINKER_FLAGS ${INITIAL_CMAKE_MODULE_LINKER_FLAGS})
if(CC_HAS_FEXCEPTIONS)
add_compile_flags("C;CXX" "-fexceptions")
endif()
if(CC_HAS_FCXX_EXCEPTIONS)
add_compile_flags("CXX" "-fcxx-exceptions -frtti")
endif()
# In C a global variable without a storage specifier (static/extern) and
# without an initialiser is called a ’tentative definition’. The
# language permits multiple tentative definitions in the single
# translation unit; i.e. int foo; int foo; is perfectly ok. GNU
# toolchain goes even further, allowing multiple tentative definitions
# in *different* translation units. Internally, variables introduced via
# tentative definitions are implemented as ‘common’ symbols. Linker
# permits multiple definitions if they are common symbols, and it picks
# one arbitrarily for inclusion in the binary being linked.
#
# -fno-common forces GNU toolchain to behave in a more
# standard-conformant way in respect to tentative definitions and it
# prevents common symbols generation. Since we are a cross-platform
# project it really makes sense. There are toolchains that don’t
# implement GNU style handling of the tentative definitions and there
# are platforms lacking proper support for common symbols (osx).
if(CC_HAS_FNO_COMMON)
add_compile_flags("C;CXX" "-fno-common")
endif()
if(CC_HAS_GGDB)
add_compile_flags("C;CXX" "-ggdb")
endif()
if(CC_HAS_WNO_UNKNOWN_PRAGMAS AND NOT HAVE_OPENMP)
add_compile_flags("C;CXX" -Wno-unknown-pragmas)
endif()
if(CC_HAS_SECTIONS)
add_compile_flags("C;CXX" -ffunction-sections -fdata-sections)
elseif(MSVC)
add_compile_flags("C;CXX" /Gy)
endif()
# We must set -fno-omit-frame-pointer here, since we rely
# on frame pointer when getting a backtrace, and it must
# be used consistently across all object files.
# The same reasoning applies to -fno-stack-protector switch.
if(ENABLE_BACKTRACE)
if(CC_HAS_FNO_OMIT_FRAME_POINTER)
add_compile_flags("C;CXX" "-fno-omit-frame-pointer")
endif()
endif()
if(MSVC)
if (MSVC_VERSION LESS 1900)
message(FATAL_ERROR "At least \"Microsoft C/C++ Compiler\" version 19.0.24234.1 (Visual Studio 2015 Update 3) is required.")
endif()
add_compile_flags("CXX" "/Zc:__cplusplus")
add_compile_flags("C;CXX" "/W4")
add_compile_flags("C;CXX" "/utf-8")
else()
if(CC_HAS_WALL)
add_compile_flags("C;CXX" "-Wall")
endif()
if(CC_HAS_WEXTRA)
add_compile_flags("C;CXX" "-Wextra")
endif()
endif()
if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG}
AND CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 5)
# G++ bug. http://gcc.gnu.org/bugzilla/show_bug.cgi?id=31488
add_compile_flags("CXX" "-Wno-invalid-offsetof")
endif()
add_definitions("-D__STDC_FORMAT_MACROS=1")
add_definitions("-D__STDC_LIMIT_MACROS=1")
add_definitions("-D__STDC_CONSTANT_MACROS=1")
add_definitions("-D_HAS_EXCEPTIONS=1")
# Only add -Werror if it's a debug build, done by developers.
# Release builds should not cause extra trouble.
if(CC_HAS_WERROR AND (CI OR CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE STREQUAL "Debug"))
if(MSVC)
add_compile_flags("C;CXX" "/WX")
elseif(CMAKE_COMPILER_IS_CLANG)
if (NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 6)
add_compile_flags("C;CXX" "-Werror")
endif()
elseif(CMAKE_COMPILER_IS_GNUCC)
if (NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 6)
add_compile_flags("C;CXX" "-Werror")
endif()
else()
add_compile_flags("C;CXX" "-Werror")
endif()
endif()
if(HAVE_OPENMP)
add_compile_flags("C;CXX" "-fopenmp")
endif()
if (ENABLE_ASAN)
add_compile_flags("C;CXX" -fsanitize=address)
endif()
if(ENABLE_GCOV)
if(NOT HAVE_GCOV)
message(FATAL_ERROR
"ENABLE_GCOV option requested but gcov library is not found")
endif()
add_compile_flags("C;CXX" "-fprofile-arcs" "-ftest-coverage")
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
# add_library(gcov SHARED IMPORTED)
endif()
if(ENABLE_GPROF)
add_compile_flags("C;CXX" "-pg")
endif()
if(CMAKE_COMPILER_IS_GNUCC AND LTO_ENABLED)
add_compile_flags("C;CXX" ${GCC_LTO_CFLAGS})
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm -fwhole-program")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)
# Pass the same optimization flags to the linker
set(compile_flags "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}}")
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${compile_flags}")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${compile_flags}")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${compile_flags}")
unset(compile_flags)
else()
add_compile_flags("CXX" "-flto-odr-type-merging")
endif()
endif()
if(MSVC AND LTO_ENABLED)
add_compile_flags("C;CXX" "/GL")
foreach(linkmode IN ITEMS EXE SHARED STATIC MODULE)
set(${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS} /LTCG")
string(REGEX REPLACE "^(.*)(/INCREMENTAL:NO *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL:YES *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}")
string(STRIP "${${linkmode}_LINKER_FLAGS}" ${linkmode}_LINKER_FLAGS)
foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES ITEMS Release MinSizeRel RelWithDebInfo Debug)
string(TOUPPER "${config}" config_uppercase)
if(DEFINED "CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL:NO *)(.*)$" "\\1\\3" altered_flags "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL:YES *)(.*)$" "\\1\\3" altered_flags "${altered_flags}")
string(REGEX REPLACE "^(.*)(/INCREMENTAL *)(.*)$" "\\1\\3" altered_flags "${altered_flags}")
string(STRIP "${altered_flags}" altered_flags)
if(NOT "${altered_flags}" STREQUAL "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}")
set(CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase} "${altered_flags}" CACHE STRING "Altered: '/INCREMENTAL' removed for LTO" FORCE)
endif()
endif()
endforeach(config)
endforeach(linkmode)
unset(linkmode)
foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES ITEMS Release MinSizeRel RelWithDebInfo)
foreach(lang IN ITEMS C CXX)
string(TOUPPER "${config}" config_uppercase)
if(DEFINED "CMAKE_${lang}_FLAGS_${config_uppercase}")
string(REPLACE "/O2" "/Ox" altered_flags "${CMAKE_${lang}_FLAGS_${config_uppercase}}")
if(NOT "${altered_flags}" STREQUAL "${CMAKE_${lang}_FLAGS_${config_uppercase}}")
set(CMAKE_${lang}_FLAGS_${config_uppercase} "${altered_flags}" CACHE STRING "Altered: '/O2' replaced by '/Ox' for LTO" FORCE)
endif()
endif()
unset(config_uppercase)
endforeach(lang)
endforeach(config)
unset(altered_flags)
unset(lang)
unset(config)
endif()
if(CMAKE_COMPILER_IS_CLANG AND OSX_ARCHITECTURES)
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} -Wl,-keep_dwarf_unwind")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} -Wl,-keep_dwarf_unwind")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} -Wl,-keep_dwarf_unwind")
endif()
if(CMAKE_COMPILER_IS_CLANG AND LTO_ENABLED)
if(CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 3.9)
set(CLANG_LTO_FLAG "-flto")
else()
set(CLANG_LTO_FLAG "-flto=thin")
endif()
add_compile_flags("C;CXX" ${CLANG_LTO_FLAG})
set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm -fwhole-program")
set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm")
set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm")
endif()
# LY: push C/CXX flags into the cache
set(CMAKE_CXX_FLAGS ${CXX_FLAGS} CACHE STRING "Flags used by the C++ compiler during all build types" FORCE)
set(CMAKE_C_FLAGS ${C_FLAGS} CACHE STRING "Flags used by the C compiler during all build types" FORCE)
set(CMAKE_EXE_LINKER_FLAGS ${EXE_LINKER_FLAGS} CACHE STRING "Flags used by the linker" FORCE)
set(CMAKE_SHARED_LINKER_FLAGS ${SHARED_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of dll's" FORCE)
set(CMAKE_STATIC_LINKER_FLAGS ${STATIC_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of static libraries" FORCE)
set(CMAKE_MODULE_LINKER_FLAGS ${MODULE_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of modules" FORCE)
unset(CXX_FLAGS)
unset(C_FLAGS)
unset(EXE_LINKER_FLAGS)
unset(SHARED_LINKER_FLAGS)
unset(STATIC_LINKER_FLAGS)
unset(MODULE_LINKER_FLAGS)
endmacro(setup_compile_flags)
# determine library for for std::filesystem
set(LIBCXX_FILESYSTEM "")
if(CMAKE_COMPILER_IS_GNUCXX)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
set(LIBCXX_FILESYSTEM "stdc++fs")
endif()
elseif (CMAKE_COMPILER_IS_CLANG)
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
set(LIBCXX_FILESYSTEM "c++experimental")
else()
set(LIBCXX_FILESYSTEM "stdc++fs")
endif()
endif()
cmake_policy(POP)

45
contrib/db/libmdbx/cmake/profile.cmake Normal file
View file

@ -0,0 +1,45 @@
## Copyright (c) 2012-2019 Leonid Yuriev <leo@yuriev.ru>.
##
## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
include(CheckLibraryExists)
check_library_exists(gcov __gcov_flush "" HAVE_GCOV)
option(ENABLE_GCOV
"Enable integration with gcov, a code coverage program" OFF)
option(ENABLE_GPROF
"Enable integration with gprof, a performance analyzing tool" OFF)
if(CMAKE_CXX_COMPILER_LOADED)
include(CheckIncludeFileCXX)
check_include_file_cxx(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H)
else()
include(CheckIncludeFile)
check_include_file(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H)
endif()
option(MDBX_USE_VALGRIND "Enable integration with valgrind, a memory analyzing tool" OFF)
if(MDBX_USE_VALGRIND AND NOT HAVE_VALGRIND_MEMCHECK_H)
message(FATAL_ERROR "MDBX_USE_VALGRIND option is set but valgrind/memcheck.h is not found")
endif()
option(ENABLE_ASAN
"Enable AddressSanitizer, a fast memory error detector based on compiler instrumentation" OFF)
cmake_policy(POP)

183
contrib/db/libmdbx/cmake/utils.cmake Normal file
View file

@ -0,0 +1,183 @@
## Copyright (c) 2012-2019 Leonid Yuriev <leo@yuriev.ru>.
##
## Licensed under the Apache License, Version 2.0 (the "License");
## you may not use this file except in compliance with the License.
## You may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
cmake_minimum_required(VERSION 3.8.2)
cmake_policy(PUSH)
cmake_policy(VERSION 3.8.2)
macro(add_compile_flags langs)
foreach(_lang ${langs})
string(REPLACE ";" " " _flags "${ARGN}")
if(CMAKE_CXX_COMPILER_LOADED AND _lang STREQUAL "CXX")
set("${_lang}_FLAGS" "${${_lang}_FLAGS} ${_flags}")
endif()
if(CMAKE_C_COMPILER_LOADED AND _lang STREQUAL "C")
set("${_lang}_FLAGS" "${${_lang}_FLAGS} ${_flags}")
endif()
endforeach()
unset(_lang)
unset(_flags)
endmacro(add_compile_flags)
macro(set_source_files_compile_flags)
foreach(file ${ARGN})
get_filename_component(_file_ext ${file} EXT)
set(_lang "")
if("${_file_ext}" STREQUAL ".m")
set(_lang OBJC)
# CMake believes that Objective C is a flavor of C++, not C,
# and uses g++ compiler for .m files.
# LANGUAGE property forces CMake to use CC for ${file}
set_source_files_properties(${file} PROPERTIES LANGUAGE C)
elseif("${_file_ext}" STREQUAL ".mm")
set(_lang OBJCXX)
endif()
if(_lang)
get_source_file_property(_flags ${file} COMPILE_FLAGS)
if("${_flags}" STREQUAL "NOTFOUND")
set(_flags "${CMAKE_${_lang}_FLAGS}")
else()
set(_flags "${_flags} ${CMAKE_${_lang}_FLAGS}")
endif()
# message(STATUS "Set (${file} ${_flags}")
set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS
"${_flags}")
endif()
endforeach()
unset(_file_ext)
unset(_lang)
endmacro(set_source_files_compile_flags)
macro(fetch_version name version_file)
set(${name}_VERSION "")
set(${name}_GIT_DESCRIBE "")
set(${name}_GIT_TIMESTAMP "")
set(${name}_GIT_TREE "")
set(${name}_GIT_COMMIT "")
set(${name}_GIT_REVISION 0)
set(${name}_GIT_VERSION "")
if(GIT)
execute_process(COMMAND ${GIT} describe --tags --long --dirty=-dirty
OUTPUT_VARIABLE ${name}_GIT_DESCRIBE
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_DESCRIBE" STREQUAL "")
message(FATAL_ERROR "Please fetch tags and/or install latest version of git ('describe --tags --long --dirty' failed)")
endif()
execute_process(COMMAND ${GIT} show --no-patch --format=%cI HEAD
OUTPUT_VARIABLE ${name}_GIT_TIMESTAMP
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_TIMESTAMP" STREQUAL "%cI")
execute_process(COMMAND ${GIT} show --no-patch --format=%ci HEAD
OUTPUT_VARIABLE ${name}_GIT_TIMESTAMP
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_TIMESTAMP" STREQUAL "%ci")
message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%cI HEAD' failed)")
endif()
endif()
execute_process(COMMAND ${GIT} show --no-patch --format=%T HEAD
OUTPUT_VARIABLE ${name}_GIT_TREE
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_TREE" STREQUAL "")
message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%T HEAD' failed)")
endif()
execute_process(COMMAND ${GIT} show --no-patch --format=%H HEAD
OUTPUT_VARIABLE ${name}_GIT_COMMIT
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_COMMIT" STREQUAL "")
message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%H HEAD' failed)")
endif()
execute_process(COMMAND ${GIT} rev-list --count --no-merges HEAD
OUTPUT_VARIABLE ${name}_GIT_REVISION
OUTPUT_STRIP_TRAILING_WHITESPACE
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE rc)
if(rc OR "${name}_GIT_REVISION" STREQUAL "")
message(FATAL_ERROR "Please install latest version of git ('rev-list --count --no-merges HEAD' failed)")
endif()
string(REGEX MATCH "^(v)?([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" git_version_valid "${${name}_GIT_DESCRIBE}")
if(git_version_valid)
string(REGEX REPLACE "^(v)?([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" "\\2;\\3;\\4" ${name}_GIT_VERSION ${${name}_GIT_DESCRIBE})
else()
string(REGEX MATCH "^(v)?([0-9]+)\\.([0-9]+)(.*)?" git_version_valid "${${name}_GIT_DESCRIBE}")
if(git_version_valid)
string(REGEX REPLACE "^(v)?([0-9]+)\\.([0-9]+)(.*)?" "\\2;\\3;0" ${name}_GIT_VERSION ${${name}_GIT_DESCRIBE})
else()
message(AUTHOR_WARNING "Bad ${name} version \"${${name}_GIT_DESCRIBE}\"; falling back to 0.0.0 (have you made an initial release?)")
set(${name}_GIT_VERSION "0;0;0")
endif()
endif()
endif()
if(NOT ${name}_GIT_VERSION OR NOT ${name}_GIT_TIMESTAMP OR NOT ${name}_GIT_REVISION)
message(WARNING "Unable to retrive ${name} version from git.")
set(${name}_GIT_VERSION "0;0;0;0")
set(${name}_GIT_TIMESTAMP "")
set(${name}_GIT_REVISION 0)
# Try to get version from VERSION file
if(EXISTS "${version_file}")
file(STRINGS "${version_file}" ${name}_VERSION)
endif()
if(NOT ${name}_VERSION)
message(WARNING "Unable to retrive ${name} version from \"${version_file}\" file.")
set(${name}_VERSION_LIST ${${name}_GIT_VERSION})
string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}")
else()
string(REPLACE "." ";" ${name}_VERSION_LIST ${${name}_VERSION})
endif()
else()
list(APPEND ${name}_GIT_VERSION ${${name}_GIT_REVISION})
set(${name}_VERSION_LIST ${${name}_GIT_VERSION})
string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}")
endif()
list(GET ${name}_VERSION_LIST 0 "${name}_VERSION_MAJOR")
list(GET ${name}_VERSION_LIST 1 "${name}_VERSION_MINOR")
list(GET ${name}_VERSION_LIST 2 "${name}_VERSION_RELEASE")
list(GET ${name}_VERSION_LIST 3 "${name}_VERSION_REVISION")
set(${name}_VERSION_MAJOR ${${name}_VERSION_MAJOR} PARENT_SCOPE)
set(${name}_VERSION_MINOR ${${name}_VERSION_MINOR} PARENT_SCOPE)
set(${name}_VERSION_RELEASE ${${name}_VERSION_RELEASE} PARENT_SCOPE)
set(${name}_VERSION_REVISION ${${name}_VERSION_REVISION} PARENT_SCOPE)
set(${name}_VERSION ${${name}_VERSION} PARENT_SCOPE)
set(${name}_GIT_DESCRIBE ${${name}_GIT_DESCRIBE} PARENT_SCOPE)
set(${name}_GIT_TIMESTAMP ${${name}_GIT_TIMESTAMP} PARENT_SCOPE)
set(${name}_GIT_TREE ${${name}_GIT_TREE} PARENT_SCOPE)
set(${name}_GIT_COMMIT ${${name}_GIT_COMMIT} PARENT_SCOPE)
set(${name}_GIT_REVISION ${${name}_GIT_REVISION} PARENT_SCOPE)
set(${name}_GIT_VERSION ${${name}_GIT_VERSION} PARENT_SCOPE)
endmacro(fetch_version)
cmake_policy(POP)

6
contrib/db/libmdbx/example/CMakeLists.txt Normal file
View file

@ -0,0 +1,6 @@
set(TARGET mdbx_example)
project(${TARGET})
add_executable(${TARGET} example-mdbx.c)
target_link_libraries(${TARGET} mdbx)

1
contrib/db/libmdbx/example/README.md Normal file
View file

@ -0,0 +1 @@
See [example-mdbx.c](example-mdbx.c) as an example of using _libmdbx_, and do a line-by-line comparison of it with the [sample-bdb.txt](sample-bdb.txt) file.

112
contrib/db/libmdbx/example/example-mdbx.c Normal file
View file

@ -0,0 +1,112 @@
/* MDBX usage examle
*
* Do a line-by-line comparison of this and sample-bdb.txt
*/
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2017 Ilya Shipitsin <chipitsine@gmail.com>.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "mdbx.h"
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char *argv[]) {
(void)argc;
(void)argv;
int rc;
MDBX_env *env = NULL;
MDBX_dbi dbi = 0;
MDBX_val key, data;
MDBX_txn *txn = NULL;
MDBX_cursor *cursor = NULL;
char sval[32];
rc = mdbx_env_create(&env);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_open(env, "./example-db",
MDBX_NOSUBDIR | MDBX_COALESCE | MDBX_LIFORECLAIM, 0664);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_dbi_open(txn, NULL, 0, &dbi);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
key.iov_len = sizeof(int);
key.iov_base = sval;
data.iov_len = sizeof(sval);
data.iov_base = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = mdbx_put(txn, dbi, &key, &data, 0);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_put: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
txn = NULL;
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_cursor_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
int found = 0;
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len,
(char *)key.iov_base, data.iov_base, (int)data.iov_len,
(char *)data.iov_base);
found += 1;
}
if (rc != MDBX_NOTFOUND || found == 0) {
fprintf(stderr, "mdbx_cursor_get: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
} else {
rc = MDBX_SUCCESS;
}
bailout:
if (cursor)
mdbx_cursor_close(cursor);
if (txn)
mdbx_txn_abort(txn);
if (dbi)
mdbx_dbi_close(env, dbi);
if (env)
mdbx_env_close(env);
return (rc != MDBX_SUCCESS) ? EXIT_FAILURE : EXIT_SUCCESS;
}

77
contrib/db/libmdbx/example/sample-bdb.txt Normal file
View file

@ -0,0 +1,77 @@
/* BerkeleyDB toy/sample
*
* Do a line-by-line comparison of this and example-mdbx.c
*/
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <string.h>
#include <db.h>
int main(int argc,char * argv[])
{
int rc;
DB_ENV *env;
DB *dbi;
DBT key, data;
DB_TXN *txn;
DBC *cursor;
char sval[32], kval[32];
/* Note: Most error checking omitted for simplicity */
#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD)
rc = db_env_create(&env, 0);
rc = env->open(env, "./testdb", FLAGS, 0664);
rc = db_create(&dbi, env, 0);
rc = env->txn_begin(env, NULL, &txn, 0);
rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664);
memset(&key, 0, sizeof(DBT));
memset(&data, 0, sizeof(DBT));
key.size = sizeof(int);
key.data = sval;
data.size = sizeof(sval);
data.data = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = dbi->put(dbi, txn, &key, &data, 0);
rc = txn->commit(txn, 0);
if (rc) {
fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc));
goto leave;
}
rc = env->txn_begin(env, NULL, &txn, 0);
rc = dbi->cursor(dbi, txn, &cursor, 0);
key.flags = DB_DBT_USERMEM;
key.data = kval;
key.ulen = sizeof(kval);
data.flags = DB_DBT_USERMEM;
data.data = sval;
data.ulen = sizeof(sval);
while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.data, (int) key.size, (char *) key.data,
data.data, (int) data.size, (char *) data.data);
}
rc = cursor->c_close(cursor);
rc = txn->abort(txn);
leave:
rc = dbi->close(dbi, 0);
rc = env->close(env, 0);
return rc;
}

3497
contrib/db/libmdbx/mdbx.h Normal file
View file

File diff suppressed because it is too large Load diff

184
contrib/db/libmdbx/packages/rpm/CMakeLists.txt Normal file
View file

@ -0,0 +1,184 @@
cmake_minimum_required(VERSION 2.8.7)
set(TARGET mdbx)
project(${TARGET})
set(MDBX_VERSION_MAJOR 0)
set(MDBX_VERSION_MINOR 3)
set(MDBX_VERSION_RELEASE 1)
set(MDBX_VERSION_REVISION 0)
set(MDBX_VERSION_STRING ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}.${MDBX_VERSION_RELEASE})
enable_language(C)
enable_language(CXX)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED on)
add_definitions(-DNDEBUG=1 -DMDBX_DEBUG=0 -DLIBMDBX_EXPORTS=1 -D_GNU_SOURCE=1)
find_package(Threads REQUIRED)
get_directory_property(hasParent PARENT_DIRECTORY)
if(hasParent)
set(STANDALONE_BUILD 0)
else()
set(STANDALONE_BUILD 1)
enable_testing()
if (CMAKE_C_COMPILER_ID MATCHES GNU)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g3")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wextra")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread")
endif()
if (CMAKE_CXX_COMPILER_ID MATCHES GNU)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpointer-arith")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat-security")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Woverloaded-virtual")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wwrite-strings")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=20")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wunused-function -Wunused-variable -Wunused-value -Wmissing-declarations")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wcast-qual")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finline-functions-called-once")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-packed-bitfield-compat")
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g3")
endif()
if (COVERAGE)
if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
message(FATAL_ERROR "Coverage requires -DCMAKE_BUILD_TYPE=Debug Current value=${CMAKE_BUILD_TYPE}")
endif()
message(STATUS "Setting coverage compiler flags")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage")
add_definitions(-DCOVERAGE_TEST)
endif()
if (NOT TRAVIS)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fstack-protector-strong -static-libasan")
endif()
endif()
set(${TARGET}_SRC
mdbx.h
src/bits.h
src/defs.h
src/lck-linux.c
src/mdbx.c
src/osal.c
src/osal.h
src/version.c
)
add_library(${TARGET}_STATIC STATIC
${${TARGET}_SRC}
)
add_library(${TARGET} ALIAS ${TARGET}_STATIC)
add_library(${TARGET}_SHARED SHARED
${${TARGET}_SRC}
)
set_target_properties(${TARGET}_SHARED PROPERTIES
VERSION ${MDBX_VERSION_STRING}
SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}
OUTPUT_NAME ${TARGET}
CLEAN_DIRECT_OUTPUT 1
)
set_target_properties(${TARGET}_STATIC PROPERTIES
VERSION ${MDBX_VERSION_STRING}
SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}
OUTPUT_NAME ${TARGET}
CLEAN_DIRECT_OUTPUT 1
)
target_include_directories(${TARGET}_STATIC PUBLIC
${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(${TARGET}_SHARED PUBLIC
${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(${TARGET}_STATIC ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(${TARGET}_SHARED ${CMAKE_THREAD_LIBS_INIT})
if(UNIX AND NOT APPLE)
target_link_libraries(${TARGET}_STATIC rt)
target_link_libraries(${TARGET}_SHARED rt)
endif()
install(TARGETS ${TARGET}_STATIC DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx)
install(TARGETS ${TARGET}_SHARED DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx)
install(FILES mdbx.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT mdbx-devel)
add_subdirectory(src/tools)
add_subdirectory(test)
add_subdirectory(test/pcrf)
add_subdirectory(tutorial)
##############################################################################
set(CPACK_GENERATOR "RPM")
set(CPACK_RPM_COMPONENT_INSTALL ON)
# Version
if (NOT "$ENV{BUILD_NUMBER}" STREQUAL "")
set(CPACK_PACKAGE_RELEASE $ENV{BUILD_NUMBER})
else()
if (NOT "$ENV{CI_PIPELINE_ID}" STREQUAL "")
set(CPACK_PACKAGE_RELEASE $ENV{CI_PIPELINE_ID})
else()
set(CPACK_PACKAGE_RELEASE 1)
endif()
endif()
set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE})
set(CPACK_PACKAGE_VERSION ${MDBX_VERSION_STRING})
set(CPACK_PACKAGE_VERSION_FULL ${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE})
set(CPACK_RPM_mdbx-devel_PACKAGE_REQUIRES "mdbx = ${CPACK_PACKAGE_VERSION}")
set(CPACK_RPM_SPEC_INSTALL_POST "/bin/true")
set(CPACK_RPM_mdbx_PACKAGE_NAME mdbx)
set(CPACK_RPM_mdbx-devel_PACKAGE_NAME mdbx-devel)
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "The revised and extended descendant of Symas LMDB")
set(CPACK_PACKAGE_VENDOR "???")
set(CPACK_PACKAGE_CONTACT "Vladimir Romanov")
set(CPACK_PACKAGE_RELOCATABLE false)
set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
set(CPACK_RPM_PACKAGE_REQUIRES "")
set(CPACK_RPM_PACKAGE_GROUP "Applications/Database")
set(CPACK_RPM_mdbx_FILE_NAME "${CPACK_RPM_mdbx_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm")
set(CPACK_RPM_mdbx-devel_FILE_NAME "${CPACK_RPM_mdbx-devel_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm")
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/usr/local
/usr/local/bin
/usr/local/lib64
/usr/local/include
/usr/local/man
/usr/local/man/man1
)
include(CPack)

18
contrib/db/libmdbx/packages/rpm/build.sh Normal file
View file

@ -0,0 +1,18 @@
#!/bin/bash
set -e
CONFIG=$1
if [[ -z "${CONFIG}" ]]; then
CONFIG=Debug
fi
if [[ -r /opt/rh/devtoolset-6/enable ]]; then
source /opt/rh/devtoolset-6/enable
fi
#rm -f -r build || true
mkdir -p cmake-build-${CONFIG}
pushd cmake-build-${CONFIG} &> /dev/null
if [[ ! -r Makefile ]]; then
cmake .. -DCMAKE_BUILD_TYPE=${CONFIG}
fi
make -j8 || exit 1
popd &> /dev/null

25
contrib/db/libmdbx/packages/rpm/package.sh Normal file
View file

@ -0,0 +1,25 @@
#!/bin/bash
set -e
CONFIG=$1
if [[ -z "${CONFIG}" ]]; then
CONFIG=Debug
fi
DIRNAME=`dirname ${BASH_SOURCE[0]}`
DIRNAME=`readlink --canonicalize ${DIRNAME}`
if [[ -r /opt/rh/devtoolset-6/enable ]]; then
source /opt/rh/devtoolset-6/enable
fi
mkdir -p cmake-build-${CONFIG}
pushd cmake-build-${CONFIG} &> /dev/null
if [[ ! -r Makefile ]]; then
cmake .. -DCMAKE_BUILD_TYPE=${CONFIG}
fi
rm -f *.rpm
make -j8 package || exit 1
rm -f *-Unspecified.rpm
popd &> /dev/null

225
contrib/db/libmdbx/src/CMakeLists.txt Normal file
View file

@ -0,0 +1,225 @@
##
## Copyright 2019 Leonid Yuriev <leo@yuriev.ru>
## and other libmdbx authors: please see AUTHORS file.
## All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted only as authorized by the OpenLDAP
## Public License.
##
## A copy of this license is available in the file LICENSE in the
## top-level directory of the distribution or, alternatively, at
## <http://www.OpenLDAP.org/license.html>.
##
# Get version
fetch_version(MDBX "${CMAKE_CURRENT_SOURCE_DIR}/../VERSION")
message(STATUS "libmdbx version is ${MDBX_VERSION}")
if(MDBX_ALLOY_MODE)
set(LIBMDBX_SOURCES alloy.c)
else()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
set(LIBMDBX_OSAL windows)
else()
set(LIBMDBX_OSAL posix)
endif()
set(LIBMDBX_SOURCES
elements/defs.h elements/internals.h elements/osal.h
elements/core.c elements/osal.c elements/lck-${LIBMDBX_OSAL}.c)
endif()
list(APPEND LIBMDBX_SOURCES ../mdbx.h
"${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c"
"${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h")
if(MDBX_BUILD_SHARED_LIBRARY)
add_library(mdbx SHARED ${LIBMDBX_SOURCES})
target_compile_definitions(mdbx PRIVATE LIBMDBX_EXPORTS INTERFACE LIBMDBX_IMPORTS)
set(MDBX_LIBDEP_MODE PRIVATE)
else()
add_library(mdbx STATIC ${LIBMDBX_SOURCES})
set(MDBX_LIBDEP_MODE PUBLIC)
endif()
if(CC_HAS_VISIBILITY AND (LTO_ENABLED OR INTERPROCEDURAL_OPTIMIZATION))
set_target_properties(mdbx PROPERTIES LINK_FLAGS "-fvisibility=hidden")
endif()
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
if(MSVC)
if(NOT MSVC_LIB_EXE)
# Find lib.exe
get_filename_component(CL_NAME ${CMAKE_C_COMPILER} NAME)
string(REPLACE cl.exe lib.exe MSVC_LIB_EXE ${CL_NAME})
find_program(MSVC_LIB_EXE ${MSVC_LIB_EXE})
endif()
if(MSVC_LIB_EXE)
message(STATUS "Found MSVC's lib tool: ${MSVC_LIB_EXE}")
set(MDBX_NTDLL_EXTRA_IMPLIB ${CMAKE_CURRENT_BINARY_DIR}/mdbx_ntdll_extra.lib)
add_custom_command(OUTPUT ${MDBX_NTDLL_EXTRA_IMPLIB}
COMMENT "Create extra-import-library for ntdll.dll"
MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def"
COMMAND ${MSVC_LIB_EXE} /def:"${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def" /out:"${MDBX_NTDLL_EXTRA_IMPLIB}" ${INITIAL_CMAKE_STATIC_LINKER_FLAGS})
else()
message(SEND_ERROR "MSVC's lib tool not found")
endif()
elseif(MINGW OR MINGW64)
if(NOT DLLTOOL)
# Find dlltool
get_filename_component(GCC_NAME ${CMAKE_C_COMPILER} NAME)
string(REPLACE gcc dlltool DLLTOOL_NAME ${GCC_NAME})
find_program(DLLTOOL NAMES ${DLLTOOL_NAME})
endif()
if(DLLTOOL)
message(STATUS "Found dlltool: ${DLLTOOL}")
set(MDBX_NTDLL_EXTRA_IMPLIB "${CMAKE_CURRENT_BINARY_DIR}/mdbx_ntdll_extra.a")
add_custom_command(OUTPUT ${MDBX_NTDLL_EXTRA_IMPLIB}
COMMENT "Create extra-import-library for ntdll.dll"
MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def"
COMMAND ${DLLTOOL} -d "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def" -l "${MDBX_NTDLL_EXTRA_IMPLIB}")
else()
message(SEND_ERROR "dlltool not found")
endif()
endif()
endif()
target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ${CMAKE_THREAD_LIBS_INIT})
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ntdll.lib)
if(MDBX_NTDLL_EXTRA_IMPLIB)
# LY: Sometimes Cmake requires a nightmarish magic for simple things.
# 1) create a target out of the library compilation result
add_custom_target(ntdll_extra_target DEPENDS ${MDBX_NTDLL_EXTRA_IMPLIB})
# 2) create an library target out of the library compilation result
add_library(ntdll_extra STATIC IMPORTED GLOBAL)
add_dependencies(ntdll_extra ntdll_extra_target)
# 3) specify where the library is (and where to find the headers)
set_target_properties(ntdll_extra
PROPERTIES
IMPORTED_LOCATION ${MDBX_NTDLL_EXTRA_IMPLIB})
target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ntdll_extra)
endif()
endif()
set_target_properties(mdbx PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON
PUBLIC_HEADER "../mdbx.h")
if(CC_HAS_FASTMATH)
target_compile_options(mdbx PRIVATE "-ffast-math")
endif()
if(BUILD_FOR_NATIVE_CPU AND CC_HAS_ARCH_NATIVE)
target_compile_options(mdbx PUBLIC "-march=native")
endif()
if(CC_HAS_VISIBILITY)
target_compile_options(mdbx PRIVATE "-fvisibility=hidden")
endif()
install(TARGETS mdbx
LIBRARY DESTINATION lib COMPONENT runtime
RUNTIME DESTINATION bin COMPONENT runtime
ARCHIVE DESTINATION lib/static COMPONENT devel
PUBLIC_HEADER DESTINATION include
INCLUDES DESTINATION include COMPONENT devel)
################################################################################
#
# library build info (used in library version output)
#
set(MDBX_BUILD_FLAGS "")
# append cmake's build-type flags and defines
if(NOT CMAKE_CONFIGURATION_TYPES)
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}})
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES_${CMAKE_BUILD_TYPE_UPPERCASE}})
endif()
# append linker dll's options
if(LIBMDBX_TYPE STREQUAL "SHARED")
list(APPEND MDBX_BUILD_FLAGS ${CMAKE_SHARED_LINKER_FLAGS})
endif()
# get definitions
get_target_property(defs_list mdbx COMPILE_DEFINITIONS)
if(defs_list)
list(APPEND MDBX_BUILD_FLAGS ${defs_list})
endif()
# get target compile options
get_target_property(options_list mdbx COMPILE_OPTIONS)
if(options_list)
list(APPEND MDBX_BUILD_FLAGS ${options_list})
endif()
list(REMOVE_DUPLICATES MDBX_BUILD_FLAGS)
string(REPLACE ";" " " MDBX_BUILD_FLAGS "${MDBX_BUILD_FLAGS}")
if(CMAKE_CONFIGURATION_TYPES)
# add dynamic part via per-configuration define
message(STATUS "MDBX Compile Flags: ${MDBX_BUILD_FLAGS} <AND CONFIGURATION DEPENDENT>")
add_definitions(-DMDBX_BUILD_FLAGS_CONFIG="$<$<CONFIG:Debug>:${CMAKE_C_FLAGS_DEBUG} ${CMAKE_C_DEFINES_DEBUG}>$<$<CONFIG:Release>:${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_DEFINES_RELEASE}>$<$<CONFIG:RelWithDebInfo>:${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_DEFINES_RELWITHDEBINFO}>$<$<CONFIG:MinSizeRel>:${CMAKE_C_FLAGS_MINSIZEREL} ${CMAKE_C_DEFINES_MINSIZEREL}>")
else()
message(STATUS "MDBX Compile Flags: ${MDBX_BUILD_FLAGS}")
endif()
# get compiler info
execute_process(COMMAND sh -c "${CMAKE_C_COMPILER} --version | head -1"
OUTPUT_VARIABLE MDBX_BUILD_COMPILER
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
RESULT_VARIABLE rc)
if(rc OR NOT MDBX_BUILD_COMPILER)
string(STRIP "${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION}" MDBX_BUILD_COMPILER)
endif()
# make a build-target triplet
if(CMAKE_C_COMPILER_TARGET)
set(MDBX_BUILD_TARGET "${CMAKE_C_COMPILER_TARGET}")
elseif(CMAKE_C_PLATFORM_ID AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME)
string(STRIP "${CMAKE_C_PLATFORM_ID}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_LIBRARY_ARCHITECTURE)
string(STRIP "${CMAKE_LIBRARY_ARCHITECTURE}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_GENERATOR_PLATFORM AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME)
string(STRIP "${CMAKE_GENERATOR_PLATFORM}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
elseif(CMAKE_SYSTEM_ARCH)
string(STRIP "${CMAKE_SYSTEM_ARCH}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
else()
string(STRIP "${CMAKE_SYSTEM_PROCESSOR}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET)
endif()
if(CMAKE_CONFIGURATION_TYPES)
add_definitions(-DMDBX_BUILD_CONFIG="$<CONFIG>")
else()
set(MDBX_BUILD_CONFIG ${CMAKE_BUILD_TYPE})
endif()
# options
string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC)
set(options VERSION C_COMPILER CXX_COMPILER)
foreach(item IN LISTS options)
if(DEFINED ${item})
set(value "${${item}}")
elseif(DEFINED MDBX_${item})
set(item MDBX_${item})
set(value "${${item}}")
elseif(DEFINED CMAKE_${item})
set(item CMAKE_${item})
set(value "${${item}}")
else()
set(value "undefined")
endif()
message(STATUS "${item}: ${value}")
endforeach(item)
# generate version and config files
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c.in"
"${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c" ESCAPE_QUOTES)
file(SHA256 "${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c" MDBX_SOURCERY_DIGEST)
string(MAKE_C_IDENTIFIER "${MDBX_GIT_DESCRIBE}" MDBX_SOURCERY_SUFFIX)
set(MDBX_BUILD_SOURCERY "${MDBX_SOURCERY_DIGEST}_${MDBX_SOURCERY_SUFFIX}")
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h.in"
"${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h" ESCAPE_QUOTES)
add_definitions(-DMDBX_CONFIG_H="config.h")
add_subdirectory(tools)

26
contrib/db/libmdbx/src/alloy.c Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
/* Amalgamated build */
#define MDBX_ALLOY 1
#include "elements/internals.h" /* must be included fisrt */
#include "elements/core.c"
#include "elements/osal.c"
#include "elements/version.c"
#if defined(_WIN32) || defined(_WIN64)
#include "elements/lck-windows.c"
#else
#include "elements/lck-posix.c"
#endif

56
contrib/db/libmdbx/src/elements/config.h.in Normal file
View file

@ -0,0 +1,56 @@
/* This is CMake-template for libmdbx's config.h
******************************************************************************/
/* *INDENT-OFF* */
/* clang-format off */
#cmakedefine HAVE_VALGRIND_MEMCHECK_H
#cmakedefine HAS_RELAXED_CONSTEXPR
#cmakedefine LTO_ENABLED
#cmakedefine MDBX_USE_VALGRIND
#cmakedefine ENABLE_GPROF
#cmakedefine ENABLE_GCOV
#cmakedefine ENABLE_ASAN
/* Common */
#cmakedefine01 MDBX_TXN_CHECKPID
#cmakedefine01 MDBX_TXN_CHECKOWNER
#cmakedefine01 MDBX_BUILD_SHARED_LIBRARY
/* Windows */
#cmakedefine01 MDBX_CONFIG_MANUAL_TLS_CALLBACK
#cmakedefine01 MDBX_AVOID_CRT
/* MacOS */
#cmakedefine01 MDBX_OSX_SPEED_INSTEADOF_DURABILITY
/* POSIX */
#cmakedefine01 MDBX_USE_ROBUST
#cmakedefine01 MDBX_USE_OFDLOCKS
#cmakedefine01 MDBX_DISABLE_GNU_SOURCE
/* Simulate "AUTO" values of tristate options */
#cmakedefine MDBX_TXN_CHECKPID_AUTO
#ifdef MDBX_TXN_CHECKPID_AUTO
#undef MDBX_TXN_CHECKPID
#endif
#cmakedefine MDBX_USE_ROBUST_AUTO
#ifdef MDBX_USE_ROBUST_AUTO
#undef MDBX_USE_ROBUST
#endif
#cmakedefine MDBX_USE_OFDLOCKS_AUTO
#ifdef MDBX_USE_OFDLOCKS_AUTO
#undef MDBX_USE_OFDLOCKS
#endif
/* Build Info */
#cmakedefine MDBX_BUILD_TIMESTAMP "@MDBX_BUILD_TIMESTAMP@"
#cmakedefine MDBX_BUILD_TARGET "@MDBX_BUILD_TARGET@"
#cmakedefine MDBX_BUILD_CONFIG "@MDBX_BUILD_CONFIG@"
#cmakedefine MDBX_BUILD_COMPILER "@MDBX_BUILD_COMPILER@"
#cmakedefine MDBX_BUILD_FLAGS "@MDBX_BUILD_FLAGS@"
#cmakedefine MDBX_BUILD_SOURCERY @MDBX_BUILD_SOURCERY@
/* *INDENT-ON* */
/* clang-format on */

16709
contrib/db/libmdbx/src/elements/core.c Normal file
View file

File diff suppressed because it is too large Load diff

450
contrib/db/libmdbx/src/elements/defs.h Normal file
View file

@ -0,0 +1,450 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
/* *INDENT-OFF* */
/* clang-format off */
#ifndef __GNUC_PREREQ
# if defined(__GNUC__) && defined(__GNUC_MINOR__)
# define __GNUC_PREREQ(maj, min) \
((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GNUC_PREREQ(maj, min) (0)
# endif
#endif /* __GNUC_PREREQ */
#ifndef __CLANG_PREREQ
# ifdef __clang__
# define __CLANG_PREREQ(maj,min) \
((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
# else
# define __CLANG_PREREQ(maj,min) (0)
# endif
#endif /* __CLANG_PREREQ */
#ifndef __GLIBC_PREREQ
# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
# define __GLIBC_PREREQ(maj, min) \
((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GLIBC_PREREQ(maj, min) (0)
# endif
#endif /* __GLIBC_PREREQ */
#ifndef __has_attribute
# define __has_attribute(x) (0)
#endif
#ifndef __has_feature
# define __has_feature(x) (0)
#endif
#ifndef __has_extension
# define __has_extension(x) (0)
#endif
#ifndef __has_builtin
# define __has_builtin(x) (0)
#endif
#ifndef __has_warning
# define __has_warning(x) (0)
#endif
#ifndef __has_include
# define __has_include(x) (0)
#endif
#if __has_feature(thread_sanitizer)
# define __SANITIZE_THREAD__ 1
#endif
#if __has_feature(address_sanitizer)
# define __SANITIZE_ADDRESS__ 1
#endif
/*----------------------------------------------------------------------------*/
#ifndef __extern_C
# ifdef __cplusplus
# define __extern_C extern "C"
# else
# define __extern_C
# endif
#endif /* __extern_C */
#ifndef __cplusplus
# ifndef bool
# define bool _Bool
# endif
# ifndef true
# define true (1)
# endif
# ifndef false
# define false (0)
# endif
#endif
#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
# define nullptr NULL
#endif
/*----------------------------------------------------------------------------*/
#ifndef __always_inline
# if defined(__GNUC__) || __has_attribute(__always_inline__)
# define __always_inline __inline __attribute__((__always_inline__))
# elif defined(_MSC_VER)
# define __always_inline __forceinline
# else
# define __always_inline
# endif
#endif /* __always_inline */
#ifndef __noinline
# if defined(__GNUC__) || __has_attribute(__noinline__)
# define __noinline __attribute__((__noinline__))
# elif defined(_MSC_VER)
# define __noinline __declspec(noinline)
# elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
# define __noinline inline
# elif !defined(__INTEL_COMPILER)
# define __noinline /* FIXME ? */
# endif
#endif /* __noinline */
#ifndef __must_check_result
# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
# define __must_check_result __attribute__((__warn_unused_result__))
# else
# define __must_check_result
# endif
#endif /* __must_check_result */
#ifndef __maybe_unused
# if defined(__GNUC__) || __has_attribute(__unused__)
# define __maybe_unused __attribute__((__unused__))
# else
# define __maybe_unused
# endif
#endif /* __maybe_unused */
#ifndef __deprecated
# if defined(__GNUC__) || __has_attribute(__deprecated__)
# define __deprecated __attribute__((__deprecated__))
# elif defined(_MSC_VER)
# define __deprecated __declspec(deprecated)
# else
# define __deprecated
# endif
#endif /* __deprecated */
#if !defined(__noop) && !defined(_MSC_VER)
# define __noop(...) do {} while(0)
#endif /* __noop */
#ifndef __fallthrough
# if __GNUC_PREREQ(7, 0) || __has_attribute(__fallthrough__)
# define __fallthrough __attribute__((__fallthrough__))
# else
# define __fallthrough __noop()
# endif
#endif /* __fallthrough */
#ifndef __unreachable
# if __GNUC_PREREQ(4,5)
# define __unreachable() __builtin_unreachable()
# elif defined(_MSC_VER)
# define __unreachable() __assume(0)
# else
# define __unreachable() __noop()
# endif
#endif /* __unreachable */
#ifndef __prefetch
# if defined(__GNUC__) || defined(__clang__)
# define __prefetch(ptr) __builtin_prefetch(ptr)
# else
# define __prefetch(ptr) __noop(ptr)
# endif
#endif /* __prefetch */
#ifndef __noreturn
# if defined(__GNUC__) || __has_attribute(__noreturn__)
# define __noreturn __attribute__((__noreturn__))
# elif defined(_MSC_VER)
# define __noreturn __declspec(noreturn)
# else
# define __noreturn
# endif
#endif /* __noreturn */
#ifndef __nothrow
# if defined(__cplusplus)
# if __cplusplus < 201703L
# define __nothrow throw()
# else
# define __nothrow noexcept(true)
# endif /* __cplusplus */
# elif defined(__GNUC__) || __has_attribute(__nothrow__)
# define __nothrow __attribute__((__nothrow__))
# elif defined(_MSC_VER) && defined(__cplusplus)
# define __nothrow __declspec(nothrow)
# else
# define __nothrow
# endif
#endif /* __nothrow */
#ifndef __pure_function
/* Many functions have no effects except the return value and their
* return value depends only on the parameters and/or global variables.
* Such a function can be subject to common subexpression elimination
* and loop optimization just as an arithmetic operator would be.
* These functions should be declared with the attribute pure. */
# if defined(__GNUC__) || __has_attribute(__pure__)
# define __pure_function __attribute__((__pure__))
# else
# define __pure_function
# endif
#endif /* __pure_function */
#ifndef __const_function
/* Many functions do not examine any values except their arguments,
* and have no effects except the return value. Basically this is just
* slightly more strict class than the PURE attribute, since function
* is not allowed to read global memory.
*
* Note that a function that has pointer arguments and examines the
* data pointed to must not be declared const. Likewise, a function
* that calls a non-const function usually must not be const.
* It does not make sense for a const function to return void. */
# if defined(__GNUC__) || __has_attribute(__const__)
# define __const_function __attribute__((__const__))
# else
# define __const_function
# endif
#endif /* __const_function */
#ifndef __hidden
# if defined(__GNUC__) || __has_attribute(__visibility__)
# define __hidden __attribute__((__visibility__("hidden")))
# else
# define __hidden
# endif
#endif /* __hidden */
#ifndef __optimize
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(__optimize__)
# define __optimize(ops)
# elif defined(__GNUC__) || __has_attribute(__optimize__)
# define __optimize(ops) __attribute__((__optimize__(ops)))
# else
# define __optimize(ops)
# endif
# else
# define __optimize(ops)
# endif
#endif /* __optimize */
#ifndef __hot
# if defined(__OPTIMIZE__)
# if defined(__e2k__)
# define __hot __attribute__((__hot__)) __optimize(3)
# elif defined(__clang__) && !__has_attribute(__hot_) \
&& __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
/* just put frequently used functions in separate section */
# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
# elif defined(__GNUC__) || __has_attribute(__hot__)
# define __hot __attribute__((__hot__)) __optimize("O3")
# else
# define __hot __optimize("O3")
# endif
# else
# define __hot
# endif
#endif /* __hot */
#ifndef __cold
# if defined(__OPTIMIZE__)
# if defined(__e2k__)
# define __cold __attribute__((__cold__)) __optimize(1)
# elif defined(__clang__) && !__has_attribute(cold) \
&& __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
/* just put infrequently used functions in separate section */
# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
# elif defined(__GNUC__) || __has_attribute(cold)
# define __cold __attribute__((__cold__)) __optimize("Os")
# else
# define __cold __optimize("Os")
# endif
# else
# define __cold
# endif
#endif /* __cold */
#ifndef __flatten
# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
# define __flatten __attribute__((__flatten__))
# else
# define __flatten
# endif
#endif /* __flatten */
#ifndef likely
# if (defined(__GNUC__) || defined(__clang__)) && !defined(__COVERITY__)
# define likely(cond) __builtin_expect(!!(cond), 1)
# else
# define likely(x) (x)
# endif
#endif /* likely */
#ifndef unlikely
# if (defined(__GNUC__) || defined(__clang__)) && !defined(__COVERITY__)
# define unlikely(cond) __builtin_expect(!!(cond), 0)
# else
# define unlikely(x) (x)
# endif
#endif /* unlikely */
/* Workaround for Coverity Scan */
#if defined(__COVERITY__) && __GNUC_PREREQ(7, 0) && !defined(__cplusplus)
typedef float _Float32;
typedef double _Float32x;
typedef double _Float64;
typedef long double _Float64x;
typedef float _Float128 __attribute__((__mode__(__TF__)));
typedef __complex__ float __cfloat128 __attribute__ ((__mode__ (__TC__)));
typedef _Complex float __cfloat128 __attribute__ ((__mode__ (__TC__)));
#endif /* Workaround for Coverity Scan */
#ifndef __printf_args
# if defined(__GNUC__) || __has_attribute(__format__)
# define __printf_args(format_index, first_arg) \
__attribute__((__format__(printf, format_index, first_arg)))
# else
# define __printf_args(format_index, first_arg)
# endif
#endif /* __printf_args */
#ifndef __anonymous_struct_extension__
# if defined(__GNUC__)
# define __anonymous_struct_extension__ __extension__
# else
# define __anonymous_struct_extension__
# endif
#endif /* __anonymous_struct_extension__ */
#ifndef __Wpedantic_format_voidptr
static __inline const void* __pure_function
__Wpedantic_format_voidptr(const void* ptr) {return ptr;}
# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG)
#endif /* __Wpedantic_format_voidptr */
/*----------------------------------------------------------------------------*/
#if defined(MDBX_USE_VALGRIND)
# include <valgrind/memcheck.h>
# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
/* LY: available since Valgrind 3.10 */
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# endif
#elif !defined(RUNNING_ON_VALGRIND)
# define VALGRIND_CREATE_MEMPOOL(h,r,z)
# define VALGRIND_DESTROY_MEMPOOL(h)
# define VALGRIND_MEMPOOL_TRIM(h,a,s)
# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
# define VALGRIND_MEMPOOL_FREE(h,a)
# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
# define VALGRIND_MAKE_MEM_DEFINED(a,s)
# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
# define RUNNING_ON_VALGRIND (0)
#endif /* MDBX_USE_VALGRIND */
#ifdef __SANITIZE_ADDRESS__
# include <sanitizer/asan_interface.h>
#elif !defined(ASAN_POISON_MEMORY_REGION)
# define ASAN_POISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
#endif /* __SANITIZE_ADDRESS__ */
/*----------------------------------------------------------------------------*/
#ifndef ARRAY_LENGTH
# ifdef __cplusplus
template <typename T, size_t N>
char (&__ArraySizeHelper(T (&array)[N]))[N];
# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
# else
# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
# endif
#endif /* ARRAY_LENGTH */
#ifndef ARRAY_END
# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
#endif /* ARRAY_END */
#ifndef STRINGIFY
# define STRINGIFY_HELPER(x) #x
# define STRINGIFY(x) STRINGIFY_HELPER(x)
#endif /* STRINGIFY */
#define CONCAT(a,b) a##b
#define XCONCAT(a,b) CONCAT(a,b)
#ifndef offsetof
# define offsetof(type, member) __builtin_offsetof(type, member)
#endif /* offsetof */
#ifndef container_of
# define container_of(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
#endif /* container_of */
#define MDBX_TETRAD(a, b, c, d) \
((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__)
#ifndef STATIC_ASSERT_MSG
# if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
|| __has_feature(c_static_assert)
# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
# elif defined(static_assert)
# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
# elif defined(_MSC_VER)
# include <crtdbg.h>
# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
# else
# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
# endif
#endif /* STATIC_ASSERT */
#ifndef STATIC_ASSERT
# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
#endif
/* *INDENT-ON* */
/* clang-format on */

1367
contrib/db/libmdbx/src/elements/internals.h Normal file
View file

File diff suppressed because it is too large Load diff

551
contrib/db/libmdbx/src/elements/lck-posix.c Normal file
View file

@ -0,0 +1,551 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "internals.h"
/*----------------------------------------------------------------------------*/
/* global constructor/destructor */
#if defined(__linux__) || defined(__gnu_linux__)
#include <sys/utsname.h>
#ifndef MDBX_ALLOY
uint32_t mdbx_linux_kernel_version;
#endif /* MDBX_ALLOY */
#endif /* Linux */
static __cold __attribute__((__constructor__)) void
mdbx_global_constructor(void) {
#if defined(__linux__) || defined(__gnu_linux__)
struct utsname buffer;
if (uname(&buffer) == 0) {
int i = 0;
char *p = buffer.release;
while (*p && i < 4) {
if (*p >= '0' && *p <= '9') {
long number = strtol(p, &p, 10);
if (number > 0) {
if (number > 255)
number = 255;
mdbx_linux_kernel_version += number << (24 - i * 8);
}
++i;
} else {
++p;
}
}
}
#endif /* Linux */
mdbx_rthc_global_init();
}
static __cold __attribute__((__destructor__)) void
mdbx_global_destructor(void) {
mdbx_rthc_global_dtor();
}
/*----------------------------------------------------------------------------*/
/* lck */
/* Описание реализации блокировок для POSIX & Linux:
*
* lck-файл отображается в память, в нём организуется таблица читателей и
* размещаются совместно используемые posix-мьютексы (futex). Посредством
* этих мьютексов (см struct MDBX_lockinfo) реализуются:
* - Блокировка таблицы читателей для регистрации,
* т.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock().
* - Блокировка БД для пишущих транзакций,
* т.е. функции mdbx_txn_lock() и mdbx_txn_unlock().
*
* Остальной функционал реализуется отдельно посредством файловых блокировок:
* - Первоначальный захват БД в режиме exclusive/shared и последующий перевод
* в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade().
* - Проверка присутствие процессов-читателей,
* т.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check().
*
* Для блокировки файлов используется fcntl(F_SETLK), так как:
* - lockf() оперирует только эксклюзивной блокировкой и требует
* открытия файла в RW-режиме.
* - flock() не гарантирует атомарности при смене блокировок
* и оперирует только всем файлом целиком.
* - Для контроля процессов-читателей используются однобайтовые
* range-блокировки lck-файла посредством fcntl(F_SETLK). При этом
* в качестве позиции используется pid процесса-читателя.
* - Для первоначального захвата и shared/exclusive выполняется блокировка
* основного файла БД и при успехе lck-файла.
*
* ----------------------------------------------------------------------------
* УДЕРЖИВАЕМЫЕ БЛОКИРОВКИ В ЗАВИСИМОСТИ ОТ РЕЖИМА И СОСТОЯНИЯ
*
* Эксклюзивный режим без lck-файла:
* = заблокирован весь dxb-файл посредством F_RDLCK или F_WRLCK,
* в зависимости от MDBX_RDONLY.
*
* Не-операционный режим на время пере-инициализации и разрушении lck-файла:
* = F_WRLCK блокировка первого байта lck-файла, другие процессы ждут её
* снятия при получении F_RDLCK через F_SETLKW.
* - блокировки dxb-файла могут меняться до снятие эксклюзивной блокировки
* lck-файла:
* + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле
* посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
* + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла
* посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
*
* ОПЕРАЦИОННЫЙ режим с lck-файлом:
* = F_RDLCK блокировка первого байта lck-файла, другие процессы не могут
* получить F_WRLCK и таким образом видят что БД используется.
* + F_WRLCK блокировка pid-байта в clk-файле после первой транзакции чтения.
* + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле
* посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
* + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла
* посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY.
*/
#if MDBX_USE_OFDLOCKS
static int op_setlk, op_setlkw, op_getlk;
static void __cold choice_fcntl() {
assert(!op_setlk && !op_setlkw && !op_getlk);
if ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0
#if defined(__linux__) || defined(__gnu_linux__)
&& mdbx_linux_kernel_version >
0x030f0000 /* OFD locks are available since 3.15, but engages here
only for 3.16 and larer kernels (LTS) for reliability reasons */
#endif /* linux */
) {
op_setlk = F_OFD_SETLK;
op_setlkw = F_OFD_SETLKW;
op_getlk = F_OFD_GETLK;
return;
}
op_setlk = F_SETLK;
op_setlkw = F_SETLKW;
op_getlk = F_GETLK;
}
#else
#define op_setlk F_SETLK
#define op_setlkw F_SETLKW
#define op_getlk F_GETLK
#endif /* MDBX_USE_OFDLOCKS */
#ifndef OFF_T_MAX
#define OFF_T_MAX \
((sizeof(off_t) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff)
#endif
static int lck_op(mdbx_filehandle_t fd, int cmd, int lck, off_t offset,
off_t len) {
mdbx_jitter4testing(true);
for (;;) {
struct flock lock_op;
memset(&lock_op, 0, sizeof(lock_op));
lock_op.l_type = lck;
lock_op.l_whence = SEEK_SET;
lock_op.l_start = offset;
lock_op.l_len = len;
int rc = fcntl(fd, cmd, &lock_op);
mdbx_jitter4testing(true);
if (rc != -1) {
if (cmd == op_getlk) {
/* Checks reader by pid. Returns:
* MDBX_RESULT_TRUE - if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE - if pid is dead (lock acquired). */
return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE
: MDBX_RESULT_TRUE;
}
return MDBX_SUCCESS;
}
rc = errno;
if (rc != EINTR || cmd == op_setlkw) {
mdbx_assert(nullptr, MDBX_IS_ERROR(rc));
return rc;
}
}
}
MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0);
if (unlikely(mdbx_getpid() != env->me_pid))
return MDBX_PANIC;
return lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1);
}
MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0);
return lck_op(env->me_lfd, op_setlk, F_UNLCK, env->me_pid, 1);
}
MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(pid > 0);
return lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1);
}
/*---------------------------------------------------------------------------*/
MDBX_INTERNAL_FUNC int __cold mdbx_lck_seize(MDBX_env *env) {
assert(env->me_fd != INVALID_HANDLE_VALUE);
if (unlikely(mdbx_getpid() != env->me_pid))
return MDBX_PANIC;
#if MDBX_USE_OFDLOCKS
if (unlikely(op_setlk == 0))
choice_fcntl();
#endif /* MDBX_USE_OFDLOCKS */
int rc;
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
rc =
lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
if (rc != MDBX_SUCCESS) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "without-lck", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
}
/* Firstly try to get exclusive locking. */
rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1);
if (rc == MDBX_SUCCESS) {
continue_dxb_exclusive:
rc =
lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX);
if (rc == MDBX_SUCCESS)
return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */;
/* the cause may be a collision with POSIX's file-lock recovery. */
if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
rc == EDEADLK)) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "dxb-exclusive", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
/* Fallback to lck-shared */
rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1);
if (rc != MDBX_SUCCESS) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "fallback-shared", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
/* Done: return with shared locking. */
return MDBX_RESULT_FALSE;
}
/* Wait for lck-shared now. */
/* Here may be await during transient processes, for instance until another
* competing process doesn't call lck_downgrade(). */
rc = lck_op(env->me_lfd, op_setlkw, F_RDLCK, 0, 1);
if (rc != MDBX_SUCCESS) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "try-shared", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
/* Lock against another process operating in without-lck or exclusive mode. */
rc =
lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->me_pid, 1);
if (rc != MDBX_SUCCESS) {
mdbx_error("%s(%s) failed: errcode %u", __func__,
"lock-against-without-lck", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
/* got shared, retry exclusive */
rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1);
if (rc == MDBX_SUCCESS)
goto continue_dxb_exclusive;
if (rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK ||
rc == EDEADLK)
return MDBX_RESULT_FALSE /* Done: exclusive is unavailable,
but shared locks are alive. */
;
mdbx_error("%s(%s) failed: errcode %u", __func__, "try-exclusive", rc);
mdbx_assert(env, MDBX_IS_ERROR(rc));
return rc;
}
MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
if (unlikely(mdbx_getpid() != env->me_pid))
return MDBX_PANIC;
int rc = MDBX_SUCCESS;
if ((env->me_flags & MDBX_EXCLUSIVE) == 0) {
rc = lck_op(env->me_fd, op_setlk, F_UNLCK, 0, env->me_pid);
if (rc == MDBX_SUCCESS)
rc = lck_op(env->me_fd, op_setlk, F_UNLCK, env->me_pid + 1,
OFF_T_MAX - env->me_pid - 1);
}
if (rc == MDBX_SUCCESS)
rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1);
if (unlikely(rc != 0)) {
mdbx_error("%s(%s) failed: errcode %u", __func__, "lck", rc);
assert(MDBX_IS_ERROR(rc));
}
return rc;
}
MDBX_INTERNAL_FUNC int __cold mdbx_lck_destroy(MDBX_env *env,
MDBX_env *inprocess_neighbor) {
if (unlikely(mdbx_getpid() != env->me_pid))
return MDBX_PANIC;
int rc = MDBX_SUCCESS;
if (env->me_lfd != INVALID_HANDLE_VALUE && !inprocess_neighbor &&
env->me_lck &&
/* try get exclusive access */
lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 &&
lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX)) {
mdbx_verbose("%s: got exclusive, drown mutexes", __func__);
rc = pthread_mutex_destroy(&env->me_lck->mti_rmutex);
if (rc == 0)
rc = pthread_mutex_destroy(&env->me_lck->mti_wmutex);
mdbx_assert(env, rc == 0);
if (rc == 0) {
memset(env->me_lck, 0x81, sizeof(MDBX_lockinfo));
msync(env->me_lck, env->me_os_psize, MS_ASYNC);
}
mdbx_jitter4testing(false);
}
/* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored
* after file was closed.
*
* 2) File locks would be released (by kernel) while the file-descriptors will
* be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel,
* locks should be released here explicitly with properly order. */
/* close dxb and restore lock */
if (env->me_fd != INVALID_HANDLE_VALUE) {
if (unlikely(close(env->me_fd) != 0) && rc == MDBX_SUCCESS)
rc = errno;
env->me_fd = INVALID_HANDLE_VALUE;
if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
/* restore file-lock */
rc = lck_op(
inprocess_neighbor->me_fd, F_SETLKW,
(inprocess_neighbor->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
(inprocess_neighbor->me_flags & MDBX_EXCLUSIVE)
? 0
: inprocess_neighbor->me_pid,
(inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1);
}
}
/* close clk and restore locks */
if (env->me_lfd != INVALID_HANDLE_VALUE) {
if (unlikely(close(env->me_lfd) != 0) && rc == MDBX_SUCCESS)
rc = errno;
env->me_lfd = INVALID_HANDLE_VALUE;
if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) {
/* restore file-locks */
rc = lck_op(inprocess_neighbor->me_lfd, F_SETLKW, F_RDLCK, 0, 1);
if (rc == MDBX_SUCCESS && inprocess_neighbor->me_live_reader)
rc = mdbx_rpid_set(inprocess_neighbor);
}
}
if (inprocess_neighbor && rc != MDBX_SUCCESS)
inprocess_neighbor->me_flags |= MDBX_FATAL_ERROR;
return rc;
}
/*---------------------------------------------------------------------------*/
static int mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
const int rc);
MDBX_INTERNAL_FUNC int __cold mdbx_lck_init(MDBX_env *env,
MDBX_env *inprocess_neighbor,
int global_uniqueness_flag) {
if (inprocess_neighbor)
return MDBX_SUCCESS /* currently don't need any initialization
if LCK already opened/used inside current process */
;
/* FIXME: Unfortunately, there is no other reliable way but to long testing
* on each platform. On the other hand, behavior like FreeBSD is incorrect
* and we can expect it to be rare. Moreover, even on FreeBSD without
* additional in-process initialization, the probability of an problem
* occurring is vanishingly small, and the symptom is a return of EINVAL
* while locking a mutex. In other words, in the worst case, the problem
* results in an EINVAL error at the start of the transaction, but NOT data
* loss, nor database corruption, nor other fatal troubles. Thus, the code
* below I am inclined to think the workaround for erroneous platforms (like
* FreeBSD), rather than a defect of libmdbx. */
#if defined(__FreeBSD__)
/* seems that shared mutexes on FreeBSD required in-process initialization */
(void)global_uniqueness_flag;
#else
/* shared mutexes on many other platforms (including Darwin and Linux's
* futexes) doesn't need any addition in-process initialization */
if (global_uniqueness_flag != MDBX_RESULT_TRUE)
return MDBX_SUCCESS;
#endif
pthread_mutexattr_t ma;
int rc = pthread_mutexattr_init(&ma);
if (rc)
return rc;
rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED);
if (rc)
goto bailout;
#if MDBX_USE_ROBUST
#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) && \
!defined(pthread_mutex_consistent) && _POSIX_C_SOURCE < 200809L
rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
#else
rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
#endif
if (rc)
goto bailout;
#endif /* MDBX_USE_ROBUST */
#if _POSIX_C_SOURCE >= 199506L && !defined(MDBX_SAFE4QEMU)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
if (rc == ENOTSUP)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
if (rc)
goto bailout;
#endif /* PTHREAD_PRIO_INHERIT */
rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_rmutex, &ma);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_wmutex, &ma);
bailout:
pthread_mutexattr_destroy(&ma);
return rc;
}
static int mdbx_robust_lock(MDBX_env *env, pthread_mutex_t *mutex) {
mdbx_jitter4testing(true);
int rc = pthread_mutex_lock(mutex);
if (unlikely(rc != 0))
rc = mdbx_mutex_failed(env, mutex, rc);
return rc;
}
static int mdbx_robust_trylock(MDBX_env *env, pthread_mutex_t *mutex) {
mdbx_jitter4testing(true);
int rc = pthread_mutex_trylock(mutex);
if (unlikely(rc != 0 && rc != EBUSY))
rc = mdbx_mutex_failed(env, mutex, rc);
return (rc != EBUSY) ? rc : MDBX_BUSY;
}
static int mdbx_robust_unlock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_unlock(mutex);
mdbx_jitter4testing(true);
if (unlikely(rc != 0))
env->me_flags |= MDBX_FATAL_ERROR;
return rc;
}
MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) {
mdbx_trace("%s", ">>");
int rc = mdbx_robust_lock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
return rc;
}
MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) {
mdbx_trace("%s", ">>");
int rc = mdbx_robust_unlock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", __func__, rc);
}
int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
mdbx_trace("%s", ">>");
int rc = dontwait ? mdbx_robust_trylock(env, env->me_wmutex)
: mdbx_robust_lock(env, env->me_wmutex);
mdbx_trace("<< rc %d", rc);
return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS;
}
void mdbx_txn_unlock(MDBX_env *env) {
mdbx_trace("%s", ">>");
int rc = mdbx_robust_unlock(env, env->me_wmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", __func__, rc);
}
static int __cold mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
const int err) {
int rc = err;
#if MDBX_USE_ROBUST
if (err == EOWNERDEAD) {
/* We own the mutex. Clean up after dead previous owner. */
int rlocked = (env->me_lck && mutex == &env->me_lck->mti_rmutex);
rc = MDBX_SUCCESS;
if (!rlocked) {
if (unlikely(env->me_txn)) {
/* env is hosed if the dead thread was ours */
env->me_flags |= MDBX_FATAL_ERROR;
env->me_txn = NULL;
rc = MDBX_PANIC;
}
}
mdbx_notice("%cmutex owner died, %s", (rlocked ? 'r' : 'w'),
(rc ? "this process' env is hosed" : "recovering"));
int check_rc = mdbx_reader_check0(env, rlocked, NULL);
check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) && \
!defined(pthread_mutex_consistent) && _POSIX_C_SOURCE < 200809L
int mreco_rc = pthread_mutex_consistent_np(mutex);
#else
int mreco_rc = pthread_mutex_consistent(mutex);
#endif
check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;
if (unlikely(mreco_rc))
mdbx_error("mutex recovery failed, %s", mdbx_strerror(mreco_rc));
rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
if (MDBX_IS_ERROR(rc))
pthread_mutex_unlock(mutex);
return rc;
}
#else
(void)mutex;
#endif /* MDBX_USE_ROBUST */
mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err));
if (rc != EDEADLK)
env->me_flags |= MDBX_FATAL_ERROR;
return rc;
}

777
contrib/db/libmdbx/src/elements/lck-windows.c Normal file
View file

@ -0,0 +1,777 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "internals.h"
/* PREAMBLE FOR WINDOWS:
*
* We are not concerned for performance here.
* If you are running Windows a performance could NOT be the goal.
* Otherwise please use Linux.
*
* Regards,
* LY
*/
static void mdbx_winnt_import(void);
#if MDBX_BUILD_SHARED_LIBRARY
#if MDBX_AVOID_CRT && defined(NDEBUG)
/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks.
*
* Define dll's entry point only for Release build when NDEBUG is defined and
* MDBX_AVOID_CRT=ON. if the entry point isn't defined then MSVC's will
* automatically use DllMainCRTStartup() from CRT library, which also
* automatically call DllMain() from our mdbx.dll */
#pragma comment(linker, "/ENTRY:DllMain")
#endif /* MDBX_AVOID_CRT */
BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved)
#else
#if !MDBX_CONFIG_MANUAL_TLS_CALLBACK
static
#endif /* !MDBX_CONFIG_MANUAL_TLS_CALLBACK */
void NTAPI
mdbx_dll_handler(PVOID module, DWORD reason, PVOID reserved)
#endif /* MDBX_BUILD_SHARED_LIBRARY */
{
(void)reserved;
switch (reason) {
case DLL_PROCESS_ATTACH:
mdbx_winnt_import();
mdbx_rthc_global_init();
break;
case DLL_PROCESS_DETACH:
mdbx_rthc_global_dtor();
break;
case DLL_THREAD_ATTACH:
break;
case DLL_THREAD_DETACH:
mdbx_rthc_thread_dtor(module);
break;
}
#if MDBX_BUILD_SHARED_LIBRARY
return TRUE;
#endif
}
#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK
/* *INDENT-OFF* */
/* clang-format off */
#if defined(_MSC_VER)
# pragma const_seg(push)
# pragma data_seg(push)
# ifdef _WIN64
/* kick a linker to create the TLS directory if not already done */
# pragma comment(linker, "/INCLUDE:_tls_used")
/* Force some symbol references. */
# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor")
/* specific const-segment for WIN64 */
# pragma const_seg(".CRT$XLB")
const
# else
/* kick a linker to create the TLS directory if not already done */
# pragma comment(linker, "/INCLUDE:__tls_used")
/* Force some symbol references. */
# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor")
/* specific data-segment for WIN32 */
# pragma data_seg(".CRT$XLB")
# endif
__declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_dll_handler;
# pragma data_seg(pop)
# pragma const_seg(pop)
#elif defined(__GNUC__)
# ifdef _WIN64
const
# endif
PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_dll_handler;
#else
# error FIXME
#endif
/* *INDENT-ON* */
/* clang-format on */
#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK */
/*----------------------------------------------------------------------------*/
#define LCK_SHARED 0
#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK
#define LCK_WAITFOR 0
#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY
static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset,
size_t bytes) {
OVERLAPPED ov;
ov.hEvent = 0;
ov.Offset = (DWORD)offset;
ov.OffsetHigh = HIGH_DWORD(offset);
return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov);
}
static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset,
size_t bytes) {
return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes,
HIGH_DWORD(bytes));
}
/*----------------------------------------------------------------------------*/
/* global `write` lock for write-txt processing,
* exclusive locking both meta-pages) */
#define LCK_MAXLEN (1u + (size_t)(MAXSSIZE_T))
#define LCK_META_OFFSET 0
#define LCK_META_LEN 0x10000u
#define LCK_BODY_OFFSET LCK_META_LEN
#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET)
#define LCK_META LCK_META_OFFSET, LCK_META_LEN
#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN
#define LCK_WHOLE 0, LCK_MAXLEN
int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
if (dontwait) {
if (!TryEnterCriticalSection(&env->me_windowsbug_lock))
return MDBX_BUSY;
} else {
EnterCriticalSection(&env->me_windowsbug_lock);
}
if ((env->me_flags & MDBX_EXCLUSIVE) ||
flock(env->me_fd,
dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT)
: (LCK_EXCLUSIVE | LCK_WAITFOR),
LCK_BODY))
return MDBX_SUCCESS;
int rc = GetLastError();
LeaveCriticalSection(&env->me_windowsbug_lock);
return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY;
}
void mdbx_txn_unlock(MDBX_env *env) {
int rc =
(env->me_flags & MDBX_EXCLUSIVE) ? TRUE : funlock(env->me_fd, LCK_BODY);
LeaveCriticalSection(&env->me_windowsbug_lock);
if (!rc)
mdbx_panic("%s failed: errcode %u", __func__, GetLastError());
}
/*----------------------------------------------------------------------------*/
/* global `read` lock for readers registration,
* exclusive locking `mti_numreaders` (second) cacheline */
#define LCK_LO_OFFSET 0
#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders)
#define LCK_UP_OFFSET LCK_LO_LEN
#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET)
#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN
#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN
MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) {
mdbx_srwlock_AcquireShared(&env->me_remap_guard);
if (env->me_lfd == INVALID_HANDLE_VALUE)
return MDBX_SUCCESS; /* readonly database in readonly filesystem */
/* transite from S-? (used) to S-E (locked), e.g. exclusive lock upper-part */
if ((env->me_flags & MDBX_EXCLUSIVE) ||
flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER))
return MDBX_SUCCESS;
int rc = GetLastError();
mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
return rc;
}
MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) {
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* transite from S-E (locked) to S-? (used), e.g. unlock upper-part */
if ((env->me_flags & MDBX_EXCLUSIVE) == 0 &&
!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s failed: errcode %u", __func__, GetLastError());
}
mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
}
static int suspend_and_append(mdbx_handle_array_t **array,
const DWORD ThreadId) {
const unsigned limit = (*array)->limit;
if ((*array)->count == limit) {
void *ptr = mdbx_realloc(
(limit > ARRAY_LENGTH((*array)->handles))
? *array
: /* don't free initial array on the stack */ NULL,
sizeof(mdbx_handle_array_t) +
sizeof(HANDLE) * (limit * 2 - ARRAY_LENGTH((*array)->handles)));
if (!ptr)
return MDBX_ENOMEM;
if (limit == ARRAY_LENGTH((*array)->handles))
memcpy(ptr, *array, sizeof(mdbx_handle_array_t));
*array = (mdbx_handle_array_t *)ptr;
(*array)->limit = limit * 2;
}
HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION,
FALSE, ThreadId);
if (hThread == NULL)
return GetLastError();
if (SuspendThread(hThread) == -1) {
int err = GetLastError();
DWORD ExitCode;
if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED ||
!GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE)
err = MDBX_SUCCESS;
CloseHandle(hThread);
return err;
}
(*array)->handles[(*array)->count++] = hThread;
return MDBX_SUCCESS;
}
MDBX_INTERNAL_FUNC int
mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) {
const size_t CurrentTid = GetCurrentThreadId();
int rc;
if (env->me_lck) {
/* Scan LCK for threads of the current process */
const MDBX_reader *const begin = env->me_lck->mti_readers;
const MDBX_reader *const end = begin + env->me_lck->mti_numreaders;
const size_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0;
for (const MDBX_reader *reader = begin; reader < end; ++reader) {
if (reader->mr_pid != env->me_pid || !reader->mr_tid) {
skip_lck:
continue;
}
if (reader->mr_tid == CurrentTid || reader->mr_tid == WriteTxnOwner)
goto skip_lck;
if (env->me_flags & MDBX_NOTLS) {
/* Skip duplicates in no-tls mode */
for (const MDBX_reader *scan = reader; --scan >= begin;)
if (scan->mr_tid == reader->mr_tid)
goto skip_lck;
}
rc = suspend_and_append(array, (mdbx_tid_t)reader->mr_tid);
if (rc != MDBX_SUCCESS) {
bailout_lck:
(void)mdbx_resume_threads_after_remap(*array);
return rc;
}
}
if (WriteTxnOwner && WriteTxnOwner != CurrentTid) {
rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner);
if (rc != MDBX_SUCCESS)
goto bailout_lck;
}
} else {
/* Without LCK (i.e. read-only mode).
* Walk thougth a snapshot of all running threads */
mdbx_assert(env,
env->me_txn0 == NULL || (env->me_flags & MDBX_EXCLUSIVE) != 0);
const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
if (hSnapshot == INVALID_HANDLE_VALUE)
return GetLastError();
THREADENTRY32 entry;
entry.dwSize = sizeof(THREADENTRY32);
if (!Thread32First(hSnapshot, &entry)) {
rc = GetLastError();
bailout_toolhelp:
CloseHandle(hSnapshot);
(void)mdbx_resume_threads_after_remap(*array);
return rc;
}
do {
if (entry.th32OwnerProcessID != env->me_pid ||
entry.th32ThreadID == CurrentTid)
continue;
rc = suspend_and_append(array, entry.th32ThreadID);
if (rc != MDBX_SUCCESS)
goto bailout_toolhelp;
} while (Thread32Next(hSnapshot, &entry));
rc = GetLastError();
if (rc != ERROR_NO_MORE_FILES)
goto bailout_toolhelp;
CloseHandle(hSnapshot);
}
return MDBX_SUCCESS;
}
MDBX_INTERNAL_FUNC int
mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) {
int rc = MDBX_SUCCESS;
for (unsigned i = 0; i < array->count; ++i) {
const HANDLE hThread = array->handles[i];
if (ResumeThread(hThread) == -1) {
const int err = GetLastError();
DWORD ExitCode;
if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED &&
GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE)
rc = err;
}
CloseHandle(hThread);
}
return rc;
}
/*----------------------------------------------------------------------------*/
/* global `initial` lock for lockfile initialization,
* exclusive/shared locking first cacheline */
/* Briefly descritpion of locking schema/algorithm:
* - Windows does not support upgrading or downgrading for file locking.
* - Therefore upgrading/downgrading is emulated by shared and exclusive
* locking of upper and lower halves.
* - In other words, we have FSM with possible 9 states,
* i.e. free/shared/exclusive x free/shared/exclusive == 9.
* Only 6 states of FSM are used, which 2 of ones are transitive.
*
* The mdbx_lck_seize() moves the locking-FSM from the initial free/unlocked
* state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible,
* or to the "used" (and returns MDBX_RESULT_FALSE).
*
* The mdbx_lck_downgrade() moves the locking-FSM from "exclusive write"
* state to the "used" (i.e. shared) state.
*
* States:
* ?-? = free, i.e. unlocked
* S-? = used, i.e. shared lock
* E-? = exclusive-read, i.e. operational exclusive
* ?-S
* ?-E = middle (transitive state)
* S-S
* S-E = locked (transitive state)
* E-S
* E-E = exclusive-write, i.e. exclusive due (re)initialization
*/
static void lck_unlock(MDBX_env *env) {
int rc;
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* double `unlock` for robustly remove overlapped shared/exclusive locks */
while (funlock(env->me_lfd, LCK_LOWER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_lfd, LCK_UPPER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
}
if (env->me_fd != INVALID_HANDLE_VALUE) {
/* explicitly unlock to avoid latency for other processes (windows kernel
* releases such locks via deferred queues) */
while (funlock(env->me_fd, LCK_BODY))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_fd, LCK_META))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_fd, LCK_WHOLE))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
}
}
MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
MDBX_env *inprocess_neighbor,
int global_uniqueness_flag) {
(void)env;
(void)inprocess_neighbor;
(void)global_uniqueness_flag;
return MDBX_SUCCESS;
}
MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
MDBX_env *inprocess_neighbor) {
(void)inprocess_neighbor;
/* LY: should unmap before releasing the locks to avoid race condition and
* STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */
if (env->me_map)
mdbx_munmap(&env->me_dxb_mmap);
if (env->me_lck)
mdbx_munmap(&env->me_lck_mmap);
lck_unlock(env);
return MDBX_SUCCESS;
}
/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE)
* or as 'used' (S-? and returns MDBX_RESULT_FALSE).
* Oherwise returns an error. */
static int internal_seize_lck(HANDLE lfd) {
int rc;
assert(lfd != INVALID_HANDLE_VALUE);
/* 1) now on ?-? (free), get ?-E (middle) */
mdbx_jitter4testing(false);
if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) {
rc = GetLastError() /* 2) something went wrong, give up */;
mdbx_error("%s(%s) failed: errcode %u", __func__,
"?-?(free) >> ?-E(middle)", rc);
return rc;
}
/* 3) now on ?-E (middle), try E-E (exclusive-write) */
mdbx_jitter4testing(false);
if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER))
return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */;
/* 5) still on ?-E (middle) */
rc = GetLastError();
mdbx_jitter4testing(false);
if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) {
/* 6) something went wrong, give up */
if (!funlock(lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"?-E(middle) >> ?-?(free)", GetLastError());
return rc;
}
/* 7) still on ?-E (middle), try S-E (locked) */
mdbx_jitter4testing(false);
rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE
: GetLastError();
mdbx_jitter4testing(false);
if (rc != MDBX_RESULT_FALSE)
mdbx_error("%s(%s) failed: errcode %u", __func__,
"?-E(middle) >> S-E(locked)", rc);
/* 8) now on S-E (locked) or still on ?-E (middle),
* transite to S-? (used) or ?-? (free) */
if (!funlock(lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"X-E(locked/middle) >> X-?(used/free)", GetLastError());
/* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */
return rc;
}
MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) {
int rc;
assert(env->me_fd != INVALID_HANDLE_VALUE);
if (env->me_flags & MDBX_EXCLUSIVE)
return MDBX_RESULT_TRUE /* nope since files were must be opened
non-shareable */
;
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. on read-only filesystem) */
mdbx_jitter4testing(false);
if (!flock(env->me_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) {
rc = GetLastError();
mdbx_error("%s(%s) failed: errcode %u", __func__, "without-lck", rc);
return rc;
}
return MDBX_RESULT_FALSE;
}
rc = internal_seize_lck(env->me_lfd);
mdbx_jitter4testing(false);
if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) {
/* Check that another process don't operates in without-lck mode.
* Doing such check by exclusive locking the body-part of db. Should be
* noted:
* - we need an exclusive lock for do so;
* - we can't lock meta-pages, otherwise other process could get an error
* while opening db in valid (non-conflict) mode. */
if (!flock(env->me_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) {
rc = GetLastError();
mdbx_error("%s(%s) failed: errcode %u", __func__,
"lock-against-without-lck", rc);
mdbx_jitter4testing(false);
lck_unlock(env);
} else {
mdbx_jitter4testing(false);
if (!funlock(env->me_fd, LCK_BODY))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"unlock-against-without-lck", GetLastError());
}
}
return rc;
}
MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) {
/* Transite from exclusive state (E-?) to used (S-?) */
assert(env->me_fd != INVALID_HANDLE_VALUE);
assert(env->me_lfd != INVALID_HANDLE_VALUE);
#if 1
if (env->me_flags & MDBX_EXCLUSIVE)
return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
;
#else
/* 1) must be at E-E (exclusive-write) */
if (env->me_flags & MDBX_EXCLUSIVE) {
/* transite from E-E to E_? (exclusive-read) */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"E-E(exclusive-write) >> E-?(exclusive-read)", GetLastError());
return MDBX_SUCCESS /* 2) now at E-? (exclusive-read), done */;
}
#endif
/* 3) now at E-E (exclusive-write), transite to ?_E (middle) */
if (!funlock(env->me_lfd, LCK_LOWER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"E-E(exclusive-write) >> ?-E(middle)", GetLastError());
/* 4) now at ?-E (middle), transite to S-E (locked) */
if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) {
int rc = GetLastError() /* 5) something went wrong, give up */;
mdbx_error("%s(%s) failed: errcode %u", __func__,
"?-E(middle) >> S-E(locked)", rc);
return rc;
}
/* 6) got S-E (locked), continue transition to S-? (used) */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", __func__,
"S-E(locked) >> S-?(used)", GetLastError());
return MDBX_SUCCESS /* 7) now at S-? (used), done */;
}
/*----------------------------------------------------------------------------*/
/* reader checking (by pid) */
MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
/* Checks reader by pid.
*
* Returns:
* MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE, if pid is dead (lock acquired)
* or otherwise the errcode. */
MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) {
(void)env;
HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid);
int rc;
if (likely(hProcess)) {
rc = WaitForSingleObject(hProcess, 0);
if (unlikely(rc == WAIT_FAILED))
rc = GetLastError();
CloseHandle(hProcess);
} else {
rc = GetLastError();
}
switch (rc) {
case ERROR_INVALID_PARAMETER:
/* pid seems invalid */
return MDBX_RESULT_FALSE;
case WAIT_OBJECT_0:
/* process just exited */
return MDBX_RESULT_FALSE;
case WAIT_TIMEOUT:
/* pid running */
return MDBX_RESULT_TRUE;
default:
/* failure */
return rc;
}
}
//----------------------------------------------------------------------------
// Stub for slim read-write lock
// Copyright (C) 1995-2002 Brad Wilson
static void WINAPI stub_srwlock_Init(MDBX_srwlock *srwl) {
srwl->readerCount = srwl->writerCount = 0;
}
static void WINAPI stub_srwlock_AcquireShared(MDBX_srwlock *srwl) {
while (true) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
// If there's a writer already, spin without unnecessarily
// interlocking the CPUs
if (srwl->writerCount != 0) {
YieldProcessor();
continue;
}
// Add to the readers list
_InterlockedIncrement(&srwl->readerCount);
// Check for writers again (we may have been pre-empted). If
// there are no writers writing or waiting, then we're done.
if (srwl->writerCount == 0)
break;
// Remove from the readers list, spin, try again
_InterlockedDecrement(&srwl->readerCount);
YieldProcessor();
}
}
static void WINAPI stub_srwlock_ReleaseShared(MDBX_srwlock *srwl) {
assert(srwl->readerCount > 0);
_InterlockedDecrement(&srwl->readerCount);
}
static void WINAPI stub_srwlock_AcquireExclusive(MDBX_srwlock *srwl) {
while (true) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
// If there's a writer already, spin without unnecessarily
// interlocking the CPUs
if (srwl->writerCount != 0) {
YieldProcessor();
continue;
}
// See if we can become the writer (expensive, because it inter-
// locks the CPUs, so writing should be an infrequent process)
if (_InterlockedExchange(&srwl->writerCount, 1) == 0)
break;
}
// Now we're the writer, but there may be outstanding readers.
// Spin until there aren't any more; new readers will wait now
// that we're the writer.
while (srwl->readerCount != 0) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
YieldProcessor();
}
}
static void WINAPI stub_srwlock_ReleaseExclusive(MDBX_srwlock *srwl) {
assert(srwl->writerCount == 1 && srwl->readerCount >= 0);
srwl->writerCount = 0;
}
MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared,
mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive,
mdbx_srwlock_ReleaseExclusive;
/*----------------------------------------------------------------------------*/
#if 0 /* LY: unused for now */
static DWORD WINAPI stub_DiscardVirtualMemory(PVOID VirtualAddress,
SIZE_T Size) {
return VirtualAlloc(VirtualAddress, Size, MEM_RESET, PAGE_NOACCESS)
? ERROR_SUCCESS
: GetLastError();
}
#endif /* unused for now */
static uint64_t WINAPI stub_GetTickCount64(void) {
LARGE_INTEGER Counter, Frequency;
return (QueryPerformanceFrequency(&Frequency) &&
QueryPerformanceCounter(&Counter))
? Counter.QuadPart * 1000ul / Frequency.QuadPart
: 0;
}
/*----------------------------------------------------------------------------*/
#ifndef MDBX_ALLOY
MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx;
MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW;
MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle;
MDBX_NtFsControlFile mdbx_NtFsControlFile;
MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
MDBX_GetTickCount64 mdbx_GetTickCount64;
#if 0 /* LY: unused for now */
MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory;
MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory;
MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory;
#endif /* unused for now */
#endif /* MDBX_ALLOY */
static void mdbx_winnt_import(void) {
const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll");
const MDBX_srwlock_function init =
(MDBX_srwlock_function)GetProcAddress(hKernel32dll, "InitializeSRWLock");
if (init != NULL) {
mdbx_srwlock_Init = init;
mdbx_srwlock_AcquireShared = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "AcquireSRWLockShared");
mdbx_srwlock_ReleaseShared = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "ReleaseSRWLockShared");
mdbx_srwlock_AcquireExclusive = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "AcquireSRWLockExclusive");
mdbx_srwlock_ReleaseExclusive = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "ReleaseSRWLockExclusive");
} else {
mdbx_srwlock_Init = stub_srwlock_Init;
mdbx_srwlock_AcquireShared = stub_srwlock_AcquireShared;
mdbx_srwlock_ReleaseShared = stub_srwlock_ReleaseShared;
mdbx_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive;
mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive;
}
#define GET_KERNEL32_PROC(ENTRY) \
mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(hKernel32dll, #ENTRY)
GET_KERNEL32_PROC(GetFileInformationByHandleEx);
GET_KERNEL32_PROC(GetVolumeInformationByHandleW);
GET_KERNEL32_PROC(GetFinalPathNameByHandleW);
GET_KERNEL32_PROC(SetFileInformationByHandle);
GET_KERNEL32_PROC(PrefetchVirtualMemory);
GET_KERNEL32_PROC(GetTickCount64);
if (!mdbx_GetTickCount64)
mdbx_GetTickCount64 = stub_GetTickCount64;
#if 0 /* LY: unused for now */
GET_KERNEL32_PROC(DiscardVirtualMemory);
if (!mdbx_DiscardVirtualMemory)
mdbx_DiscardVirtualMemory = stub_DiscardVirtualMemory;
GET_KERNEL32_PROC(OfferVirtualMemory);
GET_KERNEL32_PROC(ReclaimVirtualMemory);
#endif /* unused for now */
#undef GET_KERNEL32_PROC
const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll");
mdbx_NtFsControlFile =
(MDBX_NtFsControlFile)GetProcAddress(hNtdll, "NtFsControlFile");
}

1244
contrib/db/libmdbx/src/elements/ntdll.def Normal file
View file

File diff suppressed because it is too large Load diff

1899
contrib/db/libmdbx/src/elements/osal.c Normal file
View file

File diff suppressed because it is too large Load diff

959
contrib/db/libmdbx/src/elements/osal.h Normal file
View file

@ -0,0 +1,959 @@
/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
/*----------------------------------------------------------------------------*/
/* Microsoft compiler generates a lot of warning for self includes... */
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
#if defined(_WIN32) || defined(_WIN64)
#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
#endif
#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \
!defined(MDBX_TOOLS)
#define _NO_CRT_STDIO_INLINE
#endif
#endif /* Windows */
/*----------------------------------------------------------------------------*/
/* C99 includes */
#include <inttypes.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#include <fcntl.h>
#include <limits.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
/* C11 stdalign.h */
#if __has_include(<stdalign.h>)
#include <stdalign.h>
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
#define alignas(N) _Alignas(N)
#elif defined(_MSC_VER)
#define alignas(N) __declspec(align(N))
#elif __has_attribute(__aligned__) || defined(__GNUC__)
#define alignas(N) __attribute__((__aligned__(N)))
#else
#error "FIXME: Required _alignas() or equivalent."
#endif
/*----------------------------------------------------------------------------*/
/* Systems includes */
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
defined(__BSD__) || defined(__NETBSD__) || defined(__bsdi__) || \
defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__)
#include <sys/cdefs.h>
#include <sys/sysctl.h>
#include <sys/types.h>
#if defined(__FreeBSD__) || defined(__DragonFly__)
#include <vm/vm_param.h>
#elif defined(__OpenBSD__) || defined(__NetBSD__)
#include <uvm/uvm_param.h>
#else
#define SYSCTL_LEGACY_NONCONST_MIB
#endif
#include <sys/vmmeter.h>
#else
#include <malloc.h>
#ifndef _POSIX_C_SOURCE
#ifdef _POSIX_SOURCE
#define _POSIX_C_SOURCE 1
#else
#define _POSIX_C_SOURCE 0
#endif
#endif
#endif /* !xBSD */
#if defined(__FreeBSD__) || defined(__OpenBSD__) || __has_include(<malloc_np.h>)
#include <malloc_np.h>
#endif
#if defined(__APPLE__) || defined(__MACH__) || __has_include(<malloc/malloc.h>)
#include <malloc/malloc.h>
#endif /* MacOS */
#if defined(__MACH__)
#include <mach/host_info.h>
#include <mach/mach_host.h>
#include <mach/mach_port.h>
#include <uuid/uuid.h>
#undef P_DIRTY
#endif
#if defined(__linux__) || defined(__gnu_linux__)
#include <linux/sysctl.h>
#include <sys/sendfile.h>
#include <sys/statvfs.h>
#endif /* Linux */
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 0
#endif
#ifndef _XOPEN_SOURCE_EXTENDED
#define _XOPEN_SOURCE_EXTENDED 0
#else
#include <utmpx.h>
#endif /* _XOPEN_SOURCE_EXTENDED */
#if defined(__sun) || defined(__SVR4) || defined(__svr4__)
#include <kstat.h>
#endif /* SunOS/Solaris */
#if defined(_WIN32) || defined(_WIN64)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <tlhelp32.h>
#include <windows.h>
#include <winnt.h>
#include <winternl.h>
#define HAVE_SYS_STAT_H
#define HAVE_SYS_TYPES_H
typedef HANDLE mdbx_thread_t;
typedef unsigned mdbx_thread_key_t;
#define MDBX_OSAL_SECTION HANDLE
#define MAP_FAILED NULL
#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
#define THREAD_CALL WINAPI
#define THREAD_RESULT DWORD
typedef struct {
HANDLE mutex;
HANDLE event;
} mdbx_condmutex_t;
typedef CRITICAL_SECTION mdbx_fastmutex_t;
#if MDBX_AVOID_CRT
#ifndef mdbx_malloc
static inline void *mdbx_malloc(size_t bytes) {
return LocalAlloc(LMEM_FIXED, bytes);
}
#endif /* mdbx_malloc */
#ifndef mdbx_calloc
static inline void *mdbx_calloc(size_t nelem, size_t size) {
return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size);
}
#endif /* mdbx_calloc */
#ifndef mdbx_realloc
static inline void *mdbx_realloc(void *ptr, size_t bytes) {
return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE);
}
#endif /* mdbx_realloc */
#ifndef mdbx_free
#define mdbx_free LocalFree
#endif /* mdbx_free */
#else
#define mdbx_malloc malloc
#define mdbx_calloc calloc
#define mdbx_realloc realloc
#define mdbx_free free
#define mdbx_strdup _strdup
#endif /* MDBX_AVOID_CRT */
#ifndef snprintf
#define snprintf _snprintf /* ntdll */
#endif
#ifndef vsnprintf
#define vsnprintf _vsnprintf /* ntdll */
#endif
#else /*----------------------------------------------------------------------*/
#include <pthread.h>
#include <signal.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/uio.h>
#include <unistd.h>
typedef pthread_t mdbx_thread_t;
typedef pthread_key_t mdbx_thread_key_t;
#define INVALID_HANDLE_VALUE (-1)
#define THREAD_CALL
#define THREAD_RESULT void *
typedef struct {
pthread_mutex_t mutex;
pthread_cond_t cond;
} mdbx_condmutex_t;
typedef pthread_mutex_t mdbx_fastmutex_t;
#define mdbx_malloc malloc
#define mdbx_calloc calloc
#define mdbx_realloc realloc
#define mdbx_free free
#define mdbx_strdup strdup
#endif /* Platform */
#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size)
/* malloc_usable_size() already provided */
#elif defined(__APPLE__)
#define malloc_usable_size(ptr) malloc_size(ptr)
#elif defined(_MSC_VER) && !MDBX_AVOID_CRT
#define malloc_usable_size(ptr) _msize(ptr)
#endif /* malloc_usable_size */
/* *INDENT-OFF* */
/* clang-format off */
#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
#include <sys/stat.h>
#endif
#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
#include <sys/types.h>
#endif
#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
#include <sys/file.h>
#endif
/* *INDENT-ON* */
/* clang-format on */
#ifndef SSIZE_MAX
#define SSIZE_MAX INTPTR_MAX
#endif
#if !defined(MADV_DODUMP) && defined(MADV_CORE)
#define MADV_DODUMP MADV_CORE
#endif /* MADV_CORE -> MADV_DODUMP */
#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
#define MADV_DONTDUMP MADV_NOCORE
#endif /* MADV_NOCORE -> MADV_DONTDUMP */
#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
defined(i486) || defined(__i486) || defined(__i486__) || \
defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
#ifndef __ia32__
/* LY: define neutral __ia32__ for x86 and x86-64 archs */
#define __ia32__ 1
#endif /* __ia32__ */
#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64) || defined(_M_X64))
/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
#define __amd64__ 1
#endif /* __amd64__ */
#endif /* all x86 */
#if !defined(MDBX_UNALIGNED_OK)
#if defined(_MSC_VER)
#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */
#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0)
#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */
#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \
!defined(__ALIGNED__)
#define MDBX_UNALIGNED_OK 1
#else
#define MDBX_UNALIGNED_OK 0
#endif
#endif /* MDBX_UNALIGNED_OK */
#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
#error \
"Sanity checking failed: Two's complement, reasonably sized integer types"
#endif
/*----------------------------------------------------------------------------*/
/* Compiler's includes for builtins/intrinsics */
#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
#include <intrin.h>
#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
#if defined(__ia32__) || defined(__e2k__)
#include <x86intrin.h>
#endif /* __ia32__ */
#if defined(__ia32__)
#include <cpuid.h>
#endif /* __ia32__ */
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
#include <mbarrier.h>
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
#include <machine/sys/inline.h>
#elif defined(__IBMC__) && defined(__powerpc)
#include <atomic.h>
#elif defined(_AIX)
#include <builtins.h>
#include <sys/atomic_op.h>
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
#include <c_asm.h>
#include <machine/builtins.h>
#elif defined(__MWERKS__)
/* CodeWarrior - troubles ? */
#pragma gcc_extensions
#elif defined(__SNC__)
/* Sony PS3 - troubles ? */
#elif defined(__hppa__) || defined(__hppa)
#include <machine/inline.h>
#else
#error Unsupported C compiler, please use GNU C 4.4 or newer
#endif /* Compiler */
/*----------------------------------------------------------------------------*/
/* Byteorder */
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
/* *INDENT-OFF* */
/* clang-format off */
#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \
defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
#include <endian.h>
#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
#include <machine/endian.h>
#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
#include <sys/isa_defs.h>
#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
(__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
#include <sys/endian.h>
#include <sys/types.h>
#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
defined(__NETBSD__) || defined(__NetBSD__) || \
defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
#include <sys/param.h>
#endif /* OS */
/* *INDENT-ON* */
/* clang-format on */
#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
#define __BYTE_ORDER__ __BYTE_ORDER
#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
#define __BYTE_ORDER__ _BYTE_ORDER
#else
#define __ORDER_LITTLE_ENDIAN__ 1234
#define __ORDER_BIG_ENDIAN__ 4321
#if defined(__LITTLE_ENDIAN__) || \
(defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
defined(__WINDOWS__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#elif defined(__BIG_ENDIAN__) || \
(defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
#else
#error __BYTE_ORDER__ should be defined.
#endif /* Arch */
#endif
#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
/*----------------------------------------------------------------------------*/
/* Memory/Compiler barriers, cache coherence */
static __maybe_unused __inline void mdbx_compiler_barrier(void) {
#if defined(__clang__) || defined(__GNUC__)
__asm__ __volatile__("" ::: "memory");
#elif defined(_MSC_VER)
_ReadWriteBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
__memory_barrier();
if (type > MDBX_BARRIER_COMPILER)
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__compiler_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__fence();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
static __maybe_unused __inline void mdbx_memory_barrier(void) {
#if __has_extension(c_atomic) || __has_extension(cxx_atomic)
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__ATOMIC_SEQ_CST)
__atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__clang__) || defined(__GNUC__)
__sync_synchronize();
#elif defined(_MSC_VER)
MemoryBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__lwsync();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
/*----------------------------------------------------------------------------*/
/* Cache coherence and invalidation */
#ifndef MDBX_CPU_WRITEBACK_IS_COHERENT
#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
defined(__hppa__)
#define MDBX_CPU_WRITEBACK_IS_COHERENT 1
#else
#define MDBX_CPU_WRITEBACK_IS_COHERENT 0
#endif
#endif /* MDBX_CPU_WRITEBACK_IS_COHERENT */
#ifndef MDBX_CACHELINE_SIZE
#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
#define MDBX_CACHELINE_SIZE 128
#else
#define MDBX_CACHELINE_SIZE 64
#endif
#endif /* MDBX_CACHELINE_SIZE */
#if MDBX_CPU_WRITEBACK_IS_COHERENT
#define mdbx_flush_noncoherent_cpu_writeback() mdbx_compiler_barrier()
#else
#define mdbx_flush_noncoherent_cpu_writeback() mdbx_memory_barrier()
#endif
#if __has_include(<sys/cachectl.h>)
#include <sys/cachectl.h>
#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
/* MIPS should have explicit cache control */
#include <sys/cachectl.h>
#endif
#ifndef MDBX_CPU_CACHE_MMAP_NONCOHERENT
#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
/* MIPS has cache coherency issues. */
#define MDBX_CPU_CACHE_MMAP_NONCOHERENT 1
#else
/* LY: assume no relevant mmap/dcache issues. */
#define MDBX_CPU_CACHE_MMAP_NONCOHERENT 0
#endif
#endif /* ndef MDBX_CPU_CACHE_MMAP_NONCOHERENT */
static __maybe_unused __inline void
mdbx_invalidate_mmap_noncoherent_cache(void *addr, size_t nbytes) {
#if MDBX_CPU_CACHE_MMAP_NONCOHERENT
#ifdef DCACHE
/* MIPS has cache coherency issues.
* Note: for any nbytes >= on-chip cache size, entire is flushed. */
cacheflush(addr, nbytes, DCACHE);
#else
#error "Oops, cacheflush() not available"
#endif /* DCACHE */
#else /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */
(void)addr;
(void)nbytes;
#endif /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */
}
/*----------------------------------------------------------------------------*/
/* libc compatibility stuff */
#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
(defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
#define mdbx_asprintf asprintf
#define mdbx_vasprintf vasprintf
#else
MDBX_INTERNAL_FUNC __printf_args(2, 3) int __maybe_unused
mdbx_asprintf(char **strp, const char *fmt, ...);
MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
#endif
/*----------------------------------------------------------------------------*/
/* OS abstraction layer stuff */
/* max bytes to write in one call */
#if defined(_WIN32) || defined(_WIN64)
#define MAX_WRITE UINT32_C(0x01000000)
#else
#define MAX_WRITE UINT32_C(0x3fff0000)
#endif
#if defined(__linux__) || defined(__gnu_linux__)
MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version;
#endif /* Linux */
/* Get the size of a memory page for the system.
* This is the basic size that the platform's memory manager uses, and is
* fundamental to the use of memory-mapped files. */
static __maybe_unused __inline size_t mdbx_syspagesize(void) {
#if defined(_WIN32) || defined(_WIN64)
SYSTEM_INFO si;
GetSystemInfo(&si);
return si.dwPageSize;
#else
return sysconf(_SC_PAGE_SIZE);
#endif
}
#ifndef mdbx_strdup
LIBMDBX_API char *mdbx_strdup(const char *str);
#endif
static __maybe_unused __inline int mdbx_get_errno(void) {
#if defined(_WIN32) || defined(_WIN64)
DWORD rc = GetLastError();
#else
int rc = errno;
#endif
return rc;
}
#ifndef mdbx_memalign_alloc
MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes,
void **result);
#endif
#ifndef mdbx_memalign_free
MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr);
#endif
MDBX_INTERNAL_FUNC int mdbx_condmutex_init(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_lock(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_unlock(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_signal(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_wait(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_condmutex_destroy(mdbx_condmutex_t *condmutex);
MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov,
int iovcnt, uint64_t offset,
size_t expected_written);
MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count,
uint64_t offset);
MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf,
size_t count, uint64_t offset);
MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf,
size_t count);
MDBX_INTERNAL_FUNC int
mdbx_thread_create(mdbx_thread_t *thread,
THREAD_RESULT(THREAD_CALL *start_routine)(void *),
void *arg);
MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread);
enum mdbx_syncmode_bits {
MDBX_SYNC_DATA = 1,
MDBX_SYNC_SIZE = 2,
MDBX_SYNC_IODQ = 4
};
MDBX_INTERNAL_FUNC int mdbx_filesync(mdbx_filehandle_t fd,
enum mdbx_syncmode_bits mode_bits);
MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
MDBX_INTERNAL_FUNC int mdbx_openfile(const char *pathname, int flags,
mode_t mode, mdbx_filehandle_t *fd,
bool exclusive);
MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd);
MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname);
MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd);
typedef struct mdbx_mmap_param {
union {
void *address;
uint8_t *dxb;
struct MDBX_lockinfo *lck;
};
mdbx_filehandle_t fd;
size_t limit; /* mapping length, but NOT a size of file nor DB */
size_t current; /* mapped region size, i.e. the size of file and DB */
#if defined(_WIN32) || defined(_WIN64)
uint64_t filesize /* in-process cache of a file size. */;
#endif
#ifdef MDBX_OSAL_SECTION
MDBX_OSAL_SECTION section;
#endif
} mdbx_mmap_t;
MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map,
const size_t must, const size_t limit,
const bool truncate);
MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map);
MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current,
size_t wanna);
#if defined(_WIN32) || defined(_WIN64)
typedef struct {
unsigned limit, count;
HANDLE handles[31];
} mdbx_handle_array_t;
MDBX_INTERNAL_FUNC int
mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array);
MDBX_INTERNAL_FUNC int
mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
#endif /* Windows */
MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset,
size_t length, int async);
MDBX_INTERNAL_FUNC int mdbx_check4nonlocal(mdbx_filehandle_t handle, int flags);
static __maybe_unused __inline uint32_t mdbx_getpid(void) {
STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t));
#if defined(_WIN32) || defined(_WIN64)
return GetCurrentProcessId();
#else
return getpid();
#endif
}
static __maybe_unused __inline size_t mdbx_thread_self(void) {
STATIC_ASSERT(sizeof(mdbx_tid_t) <= sizeof(size_t));
#if defined(_WIN32) || defined(_WIN64)
return GetCurrentThreadId();
#else
return (size_t)pthread_self();
#endif
}
MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny);
MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void);
MDBX_INTERNAL_FUNC uint64_t
mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime);
typedef union bin128 {
__anonymous_struct_extension__ struct { uint64_t x, y; };
__anonymous_struct_extension__ struct { uint32_t a, b, c, d; };
} bin128_t;
MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void);
/*----------------------------------------------------------------------------*/
/* lck stuff */
#if defined(_WIN32) || defined(_WIN64)
#undef MDBX_OSAL_LOCK
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0xF10C)
#else
#define MDBX_OSAL_LOCK pthread_mutex_t
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0x8017)
#endif /* MDBX_OSAL_LOCK */
/// \brief Initialization of synchronization primitives linked with MDBX_env
/// instance both in LCK-file and within the current process.
/// \param
/// global_uniqueness_flag = true - denotes that there are no other processes
/// working with DB and LCK-file. Thus the function MUST initialize
/// shared synchronization objects in memory-mapped LCK-file.
/// global_uniqueness_flag = false - denotes that at least one process is
/// already working with DB and LCK-file, including the case when DB
/// has already been opened in the current process. Thus the function
/// MUST NOT initialize shared synchronization objects in memory-mapped
/// LCK-file that are already in use.
/// \return Error code or zero on success.
MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env,
MDBX_env *inprocess_neighbor,
int global_uniqueness_flag);
/// \brief Disconnects from shared interprocess objects and destructs
/// synchronization objects linked with MDBX_env instance
/// within the current process.
/// \param
/// inprocess_neighbor = NULL - if the current process does not have other
/// instances of MDBX_env linked with the DB being closed.
/// Thus the function MUST check for other processes working with DB or
/// LCK-file, and keep or destroy shared synchronization objects in
/// memory-mapped LCK-file depending on the result.
/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env
/// (anyone of there is several) working with DB or LCK-file within the
/// current process. Thus the function MUST NOT try to acquire exclusive
/// lock and/or try to destruct shared synchronization objects linked with
/// DB or LCK-file. Moreover, the implementation MUST ensure correct work
/// of other instances of MDBX_env within the current process, e.g.
/// restore POSIX-fcntl locks after the closing of file descriptors.
/// \return Error code (MDBX_PANIC) or zero on success.
MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env,
MDBX_env *inprocess_neighbor);
/// \brief Connects to shared interprocess locking objects and tries to acquire
/// the maximum lock level (shared if exclusive is not available)
/// Depending on implementation or/and platform (Windows) this function may
/// acquire the non-OS super-level lock (e.g. for shared synchronization
/// objects initialization), which will be downgraded to OS-exclusive or
/// shared via explicit calling of mdbx_lck_downgrade().
/// \return
/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus
/// the current process is the first and only after the last use of DB.
/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus
/// DB has already been opened and now is used by other processes.
/// Otherwise (not 0 and not -1) - error code.
MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env);
/// \brief Downgrades the level of initially acquired lock to
/// operational level specified by agrument. The reson for such downgrade:
/// - unblocking of other processes that are waiting for access, i.e.
/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes
/// should be made aware that access is unavailable rather than
/// wait for it.
/// - freeing locks that interfere file operation (expecially for Windows)
/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock.
/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive
/// operational lock.
/// \return Error code or zero on success
MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env);
/// \brief Locks LCK-file or/and table of readers for (de)registering.
/// \return Error code or zero on success
MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env);
/// \brief Unlocks LCK-file or/and table of readers after (de)registering.
MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env);
/// \brief Acquires lock for DB change (on writing transaction start)
/// Reading transactions will not be blocked.
/// Declared as LIBMDBX_API because it is used in mdbx_chk.
/// \return Error code or zero on success
LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dontwait);
/// \brief Releases lock once DB changes is made (after writing transaction
/// has finished).
/// Declared as LIBMDBX_API because it is used in mdbx_chk.
LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
/// \brief Sets alive-flag of reader presence (indicative lock) for PID of
/// the current process. The function does no more than needed for
/// the correct working of mdbx_rpid_check() in other processes.
/// \return Error code or zero on success
MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env);
/// \brief Resets alive-flag of reader presence (indicative lock)
/// for PID of the current process. The function does no more than needed
/// for the correct working of mdbx_rpid_check() in other processes.
/// \return Error code or zero on success
MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env);
/// \brief Checks for reading process status with the given pid with help of
/// alive-flag of presence (indicative lock) or using another way.
/// \return
/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive
/// and working with DB (indicative lock is present).
/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent
/// or not working with DB (indicative lock is not present).
/// Otherwise (not 0 and not -1) - error code.
MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid);
#if defined(_WIN32) || defined(_WIN64)
typedef union MDBX_srwlock {
struct {
long volatile readerCount;
long volatile writerCount;
};
RTL_SRWLOCK native;
} MDBX_srwlock;
typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init,
mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared,
mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive;
typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
_In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx
mdbx_GetFileInformationByHandleEx;
typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
_In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
_In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
_Out_opt_ LPDWORD lpMaximumComponentLength,
_Out_opt_ LPDWORD lpFileSystemFlags,
_Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW
mdbx_GetVolumeInformationByHandleW;
typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
_Out_ LPWSTR lpszFilePath,
_In_ DWORD cchFilePath,
_In_ DWORD dwFlags);
MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
_In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle
mdbx_SetFileInformationByHandle;
typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
IN HANDLE FileHandle, IN OUT HANDLE Event,
IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile;
typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void);
MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64;
#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8
typedef struct _WIN32_MEMORY_RANGE_ENTRY {
PVOID VirtualAddress;
SIZE_T NumberOfBytes;
} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
#endif /* Windows 8.x */
typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
HANDLE hProcess, ULONG_PTR NumberOfEntries,
PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
#if 0 /* LY: unused for now */
#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81
typedef enum OFFER_PRIORITY {
VmOfferPriorityVeryLow = 1,
VmOfferPriorityLow,
VmOfferPriorityBelowNormal,
VmOfferPriorityNormal
} OFFER_PRIORITY;
#endif /* Windows 8.1 */
typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress,
SIZE_T Size);
MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory;
typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress,
SIZE_T Size);
MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory;
typedef DWORD(WINAPI *MDBX_OfferVirtualMemory(
PVOID VirtualAddress,
SIZE_T Size,
OFFER_PRIORITY Priority
);
MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory;
#endif /* unused for now */
#endif /* Windows */
/*----------------------------------------------------------------------------*/
/* Atomics */
#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \
!defined(__STDC_NO_ATOMICS__) && \
(__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
!(defined(__GNUC__) || defined(__clang__)))
#include <stdatomic.h>
#elif defined(__GNUC__) || defined(__clang__)
/* LY: nothing required */
#elif defined(_MSC_VER)
#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
#pragma warning(disable : 4133) /* 'function': incompatible types - from \
'size_t' to 'LONGLONG' */
#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
'std::size_t', possible loss of data */
#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
'long', possible loss of data */
#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
#elif defined(__APPLE__)
#include <libkern/OSAtomic.h>
#else
#error FIXME atomic-ops
#endif
/*----------------------------------------------------------------------------*/
#if defined(_MSC_VER) && _MSC_VER >= 1900
/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
* for internal format-args checker. */
#undef PRIuPTR
#undef PRIiPTR
#undef PRIdPTR
#undef PRIxPTR
#define PRIuPTR "Iu"
#define PRIiPTR "Ii"
#define PRIdPTR "Id"
#define PRIxPTR "Ix"
#define PRIuSIZE "zu"
#define PRIiSIZE "zi"
#define PRIdSIZE "zd"
#define PRIxSIZE "zx"
#endif /* fix PRI*PTR for _MSC_VER */
#ifndef PRIuSIZE
#define PRIuSIZE PRIuPTR
#define PRIiSIZE PRIiPTR
#define PRIdSIZE PRIdPTR
#define PRIxSIZE PRIxPTR
#endif /* PRI*SIZE macros for MSVC */
#ifdef _MSC_VER
#pragma warning(pop)
#endif

46
contrib/db/libmdbx/src/elements/version.c.in Normal file
View file

@ -0,0 +1,46 @@
/* This is CMake-template for libmdbx's version.c
******************************************************************************/
#include "internals.h"
#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || \
MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR}
#error "API version mismatch! Had `git fetch --tags` done?"
#endif
static const char sourcery[] = STRINGIFY(MDBX_BUILD_SOURCERY);
__dll_export
#ifdef __attribute_used__
__attribute_used__
#elif defined(__GNUC__) || __has_attribute(__used__)
__attribute__((__used__))
#endif
#ifdef __attribute_externally_visible__
__attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) || \
__has_attribute(__externally_visible__)
__attribute__((__externally_visible__))
#endif
const mdbx_version_info mdbx_version = {
${MDBX_VERSION_MAJOR},
${MDBX_VERSION_MINOR},
${MDBX_VERSION_RELEASE},
${MDBX_VERSION_REVISION},
{"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@",
"@MDBX_GIT_DESCRIBE@"},
sourcery};
__dll_export
#ifdef __attribute_used__
__attribute_used__
#elif defined(__GNUC__) || __has_attribute(__used__)
__attribute__((__used__))
#endif
#ifdef __attribute_externally_visible__
__attribute_externally_visible__
#elif (defined(__GNUC__) && !defined(__clang__)) || \
__has_attribute(__externally_visible__)
__attribute__((__externally_visible__))
#endif
const char *const mdbx_sourcery_anchor = sourcery;

87
contrib/db/libmdbx/src/man1/mdbx_chk.1 Normal file
View file

@ -0,0 +1,87 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_CHK 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_chk \- MDBX checking tool
.SH SYNOPSIS
.B mdbx_chk
[\c
.BR \-V ]
[\c
.BR \-v [ v [ v ]]]
[\c
.BR \-n ]
[\c
.BR \-q ]
[\c
.BR \-w ]
[\c
.BR \-d ]
[\c
.BI \-s \ subdb\fR]
[\c
.BR \-c ]
[\c
.BR \-i ]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_chk
utility intended to check an MDBX database file.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-v
Produce verbose output, including summarize space and page usage statistics.
If \fB\-vv\fP is given, be more verbose, show summarized B-tree info
and space allocation.
If \fB\-vvv\fP is given, be more verbose, include summarized statistics
of leaf B-tree pages.
If \fB\-vvvv\fP is given, be even more verbose, show info of each page
during B-tree traversal and basic info of each GC record.
If \fB\-vvvvv\fP is given, turn maximal verbosity, display the full list
of page IDs in the GC records and size of each key-value pair of database(s).
.TP
.BR \-n
Open MDBX environment(s) which do not use subdirectories.
.TP
.BR \-q
Be quiet; do not output anything even if an error was detected.
.TP
.BR \-w
Open environment in read-write mode and lock for writing while checking.
This could be impossible if environment already used by another process(s)
in an incompatible read-write mode. This allow rollback to last steady commit
(in case environment was not closed properly) and then check transaction IDs
of meta-pages. Otherwise, without \fB\-w\fP option environment will be
opened in read-only mode.
.TP
.BR \-d
Disable page-by-page traversal of B-tree. In this case, without B-tree
traversal, it is unable to check for lost-unused pages nor for double-used
pages.
.TP
.BR \-s \ subdb
Verify and show info only for a specific subdatabase.
.TP
.BR \-c
Force using cooperative mode while opening environment, i.e. don't try to open
in exclusive/monopolistic mode. Only exclusive/monopolistic mode allow complete
check, including full check of all meta-pages and actual size of database file.
.TP
.BR \-i
Ignore wrong order errors, which will likely false-positive if custom
comparator(s) was used.
.SH DIAGNOSTICS
Exit status is zero if no errors occur. Errors result in a non-zero exit status
and a diagnostic message being written to standard error
if no quiet mode was requested.
.SH "SEE ALSO"
.BR mdbx_stat (1),
.BR mdbx_copy (1),
.BR mdbx_dump (1),
.BR mdbx_load (1)
.SH AUTHOR
Leonid Yuriev <https://github.com/leo-yuriev>

60
contrib/db/libmdbx/src/man1/mdbx_copy.1 Normal file
View file

@ -0,0 +1,60 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_COPY 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_copy \- MDBX environment copy tool
.SH SYNOPSIS
.B mdbx_copy
[\c
.BR \-V ]
[\c
.BR \-c ]
[\c
.BR \-n ]
.B srcpath
[\c
.BR dstpath ]
.SH DESCRIPTION
The
.B mdbx_copy
utility copies an MDBX environment. The environment can
be copied regardless of whether it is currently in use.
No lockfile is created, since it gets recreated at need.
If
.I dstpath
is specified it must be the path of an empty directory
for storing the backup. Otherwise, the backup will be
written to stdout.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-c
Compact while copying. Only current data pages will be copied; freed
or unused pages will be omitted from the copy. This option will
slow down the backup process as it is more CPU-intensive.
Currently it fails if the environment has suffered a page leak.
.TP
.BR \-n
Open MDBX environment(s) which do not use subdirectories.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH CAVEATS
This utility can trigger significant file size growth if run
in parallel with write transactions, because pages which they
free during copying cannot be reused until the copy is done.
.SH "SEE ALSO"
.BR mdbx_dump (1),
.BR mdbx_chk (1),
.BR mdbx_stat (1),
.BR mdbx_load (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

80
contrib/db/libmdbx/src/man1/mdbx_dump.1 Normal file
View file

@ -0,0 +1,80 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_DUMP 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_dump \- MDBX environment export tool
.SH SYNOPSIS
.B mdbx_dump
[\c
.BR \-V ]
[\c
.BI \-f \ file\fR]
[\c
.BR \-l ]
[\c
.BR \-n ]
[\c
.BR \-p ]
[\c
.BR \-a \ |
.BI \-s \ subdb\fR]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_dump
utility reads a database and writes its contents to the
standard output using a portable flat-text format
understood by the
.BR mdbx_load (1)
utility.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-f \ file
Write to the specified file instead of to the standard output.
.TP
.BR \-l
List the databases stored in the environment. Just the
names will be listed, no data will be output.
.TP
.BR \-n
Dump an MDBX database which does not use subdirectories.
.TP
.BR \-p
If characters in either the key or data items are printing characters (as
defined by isprint(3)), output them directly. This option permits users to
use standard text editors and tools to modify the contents of databases.
Note: different systems may have different notions about what characters
are considered printing characters, and databases dumped in this manner may
be less portable to external systems.
.TP
.BR \-a
Dump all of the subdatabases in the environment.
.TP
.BR \-s \ subdb
Dump a specific subdatabase. If no database is specified, only the main database is dumped.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
Dumping and reloading databases that use user-defined comparison functions
will result in new databases that use the default comparison functions.
\fBIn this case it is quite likely that the reloaded database will be
damaged beyond repair permitting neither record storage nor retrieval.\fP
The only available workaround is to modify the source for the
.BR mdbx_load (1)
utility to load the database using the correct comparison functions.
.SH "SEE ALSO"
.BR mdbx_load (1),
.BR mdbx_copy (1),
.BR mdbx_chk (1),
.BR mdbx_stat (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

89
contrib/db/libmdbx/src/man1/mdbx_load.1 Normal file
View file

@ -0,0 +1,89 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_LOAD 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_load \- MDBX environment import tool
.SH SYNOPSIS
.B mdbx_load
[\c
.BR \-V ]
[\c
.BI \-f \ file\fR]
[\c
.BR \-n ]
[\c
.BI \-s \ subdb\fR]
[\c
.BR \-N ]
[\c
.BR \-T ]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_load
utility reads from the standard input and loads it into the
MDBX environment
.BR envpath .
The input to
.B mdbx_load
must be in the output format specified by the
.BR mdbx_dump (1)
utility or as specified by the
.B -T
option below.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-a
Append all records in the order they appear in the input. The input is assumed to already be
in correctly sorted order and no sorting or checking for redundant values will be performed.
This option must be used to reload data that was produced by running
.B mdbx_dump
on a database that uses custom compare functions.
.TP
.BR \-f \ file
Read from the specified file instead of from the standard input.
.TP
.BR \-n
Load an MDBX database which does not use subdirectories.
.TP
.BR \-s \ subdb
Load a specific subdatabase. If no database is specified, data is loaded into the main database.
.TP
.BR \-N
Don't overwrite existing records when loading into an already existing database; just skip them.
.TP
.BR \-T
Load data from simple text files. The input must be paired lines of text, where the first
line of the pair is the key item, and the second line of the pair is its corresponding
data item.
A simple escape mechanism, where newline and backslash (\\) characters are special, is
applied to the text input. Newline characters are interpreted as record separators.
Backslash characters in the text will be interpreted in one of two ways: If the backslash
character precedes another backslash character, the pair will be interpreted as a literal
backslash. If the backslash character precedes any other character, the two characters
following the backslash will be interpreted as a hexadecimal specification of a single
character; for example, \\0a is a newline character in the ASCII character set.
For this reason, any backslash or newline characters that naturally occur in the text
input must be escaped to avoid misinterpretation by
.BR mdbx_load .
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdbx_dump (1),
.BR mdbx_chk (1),
.BR mdbx_stat (1),
.BR mdbx_copy (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

69
contrib/db/libmdbx/src/man1/mdbx_stat.1 Normal file
View file

@ -0,0 +1,69 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_STAT 1 "2019-09-10" "MDBX 0.x"
.SH NAME
mdbx_stat \- MDBX environment status tool
.SH SYNOPSIS
.B mdbx_stat
[\c
.BR \-V ]
[\c
.BR \-e ]
[\c
.BR \-f [ f [ f ]]]
[\c
.BR \-n ]
[\c
.BR \-r [ r ]]
[\c
.BR \-a \ |
.BI \-s \ subdb\fR]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_stat
utility displays the status of an MDBX environment.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-e
Display information about the database environment.
.TP
.BR \-f
Display information about the environment freelist.
If \fB\-ff\fP is given, summarize each freelist entry.
If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
.TP
.BR \-n
Display the status of an MDBX database which does not use subdirectories.
.TP
.BR \-r
Display information about the environment reader table.
Shows the process ID, thread ID, and transaction ID for each active
reader slot. The process ID and transaction ID are in decimal, the
thread ID is in hexadecimal. The transaction ID is displayed as "-"
if the reader does not currently have a read transaction open.
If \fB\-rr\fP is given, check for stale entries in the reader
table and clear them. The reader table will be printed again
after the check is performed.
.TP
.BR \-a
Display the status of all of the subdatabases in the environment.
.TP
.BR \-s \ subdb
Display the status of a specific subdatabase.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdbx_chk (1),
.BR mdbx_copy (1),
.BR mdbx_dump (1),
.BR mdbx_load (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

42
contrib/db/libmdbx/src/tools/CMakeLists.txt Normal file
View file

@ -0,0 +1,42 @@
set(MDBX_TOOLS mdbx_chk mdbx_copy mdbx_dump mdbx_load mdbx_stat)
# use, i.e. don't skip the full RPATH for the build tree
set(CMAKE_SKIP_BUILD_RPATH FALSE)
# when building, don't use the install RPATH already (but later on when installing)
set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
# add the automatically determined parts of the RPATH
# which point to directories outside the build tree to the install RPATH
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
# the RPATH to be used when installing, but only if it's not a system directory
list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
if(isSystemDir EQUAL -1)
if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin")
set(CMAKE_INSTALL_RPATH "@executable_path/../lib")
else()
set(CMAKE_INSTALL_RPATH "\$ORIGIN/../lib")
endif()
endif()
foreach(TOOL ${MDBX_TOOLS})
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
add_executable(${TOOL} ${TOOL}.c wingetopt.c wingetopt.h)
else()
add_executable(${TOOL} ${TOOL}.c)
endif()
target_link_libraries(${TOOL} mdbx ${CMAKE_THREAD_LIBS_INIT})
set_target_properties(${TOOL} PROPERTIES
C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>)
install(TARGETS ${TOOL} DESTINATION ${CMAKE_INSTALL_PREFIX}/bin COMPONENT mdbx)
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../man1/${TOOL}.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/man/man1 COMPONENT mdbx)
endforeach()
if(LIB_MATH)
target_link_libraries(mdbx_chk ${LIB_MATH})
target_link_libraries(mdbx_stat ${LIB_MATH})
endif()

1430
contrib/db/libmdbx/src/tools/mdbx_chk.c Normal file
View file

File diff suppressed because it is too large Load diff

130
contrib/db/libmdbx/src/tools/mdbx_copy.c Normal file
View file

@ -0,0 +1,130 @@
/* mdbx_copy.c - memory-mapped database backup tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../elements/internals.h"
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
int main(int argc, char *argv[]) {
int rc;
MDBX_env *env = NULL;
const char *progname = argv[0], *act;
unsigned flags = MDBX_RDONLY;
unsigned cpflags = 0;
bool quiet = false;
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
if (argv[1][1] == 'n' && argv[1][2] == '\0')
flags |= MDBX_NOSUBDIR;
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
cpflags |= MDBX_CP_COMPACT;
else if (argv[1][1] == 'q' && argv[1][2] == '\0')
quiet = true;
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
printf("mdbx_copy version %d.%d.%d.%d\n"
" - source: %s %s, commit %s, tree %s\n"
" - anchor: %s\n"
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.release,
mdbx_version.revision, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
return EXIT_SUCCESS;
} else
argc = 0;
}
if (argc < 2 || argc > 3) {
fprintf(stderr, "usage: %s [-V] [-q] [-c] [-n] srcpath [dstpath]\n",
progname);
exit(EXIT_FAILURE);
}
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
if (!quiet) {
fprintf((argc == 2) ? stderr : stdout,
"mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]);
fflush(NULL);
}
act = "opening environment";
rc = mdbx_env_create(&env);
if (rc == MDBX_SUCCESS) {
rc = mdbx_env_open(env, argv[1], flags, 0640);
}
if (rc == MDBX_SUCCESS) {
act = "copying";
if (argc == 2) {
mdbx_filehandle_t fd;
#if defined(_WIN32) || defined(_WIN64)
fd = GetStdHandle(STD_OUTPUT_HANDLE);
#else
fd = fileno(stdout);
#endif
rc = mdbx_env_copy2fd(env, fd, cpflags);
} else
rc = mdbx_env_copy(env, argv[2], cpflags);
}
if (rc)
fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc,
mdbx_strerror(rc));
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

352
contrib/db/libmdbx/src/tools/mdbx_dump.c Normal file
View file

@ -0,0 +1,352 @@
/* mdbx_dump.c - memory-mapped database dump tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../elements/internals.h"
#include <ctype.h>
#define PRINT 1
static int mode;
typedef struct flagbit {
int bit;
char *name;
} flagbit;
flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"},
{MDBX_DUPSORT, "dupsort"},
{MDBX_INTEGERKEY, "integerkey"},
{MDBX_DUPFIXED, "dupfixed"},
{MDBX_INTEGERDUP, "integerdup"},
{MDBX_REVERSEDUP, "reversedup"},
{0, NULL}};
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
static const char hexc[] = "0123456789abcdef";
static void dumpbyte(unsigned char c) {
putchar(hexc[c >> 4]);
putchar(hexc[c & 0xf]);
}
static void text(MDBX_val *v) {
unsigned char *c, *end;
putchar(' ');
c = v->iov_base;
end = c + v->iov_len;
while (c < end) {
if (isprint(*c) && *c != '\\') {
putchar(*c);
} else {
putchar('\\');
dumpbyte(*c);
}
c++;
}
putchar('\n');
}
static void dumpval(MDBX_val *v) {
unsigned char *c, *end;
putchar(' ');
c = v->iov_base;
end = c + v->iov_len;
while (c < end) {
dumpbyte(*c++);
}
putchar('\n');
}
/* Dump in BDB-compatible format */
static int dumpit(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
MDBX_cursor *mc;
MDBX_stat ms;
MDBX_val key, data;
MDBX_envinfo info;
unsigned int flags;
int rc, i;
rc = mdbx_dbi_flags(txn, dbi, &flags);
if (rc)
return rc;
rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms));
if (rc)
return rc;
rc = mdbx_env_info(mdbx_txn_env(txn), &info, sizeof(info));
if (rc)
return rc;
printf("VERSION=3\n");
printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
if (name)
printf("database=%s\n", name);
printf("type=btree\n");
printf("mapsize=%" PRIu64 "\n", info.mi_mapsize);
printf("maxreaders=%u\n", info.mi_maxreaders);
for (i = 0; dbflags[i].bit; i++)
if (flags & dbflags[i].bit)
printf("%s=1\n", dbflags[i].name);
printf("db_pagesize=%d\n", ms.ms_psize);
printf("HEADER=END\n");
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc)
return rc;
while ((rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT)) == MDBX_SUCCESS) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
if (mode & PRINT) {
text(&key);
text(&data);
} else {
dumpval(&key);
dumpval(&data);
}
}
printf("DATA=END\n");
if (rc == MDBX_NOTFOUND)
rc = MDBX_SUCCESS;
return rc;
}
static void usage(char *prog) {
fprintf(stderr,
"usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env;
MDBX_txn *txn;
MDBX_dbi dbi;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envflags = 0, list = 0;
if (argc < 2)
usage(prog);
/* -a: dump main DB and all subDBs
* -s: dump only the named subDB
* -n: use NOSUBDIR flag on env_open
* -p: use printable characters
* -f: write to file instead of stdout
* -V: print version and exit
* (default) dump only the main DB
*/
while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) {
switch (i) {
case 'V':
printf("mdbx_dump version %d.%d.%d.%d\n"
" - source: %s %s, commit %s, tree %s\n"
" - anchor: %s\n"
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.release,
mdbx_version.revision, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
return EXIT_SUCCESS;
case 'l':
list = 1;
/*FALLTHROUGH*/;
__fallthrough;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'f':
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 'p':
mode |= PRINT;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
printf("mdbx_dump %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fflush(NULL);
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdbx_env_set_maxdbs(env, 2);
}
rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_dbi_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
if (alldbs) {
MDBX_cursor *cursor;
MDBX_val key;
int count = 0;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
char *str;
MDBX_dbi db2;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;
count++;
str = mdbx_malloc(key.iov_len + 1);
memcpy(str, key.iov_base, key.iov_len);
str[key.iov_len] = '\0';
rc = mdbx_dbi_open(txn, str, 0, &db2);
if (rc == MDBX_SUCCESS) {
if (list) {
printf("%s\n", str);
list++;
} else {
rc = dumpit(txn, db2, str);
if (rc)
break;
}
mdbx_dbi_close(env, db2);
}
mdbx_free(str);
if (rc)
continue;
}
mdbx_cursor_close(cursor);
if (!count) {
fprintf(stderr, "%s: %s does not contain multiple databases\n", prog,
envname);
rc = MDBX_NOTFOUND;
} else if (rc == MDBX_INCOMPATIBLE) {
/* LY: the record it not a named sub-db. */
rc = MDBX_SUCCESS;
}
} else {
rc = dumpit(txn, dbi, subname);
}
if (rc && rc != MDBX_NOTFOUND)
fprintf(stderr, "%s: %s: %s\n", prog, envname, mdbx_strerror(rc));
mdbx_dbi_close(env, dbi);
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

567
contrib/db/libmdbx/src/tools/mdbx_load.c Normal file
View file

@ -0,0 +1,567 @@
/* mdbx_load.c - memory-mapped database load tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../elements/internals.h"
#include <ctype.h>
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
#define PRINT 1
#define NOHDR 2
static int mode;
static char *subname = NULL;
static size_t lineno;
static int version;
static int dbi_flags;
static char *prog;
static int Eof;
static MDBX_envinfo envinfo;
static MDBX_val kbuf, dbuf;
static MDBX_val k0buf;
#define STRLENOF(s) (sizeof(s) - 1)
typedef struct flagbit {
int bit;
char *name;
int len;
} flagbit;
#define S(s) s, STRLENOF(s)
flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")},
{MDBX_DUPSORT, S("dupsort")},
{MDBX_INTEGERKEY, S("integerkey")},
{MDBX_DUPFIXED, S("dupfixed")},
{MDBX_INTEGERDUP, S("integerdup")},
{MDBX_REVERSEDUP, S("reversedup")},
{0, NULL, 0}};
static void readhdr(void) {
char *ptr;
dbi_flags = 0;
while (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) {
lineno++;
if (!strncmp(dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) ||
!strncmp(dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) {
/* LY: silently ignore information fields. */
continue;
} else if (!strncmp(dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) {
version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION="));
if (version > 3) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n",
prog, lineno, version);
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) {
break;
} else if (!strncmp(dbuf.iov_base, "format=", STRLENOF("format="))) {
if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print",
STRLENOF("print")))
mode |= PRINT;
else if (strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "bytevalue",
STRLENOF("bytevalue"))) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported FORMAT %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "database=", STRLENOF("database="))) {
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
if (subname)
mdbx_free(subname);
subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database="));
} else if (!strncmp(dbuf.iov_base, "type=", STRLENOF("type="))) {
if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree",
STRLENOF("btree"))) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("type="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
void *unused;
i = sscanf((char *)dbuf.iov_base + STRLENOF("mapaddr="), "%p", &unused);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapaddr %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
i = sscanf((char *)dbuf.iov_base + STRLENOF("mapsize="), "%" PRIu64,
&envinfo.mi_mapsize);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapsize %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("mapsize="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base,
"maxreaders=", STRLENOF("maxreaders="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
i = sscanf((char *)dbuf.iov_base + STRLENOF("maxreaders="), "%u",
&envinfo.mi_maxreaders);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid maxreaders %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders="));
exit(EXIT_FAILURE);
}
} else {
int i;
for (i = 0; dbflags[i].bit; i++) {
if (!strncmp(dbuf.iov_base, dbflags[i].name, dbflags[i].len) &&
((char *)dbuf.iov_base)[dbflags[i].len] == '=') {
if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1')
dbi_flags |= dbflags[i].bit;
break;
}
}
if (!dbflags[i].bit) {
ptr = memchr(dbuf.iov_base, '=', dbuf.iov_len);
if (!ptr) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog,
lineno);
exit(EXIT_FAILURE);
} else {
*ptr = '\0';
fprintf(stderr,
"%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
prog, lineno, (char *)dbuf.iov_base);
}
}
}
}
}
static void badend(void) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog,
lineno);
}
static int unhex(unsigned char *c2) {
int x, c;
x = *c2++ & 0x4f;
if (x & 0x40)
x -= 55;
c = x << 4;
x = *c2 & 0x4f;
if (x & 0x40)
x -= 55;
c |= x;
return c;
}
static int readline(MDBX_val *out, MDBX_val *buf) {
unsigned char *c1, *c2, *end;
size_t len, l2;
int c;
if (!(mode & NOHDR)) {
c = fgetc(stdin);
if (c == EOF) {
Eof = 1;
return EOF;
}
if (c != ' ') {
lineno++;
if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
badend:
Eof = 1;
badend();
return EOF;
}
if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END")))
return EOF;
goto badend;
}
}
if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
Eof = 1;
return EOF;
}
lineno++;
c1 = buf->iov_base;
len = strlen((char *)c1);
l2 = len;
/* Is buffer too short? */
while (c1[len - 1] != '\n') {
buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2);
if (!buf->iov_base) {
Eof = 1;
fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n",
prog, lineno);
return EOF;
}
c1 = buf->iov_base;
c1 += l2;
if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) {
Eof = 1;
badend();
return EOF;
}
buf->iov_len *= 2;
len = strlen((char *)c1);
l2 += len;
}
c1 = c2 = buf->iov_base;
len = l2;
c1[--len] = '\0';
end = c1 + len;
if (mode & PRINT) {
while (c2 < end) {
if (unlikely(*c2 == '\\')) {
if (c2[1] == '\\') {
*c1++ = '\\';
} else {
if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = (char)unhex(++c2);
}
c2 += 2;
} else {
/* copies are redundant when no escapes were used */
*c1++ = *c2++;
}
}
} else {
/* odd length not allowed */
if (len & 1) {
Eof = 1;
badend();
return EOF;
}
while (c2 < end) {
if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = (char)unhex(c2);
c2 += 2;
}
}
c2 = out->iov_base = buf->iov_base;
out->iov_len = c1 - c2;
return 0;
}
static void usage(void) {
fprintf(stderr,
"usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
static int anyway_greater(const MDBX_val *a, const MDBX_val *b) {
(void)a;
(void)b;
return 1;
}
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env = NULL;
MDBX_txn *txn = NULL;
MDBX_cursor *mc = NULL;
MDBX_dbi dbi;
char *envname = NULL;
int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0;
int append = 0;
MDBX_val prevk;
prog = argv[0];
if (argc < 2)
usage();
/* -a: append records in input order
* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
switch (i) {
case 'V':
printf("mdbx_load version %d.%d.%d.%d\n"
" - source: %s %s, commit %s, tree %s\n"
" - anchor: %s\n"
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.release,
mdbx_version.revision, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
return EXIT_SUCCESS;
case 'a':
append = 1;
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 's':
subname = mdbx_strdup(optarg);
break;
case 'N':
putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fflush(NULL);
dbuf.iov_len = 4096;
dbuf.iov_base = mdbx_malloc(dbuf.iov_len);
/* read first header for mapsize= */
if (!(mode & NOHDR))
readhdr();
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
mdbx_env_set_maxdbs(env, 2);
if (envinfo.mi_maxreaders)
mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders);
if (envinfo.mi_mapsize) {
if (envinfo.mi_mapsize > SIZE_MAX) {
fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc,
mdbx_strerror(MDBX_TOO_LARGE));
return EXIT_FAILURE;
}
mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize);
}
#ifdef MDBX_FIXEDMAP
if (info.mi_mapaddr)
envflags |= MDBX_FIXEDMAP;
#endif
rc = mdbx_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
kbuf.iov_len = mdbx_env_get_maxkeysize(env);
if (kbuf.iov_len >= SIZE_MAX / 4) {
fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n",
kbuf.iov_len);
goto env_close;
}
kbuf.iov_len = (kbuf.iov_len + 1) * 2;
kbuf.iov_base = malloc(kbuf.iov_len * 2);
k0buf.iov_len = kbuf.iov_len;
k0buf.iov_base = (char *)kbuf.iov_base + kbuf.iov_len;
prevk.iov_base = k0buf.iov_base;
while (!Eof) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi,
append ? anyway_greater : NULL,
append ? anyway_greater : NULL);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
int batch = 0;
prevk.iov_len = 0;
while (1) {
MDBX_val key;
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
MDBX_val data;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n",
prog, lineno);
goto txn_abort;
}
int appflag = 0;
if (append) {
appflag = MDBX_APPEND;
if (dbi_flags & MDBX_DUPSORT) {
if (prevk.iov_len == key.iov_len &&
memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0)
appflag = MDBX_APPEND | MDBX_APPENDDUP;
else
memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len);
}
}
rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag);
if (rc == MDBX_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdbx_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog,
lineno, mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdbx_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog, lineno,
mdbx_strerror(rc));
goto env_close;
}
mdbx_dbi_close(env, dbi);
/* try read next header */
if (!(mode & NOHDR))
readhdr();
}
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

436
contrib/db/libmdbx/src/tools/mdbx_stat.c Normal file
View file

@ -0,0 +1,436 @@
/* mdbx_stat.c - memory-mapped database status tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../elements/internals.h"
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
static void prstat(MDBX_stat *ms) {
printf(" Pagesize: %u\n", ms->ms_psize);
printf(" Tree depth: %u\n", ms->ms_depth);
printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages);
printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages);
printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages);
printf(" Entries: %" PRIu64 "\n", ms->ms_entries);
}
static void usage(char *prog) {
fprintf(stderr,
"usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid,
mdbx_tid_t thread, uint64_t txnid, uint64_t lag,
size_t bytes_used, size_t bytes_retained) {
(void)ctx;
if (num == 1)
printf("Reader Table Status\n"
" #\tslot\t%6s %*s %20s %10s %13s %13s\n",
"pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used",
"retained");
printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxSIZE, num, slot, (size_t)pid,
(int)sizeof(size_t) * 2, (size_t)thread);
if (txnid)
printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag,
bytes_used / 1048576.0, bytes_retained / 1048576.0);
else
printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0");
return user_break ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE;
}
int main(int argc, char *argv[]) {
int o, rc;
MDBX_env *env;
MDBX_txn *txn;
MDBX_dbi dbi;
MDBX_stat mst;
MDBX_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2)
usage(prog);
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((o = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch (o) {
case 'V':
printf("mdbx_stat version %d.%d.%d.%d\n"
" - source: %s %s, commit %s, tree %s\n"
" - anchor: %s\n"
" - build: %s for %s by %s\n"
" - flags: %s\n"
" - options: %s\n",
mdbx_version.major, mdbx_version.minor, mdbx_version.release,
mdbx_version.revision, mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_version.git.commit,
mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime,
mdbx_build.target, mdbx_build.compiler, mdbx_build.flags,
mdbx_build.options);
return EXIT_SUCCESS;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
envname = argv[optind];
printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n",
mdbx_version.git.describe, mdbx_version.git.datetime,
mdbx_version.git.tree, envname);
fflush(NULL);
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname)
mdbx_env_set_maxdbs(env, 4);
rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (envinfo || freinfo) {
(void)mdbx_env_info(env, &mei, sizeof(mei));
} else {
/* LY: zap warnings from gcc */
memset(&mei, 0, sizeof(mei));
}
if (envinfo) {
(void)mdbx_env_stat(env, &mst, sizeof(mst));
printf("Environment Info\n");
printf(" Pagesize: %u\n", mst.ms_psize);
if (mei.mi_geo.lower != mei.mi_geo.upper) {
printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
"/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
"/-%" PRIu64 ")\n",
mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize,
mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize,
mei.mi_geo.shrink / mst.ms_psize);
printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
} else {
printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
}
printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize);
printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1);
printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
mei.mi_latter_reader_txnid,
mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
printf(" Max readers: %u\n", mei.mi_maxreaders);
printf(" Number of readers used: %u\n", mei.mi_numreaders);
} else {
/* LY: zap warnings from gcc */
memset(&mst, 0, sizeof(mst));
}
if (rdrinfo) {
rc = mdbx_reader_list(env, reader_list_func, nullptr);
if (rc == MDBX_RESULT_TRUE)
printf("Reader Table is empty\n");
else if (rc == MDBX_SUCCESS && rdrinfo > 1) {
int dead;
rc = mdbx_reader_check(env, &dead);
if (rc == MDBX_RESULT_TRUE) {
printf(" %d stale readers cleared.\n", dead);
rc = mdbx_reader_list(env, reader_list_func, nullptr);
if (rc == MDBX_RESULT_TRUE)
printf(" Now Reader Table is empty\n");
} else
printf(" No stale readers.\n");
}
if (MDBX_IS_ERROR(rc)) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (freinfo) {
MDBX_cursor *cursor;
MDBX_val key, data;
pgno_t pages = 0, *iptr;
pgno_t reclaimable = 0;
printf("Freelist Status\n");
dbi = 0;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) ==
MDBX_SUCCESS) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
iptr = data.iov_base;
const pgno_t number = *iptr++;
pages += number;
if (envinfo && mei.mi_latter_reader_txnid > *(size_t *)key.iov_base)
reclaimable += number;
if (freinfo > 1) {
char *bad = "";
pgno_t prev =
MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1;
pgno_t span = 1;
for (unsigned i = 0; i < number; ++i) {
pgno_t pg = iptr[i];
if (MDBX_PNL_DISORDERED(prev, pg))
bad = " [bad sequence]";
prev = pg;
while (i + span < number &&
iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
: pgno_sub(pg, span)))
++span;
}
printf(" Transaction %" PRIaTXN ", %" PRIaPGNO
" pages, maxspan %" PRIaPGNO "%s\n",
*(txnid_t *)key.iov_base, number, span, bad);
if (freinfo > 2) {
for (unsigned i = 0; i < number; i += span) {
const pgno_t pg = iptr[i];
for (span = 1;
i + span < number &&
iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
: pgno_sub(pg, span));
++span)
;
if (span > 1)
printf(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pg, span);
else
printf(" %9" PRIaPGNO "\n", pg);
}
}
}
}
mdbx_cursor_close(cursor);
switch (rc) {
case MDBX_SUCCESS:
case MDBX_NOTFOUND:
break;
case MDBX_EINTR:
fprintf(stderr, "Interrupted by signal/user\n");
goto txn_abort;
default:
fprintf(stderr, "mdbx_cursor_get failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
if (envinfo) {
uint64_t value = mei.mi_mapsize / mst.ms_psize;
double percent = value / 100.0;
printf("Page Allocation Info\n");
printf(" Max pages: %" PRIu64 " 100%%\n", value);
value = mei.mi_last_pgno + 1;
printf(" Pages used: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1);
printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_last_pgno + 1 - pages;
printf(" Used now: %" PRIu64 " %.1f%%\n", value, value / percent);
value = pages;
printf(" Unallocated: %" PRIu64 " %.1f%%\n", value, value / percent);
value = pages - reclaimable;
printf(" Detained: %" PRIu64 " %.1f%%\n", value, value / percent);
value = reclaimable;
printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent);
value =
mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1) + reclaimable;
printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent);
} else
printf(" Free pages: %" PRIaPGNO "\n", pages);
}
rc = mdbx_dbi_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDBX_cursor *cursor;
MDBX_val key;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) {
char *str;
MDBX_dbi db2;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;
str = mdbx_malloc(key.iov_len + 1);
memcpy(str, key.iov_base, key.iov_len);
str[key.iov_len] = '\0';
rc = mdbx_dbi_open(txn, str, 0, &db2);
if (rc == MDBX_SUCCESS)
printf("Status of %s\n", str);
mdbx_free(str);
if (rc)
continue;
rc = mdbx_dbi_stat(txn, db2, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdbx_dbi_close(env, db2);
}
mdbx_cursor_close(cursor);
}
if (rc == MDBX_NOTFOUND)
rc = MDBX_SUCCESS;
mdbx_dbi_close(env, dbi);
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

95
contrib/db/libmdbx/src/tools/wingetopt.c Normal file
View file

@ -0,0 +1,95 @@
/*
* POSIX getopt for Windows
*
* AT&T Public License
*
* Code given out at the 1985 UNIFORUM conference in Dallas.
*/
/*----------------------------------------------------------------------------*/
/* Microsoft compiler generates a lot of warning for self includes... */
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
#endif
#endif /* _MSC_VER (warnings) */
#include "wingetopt.h"
#include <stdio.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(pop)
#endif
/*----------------------------------------------------------------------------*/
#ifndef NULL
#define NULL 0
#endif
#ifndef EOF
#define EOF (-1)
#endif
#define ERR(s, c) \
if (opterr) { \
fputs(argv[0], stderr); \
fputs(s, stderr); \
fputc(c, stderr); \
}
int opterr = 1;
int optind = 1;
int optopt;
char *optarg;
int getopt(int argc, char *const argv[], const char *opts) {
static int sp = 1;
int c;
const char *cp;
if (sp == 1) {
if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
return EOF;
else if (strcmp(argv[optind], "--") == 0) {
optind++;
return EOF;
}
}
optopt = c = argv[optind][sp];
if (c == ':' || (cp = strchr(opts, c)) == NULL) {
ERR(": illegal option -- ", c);
if (argv[optind][++sp] == '\0') {
optind++;
sp = 1;
}
return '?';
}
if (*++cp == ':') {
if (argv[optind][sp + 1] != '\0')
optarg = &argv[optind++][sp + 1];
else if (++optind >= argc) {
ERR(": option requires an argument -- ", c);
sp = 1;
return '?';
} else
optarg = argv[optind++];
sp = 1;
} else {
if (argv[optind][++sp] == '\0') {
sp = 1;
optind++;
}
optarg = NULL;
}
return c;
}

30
contrib/db/libmdbx/src/tools/wingetopt.h Normal file
View file

@ -0,0 +1,30 @@
/*
* POSIX getopt for Windows
*
* AT&T Public License
*
* Code given out at the 1985 UNIFORUM conference in Dallas.
*/
#ifndef _WINGETOPT_H_
#define _WINGETOPT_H_
/* Bit of madness for Windows console */
#define mdbx_strerror mdbx_strerror_ANSI2OEM
#define mdbx_strerror_r mdbx_strerror_r_ANSI2OEM
#ifdef __cplusplus
extern "C" {
#endif
extern int opterr;
extern int optind;
extern int optopt;
extern char *optarg;
int getopt(int argc, char *const argv[], const char *optstring);
#ifdef __cplusplus
}
#endif
#endif /* _GETOPT_H_ */

53
contrib/db/libmdbx/test/CMakeLists.txt Normal file
View file

@ -0,0 +1,53 @@
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
set(TEST_OSAL windows)
else()
set(TEST_OSAL unix)
endif()
add_executable(mdbx_test
base.h
cases.cc
chrono.cc
chrono.h
config.cc
config.h
copy.cc
dead.cc
hill.cc
jitter.cc
keygen.cc
keygen.h
log.cc
log.h
main.cc
osal.h
osal-${TEST_OSAL}.cc
test.cc
test.h
try.cc
utils.cc
utils.h
append.cc
ttl.cc
nested.cc
)
set_target_properties(mdbx_test PROPERTIES
INTERPROCEDURAL_OPTIMIZATION $<BOOL:${INTERPROCEDURAL_OPTIMIZATION}>
CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON)
if(CC_HAS_FASTMATH)
target_compile_options(mdbx_test PRIVATE "-ffast-math")
endif()
if(CC_HAS_VISIBILITY AND (LTO_ENABLED OR INTERPROCEDURAL_OPTIMIZATION))
set_target_properties(mdbx_test PROPERTIES LINK_FLAGS "-fvisibility=hidden")
endif()
target_link_libraries(mdbx_test mdbx ${LIB_MATH} ${CMAKE_THREAD_LIBS_INIT})
if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
target_link_libraries(mdbx_test winmm.lib)
endif()
if(UNIX AND NOT SUBPROJECT)
add_subdirectory(pcrf)
endif()

164
contrib/db/libmdbx/test/append.cc Normal file
View file

@ -0,0 +1,164 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_append::run() {
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
/* LY: тест наполнения таблиц в append-режиме,
* при котором записи добавляются строго в конец (в порядке сортировки) */
const unsigned flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_APPEND | MDBX_APPENDDUP
: MDBX_APPEND;
keyvalue_maker.make_ordered();
key = keygen::alloc(config.params.keylen_max);
data = keygen::alloc(config.params.datalen_max);
keygen::buffer last_key = keygen::alloc(config.params.keylen_max);
keygen::buffer last_data = keygen::alloc(config.params.datalen_max);
last_key->value.iov_base = last_key->bytes;
last_key->value.iov_len = 0;
last_data->value.iov_base = last_data->bytes;
last_data->value.iov_len = 0;
simple_checksum inserted_checksum;
uint64_t inserted_number = 0;
uint64_t serial_count = 0;
unsigned txn_nops = 0;
uint64_t commited_inserted_number = inserted_number;
simple_checksum commited_inserted_checksum = inserted_checksum;
while (should_continue()) {
const keygen::serial_t serial = serial_count;
if (!keyvalue_maker.increment(serial_count, 1)) {
// дошли до границы пространства ключей
break;
}
log_trace("append: append-a %" PRIu64, serial);
generate_pair(serial, key, data);
int cmp = inserted_number ? mdbx_cmp(txn_guard.get(), dbi, &key->value,
&last_key->value)
: 1;
if (cmp == 0 && (config.params.table_flags & MDBX_DUPSORT))
cmp = mdbx_dcmp(txn_guard.get(), dbi, &data->value, &last_data->value);
err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags);
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("append: bailout-insert due '%s'", mdbx_strerror(err));
txn_end(true);
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
break;
}
if (cmp > 0) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_put(appenda-a)", err);
memcpy(last_key->value.iov_base, key->value.iov_base,
last_key->value.iov_len = key->value.iov_len);
memcpy(last_data->value.iov_base, data->value.iov_base,
last_data->value.iov_len = data->value.iov_len);
++inserted_number;
inserted_checksum.push((uint32_t)inserted_number, key->value);
inserted_checksum.push(10639, data->value);
} else {
if (unlikely(err != MDBX_EKEYMISMATCH))
failure_perror("mdbx_put(appenda-a) != MDBX_EKEYMISMATCH", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-commit due '%s'", mdbx_strerror(err));
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
break;
}
commited_inserted_number = inserted_number;
commited_inserted_checksum = inserted_checksum;
txn_nops = 0;
}
report(1);
}
if (txn_guard) {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-commit due '%s'", mdbx_strerror(err));
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
}
}
//----------------------------------------------------------------------------
txn_begin(true);
cursor_open(dbi);
MDBX_val check_key, check_data;
err =
mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_FIRST);
if (likely(inserted_number)) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_cursor_get(MDBX_FIRST)", err);
}
simple_checksum read_checksum;
uint64_t read_count = 0;
while (err == MDBX_SUCCESS) {
++read_count;
read_checksum.push((uint32_t)read_count, check_key);
read_checksum.push(10639, check_data);
err =
mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_NEXT);
}
if (unlikely(err != MDBX_NOTFOUND))
failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", err);
if (unlikely(read_count != inserted_number))
failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")",
read_count, inserted_number);
if (unlikely(read_checksum.value != inserted_checksum.value))
failure("read_checksum(0x%016" PRIu64 ") "
"!= inserted_checksum(0x%016" PRIu64 ")",
read_checksum.value, inserted_checksum.value);
cursor_close();
txn_end(true);
//----------------------------------------------------------------------------
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-clean due '%s'", mdbx_strerror(err));
return true;
}
} else
db_table_close(dbi);
}
return true;
}

116
contrib/db/libmdbx/test/base.h Normal file
View file

@ -0,0 +1,116 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#ifndef NOMINMAX
#define NOMINMAX
#endif
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
#ifdef _MSC_VER
#define _CRT_SECURE_NO_WARNINGS
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
mode specified; termination on exception \
is not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
/* If you wish to build your application for a previous Windows platform,
* include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you
* wish to support before including SDKDDKVer.h.
*
* TODO: #define _WIN32_WINNT WIN32_MUSTDIE */
#include <SDKDDKVer.h>
#endif /* WINDOWS */
#ifdef __APPLE__
#define _DARWIN_C_SOURCE
#endif
#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
#include <io.h>
#else
#include <fcntl.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#endif
#ifdef _BSD_SOURCE
#include <endian.h>
#endif
#include <algorithm>
#include <cassert>
#include <cinttypes> // for PRId64, PRIu64
#include <cstdarg>
#include <cstddef>
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#define MDBX_INTERNAL_FUNC
#define MDBX_INTERNAL_VAR extern
#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */
#include "../mdbx.h"
#include "../src/elements/defs.h"
#include "../src/elements/osal.h"
#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__))
#define __thread __declspec(thread)
#endif /* __thread */
#ifdef _MSC_VER
#pragma warning(pop)
#pragma warning(disable : 4201) /* nonstandard extension used : \
nameless struct / union */
#pragma warning(disable : 4127) /* conditional expression is constant */
#if _MSC_VER < 1900
#pragma warning(disable : 4510) /* default constructor could \
not be generated */
#pragma warning(disable : 4512) /* assignment operator could \
not be generated */
#pragma warning(disable : 4610) /* user-defined constructor required */
#ifndef snprintf
#define snprintf(buffer, buffer_size, format, ...) \
_snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__)
#endif
#ifndef vsnprintf
#define vsnprintf(buffer, buffer_size, format, args) \
_vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args)
#endif
#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \
may be unsafe */
#endif
#endif /* _MSC_VER */

99
contrib/db/libmdbx/test/cases.cc Normal file
View file

@ -0,0 +1,99 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
const char *space_id_cstr, const actor_params &params) {
unsigned wait4id = 0;
if (params.waitfor_nops) {
for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) {
if (i->is_waitable(params.waitfor_nops)) {
if (i->signal_nops && i->signal_nops != params.waitfor_nops)
failure("Previous waitable actor (id=%u) already linked on %u-ops\n",
i->actor_id, i->signal_nops);
wait4id = i->actor_id;
i->signal_nops = params.waitfor_nops;
break;
}
}
if (!wait4id)
failure("No previous waitable actor for %u-ops\n", params.waitfor_nops);
}
unsigned space_id = 0;
if (!space_id_cstr || strcmp(space_id_cstr, "auto") == 0)
space_id = last_space_id + 1;
else {
char *end = nullptr;
errno = 0;
space_id = strtoul(space_id_cstr, &end, 0);
if (errno)
failure_perror("Expects an integer value for space-id\n", errno);
if (end && *end)
failure("The '%s' is unexpected for space-id\n", end);
}
if (space_id > ACTOR_ID_MAX)
failure("Invalid space-id %u\n", space_id);
last_space_id = space_id;
log_trace("configure_actor: space %u for %s", space_id,
testcase2str(testcase));
global::actors.emplace_back(
actor_config(testcase, params, space_id, wait4id));
global::databases.insert(params.pathname_db);
}
void testcase_setup(const char *casename, actor_params &params,
unsigned &last_space_id) {
if (strcmp(casename, "basic") == 0) {
log_notice(">>> testcase_setup(%s)", casename);
configure_actor(last_space_id, ac_nested, nullptr, params);
configure_actor(last_space_id, ac_hill, nullptr, params);
configure_actor(last_space_id, ac_ttl, nullptr, params);
configure_actor(last_space_id, ac_copy, nullptr, params);
configure_actor(last_space_id, ac_append, nullptr, params);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_try, nullptr, params);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_try, nullptr, params);
log_notice("<<< testcase_setup(%s): done", casename);
} else {
failure("unknown testcase `%s`", casename);
}
}
void keycase_setup(const char *casename, actor_params &params) {
if (strcmp(casename, "random") == 0 || strcmp(casename, "prng") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_random;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "dashes") == 0 ||
strcmp(casename, "aside") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_dashes;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "custom") == 0) {
log_notice("=== keycase_setup(%s): skip", casename);
params.keygen.keycase = kc_custom;
} else {
failure("unknown keycase `%s`", casename);
}
}
/* TODO */

136
contrib/db/libmdbx/test/chrono.cc Normal file
View file

@ -0,0 +1,136 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
namespace chrono {
#ifndef NSEC_PER_SEC
#define NSEC_PER_SEC 1000000000u
#endif /* NSEC_PER_SEC */
uint32_t ns2fractional(uint32_t ns) {
assert(ns < NSEC_PER_SEC);
/* LY: здесь и далее используется "длинное деление", которое
* для ясности кода оставлено как есть (без ручной оптимизации). Так как
* GCC, Clang и даже MSVC сами давно умеют конвертировать деление на
* константу в быструю reciprocal-форму. */
return ((uint64_t)ns << 32) / NSEC_PER_SEC;
}
uint32_t fractional2ns(uint32_t fractional) {
return (fractional * (uint64_t)NSEC_PER_SEC) >> 32;
}
#ifndef USEC_PER_SEC
#define USEC_PER_SEC 1000000u
#endif /* USEC_PER_SEC */
uint32_t us2fractional(uint32_t us) {
assert(us < USEC_PER_SEC);
return ((uint64_t)us << 32) / USEC_PER_SEC;
}
uint32_t fractional2us(uint32_t fractional) {
return (fractional * (uint64_t)USEC_PER_SEC) >> 32;
}
#ifndef MSEC_PER_SEC
#define MSEC_PER_SEC 1000u
#endif /* MSEC_PER_SEC */
uint32_t ms2fractional(uint32_t ms) {
assert(ms < MSEC_PER_SEC);
return ((uint64_t)ms << 32) / MSEC_PER_SEC;
}
uint32_t fractional2ms(uint32_t fractional) {
return (fractional * (uint64_t)MSEC_PER_SEC) >> 32;
}
time from_ns(uint64_t ns) {
time result;
result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) |
ns2fractional((uint32_t)(ns % NSEC_PER_SEC));
return result;
}
time from_us(uint64_t us) {
time result;
result.fixedpoint = ((us / USEC_PER_SEC) << 32) |
us2fractional((uint32_t)(us % USEC_PER_SEC));
return result;
}
time from_ms(uint64_t ms) {
time result;
result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) |
ms2fractional((uint32_t)(ms % MSEC_PER_SEC));
return result;
}
time now_realtime() {
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
static void(WINAPI * query_time)(LPFILETIME);
if (!query_time) {
query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress(
GetModuleHandle(TEXT("kernel32.dll")),
"GetSystemTimePreciseAsFileTime");
if (!query_time)
query_time = GetSystemTimeAsFileTime;
}
FILETIME filetime;
query_time(&filetime);
uint64_t ns100 =
(uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime;
return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u);
#else
struct timespec ts;
if (unlikely(clock_gettime(CLOCK_REALTIME, &ts)))
failure_perror("clock_gettime(CLOCK_REALTIME", errno);
return from_timespec(ts);
#endif
}
time now_motonic() {
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
static uint64_t reciprocal;
static LARGE_INTEGER Frequency;
if (reciprocal == 0) {
if (!QueryPerformanceFrequency(&Frequency))
failure_perror("QueryPerformanceFrequency()", GetLastError());
reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) /
Frequency.QuadPart);
assert(reciprocal);
}
LARGE_INTEGER Counter;
if (!QueryPerformanceCounter(&Counter))
failure_perror("QueryPerformanceCounter()", GetLastError());
time result;
result.fixedpoint = (Counter.QuadPart / Frequency.QuadPart) << 32;
uint64_t mod = Counter.QuadPart % Frequency.QuadPart;
result.fixedpoint += (mod * reciprocal) >> 16;
return result;
#else
struct timespec ts;
if (unlikely(clock_gettime(CLOCK_MONOTONIC, &ts)))
failure_perror("clock_gettime(CLOCK_MONOTONIC)", errno);
return from_timespec(ts);
#endif
}
} /* namespace chrono */

99
contrib/db/libmdbx/test/chrono.h Normal file
View file

@ -0,0 +1,99 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "utils.h"
namespace chrono {
#pragma pack(push, 1)
typedef union time {
uint64_t fixedpoint;
__anonymous_struct_extension__ struct {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
uint32_t fractional;
union {
uint32_t utc;
uint32_t integer;
};
#else
union {
uint32_t utc;
uint32_t integer;
};
uint32_t fractional;
#endif
};
void reset() { fixedpoint = 0; }
uint32_t seconds() const { return utc; }
} time;
#pragma pack(pop)
uint32_t ns2fractional(uint32_t);
uint32_t fractional2ns(uint32_t);
uint32_t us2fractional(uint32_t);
uint32_t fractional2us(uint32_t);
uint32_t ms2fractional(uint32_t);
uint32_t fractional2ms(uint32_t);
time from_ns(uint64_t us);
time from_us(uint64_t ns);
time from_ms(uint64_t ms);
inline time from_seconds(uint64_t seconds) {
assert(seconds < UINT32_MAX);
time result;
result.fixedpoint = seconds << 32;
return result;
}
inline time from_utc(time_t utc) {
assert(utc >= 0);
return from_seconds((uint64_t)utc);
}
inline time infinite() {
time result;
result.fixedpoint = UINT64_MAX;
return result;
}
#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \
defined(CLOCK_REALTIME)
inline time from_timespec(const struct timespec &ts) {
time result;
result.fixedpoint =
((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec);
return result;
}
#endif /* HAVE_TIMESPEC_TV_NSEC */
#if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL)
inline time from_timeval(const struct timeval &tv) {
time result;
result.fixedpoint =
((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec);
return result;
}
#endif /* HAVE_TIMEVAL_TV_USEC */
time now_realtime();
time now_motonic();
} /* namespace chrono */

602
contrib/db/libmdbx/test/config.cc Normal file
View file

@ -0,0 +1,602 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#if defined(_MSC_VER) && !defined(strcasecmp)
#define strcasecmp(str, len) _stricmp(str, len)
#endif /* _MSC_VER && strcasecmp() */
namespace config {
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value) {
assert(narg < argc);
const char *current = argv[narg];
const size_t optlen = strlen(option);
if (strncmp(current, "--", 2) || strncmp(current + 2, option, optlen))
return false;
if (!value) {
if (current[optlen + 2] == '=')
failure("Option '--%s' doen't accept any value\n", option);
return true;
}
*value = nullptr;
if (current[optlen + 2] == '=') {
*value = &current[optlen + 3];
return true;
}
if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2) != 0) {
*value = argv[narg + 1];
if (strcmp(*value, "default") == 0) {
if (!default_value)
failure("Option '--%s' doen't accept default value\n", option);
*value = default_value;
}
++narg;
return true;
}
if (default_value) {
*value = default_value;
return true;
}
failure("No value given for '--%s' option\n", option);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty) {
return parse_option(argc, argv, narg, option, value, allow_empty,
allow_empty ? "" : nullptr);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty,
const char *default_value) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr, default_value))
return false;
if (!allow_empty && strlen(value_cstr) == 0)
failure("Value for option '--%s' could't be empty\n", option);
value = value_cstr;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs) {
const char *list;
if (!parse_option(argc, argv, narg, option, &list))
return false;
unsigned clear = 0;
while (*list) {
if (*list == ',' || *list == ' ' || *list == '\t') {
++list;
continue;
}
const char *const comma = strchr(list, ',');
const bool strikethrough = *list == '-' || *list == '~';
if (strikethrough || *list == '+')
++list;
else
mask = clear;
const size_t len = (comma) ? comma - list : strlen(list);
const option_verb *scan = verbs;
while (true) {
if (!scan->verb)
failure("Unknown verb '%.*s', for option '==%s'\n", (int)len, list,
option);
if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) {
mask = strikethrough ? mask & ~scan->mask : mask | scan->mask;
clear = strikethrough ? clear & ~scan->mask : clear | scan->mask;
list += len;
break;
}
++scan;
}
}
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval, const uint64_t maxval,
const uint64_t default_value) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr))
return false;
if (default_value && strcmp(value_cstr, "default") == 0) {
value = default_value;
return true;
}
if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0) {
value = minval;
return true;
}
if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0) {
value = maxval;
return true;
}
char *suffix = nullptr;
errno = 0;
unsigned long long raw = strtoull(value_cstr, &suffix, 0);
if ((suffix && *suffix) || errno) {
suffix = nullptr;
errno = 0;
raw = strtoull(value_cstr, &suffix, 10);
}
if (errno)
failure("Option '--%s' expects a numeric value (%s)\n", option,
test_strerror(errno));
uint64_t multipler = 1;
if (suffix && *suffix) {
if (scale == no_scale)
failure("Option '--%s' doen't accepts suffixes, so '%s' is unexpected\n",
option, suffix);
if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024);
else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0)
multipler =
(scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024;
else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024;
else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024 * 1024;
else if (scale == duration &&
(strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0))
multipler = 1;
else if (scale == duration &&
(strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0))
multipler = 60;
else if (scale == duration &&
(strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0))
multipler = 3600;
else if (scale == duration &&
(strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0))
multipler = 3600 * 24;
else
failure(
"Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s"
"suffixes, but '%s' is unexpected\n",
option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "",
suffix);
}
if (raw >= UINT64_MAX / multipler)
failure("The value for option '--%s' is too huge\n", option);
value = raw * multipler;
if (maxval && value > maxval)
failure("The maximal value for option '--%s' is %" PRIu64 "\n", option,
maxval);
if (value < minval)
failure("The minimal value for option '--%s' is %" PRIu64 "\n", option,
minval);
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval, const unsigned maxval,
const unsigned default_value) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval,
default_value))
return false;
value = (unsigned)huge;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval, const uint8_t maxval,
const uint8_t default_value) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval,
default_value))
return false;
value = (uint8_t)huge;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value) {
uint64_t proxy = (uint64_t)value;
if (parse_option(argc, argv, narg, option, proxy, config::binary,
(uint64_t)minval, (uint64_t)maxval,
(uint64_t)default_value)) {
value = (int64_t)proxy;
return true;
}
return false;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value) {
uint64_t proxy = (uint64_t)value;
if (parse_option(argc, argv, narg, option, proxy, config::binary,
(uint64_t)minval, (uint64_t)maxval,
(uint64_t)default_value)) {
value = (int32_t)proxy;
return true;
}
return false;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value) {
const char *value_cstr = nullptr;
if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) {
const char *current = argv[narg];
if (strncmp(current, "--no-", 5) == 0 && strcmp(current + 5, option) == 0) {
value = false;
return true;
}
if (strncmp(current, "--dont-", 7) == 0 &&
strcmp(current + 7, option) == 0) {
value = false;
return true;
}
return false;
}
if (!value_cstr) {
value = true;
return true;
}
if (strcasecmp(value_cstr, "yes") == 0 || strcasecmp(value_cstr, "1") == 0) {
value = true;
return true;
}
if (strcasecmp(value_cstr, "no") == 0 || strcasecmp(value_cstr, "0") == 0) {
value = false;
return true;
}
failure(
"Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n",
option, value_cstr);
}
//-----------------------------------------------------------------------------
const struct option_verb mode_bits[] = {
{"rdonly", MDBX_RDONLY}, {"mapasync", MDBX_MAPASYNC},
{"utterly", MDBX_UTTERLY_NOSYNC}, {"nosubdir", MDBX_NOSUBDIR},
{"nosync", MDBX_NOSYNC}, {"nometasync", MDBX_NOMETASYNC},
{"writemap", MDBX_WRITEMAP}, {"notls", MDBX_NOTLS},
{"nordahead", MDBX_NORDAHEAD}, {"nomeminit", MDBX_NOMEMINIT},
{"coalesce", MDBX_COALESCE}, {"lifo", MDBX_LIFORECLAIM},
{"perturb", MDBX_PAGEPERTURB}, {nullptr, 0}};
const struct option_verb table_bits[] = {
{"key.reverse", MDBX_REVERSEKEY},
{"key.integer", MDBX_INTEGERKEY},
{"data.integer", MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.fixed", MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.reverse", MDBX_REVERSEDUP | MDBX_DUPSORT},
{"data.dups", MDBX_DUPSORT},
{nullptr, 0}};
static void dump_verbs(const char *caption, size_t bits,
const struct option_verb *verbs) {
log_verbose("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits);
const char *comma = "";
while (verbs->mask && bits) {
if ((bits & verbs->mask) == verbs->mask) {
logging::feed("%s%s", comma, verbs->verb);
bits -= verbs->mask;
comma = ", ";
}
++verbs;
}
logging::feed("%s\n", (*comma == '\0') ? "none" : "");
}
static void dump_duration(const char *caption, unsigned duration) {
log_verbose("%s: ", caption);
if (duration) {
if (duration > 24 * 3600)
logging::feed("%u_", duration / (24 * 3600));
if (duration > 3600)
logging::feed("%02u:", (duration % (24 * 3600)) / 3600);
logging::feed("%02u:%02u", (duration % 3600) / 60, duration % 60);
} else {
logging::feed("INFINITE");
}
logging::feed("\n");
}
void dump(const char *title) {
logging::local_suffix indent(title);
for (auto i = global::actors.begin(); i != global::actors.end(); ++i) {
log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id,
testcase2str(i->testcase), i->space_id);
indent.push();
if (i->params.loglevel) {
log_verbose("log: level %u, %s\n", i->params.loglevel,
i->params.pathname_log.empty()
? "console"
: i->params.pathname_log.c_str());
}
log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR
", %i %i, %i]\n",
i->params.pathname_db.c_str(), i->params.size_now,
i->params.size_lower, i->params.size_upper,
i->params.shrink_threshold, i->params.growth_step,
i->params.pagesize);
dump_verbs("mode", i->params.mode_flags, mode_bits);
dump_verbs("table", i->params.table_flags, table_bits);
if (i->params.test_nops)
log_verbose("iterations/records %u\n", i->params.test_nops);
else
dump_duration("duration", i->params.test_duration);
if (i->params.nrepeat)
log_verbose("repeat %u\n", i->params.nrepeat);
else
log_verbose("repeat ETERNALLY\n");
log_verbose("threads %u\n", i->params.nthreads);
log_verbose(
"keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64
", split %u/%u\n",
keygencase2str(i->params.keygen.keycase), i->params.keygen.width,
i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset,
i->params.keygen.split,
i->params.keygen.width - i->params.keygen.split);
log_verbose("keygen.seed: %u\n", i->params.keygen.seed);
log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min,
i->params.keylen_max);
log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min,
i->params.datalen_max);
log_verbose("batch: read %u, write %u\n", i->params.batch_read,
i->params.batch_write);
if (i->params.waitfor_nops)
log_verbose("wait: actor %u for %u ops\n", i->wait4id,
i->params.waitfor_nops);
else if (i->params.delaystart)
dump_duration("delay", i->params.delaystart);
else
log_verbose("no-delay\n");
if (i->params.inject_writefaultn)
log_verbose("inject-writefault on %u ops\n",
i->params.inject_writefaultn);
else
log_verbose("no-inject-writefault\n");
log_verbose("limits: readers %u, tables %u\n", i->params.max_readers,
i->params.max_tables);
log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No");
log_verbose("ignore MDBX_MAP_FULL error: %s\n",
i->params.ignore_dbfull ? "Yes" : "No");
log_verbose("verifying by speculum: %s\n",
i->params.speculum ? "Yes" : "No");
indent.pop();
}
dump_duration("timeout", global::config::timeout_duration_seconds);
log_verbose("cleanup: before %s, after %s\n",
global::config::cleanup_before ? "Yes" : "No",
global::config::cleanup_after ? "Yes" : "No");
log_verbose("failfast: %s\n", global::config::failfast ? "Yes" : "No");
log_verbose("progress indicator: %s\n",
global::config::progress_indicator ? "Yes" : "No");
log_verbose("console mode: %s\n",
global::config::console_mode ? "Yes" : "No");
}
} /* namespace config */
//-----------------------------------------------------------------------------
using namespace config;
actor_config::actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id)
: params(params) {
this->space_id = space_id;
this->actor_id = 1 + (unsigned)global::actors.size();
this->testcase = testcase;
this->wait4id = wait4id;
signal_nops = 0;
}
const std::string actor_config::serialize(const char *prefix) const {
simple_checksum checksum;
std::string result;
if (prefix)
result.append(prefix);
checksum.push(params.pathname_db);
result.append(params.pathname_db);
result.push_back('|');
checksum.push(params.pathname_log);
result.append(params.pathname_log);
result.push_back('|');
static_assert(std::is_pod<actor_params_pod>::value,
"actor_params_pod should by POD");
result.append(data2hex(static_cast<const actor_params_pod *>(&params),
sizeof(actor_params_pod), checksum));
result.push_back('|');
static_assert(std::is_pod<actor_config_pod>::value,
"actor_config_pod should by POD");
result.append(data2hex(static_cast<const actor_config_pod *>(this),
sizeof(actor_config_pod), checksum));
result.push_back('|');
result.push_back(global::config::progress_indicator ? 'Y' : 'N');
checksum.push(global::config::progress_indicator);
result.push_back(global::config::console_mode ? 'Y' : 'N');
checksum.push(global::config::console_mode);
result.push_back('|');
result.append(osal_serialize(checksum));
result.push_back('|');
result.append(std::to_string(checksum.value));
return result;
}
bool actor_config::deserialize(const char *str, actor_config &config) {
simple_checksum checksum;
TRACE(">> actor_config::deserialize: %s\n", str);
const char *slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-1\n");
return false;
}
config.params.pathname_db.assign(str, slash - str);
checksum.push(config.params.pathname_db);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-2\n");
return false;
}
config.params.pathname_log.assign(str, slash - str);
checksum.push(config.params.pathname_log);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-3\n");
return false;
}
static_assert(std::is_pod<actor_params_pod>::value,
"actor_params_pod should by POD");
if (!hex2data(str, slash, static_cast<actor_params_pod *>(&config.params),
sizeof(actor_params_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n",
(int)(slash - str), str);
return false;
}
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-4\n");
return false;
}
static_assert(std::is_pod<actor_config_pod>::value,
"actor_config_pod should by POD");
if (!hex2data(str, slash, static_cast<actor_config_pod *>(&config),
sizeof(actor_config_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n",
(int)(slash - str), str);
return false;
}
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-5\n");
return false;
}
if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N')) {
global::config::progress_indicator = str[0] == 'Y';
checksum.push(global::config::progress_indicator);
global::config::console_mode = str[1] == 'Y';
checksum.push(global::config::console_mode);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-6\n");
return false;
}
}
if (!config.osal_deserialize(str, slash, checksum)) {
TRACE("<< actor_config::deserialize: osal\n");
return false;
}
str = slash + 1;
uint64_t verify = std::stoull(std::string(str));
if (checksum.value != verify) {
TRACE("<< actor_config::deserialize: checksum mismatch\n");
return false;
}
TRACE("<< actor_config::deserialize: OK\n");
return true;
}
unsigned actor_params::mdbx_keylen_min() const {
return (table_flags & MDBX_INTEGERKEY) ? 4 : 0;
}
unsigned actor_params::mdbx_keylen_max() const {
return (table_flags & MDBX_INTEGERKEY)
? 8
: std::min((unsigned)mdbx_limits_keysize_max(pagesize),
(unsigned)UINT16_MAX);
}
unsigned actor_params::mdbx_datalen_min() const {
return (table_flags & MDBX_INTEGERDUP) ? 4 : 0;
}
unsigned actor_params::mdbx_datalen_max() const {
return (table_flags & MDBX_INTEGERDUP)
? 8
: std::min((table_flags & MDBX_DUPSORT)
? (unsigned)mdbx_limits_keysize_max(pagesize)
: (unsigned)MDBX_MAXDATASIZE,
(unsigned)UINT16_MAX);
}

326
contrib/db/libmdbx/test/config.h Normal file
View file

@ -0,0 +1,326 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "log.h"
#include "utils.h"
#define ACTOR_ID_MAX INT16_MAX
enum actor_testcase {
ac_none,
ac_hill,
ac_deadread,
ac_deadwrite,
ac_jitter,
ac_try,
ac_copy,
ac_append,
ac_ttl,
ac_nested
};
enum actor_status {
as_unknown,
as_debuging,
as_running,
as_successful,
as_killed,
as_failed,
as_coredump,
};
const char *testcase2str(const actor_testcase);
const char *status2str(actor_status status);
enum keygen_case {
kc_random, /* [ 6.. 2.. 7.. 4.. 0.. 1.. 5.. 3.. ] */
kc_dashes, /* [ 0123.. 4567.. ] */
kc_custom,
/* TODO: more cases */
};
const char *keygencase2str(const keygen_case);
//-----------------------------------------------------------------------------
namespace config {
enum scale_mode { no_scale, decimal, binary, duration };
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value = nullptr);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty = false);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty,
const char *default_value);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value);
struct option_verb {
const char *const verb;
unsigned mask;
};
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval = 0, const uint64_t maxval = INT64_MAX,
const uint64_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval = 0, const unsigned maxval = INT32_MAX,
const unsigned default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval = 0,
const uint8_t maxval = 255, const uint8_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value = -1);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value = -1);
inline bool parse_option_intptr(int argc, char *const argv[], int &narg,
const char *option, intptr_t &value,
const intptr_t minval, const intptr_t maxval,
const intptr_t default_value = -1) {
static_assert(sizeof(intptr_t) == 4 || sizeof(intptr_t) == 8, "WTF?");
if (sizeof(intptr_t) == 8)
return parse_option(argc, argv, narg, option,
*reinterpret_cast<int64_t *>(&value), int64_t(minval),
int64_t(maxval), int64_t(default_value));
else
return parse_option(argc, argv, narg, option,
*reinterpret_cast<int32_t *>(&value), int32_t(minval),
int32_t(maxval), int32_t(default_value));
}
//-----------------------------------------------------------------------------
#pragma pack(push, 1)
struct keygen_params_pod {
/* Параметры генератора пар key-value.
*
* Ключи и значения генерируются по задаваемым параметрам на основе "плоской"
* исходной координаты. При этом, в общем случае, в процессе тестов исходная
* координата последовательно итерируется в заданном диапазоне, а необходимые
* паттерны/последовательности/узоры получаются за счет преобразования
* исходной координаты, согласно описанным ниже параметрам.
*
* Стоит отметить, что порядок описания параметров для удобства совпадает с
* порядком их использования, т.е. с порядком соответствующих преобразований.
*
* Второе важное замечание касается ограничений одновременной координированной
* генерации паттеров как для ключей, так и для значений. Суть в том, что
* такая возможность не нужна по следующим причинам:
* - libmdbx поддерживает два существенно различающихся вида таблиц,
* "уникальные" (без дубликатов и без multi-value), и так называемые
* "с дубликатами" (c multi-value).
* - Для таблиц "без дубликатов" только размер связанных к ключами значений
* (данных) оказывает влияния на работу движка, непосредственно содержимое
* данных не анализируется движком и не оказывает влияния на его работу.
* - Для таблиц "с дубликатами", при наличии более одного значения для
* некоторого ключа, формируется дочернее btree-поддерево. Это дерево
* формируется в отдельном "кусте" страниц и обслуживается независимо
* от окружения родительского ключа.
* - Таким образом, паттерн генерации значений имеет смысл только для
* таблиц "с дубликатами" и только в контексте одного значения ключа.
* Иначе говоря, нет смысла в со-координации генерации паттернов для
* ключей и значений. Более того, генерацию значений всегда необходимо
* рассматривать в контексте связки с одним значением ключа.
* - Тем не менее, во всех случаях достаточно важным является равномерная
* всех возможных сочетаний длин ключей и данных.
*
* width:
* Большинство тестов предполагают создание или итерирование некоторого
* количества записей. При этом требуется итерирование или генерация
* значений и ключей из некоторого ограниченного пространства вариантов.
*
* Параметр width задает такую ширину пространства вариантов в битах.
* Таким образом мощность пространства вариантов (пока) всегда равна
* степени двойки. Это ограничение можно снять, но ценой увеличения
* вычислительной сложности, включая потерю простоты и прозрачности.
*
* С другой стороны, не-битовый width может быть полезен:
* - Позволит генерировать ключи/значения в точно задаваемом диапазоне.
* Например, перебрать в псевдо-случайном порядке 10001 значение.
* - Позволит поровну разделять заданное пространство (диапазон)
* ключей/значений между количеством потоков некратным степени двойки.
*
* mesh и seed:
* Позволяют получить псевдо-случайные последовательности ключей/значений.
* Параметр mesh задает сколько младших бит исходной плоской координаты
* будет "перемешано" (инъективно отображено), а параметр seed позволяет
* выбрать конкретный вариант "перемешивания".
*
* Перемешивание выполняется при ненулевом значении mesh. Перемешивание
* реализуется посредством применения двух инъективных функций для
* заданного количества бит:
* - применяется первая инъективная функция;
* - к результату добавляется salt полученный из seed;
* - применяется вторая инъективная функция;
*
* Следует отметить, что mesh умышленно позволяет перемешать только младшую
* часть, что при ненулевом значении split (см далее) не позволяет получать
* псевдо-случайные значений ключей без псевдо-случайности в значениях.
*
* Такое ограничение соответствуют внутренней алгоритмике libmdbx. Проще
* говоря, мы можем проверить движок псевдо-случайной последовательностью
* ключей на таблицах без дубликатов (без multi-value), а затем проверить
* корректность работу псевдо-случайной последовательностью значений на
* таблицах с дубликатами (с multi-value), опционально добавляя
* псевдо-случайности к последовательности ключей. Однако, нет смысла
* генерировать псевдо-случайные ключи, одновременно с формированием
* какого-либо паттерна в значениях, так как содержимое в данных либо
* не будет иметь значения (для таблиц без дубликатов), либо будет
* обрабатываться в отдельных btree-поддеревьях.
*
* rotate и offset:
* Для проверки слияния и разделения страниц внутри движка требуются
* генерация ключей/значений в виде не-смежных последовательностей, как-бы
* в виде "пунктира", который постепенно заполняет весь заданных диапазон.
*
* Параметры позволяют генерировать такой "пунктир". Соответственно rotate
* задает циклический сдвиг вправо, а offset задает смещение, точнее говоря
* сложение по модулю внутри диапазона заданного посредством width.
*
* Например, при rotate равном 1 (циклический сдвиг вправо на 1 бит),
* четные и нечетные исходные значения сложатся в две линейные
* последовательности, которые постепенно закроют старшую и младшую
* половины диапазона.
*
* split:
* Для таблиц без дубликатов (без multi-value ключей) фактически требуется
* генерация только ключей, а данные могут быть постоянным. Но для таблиц с
* дубликатами (с multi-value ключами) также требуется генерация значений.
*
* Ненулевое значение параметра split фактически включает генерацию значений,
* при этом значение split определяет сколько бит исходного абстрактного
* номера будет отрезано для генерации значения.
*/
uint8_t width;
uint8_t mesh;
uint8_t rotate;
uint8_t split;
uint32_t seed;
uint64_t offset;
keygen_case keycase;
};
struct actor_params_pod {
unsigned mode_flags;
unsigned table_flags;
intptr_t size_lower;
intptr_t size_now;
intptr_t size_upper;
int shrink_threshold;
int growth_step;
int pagesize;
unsigned test_duration;
unsigned test_nops;
unsigned nrepeat;
unsigned nthreads;
unsigned keylen_min, keylen_max;
unsigned datalen_min, datalen_max;
unsigned batch_read;
unsigned batch_write;
unsigned delaystart;
unsigned waitfor_nops;
unsigned inject_writefaultn;
unsigned max_readers;
unsigned max_tables;
keygen_params_pod keygen;
uint8_t loglevel;
bool drop_table;
bool ignore_dbfull;
bool speculum;
};
struct actor_config_pod {
unsigned actor_id, space_id;
actor_testcase testcase;
unsigned wait4id;
unsigned signal_nops;
};
#pragma pack(pop)
extern const struct option_verb mode_bits[];
extern const struct option_verb table_bits[];
void dump(const char *title = "config-dump: ");
} /* namespace config */
struct actor_params : public config::actor_params_pod {
std::string pathname_log;
std::string pathname_db;
void set_defaults(const std::string &tmpdir);
unsigned mdbx_keylen_min() const;
unsigned mdbx_keylen_max() const;
unsigned mdbx_datalen_min() const;
unsigned mdbx_datalen_max() const;
};
struct actor_config : public config::actor_config_pod {
actor_params params;
bool wanna_event4signalling() const { return true /* TODO ? */; }
actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id);
actor_config(const char *str) {
if (!deserialize(str, *this))
failure("Invalid internal parameter '%s'\n", str);
}
const std::string osal_serialize(simple_checksum &) const;
bool osal_deserialize(const char *str, const char *end, simple_checksum &);
const std::string serialize(const char *prefix) const;
static bool deserialize(const char *str, actor_config &config);
bool is_waitable(size_t nops) const {
switch (testcase) {
case ac_hill:
if (!params.test_nops || params.test_nops >= nops)
return true;
__fallthrough;
default:
return false;
}
}
};

26
contrib/db/libmdbx/test/copy.cc Normal file
View file

@ -0,0 +1,26 @@
#include "test.h"
void testcase_copy::copy_db(const bool with_compaction) {
int err = osal_removefile(copy_pathname);
if (err != MDBX_SUCCESS && err != MDBX_ENOFILE)
failure_perror("mdbx_removefile()", err);
err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(),
with_compaction ? MDBX_CP_COMPACT : 0);
if (unlikely(err != MDBX_SUCCESS))
failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)"
: "mdbx_env_copy(MDBX_CP_ASIS)",
err);
}
bool testcase_copy::run() {
jitter_delay();
db_open();
assert(!txn_guard);
const bool order = flipcoin();
jitter_delay();
copy_db(order);
jitter_delay();
copy_db(!order);
return true;
}

24
contrib/db/libmdbx/test/darwin/LICENSE Normal file
View file

@ -0,0 +1,24 @@
Copyright (c) 2015, Aleksey Demakov
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

8
contrib/db/libmdbx/test/darwin/README.md Normal file
View file

@ -0,0 +1,8 @@
# DarwinPthreadBarrier
A pthread_barrier_t implementation for Mac OS/X
There is no pthread_barrier_t in Mac OS/X pthreads. This project fixes
this omission by providing a simple-minded barrier implementation based
on a pair of pthread_mutex_t and pthread_cond_t.

110
contrib/db/libmdbx/test/darwin/pthread_barrier.c Normal file
View file

@ -0,0 +1,110 @@
/*
* Copyright (c) 2015, Aleksey Demakov
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "pthread_barrier.h"
#include <errno.h>
#ifdef __APPLE__
int pthread_barrierattr_init(pthread_barrierattr_t *attr) {
memset(attr, 0, sizeof(pthread_barrierattr_t));
int m = pthread_mutexattr_init(&attr->mattr);
int c = pthread_condattr_init(&attr->cattr);
return m ? m : c;
}
int pthread_barrierattr_destroy(pthread_barrierattr_t *attr) {
int c = pthread_condattr_destroy(&attr->cattr);
int m = pthread_mutexattr_destroy(&attr->mattr);
return m ? m : c;
}
int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr,
int *__restrict pshared) {
return pthread_condattr_getpshared(&attr->cattr, pshared);
}
int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) {
int m = pthread_mutexattr_setpshared(&attr->mattr, pshared);
int c = pthread_condattr_setpshared(&attr->cattr, pshared);
return m ? m : c;
}
int pthread_barrier_init(pthread_barrier_t *__restrict barrier,
const pthread_barrierattr_t *__restrict attr,
unsigned count) {
if (count == 0)
return errno = EINVAL;
int rc = pthread_mutex_init(&barrier->mutex, attr ? &attr->mattr : 0);
if (rc)
return rc;
rc = pthread_cond_init(&barrier->cond, attr ? &attr->cattr : 0);
if (rc) {
int errno_save = errno;
pthread_mutex_destroy(&barrier->mutex);
errno = errno_save;
return rc;
}
barrier->limit = count;
barrier->count = 0;
barrier->phase = 0;
return 0;
}
int pthread_barrier_destroy(pthread_barrier_t *barrier) {
pthread_mutex_destroy(&barrier->mutex);
pthread_cond_destroy(&barrier->cond);
return 0;
}
int pthread_barrier_wait(pthread_barrier_t *barrier) {
int rc = pthread_mutex_lock(&barrier->mutex);
if (rc)
return rc;
barrier->count++;
if (barrier->count >= barrier->limit) {
barrier->phase++;
barrier->count = 0;
pthread_cond_broadcast(&barrier->cond);
pthread_mutex_unlock(&barrier->mutex);
return PTHREAD_BARRIER_SERIAL_THREAD;
} else {
unsigned phase = barrier->phase;
do
pthread_cond_wait(&barrier->cond, &barrier->mutex);
while (phase == barrier->phase);
pthread_mutex_unlock(&barrier->mutex);
return 0;
}
}
#endif /* __APPLE__ */

83
contrib/db/libmdbx/test/darwin/pthread_barrier.h Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright (c) 2015, Aleksey Demakov
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef PTHREAD_BARRIER_H
#define PTHREAD_BARRIER_H
#include <pthread.h>
#ifdef __APPLE__
#ifdef __cplusplus
extern "C" {
#endif
#if !defined(PTHREAD_BARRIER_SERIAL_THREAD)
#define PTHREAD_BARRIER_SERIAL_THREAD (1)
#endif
#if !defined(PTHREAD_PROCESS_PRIVATE)
#define PTHREAD_PROCESS_PRIVATE (42)
#endif
#if !defined(PTHREAD_PROCESS_SHARED)
#define PTHREAD_PROCESS_SHARED (43)
#endif
typedef struct {
pthread_mutexattr_t mattr;
pthread_condattr_t cattr;
} pthread_barrierattr_t;
typedef struct {
pthread_mutex_t mutex;
pthread_cond_t cond;
unsigned int limit;
unsigned int count;
unsigned int phase;
} pthread_barrier_t;
int pthread_barrierattr_init(pthread_barrierattr_t *attr);
int pthread_barrierattr_destroy(pthread_barrierattr_t *attr);
int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr,
int *__restrict pshared);
int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared);
int pthread_barrier_init(pthread_barrier_t *__restrict barrier,
const pthread_barrierattr_t *__restrict attr,
unsigned int count);
int pthread_barrier_destroy(pthread_barrier_t *barrier);
int pthread_barrier_wait(pthread_barrier_t *barrier);
#ifdef __cplusplus
}
#endif
#endif /* __APPLE__ */
#endif /* PTHREAD_BARRIER_H */

35
contrib/db/libmdbx/test/dead.cc Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_deadread::run() {
db_open();
txn_begin(true);
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
return true;
}
//-----------------------------------------------------------------------------
bool testcase_deadwrite::run() {
db_open();
txn_begin(false);
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
return true;
}

409
contrib/db/libmdbx/test/hill.cc Normal file
View file

@ -0,0 +1,409 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_hill::run() {
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err));
return false;
}
speculum.clear();
speculum_commited.clear();
/* LY: тест "холмиком":
* - сначала наполняем таблицу циклическими CRUD-манипуляциями,
* которые в каждом цикле делают несколько операций, включая удаление,
* но в результате добавляют записи.
* - затем очищаем таблицу также CRUD-манипуляциями, но уже с другой
* пропорцией удалений.
*
* При этом очень многое зависит от порядка перебора ключей:
* - (псевдо)случайное распределение требуется лишь для полноты картины,
* но в целом не покрывает важных кейсов.
* - кроме (псевдо)случайного перебора требуется последовательное
* итерирование ключей интервалами различной ширины, с тем чтобы
* проверить различные варианты как разделения, так и слияния страниц
* внутри движка.
* - при не-уникальных ключах (MDBX_DUPSORT с подвариантами), для каждого
* повтора внутри движка формируется вложенное btree-дерево,
* соответственно требуется соблюдение аналогичных принципов
* итерирования для значений.
*/
/* TODO: работа в несколько потоков */
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
keygen::buffer a_key = keygen::alloc(config.params.keylen_max);
keygen::buffer a_data_0 = keygen::alloc(config.params.datalen_max);
keygen::buffer a_data_1 = keygen::alloc(config.params.datalen_max);
keygen::buffer b_key = keygen::alloc(config.params.keylen_max);
keygen::buffer b_data = keygen::alloc(config.params.datalen_max);
const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_NODUPDATA
: MDBX_NODUPDATA | MDBX_NOOVERWRITE;
const unsigned update_flags =
(config.params.table_flags & MDBX_DUPSORT)
? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE
: MDBX_NODUPDATA;
uint64_t serial_count = 0;
uint64_t commited_serial = serial_count;
unsigned txn_nops = 0;
bool rc = false;
while (should_continue()) {
const keygen::serial_t a_serial = serial_count;
if (unlikely(!keyvalue_maker.increment(serial_count, 1))) {
log_notice("uphill: unexpected key-space overflow");
break;
}
const keygen::serial_t b_serial = serial_count;
assert(b_serial > a_serial);
// создаем первую запись из пары
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_1, age_shift);
err = insert(a_key, a_data_1, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-a.1)", err);
}
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-a, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-a, after commit");
goto bailout;
}
}
// создаем вторую запись из пары
log_trace("uphill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
err = insert(b_key, b_data, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-b, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("uphill: bailout after insert-b, after commit");
goto bailout;
}
}
// обновляем данные в первой записи
log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value);
err = replace(a_key, a_data_0, a_data_1, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_replace(update-a: 1->0)", err);
}
if (!speculum_verify()) {
log_notice("uphill: bailout after update-a, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("uphill: bailout after update-a, after commit");
goto bailout;
}
}
// удаляем вторую запись
log_trace("uphill: delete-b %" PRIu64, b_serial);
checkdata("uphill: delete-b", dbi, b_key->value, b_data->value);
err = remove(b_key, b_data);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (!speculum_verify()) {
log_notice("uphill: bailout after delete-b, before commit");
goto bailout;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
commited_serial = a_serial;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("uphill: bailout after delete-b, after commit");
goto bailout;
}
}
report(1);
if (!keyvalue_maker.increment(serial_count, 1)) {
// дошли до границы пространства ключей
serial_count = a_serial;
goto overflow;
}
}
while (serial_count > 1) {
if (unlikely(!keyvalue_maker.increment(serial_count, -2)))
failure("downhill: unexpected key-space underflow");
overflow:
const keygen::serial_t a_serial = serial_count;
const keygen::serial_t b_serial = a_serial + 1;
assert(b_serial > a_serial);
// обновляем первую запись из пары
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
generate_pair(a_serial, a_key, a_data_1, age_shift);
checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value);
err = replace(a_key, a_data_1, a_data_0, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at update-a due '%s'",
mdbx_strerror(err));
txn_end(true);
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(update-a: 0->1)", err);
}
if (!speculum_verify()) {
log_notice("downhill: bailout after update-a, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("downhill: bailout after update-a, after commit");
break;
}
}
// создаем вторую запись из пары
log_trace("downhill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
err = insert(b_key, b_data, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at insert-a due '%s'",
mdbx_strerror(err));
txn_end(true);
speculum = speculum_commited;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (!speculum_verify()) {
log_notice("downhill: bailout after insert-b, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("downhill: bailout after insert-b, after commit");
break;
}
}
// удаляем первую запись
log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value);
err = remove(a_key, a_data_1);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-a due '%s'",
mdbx_strerror(err));
txn_end(true);
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(a)", err);
}
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-a, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-a, after commit");
break;
}
}
// удаляем вторую запись
log_trace("downhill: delete-b %" PRIu64, b_serial);
checkdata("downhill: delete-b", dbi, b_key->value, b_data->value);
err = remove(b_key, b_data);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-b due '%s'",
mdbx_strerror(err));
txn_end(true);
speculum = speculum_commited;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-b, before commit");
break;
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
speculum = speculum_commited;
break;
}
speculum_commited = speculum;
txn_nops = 0;
if (!speculum_verify()) {
log_notice("downhill: bailout after delete-b, after commit");
goto bailout;
}
}
report(1);
}
rc = speculum_verify();
bailout:
if (txn_guard) {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS))
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
}
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err));
return rc;
}
} else
db_table_close(dbi);
}
return rc;
}

59
contrib/db/libmdbx/test/jitter.cc Normal file
View file

@ -0,0 +1,59 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_jitter::run() {
while (should_continue()) {
jitter_delay();
db_open();
if (flipcoin()) {
jitter_delay();
txn_begin(true);
fetch_canary();
jitter_delay();
txn_end(flipcoin());
}
jitter_delay();
txn_begin(mode_readonly());
jitter_delay();
if (!mode_readonly()) {
fetch_canary();
update_canary(1);
/* TODO:
* - db_setsize()
* ...
*/
}
txn_end(flipcoin());
if (flipcoin()) {
jitter_delay();
txn_begin(true);
jitter_delay();
txn_end(flipcoin());
}
jitter_delay();
db_close();
/* just 'align' nops with other tests with batching */
const auto batching =
std::max(config.params.batch_read, config.params.batch_write);
report(std::max(1u, batching / 2));
}
return true;
}

275
contrib/db/libmdbx/test/keygen.cc Normal file
View file

@ -0,0 +1,275 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
namespace keygen {
static inline __pure_function serial_t mask(unsigned bits) {
assert(bits > 0 && bits <= serial_maxwith);
return serial_allones >> (serial_maxwith - bits);
}
/* LY: https://en.wikipedia.org/wiki/Injective_function */
serial_t injective(const serial_t serial,
const unsigned bits /* at least serial_minwith (8) */,
const serial_t salt) {
assert(bits > serial_minwith && bits <= serial_maxwith);
/* LY: All these "magic" prime numbers were found
* and verified with a bit of brute force. */
static const uint64_t m[64 - serial_minwith + 1] = {
/* 8 - 24 */
113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937,
309277, 834497, 1499933, 4373441, 10184137,
/* 25 - 64 */
10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573,
6924051577, 19204053433, 45840188887, 53625693977, 73447827913,
141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197,
5871903036137, 11238507001417, 45264020802263, 105008404482889,
81921776907059, 199987980256399, 307207457507641, 946769023178273,
2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153,
23171543400565993, 53353226456762893, 155627817337932409,
227827205384840249, 816509268558278821, 576933057762605689,
2623957345935638441, 5048241705479929949, 4634245581946485653,
4613509448041658233, 4952535426879925961};
static const uint8_t s[64 - serial_minwith + 1] = {
/* 8 - 24 */
2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11,
/* 25 - 64 */
11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20,
10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20,
19};
const auto mult = m[bits - 8];
const auto shift = s[bits - 8];
serial_t result = serial * mult;
if (salt) {
const unsigned left = bits / 2;
const unsigned right = bits - left;
result = (result << left) | ((result & mask(bits)) >> right);
result = (result ^ salt) * mult;
}
result ^= result << shift;
result &= mask(bits);
log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64
" => %" PRIu64 "/%u",
serial, bits, mult, shift, salt, result, bits);
return result;
}
void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
serial_t value_age) {
assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith);
assert(mapping.split <= mapping.width);
assert(mapping.mesh <= mapping.width);
assert(mapping.rotate <= mapping.width);
assert(mapping.offset <= mask(mapping.width));
assert(!(key_essentials.flags &
~(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT)));
assert(!(value_essentials.flags & ~(MDBX_INTEGERDUP | MDBX_REVERSEDUP)));
log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial,
value_age);
if (mapping.mesh >= serial_minwith) {
serial =
(serial & ~mask(mapping.mesh)) | injective(serial, mapping.mesh, salt);
log_trace("keygen-pair: mesh@%u => %" PRIu64, mapping.mesh, serial);
}
if (mapping.rotate) {
const unsigned right = mapping.rotate;
const unsigned left = mapping.width - right;
serial = (serial << left) | ((serial & mask(mapping.width)) >> right);
log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64,
mapping.rotate, serial, serial);
}
if (mapping.offset) {
serial = (serial + mapping.offset) & mask(mapping.width);
log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset,
serial);
}
if (base) {
serial += base;
log_trace("keygen-pair: base@%" PRIu64 " => %" PRIu64, base, serial);
}
serial_t key_serial = serial;
serial_t value_serial = value_age << mapping.split;
if (mapping.split) {
if (key_essentials.flags & MDBX_DUPSORT) {
key_serial >>= mapping.split;
value_serial += serial & mask(mapping.split);
} else {
/* Без MDBX_DUPSORT требуется уникальность ключей, а для этого нельзя
* отбрасывать какие-либо биты serial после инъективного преобразования.
* Поэтому key_serial не трогаем, а в value_serial нелинейно вмешиваем
* запрошенное количество бит из serial */
value_serial +=
(serial ^ (serial >> mapping.split)) & mask(mapping.split);
}
value_serial |= value_age << mapping.split;
log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split,
key_serial, value_serial);
}
log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial,
value_serial);
mk(key_serial, key_essentials, *key);
mk(value_serial, value_essentials, *value);
if (log_enabled(logging::trace)) {
char dump_key[128], dump_value[128];
log_trace("keygen-pair: key %s, value %s",
mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)),
mdbx_dump_val(&value->value, dump_value, sizeof(dump_value)));
}
}
void maker::setup(const config::actor_params_pod &actor, unsigned actor_id,
unsigned thread_number) {
key_essentials.flags =
actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT);
assert(actor.keylen_min <= UINT8_MAX);
key_essentials.minlen = (uint8_t)actor.keylen_min;
assert(actor.keylen_max <= UINT16_MAX);
key_essentials.maxlen = (uint16_t)actor.keylen_max;
value_essentials.flags =
actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP);
assert(actor.datalen_min <= UINT8_MAX);
value_essentials.minlen = (uint8_t)actor.datalen_min;
assert(actor.datalen_max <= UINT16_MAX);
value_essentials.maxlen = (uint16_t)actor.datalen_max;
assert(thread_number < 2);
(void)thread_number;
mapping = actor.keygen;
salt = (actor.keygen.seed + actor_id) * UINT64_C(14653293970879851569);
// FIXME: TODO
base = 0;
}
void maker::make_ordered() {
mapping.mesh = 0;
mapping.rotate = 0;
}
bool maker::is_unordered() const {
return (mapping.mesh >= serial_minwith || mapping.rotate) != 0;
}
bool maker::increment(serial_t &serial, int delta) const {
if (serial > mask(mapping.width)) {
log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial,
mask(mapping.width));
return false;
}
serial_t target = serial + (int64_t)delta;
if (target > mask(mapping.width) ||
((delta > 0) ? target < serial : target > serial)) {
log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", overflow",
serial, delta, target);
return false;
}
log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", continue", serial,
delta, target);
serial = target;
return true;
}
//-----------------------------------------------------------------------------
static size_t length(serial_t serial) {
size_t n = 0;
if (serial > UINT32_MAX) {
n = 4;
serial >>= 32;
}
if (serial > UINT16_MAX) {
n += 2;
serial >>= 16;
}
if (serial > UINT8_MAX) {
n += 1;
serial >>= 8;
}
return (serial > 0) ? n + 1 : n;
}
buffer alloc(size_t limit) {
result *ptr = (result *)malloc(sizeof(result) + limit);
if (unlikely(ptr == nullptr))
failure_perror("malloc(keyvalue_buffer)", errno);
ptr->value.iov_base = ptr->bytes;
ptr->value.iov_len = 0;
ptr->limit = limit;
return buffer(ptr);
}
void __hot maker::mk(const serial_t serial, const essentials &params,
result &out) {
assert(out.limit >= params.maxlen);
assert(params.maxlen >= params.minlen);
assert(params.maxlen >= length(serial));
out.value.iov_base = out.bytes;
out.value.iov_len =
(params.maxlen > params.minlen)
? params.minlen + serial % (params.maxlen - params.minlen)
: params.minlen;
if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) {
assert(params.maxlen == params.minlen);
assert(params.minlen == 4 || params.minlen == 8);
if (is_byteorder_le() || params.minlen == 8)
out.u64 = serial;
else
out.u32 = (uint32_t)serial;
} else if (params.flags & (MDBX_REVERSEKEY | MDBX_REVERSEDUP)) {
if (out.value.iov_len > 8) {
memset(out.bytes, '\0', out.value.iov_len - 8);
unaligned::store(out.bytes + out.value.iov_len - 8, htobe64(serial));
} else {
out.u64 = htobe64(serial);
if (out.value.iov_len < 8) {
out.value.iov_len = std::max(length(serial), out.value.iov_len);
out.value.iov_base = out.bytes + 8 - out.value.iov_len;
}
}
} else {
out.u64 = htole64(serial);
if (out.value.iov_len > 8)
memset(out.bytes + 8, '\0', out.value.iov_len - 8);
else
out.value.iov_len = std::max(length(serial), out.value.iov_len);
}
assert(out.value.iov_len >= params.minlen);
assert(out.value.iov_len <= params.maxlen);
assert(out.value.iov_len >= length(serial));
assert(out.value.iov_base >= out.bytes);
assert((uint8_t *)out.value.iov_base + out.value.iov_len <=
out.bytes + out.limit);
}
} /* namespace keygen */

130
contrib/db/libmdbx/test/keygen.h Normal file
View file

@ -0,0 +1,130 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "config.h"
#include "log.h"
#include "utils.h"
namespace keygen {
/* Под "генерацией ключей" здесь понимается генерация обоих значений для
* пар key-value, т.е. не только ключей, но и ассоциированных с ними данных.
*/
/* Генерацию ключей нельзя отнести к простым задачам, так как требования
* примерно следующие:
* - генерация разного количества уникальных ключей различной длины
* в задаваемом диапазоне;
* - возможность выбора как псевдо-случайного порядка ключей,
* так и по некоторым специфическим законам (ограниченными упорядоченными
* последовательностями, в шахматном порядке по граница диапазона и т.д.);
* - возможность генерации дубликатов с задаваемым законом распределения;
* - возможность генерации непересекающимися кластерами для параллельного
* использования в нескольких потоках;
* - использовать минимум ресурсов, как CPU, так и RAM, в том числе
* включая cache pollution и ram bandwidth.
*
* При этом заведомо известно, что для MDBX не имеет значения:
* - используемый алфавит (значения байтов);
* - частотное распределение по алфавиту;
* - абсолютное значение ключей или разность между отдельными значениями;
*
* Соответственно, в общих чертах, схема генерации следующая:
* - вводится плоская одномерная "координата" serial (uint64_t);
* - генерация специфических паттернов (последовательностей)
* реализуется посредством соответствующих преобразований "координат", при
* этом все подобные преобразования выполняются только над "координатой";
* - итоговая "координата" преобразуется в 8-байтное суррогатное значение
* ключа;
* - для получения ключей длиной МЕНЕЕ 8 байт суррогат может усекаться
* до ненулевых байт, в том числе до нулевой длины;
* - для получения ключей длиной БОЛЕЕ 8 байт суррогат дополняется
* нулями или псевдослучайной последовательностью;
*
* Механизм генерации паттернов:
* - реализованный механизм является компромиссом между скоростью/простотой
* и гибкостью, необходимой для получения последовательностей, которых
* будет достаточно для проверки сценариев разделения и слияния страниц
* с данными внутри mdbx;
* - псевдо-случайные паттерны реализуются посредством набора инъективных
* отображающих функций;
* - не-псевдо-случайные паттерны реализуются посредством параметризируемого
* трех-этапного преобразования:
* 1) смещение (сложение) по модулю;
* 2) циклический сдвиг;
* 3) добавление абсолютного смещения (базы);
*/
typedef uint64_t serial_t;
enum : serial_t {
serial_minwith = 8,
serial_maxwith = sizeof(serial_t) * 8,
serial_allones = ~(serial_t)0u
};
struct result {
MDBX_val value;
size_t limit;
union {
uint8_t bytes[sizeof(uint64_t)];
uint32_t u32;
uint64_t u64;
};
std::string as_string() const {
return std::string((const char *)value.iov_base, value.iov_len);
}
};
//-----------------------------------------------------------------------------
struct buffer_deleter /* : public std::unary_function<void, result *> */ {
void operator()(result *buffer) const { free(buffer); }
};
typedef std::unique_ptr<result, buffer_deleter> buffer;
buffer alloc(size_t limit);
class maker {
config::keygen_params_pod mapping;
serial_t base;
serial_t salt;
struct essentials {
uint8_t minlen;
uint8_t flags;
uint16_t maxlen;
} key_essentials, value_essentials;
static void mk(const serial_t serial, const essentials &params, result &out);
public:
maker() { memset(this, 0, sizeof(*this)); }
void pair(serial_t serial, const buffer &key, buffer &value,
serial_t value_age);
void setup(const config::actor_params_pod &actor, unsigned actor_id,
unsigned thread_number);
void make_ordered();
bool is_unordered() const;
bool increment(serial_t &serial, int delta) const;
};
} /* namespace keygen */

371
contrib/db/libmdbx/test/log.cc Normal file
View file

@ -0,0 +1,371 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
static void fflushall() { fflush(nullptr); }
void failure(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
fflushall();
logging::output_nocheckloglevel_ap(logging::failure, fmt, ap);
va_end(ap);
fflushall();
exit(EXIT_FAILURE);
}
const char *test_strerror(int errnum) {
static __thread char buf[1024];
return mdbx_strerror_r(errnum, buf, sizeof(buf));
}
void __noreturn failure_perror(const char *what, int errnum) {
failure("%s failed: %s (%d)\n", what, test_strerror(errnum), errnum);
}
//-----------------------------------------------------------------------------
static void mdbx_logger(int priority, const char *function, int line,
const char *msg, va_list args) {
if (!function)
function = "unknown";
if (priority == MDBX_LOG_FATAL)
log_error("mdbx: fatal failure: %s, %d", function, line);
logging::output_nocheckloglevel(
logging::loglevel(priority),
strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx %s: ", function);
logging::feed_ap(msg, args);
}
namespace logging {
static std::string prefix;
static std::string suffix;
static loglevel level;
static FILE *last;
void setlevel(loglevel priority) {
level = priority;
int rc = mdbx_setup_debug(priority,
MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER |
MDBX_DBG_DUMP,
mdbx_logger);
log_trace("set mdbx debug-opts: 0x%02x", rc);
}
void setup(loglevel priority, const std::string &_prefix) {
setlevel(priority);
prefix = _prefix;
}
void setup(const std::string &_prefix) { prefix = _prefix; }
const char *level2str(const loglevel alevel) {
switch (alevel) {
default:
return "invalid/unknown";
case extra:
return "extra";
case trace:
return "trace";
case debug:
return "debug";
case verbose:
return "verbose";
case notice:
return "notice";
case warning:
return "warning";
case error:
return "error";
case failure:
return "failure";
}
}
bool output(const loglevel priority, const char *format, ...) {
if (lower(priority, level))
return false;
va_list ap;
va_start(ap, format);
output_nocheckloglevel_ap(priority, format, ap);
va_end(ap);
return true;
}
void output_nocheckloglevel_ap(const logging::loglevel priority,
const char *format, va_list ap) {
if (last) {
putc('\n', last);
fflush(last);
if (last == stderr) {
putc('\n', stdout);
fflush(stdout);
}
last = nullptr;
}
chrono::time now = chrono::now_realtime();
struct tm tm;
#ifdef _MSC_VER
int rc = _localtime32_s(&tm, (const __time32_t *)&now.utc);
#else
time_t time = now.utc;
int rc = localtime_r(&time, &tm) ? MDBX_SUCCESS : errno;
#endif
if (rc != MDBX_SUCCESS)
failure_perror("localtime_r()", rc);
last = stdout;
fprintf(last,
"[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05u %-10s %.4s ] %s" /* TODO */,
tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min,
tm.tm_sec, chrono::fractional2us(now.fractional), osal_getpid(),
prefix.c_str(), level2str(priority), suffix.c_str());
va_list ones;
memset(&ones, 0, sizeof(ones)) /* zap MSVC and other stupid compilers */;
if (same_or_higher(priority, error))
va_copy(ones, ap);
vfprintf(last, format, ap);
size_t len = strlen(format);
char end = len ? format[len - 1] : '\0';
switch (end) {
default:
putc('\n', last);
// fall through
case '\n':
fflush(last);
last = nullptr;
// fall through
case ' ':
case '_':
case ':':
case '|':
case ',':
case '\t':
case '\b':
case '\r':
case '\0':
break;
}
if (same_or_higher(priority, error)) {
if (last != stderr) {
fprintf(stderr, "[ %05u %-10s %.4s ] %s", osal_getpid(), prefix.c_str(),
level2str(priority), suffix.c_str());
vfprintf(stderr, format, ones);
if (end == '\n')
fflush(stderr);
else
last = stderr;
}
va_end(ones);
}
}
bool feed_ap(const char *format, va_list ap) {
if (!last)
return false;
if (last == stderr) {
va_list ones;
va_copy(ones, ap);
vfprintf(stdout, format, ones);
va_end(ones);
}
vfprintf(last, format, ap);
size_t len = strlen(format);
if (len && format[len - 1] == '\n') {
fflush(last);
if (last == stderr)
fflush(stdout);
last = nullptr;
}
return true;
}
bool feed(const char *format, ...) {
if (!last)
return false;
va_list ap;
va_start(ap, format);
feed_ap(format, ap);
va_end(ap);
return true;
}
local_suffix::local_suffix(const char *c_str)
: trim_pos(suffix.size()), indent(0) {
suffix.append(c_str);
}
local_suffix::local_suffix(const std::string &str)
: trim_pos(suffix.size()), indent(0) {
suffix.append(str);
}
void local_suffix::push() {
indent += 1;
suffix.push_back('\t');
}
void local_suffix::pop() {
assert(indent > 0);
if (indent > 0) {
indent -= 1;
suffix.pop_back();
}
}
local_suffix::~local_suffix() { suffix.erase(trim_pos); }
void progress_canary(bool active) {
static chrono::time progress_timestamp;
chrono::time now = chrono::now_motonic();
if (now.fixedpoint - progress_timestamp.fixedpoint <
chrono::from_ms(42).fixedpoint)
return;
if (osal_progress_push(active)) {
progress_timestamp = now;
return;
}
if (progress_timestamp.fixedpoint == 0) {
putc('>', stderr);
progress_timestamp = now;
} else if (global::config::console_mode) {
if (active) {
static int last_point = -1;
int point = (now.fixedpoint >> 29) & 3;
if (point != last_point) {
progress_timestamp = now;
fprintf(stderr, "%c\b", "-\\|/"[last_point = point]);
}
} else if (now.fixedpoint - progress_timestamp.fixedpoint >
chrono::from_seconds(2).fixedpoint) {
progress_timestamp = now;
fprintf(stderr, "%c\b", "@*"[now.utc & 1]);
}
} else {
static int count;
if (active && now.fixedpoint - progress_timestamp.fixedpoint >
chrono::from_seconds(1).fixedpoint) {
putc('.', stderr);
progress_timestamp = now;
++count;
} else if (now.fixedpoint - progress_timestamp.fixedpoint >
chrono::from_seconds(5).fixedpoint) {
putc("@*"[now.utc & 1], stderr);
progress_timestamp = now;
++count;
}
if (count == 60) {
count = 0;
putc('\n', stderr);
}
}
fflush(stderr);
}
} // namespace logging
void log_extra(const char *msg, ...) {
if (logging::same_or_higher(logging::extra, logging::level)) {
va_list ap;
va_start(ap, msg);
logging::output_nocheckloglevel_ap(logging::extra, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_trace(const char *msg, ...) {
if (logging::same_or_higher(logging::trace, logging::level)) {
va_list ap;
va_start(ap, msg);
logging::output_nocheckloglevel_ap(logging::trace, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_debug(const char *msg, ...) {
if (logging::same_or_higher(logging::debug, logging::level)) {
va_list ap;
va_start(ap, msg);
logging::output_nocheckloglevel_ap(logging::debug, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_verbose(const char *msg, ...) {
if (logging::same_or_higher(logging::verbose, logging::level)) {
va_list ap;
va_start(ap, msg);
logging::output_nocheckloglevel_ap(logging::verbose, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_notice(const char *msg, ...) {
if (logging::same_or_higher(logging::notice, logging::level)) {
va_list ap;
va_start(ap, msg);
logging::output_nocheckloglevel_ap(logging::notice, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_warning(const char *msg, ...) {
if (logging::same_or_higher(logging::warning, logging::level)) {
va_list ap;
va_start(ap, msg);
logging::output_nocheckloglevel_ap(logging::warning, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_error(const char *msg, ...) {
if (logging::same_or_higher(logging::error, logging::level)) {
va_list ap;
va_start(ap, msg);
logging::output_nocheckloglevel_ap(logging::error, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_trouble(const char *where, const char *what, int errnum) {
log_error("%s: %s %s", where, what, test_strerror(errnum));
}
bool log_enabled(const logging::loglevel priority) {
return logging::same_or_higher(priority, logging::level);
}
void log_flush(void) { fflushall(); }

104
contrib/db/libmdbx/test/log.h Normal file
View file

@ -0,0 +1,104 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "chrono.h"
void __noreturn usage(void);
void __noreturn __printf_args(1, 2) failure(const char *fmt, ...);
void __noreturn failure_perror(const char *what, int errnum);
const char *test_strerror(int errnum);
namespace logging {
enum loglevel {
extra = MDBX_LOG_EXTRA,
trace = MDBX_LOG_TRACE,
debug = MDBX_LOG_DEBUG,
verbose = MDBX_LOG_VERBOSE,
notice = MDBX_LOG_NOTICE,
warning = MDBX_LOG_WARN,
error = MDBX_LOG_ERROR,
failure = MDBX_LOG_FATAL
};
inline bool lower(loglevel left, loglevel right) {
static_assert(MDBX_LOG_EXTRA > MDBX_LOG_FATAL, "WTF?");
return left > right;
}
inline bool same_or_higher(loglevel left, loglevel right) {
return left <= right;
}
const char *level2str(const loglevel level);
void setup(loglevel priority, const std::string &prefix);
void setup(const std::string &prefix);
void setlevel(loglevel priority);
void output_nocheckloglevel_ap(const loglevel priority, const char *format,
va_list ap);
bool __printf_args(2, 3)
output(const loglevel priority, const char *format, ...);
bool feed_ap(const char *format, va_list ap);
bool __printf_args(1, 2) feed(const char *format, ...);
void inline __printf_args(2, 3)
output_nocheckloglevel(const loglevel priority, const char *format, ...) {
va_list ap;
va_start(ap, format);
output_nocheckloglevel_ap(priority, format, ap);
va_end(ap);
}
void progress_canary(bool active);
class local_suffix {
protected:
size_t trim_pos;
int indent;
public:
local_suffix(const local_suffix &) = delete;
local_suffix(const local_suffix &&) = delete;
const local_suffix &operator=(const local_suffix &) = delete;
local_suffix(const char *c_str);
local_suffix(const std::string &str);
void push();
void pop();
~local_suffix();
};
} // namespace logging
void __printf_args(1, 2) log_extra(const char *msg, ...);
void __printf_args(1, 2) log_trace(const char *msg, ...);
void __printf_args(1, 2) log_debug(const char *msg, ...);
void __printf_args(1, 2) log_verbose(const char *msg, ...);
void __printf_args(1, 2) log_notice(const char *msg, ...);
void __printf_args(1, 2) log_warning(const char *msg, ...);
void __printf_args(1, 2) log_error(const char *msg, ...);
void log_trouble(const char *where, const char *what, int errnum);
void log_flush(void);
bool log_enabled(const logging::loglevel priority);
#ifdef _DEBUG
#define TRACE(...) log_trace(__VA_ARGS__)
#else
#define TRACE(...) __noop(__VA_ARGS__)
#endif

220
contrib/db/libmdbx/test/long_stochastic.sh Normal file
View file

@ -0,0 +1,220 @@
#!/usr/bin/env bash
if ! which make cc c++ tee lz4 >/dev/null; then
echo "Please install the following prerequisites: make cc c++ tee lz4" >&2
exit 1
fi
set -euo pipefail
UNAME="$(uname -s 2>/dev/null || echo Unknown)"
## NOTE: Valgrind could produce some false-positive warnings
## in multi-process environment with shared memory.
## For instance, when the process "A" explicitly marks a memory
## region as "undefined", the process "B" fill it,
## and after this process "A" read such region, etc.
#VALGRIND="valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt"
###############################################################################
# 1. clean data from prev runs and examine available RAM
if [[ -v VALGRIND && ! -z "$VALGRIND" ]]; then
rm -f valgrind-*.log
else
VALGRIND=time
fi
WANNA_MOUNT=0
case ${UNAME} in
Linux)
MAKE=make
if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then
for old_test_dir in $(ls -d /dev/shm/mdbx-test.[0-9]*); do
rm -rf $old_test_dir
done
TESTDB_DIR="/dev/shm/mdbx-test.$$"
fi
mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
if LC_ALL=C free | grep -q -i available; then
ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 7) / 1024))
else
ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 4) / 1024))
fi
;;
FreeBSD)
MAKE=gmake
if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then
for old_test_dir in $(ls -d /tmp/mdbx-test.[0-9]*); do
umount $old_test_dir && rm -r $old_test_dir
done
TESTDB_DIR="/tmp/mdbx-test.$$"
rm -rf $TESTDB_DIR && mkdir -p $TESTDB_DIR
WANNA_MOUNT=1
else
mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
fi
ram_avail_mb=$(($(LC_ALL=C vmstat -s | grep -ie '[0-9] pages free$' | cut -d p -f 1) * ($(LC_ALL=C vmstat -s | grep -ie '[0-9] bytes per page$' | cut -d b -f 1) / 1024) / 1024))
;;
Darwin)
MAKE=make
if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then
for vol in $(ls -d /Volumes/mdx[0-9]*[0-9]tst); do
disk=$(mount | grep $vol | cut -d ' ' -f 1)
echo "umount: volume $vol disk $disk"
hdiutil unmount $vol -force
hdiutil detach $disk
done
TESTDB_DIR="/Volumes/mdx$$tst"
WANNA_MOUNT=1
else
mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
fi
pagesize=$(($(LC_ALL=C vm_stat | grep -o 'page size of [0-9]\+ bytes' | cut -d' ' -f 4) / 1024))
freepages=$(LC_ALL=C vm_stat | grep '^Pages free:' | grep -o '[0-9]\+\.$' | cut -d'.' -f 1)
ram_avail_mb=$((pagesize * freepages / 1024))
echo "pagesize ${pagesize}K, freepages ${freepages}, ram_avail_mb ${ram_avail_mb}"
;;
*)
echo "FIXME: ${UNAME} not supported by this script"
exit 2
;;
esac
###############################################################################
# 2. estimate reasonable RAM space for test-db
echo "=== ${ram_avail_mb}M RAM available"
ram_reserve4logs_mb=1234
if [ $ram_avail_mb -lt $ram_reserve4logs_mb ]; then
echo "=== At least ${ram_reserve4logs_mb}Mb RAM required"
exit 3
fi
#
# В режимах отличных от MDBX_WRITEMAP изменения до записи в файл
# будут накапливаться в памяти, что может потребовать свободной
# памяти размером с БД. Кроме этого, в тест входит сценарий
# создания копия БД на ходу. Поэтому БД не может быть больше 1/3
# от доступной памяти. Однако, следует учесть что malloc() будет
# не сразу возвращать выделенную память системе, а также
# предусмотреть места для логов.
#
# In non-MDBX_WRITEMAP modes, updates (dirty pages) will
# accumulate in memory before writing to the disk, which may
# require a free memory up to the size of a whole database. In
# addition, the test includes a script create a copy of the
# database on the go. Therefore, the database cannot be more 1/3
# of available memory. Moreover, should be taken into account
# that malloc() will not return the allocated memory to the
# system immediately, as well some space is required for logs.
#
db_size_mb=$(((ram_avail_mb - ram_reserve4logs_mb) / 4))
if [ $db_size_mb -gt 3072 ]; then
db_size_mb=3072
fi
echo "=== use ${db_size_mb}M for DB"
###############################################################################
# 3. Create test-directory in ramfs/tmpfs, i.e. create/format/mount if required
case ${UNAME} in
Linux)
;;
FreeBSD)
if [[ WANNA_MOUNT ]]; then
mount -t tmpfs tmpfs $TESTDB_DIR
fi
;;
Darwin)
if [[ WANNA_MOUNT ]]; then
ramdisk_size_mb=$((42 + db_size_mb * 2 + ram_reserve4logs_mb))
number_of_sectors=$((ramdisk_size_mb * 2048))
ramdev=$(hdiutil attach -nomount ram://${number_of_sectors})
diskutil erasevolume ExFAT "mdx$$tst" ${ramdev}
fi
;;
*)
echo "FIXME: ${UNAME} not supported by this script"
exit 2
;;
esac
###############################################################################
# 4. Run basic test, i.e. `make check`
${MAKE} TEST_DB=${TESTDB_DIR}/smoke.db TEST_LOG=${TESTDB_DIR}/smoke.log check
rm -f ${TESTDB_DIR}/*
###############################################################################
# 5. run stochastic iterations
function rep9 { printf "%*s" $1 '' | tr ' ' '9'; }
function join { local IFS="$1"; shift; echo "$*"; }
function bit2option { local -n arr=$1; (( ($2&(1<<$3)) != 0 )) && echo -n '+' || echo -n '-'; echo "${arr[$3]}"; }
options=(writemap coalesce lifo)
function bits2list {
local -n arr=$1
local i
local list=()
for ((i=0; i<${#arr[@]}; ++i)) do
list[$i]=$(bit2option $1 $2 $i)
done
join , "${list[@]}"
}
function probe {
echo "=============================================== $(date)"
echo "${caption}: $*"
rm -f ${TESTDB_DIR}/* \
&& ${VALGRIND} ./mdbx_test --ignore-dbfull --repeat=42 --pathname=${TESTDB_DIR}/long.db "$@" | lz4 > ${TESTDB_DIR}/long.log.lz4 \
&& ${VALGRIND} ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db | tee ${TESTDB_DIR}/long-chk.log \
&& ([ ! -e ${TESTDB_DIR}/long.db-copy ] || ${VALGRIND} ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db-copy | tee ${TESTDB_DIR}/long-chk-copy.log) \
|| (echo "FAILED"; exit 1)
}
#------------------------------------------------------------------------------
count=0
for nops in $(seq 2 6); do
for ((wbatch=nops-1; wbatch > 0; --wbatch)); do
loops=$(((111 >> nops) / nops + 3))
for ((rep=0; rep++ < loops; )); do
for ((bits=2**${#options[@]}; --bits >= 0; )); do
seed=$(($(date +%s) + RANDOM))
caption="Probe #$((++count)) int-key,w/o-dups, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
caption="Probe #$((++count)) int-key,with-dups, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
caption="Probe #$((++count)) int-key,int-data, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.integer --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
caption="Probe #$((++count)) w/o-dups, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
caption="Probe #$((++count)) with-dups, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
done
done
done
done
echo "=== ALL DONE ====================== $(date)"

617
contrib/db/libmdbx/test/main.cc Normal file
View file

@ -0,0 +1,617 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#if !(defined(_WIN32) || defined(_WIN64))
#include <sys/resource.h>
#include <sys/time.h>
#endif /* !Windows */
void __noreturn usage(void) {
puts(
"usage:\n"
" --help or -h Show this text\n"
"Common parameters:\n"
" --pathname=... Path and/or name of database files\n"
" --repeat=N Set repeat counter\n"
" --threads=N Number of thread (unsunpported for now)\n"
" --timeout=N[s|m|h|d] Set timeout in seconds/minutes/hours/days\n"
" --failfast[=YES/no] Lill all actors on first failure/error\n"
" --max-readers=N See mdbx_env_set_maxreaders() description\n"
" --max-tables=N Se mdbx_env_set_maxdbs() description\n"
" --dump-config[=YES/no] Dump entire test config before run\n"
" --progress[=YES/no] Enable/disable progress `canary`\n"
" --console[=yes/no] Enable/disable console-like output\n"
" --cleanup-before[=YES/no] Cleanup/remove and re-create database\n"
" --cleanup-after[=YES/no] Cleanup/remove database after completion\n"
"Database size control:\n"
" --pagesize=... Database page size: min, max, 256..65536\n"
" --size-lower=N[K|M|G|T] Lower-bound of size in Kb/Mb/Gb/Tb\n"
" --size-upper Upper-bound of size in Kb/Mb/Gb/Tb\n"
" --size Initial size in Kb/Mb/Gb/Tb\n"
" --shrink-threshold Shrink threshold in Kb/Mb/Gb/Tb\n"
" --growth-step Grow step in Kb/Mb/Gb/Tb\n"
"Predefined complext scenarios/cases:\n"
" --case=... Only `basic` scenario implemented for now\n"
" basic == Simultaneous multi-process execution\n"
" of test-actors: nested,hill,ttl,copy,append,jitter,try\n"
"Test actors:\n"
" --hill Fill-up and empty-down\n"
" by CRUD-operation quads\n"
" --ttl Stochastic time-to-live simulation\n"
" --nested Nested transactionы\n"
" with stochastic-size bellows\n"
" --jitter Jitter/delays simulation\n"
" --try Try write-transaction, no more\n"
" --copy Online copy/backup\n"
" --append Append-mode insertions\n"
" --dead.reader Dead-reader simulator\n"
" --dead.writer Dead-writer simulator\n"
"Actor options:\n"
" --batch.read=N Read-operations batch size\n"
" --batch.write=N Write-operations batch size\n"
" --delay=N | --no-delay (no)Delay test-actor before start\n"
" --wait4ops=N | --no-wait4ops (no)Wait for previous test-actor\n"
" completes # ops before start\n"
" --duration=N[s|m|h|d] Define running duration\n"
" --nops=N[K|M|G|T] Define number of operations/steps\n"
" --inject-writefault[=yes|NO] TBD (see the source code)\n"
" --drop[=yes|NO] Drop key-value space/table on "
"completion\n"
" --ignore-dbfull[=yes|NO] Ignore MDBX_MAP_FULL error\n"
" --speculum[=yes|NO] Use internal `speculum` to check "
"dataset\n"
"Keys and Value:\n"
" --keygen.min=N Minimal keys length\n"
" --keygen.max=N Miximal keys length\n"
" --datalen.min=N Minimal data length\n"
" --datalen.max=N Miximal data length\n"
" --keygen.width=N TBD (see the source code)\n"
" --keygen.mesh=N TBD (see the source code)\n"
" --keygen.seed=N TBD (see the source code)\n"
" --keygen.split=N TBD (see the source code)\n"
" --keygen.rotate=N TBD (see the source code)\n"
" --keygen.offset=N TBD (see the source code)\n"
" --keygen.case=random Generator case (only `random` for now)\n"
"Database operation mode:\n"
" --mode={[+-]FLAG}[,[+-]FLAG]...\n"
" nosubdir == MDBX_NOSUBDIR\n"
" rdonly == MDBX_RDONLY\n"
" nometasync == MDBX_NOMETASYNC\n"
" lifo == MDBX_LIFORECLAIM\n"
" coalesce == MDBX_COALESCE\n"
" nosync == MDBX_NOSYNC\n"
" writemap == MDBX_WRITEMAP\n"
" mapasync == MDBX_MAPASYNC\n"
" utterly == MDBX_UTTERLY_NOSYNC\n"
" perturb == MDBX_PAGEPERTURB\n"
" notls == MDBX_NOTLS\n"
" nordahead == MDBX_NORDAHEAD\n"
" nomeminit == MDBX_NOMEMINIT\n"
"Key-value space/table options:\n"
" --table={[+-]FLAG}[,[+-]FLAG]...\n"
" key.reverse == MDBX_REVERSEKEY\n"
" key.integer == MDBX_INTEGERKEY\n"
" data.dups == MDBX_DUPSORT\n"
" data.integer == MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT\n"
" data.fixed == MDBX_DUPFIXED | MDBX_DUPSORT\n"
" data.reverse == MDBX_REVERSEDUP | MDBX_DUPSORT\n");
exit(EXIT_FAILURE);
}
//-----------------------------------------------------------------------------
void actor_params::set_defaults(const std::string &tmpdir) {
pathname_log = "";
loglevel =
#if defined(NDEBUG) || defined(_WIN32) || defined(_WIN64)
logging::verbose;
#else
logging::trace;
#endif
pathname_db = tmpdir + "mdbx-test.db";
mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NOMEMINIT |
MDBX_COALESCE | MDBX_LIFORECLAIM;
table_flags = MDBX_DUPSORT;
size_lower = -1;
size_now =
intptr_t(1024) * 1024 * ((table_flags & MDBX_DUPSORT) ? 256 : 1024);
size_upper = -1;
shrink_threshold = -1;
growth_step = -1;
pagesize = -1;
keygen.seed = 1;
keygen.keycase = kc_random;
keygen.width = (table_flags & MDBX_DUPSORT) ? 32 : 64;
keygen.mesh = keygen.width;
keygen.split = keygen.width / 2;
keygen.rotate = 3;
keygen.offset = 41;
test_duration = 0;
test_nops = 1000;
nrepeat = 1;
nthreads = 1;
keylen_min = mdbx_keylen_min();
keylen_max = mdbx_keylen_max();
datalen_min = mdbx_datalen_min();
datalen_max = std::min(mdbx_datalen_max(), 256u * 1024 + 42);
batch_read = 42;
batch_write = 42;
delaystart = 0;
waitfor_nops = 0;
inject_writefaultn = 0;
drop_table = false;
ignore_dbfull = false;
speculum = false;
max_readers = 42;
max_tables = 42;
global::config::timeout_duration_seconds = 0 /* infinite */;
global::config::dump_config = true;
global::config::cleanup_before = true;
global::config::cleanup_after = true;
global::config::failfast = true;
global::config::progress_indicator = true;
global::config::console_mode = osal_istty(STDERR_FILENO);
}
namespace global {
std::vector<actor_config> actors;
std::unordered_map<unsigned, actor_config *> events;
std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
std::set<std::string> databases;
unsigned nactors;
chrono::time start_motonic;
chrono::time deadline_motonic;
bool singlemode;
namespace config {
unsigned timeout_duration_seconds;
bool dump_config;
bool cleanup_before;
bool cleanup_after;
bool failfast;
bool progress_indicator;
bool console_mode;
} /* namespace config */
} /* namespace global */
//-----------------------------------------------------------------------------
const char global::thunk_param_prefix[] = "--execute=";
std::string thunk_param(const actor_config &config) {
return config.serialize(global::thunk_param_prefix);
}
void cleanup() {
log_trace(">> cleanup");
/* TODO: remove each database */
log_trace("<< cleanup");
}
int main(int argc, char *const argv[]) {
#ifdef _DEBUG
log_trace("#argc = %d", argc);
for (int i = 0; i < argc; ++i)
log_trace("#argv[%d] = %s", i, argv[i]);
#endif /* _DEBUG */
if (argc < 2)
failure("No parameters given. Try --help\n");
if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix,
strlen(global::thunk_param_prefix)) == 0)
return test_execute(
actor_config(argv[1] + strlen(global::thunk_param_prefix)))
? EXIT_SUCCESS
: EXIT_FAILURE;
if (argc == 2 &&
(strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0))
usage();
actor_params params;
params.set_defaults(osal_tempdir());
global::config::dump_config = true;
logging::setup((logging::loglevel)params.loglevel, "main");
unsigned last_space_id = 0;
for (int narg = 1; narg < argc; ++narg) {
const char *value = nullptr;
if (config::parse_option(argc, argv, narg, "case", &value)) {
testcase_setup(value, params, last_space_id);
continue;
}
if (config::parse_option(argc, argv, narg, "pathname", params.pathname_db))
continue;
if (config::parse_option(argc, argv, narg, "mode", params.mode_flags,
config::mode_bits))
continue;
if (config::parse_option(argc, argv, narg, "table", params.table_flags,
config::table_bits)) {
if ((params.table_flags & MDBX_DUPFIXED) == 0)
params.table_flags &= ~MDBX_INTEGERDUP;
if ((params.table_flags & MDBX_DUPSORT) == 0)
params.table_flags &=
~(MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP);
continue;
}
if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize,
int(mdbx_limits_pgsize_min()),
int(mdbx_limits_pgsize_max()))) {
const unsigned keylen_max = params.mdbx_keylen_max();
if (params.keylen_min > keylen_max)
params.keylen_min = keylen_max;
if (params.keylen_max > keylen_max)
params.keylen_max = keylen_max;
const unsigned datalen_max = params.mdbx_datalen_max();
if (params.datalen_min > datalen_max)
params.datalen_min = datalen_max;
if (params.datalen_max > datalen_max)
params.datalen_max = datalen_max;
continue;
}
if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat,
config::no_scale))
continue;
if (config::parse_option(argc, argv, narg, "threads", params.nthreads,
config::no_scale, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "timeout",
global::config::timeout_duration_seconds,
config::duration, 1))
continue;
if (config::parse_option_intptr(argc, argv, narg, "size-lower",
params.size_lower,
mdbx_limits_dbsize_min(params.pagesize),
mdbx_limits_dbsize_max(params.pagesize)))
continue;
if (config::parse_option_intptr(argc, argv, narg, "size-upper",
params.size_upper,
mdbx_limits_dbsize_min(params.pagesize),
mdbx_limits_dbsize_max(params.pagesize)))
continue;
if (config::parse_option_intptr(argc, argv, narg, "size", params.size_now,
mdbx_limits_dbsize_min(params.pagesize),
mdbx_limits_dbsize_max(params.pagesize)))
continue;
if (config::parse_option(
argc, argv, narg, "shrink-threshold", params.shrink_threshold, 0,
(int)std::min((intptr_t)INT_MAX,
mdbx_limits_dbsize_max(params.pagesize) -
mdbx_limits_dbsize_min(params.pagesize))))
continue;
if (config::parse_option(
argc, argv, narg, "growth-step", params.growth_step, 0,
(int)std::min((intptr_t)INT_MAX,
mdbx_limits_dbsize_max(params.pagesize) -
mdbx_limits_dbsize_min(params.pagesize))))
continue;
if (config::parse_option(argc, argv, narg, "keygen.width",
params.keygen.width, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.mesh",
params.keygen.mesh, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.seed",
params.keygen.seed, config::no_scale))
continue;
if (config::parse_option(argc, argv, narg, "keygen.split",
params.keygen.split, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.rotate",
params.keygen.rotate, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.offset",
params.keygen.offset, config::binary))
continue;
if (config::parse_option(argc, argv, narg, "keygen.case", &value)) {
keycase_setup(value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min,
config::no_scale, params.mdbx_keylen_min(),
params.mdbx_keylen_max())) {
if ((params.table_flags & MDBX_INTEGERKEY) ||
params.keylen_max < params.keylen_min)
params.keylen_max = params.keylen_min;
continue;
}
if (config::parse_option(argc, argv, narg, "keylen.max", params.keylen_max,
config::no_scale, params.mdbx_keylen_min(),
params.mdbx_keylen_max())) {
if ((params.table_flags & MDBX_INTEGERKEY) ||
params.keylen_min > params.keylen_max)
params.keylen_min = params.keylen_max;
continue;
}
if (config::parse_option(argc, argv, narg, "datalen.min",
params.datalen_min, config::no_scale,
params.mdbx_datalen_min(),
params.mdbx_datalen_max())) {
if ((params.table_flags & MDBX_DUPFIXED) ||
params.datalen_max < params.datalen_min)
params.datalen_max = params.datalen_min;
continue;
}
if (config::parse_option(argc, argv, narg, "datalen.max",
params.datalen_max, config::no_scale,
params.mdbx_datalen_min(),
params.mdbx_datalen_max())) {
if ((params.table_flags & MDBX_DUPFIXED) ||
params.datalen_min > params.datalen_max)
params.datalen_min = params.datalen_max;
continue;
}
if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read,
config::no_scale, 1))
continue;
if (config::parse_option(argc, argv, narg, "batch.write",
params.batch_write, config::no_scale, 1))
continue;
if (config::parse_option(argc, argv, narg, "delay", params.delaystart,
config::duration))
continue;
if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops,
config::decimal))
continue;
if (config::parse_option(argc, argv, narg, "inject-writefault",
params.inject_writefaultn, config::decimal))
continue;
if (config::parse_option(argc, argv, narg, "drop", params.drop_table))
continue;
if (config::parse_option(argc, argv, narg, "ignore-dbfull",
params.ignore_dbfull))
continue;
if (config::parse_option(argc, argv, narg, "speculum", params.speculum))
continue;
if (config::parse_option(argc, argv, narg, "dump-config",
global::config::dump_config))
continue;
if (config::parse_option(argc, argv, narg, "cleanup-before",
global::config::cleanup_before))
continue;
if (config::parse_option(argc, argv, narg, "cleanup-after",
global::config::cleanup_after))
continue;
if (config::parse_option(argc, argv, narg, "max-readers",
params.max_readers, config::no_scale, 1, 255))
continue;
if (config::parse_option(argc, argv, narg, "max-tables", params.max_tables,
config::no_scale, 1, INT16_MAX))
continue;
if (config::parse_option(argc, argv, narg, "no-delay", nullptr)) {
params.delaystart = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "no-wait4ops", nullptr)) {
params.waitfor_nops = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "duration", params.test_duration,
config::duration, 1)) {
params.test_nops = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "nops", params.test_nops,
config::decimal, 1)) {
params.test_duration = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "hill", &value, "auto")) {
configure_actor(last_space_id, ac_hill, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "jitter", nullptr)) {
configure_actor(last_space_id, ac_jitter, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "dead.reader", nullptr)) {
configure_actor(last_space_id, ac_deadread, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "dead.writer", nullptr)) {
configure_actor(last_space_id, ac_deadwrite, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "try", nullptr)) {
configure_actor(last_space_id, ac_try, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "copy", nullptr)) {
configure_actor(last_space_id, ac_copy, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "append", nullptr)) {
configure_actor(last_space_id, ac_append, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "ttl", nullptr)) {
configure_actor(last_space_id, ac_ttl, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "nested", nullptr)) {
configure_actor(last_space_id, ac_nested, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "failfast",
global::config::failfast))
continue;
if (config::parse_option(argc, argv, narg, "progress",
global::config::progress_indicator))
continue;
if (config::parse_option(argc, argv, narg, "console",
global::config::console_mode))
continue;
if (*argv[narg] != '-')
testcase_setup(argv[narg], params, last_space_id);
else
failure("Unknown option '%s'. Try --help\n", argv[narg]);
}
if (global::config::dump_config)
config::dump();
//--------------------------------------------------------------------------
if (global::actors.empty()) {
log_notice("no testcase(s) configured, exiting");
return EXIT_SUCCESS;
}
bool failed = false;
global::start_motonic = chrono::now_motonic();
global::deadline_motonic.fixedpoint =
(global::config::timeout_duration_seconds == 0)
? chrono::infinite().fixedpoint
: global::start_motonic.fixedpoint +
chrono::from_seconds(global::config::timeout_duration_seconds)
.fixedpoint;
if (global::config::cleanup_before)
cleanup();
log_trace(">> probe entropy_ticks()");
entropy_ticks();
log_trace("<< probe entropy_ticks()");
if (global::actors.size() == 1) {
logging::setup("main");
global::singlemode = true;
if (!test_execute(global::actors.front()))
failed = true;
} else {
logging::setup("overlord");
log_trace("=== preparing...");
log_trace(">> osal_setup");
osal_setup(global::actors);
log_trace("<< osal_setup");
for (auto &a : global::actors) {
mdbx_pid_t pid;
log_trace(">> actor_start");
int rc = osal_actor_start(a, pid);
log_trace("<< actor_start");
if (rc) {
log_trace(">> killall_actors: (%s)", "start failed");
osal_killall_actors();
log_trace("<< killall_actors");
failure("Failed to start actor #%u (%s)\n", a.actor_id,
test_strerror(rc));
}
global::pid2actor[pid] = &a;
}
log_trace("=== ready to start...");
atexit(osal_killall_actors);
log_trace(">> wait4barrier");
osal_wait4barrier();
log_trace("<< wait4barrier");
size_t left = global::actors.size();
log_trace("=== polling...");
while (left > 0) {
unsigned timeout_seconds_left = INT_MAX;
chrono::time now_motonic = chrono::now_motonic();
if (now_motonic.fixedpoint >= global::deadline_motonic.fixedpoint)
timeout_seconds_left = 0;
else {
chrono::time left_motonic;
left_motonic.fixedpoint =
global::deadline_motonic.fixedpoint - now_motonic.fixedpoint;
timeout_seconds_left = left_motonic.seconds();
}
mdbx_pid_t pid;
int rc = osal_actor_poll(pid, timeout_seconds_left);
if (rc)
failure("Poll error: %s (%d)\n", test_strerror(rc), rc);
if (pid) {
actor_status status = osal_actor_info(pid);
actor_config *actor = global::pid2actor.at(pid);
if (!actor)
continue;
log_verbose("actor #%u, id %d, pid %u: %s\n", actor->actor_id,
actor->space_id, pid, status2str(status));
if (status > as_running) {
left -= 1;
if (status != as_successful) {
if (global::config::failfast && !failed) {
log_trace(">> killall_actors: (%s)", "failfast");
osal_killall_actors();
log_trace("<< killall_actors");
}
failed = true;
}
}
} else {
if (timeout_seconds_left == 0)
failure("Timeout\n");
}
}
log_trace("=== done...");
}
log_notice("RESULT: %s\n", failed ? "Failed" : "Successful");
if (global::config::cleanup_before) {
if (failed)
log_verbose("skip cleanup");
else
cleanup();
}
#if !(defined(_WIN32) || defined(_WIN64))
struct rusage spent;
if (getrusage(RUSAGE_CHILDREN, &spent) == 0) {
log_notice("%6s: user %f, system %f", "CPU",
spent.ru_utime.tv_sec + spent.ru_utime.tv_usec * 1e-6,
spent.ru_stime.tv_sec + spent.ru_stime.tv_usec * 1e-6);
#if defined(__linux__) || defined(__gnu_linux__) || defined(__FreeBSD__) || \
defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || \
defined(__NETBSD__) || defined(__bsdi__) || defined(__DragonFly__) || \
defined(__APPLE__) || defined(__MACH__)
log_notice("%6s: read %ld, write %ld", "IOPs", spent.ru_inblock,
spent.ru_oublock);
log_notice("%6s: %ld Kb", "RAM", spent.ru_maxrss);
log_notice("%6s: reclaims %ld, faults %ld, swaps %ld", "Paging",
spent.ru_minflt, spent.ru_majflt, spent.ru_nswap);
#endif /* Linux */
}
#endif /* !Windows */
return failed ? EXIT_FAILURE : EXIT_SUCCESS;
}

284
contrib/db/libmdbx/test/nested.cc Normal file
View file

@ -0,0 +1,284 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <cmath>
bool testcase_nested::setup() {
if (!inherited::setup())
return false;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("nested: bailout-prepare due '%s'", mdbx_strerror(err));
return false;
}
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
key = keygen::alloc(config.params.keylen_max);
data = keygen::alloc(config.params.datalen_max);
serial = 0;
fifo.clear();
speculum.clear();
assert(stack.empty());
stack.emplace(nullptr, serial, fifo, speculum);
return true;
}
bool testcase_nested::teardown() {
while (!stack.empty())
pop_txn(true);
bool ok = true;
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
int err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("nested: bailout-clean due '%s'", mdbx_strerror(err));
ok = false;
}
} else
db_table_close(dbi);
dbi = 0;
}
return inherited::teardown() && ok;
}
static unsigned edge2window(uint64_t edge, unsigned window_max) {
const double rnd = u64_to_double1(bleach64(edge));
const unsigned window = window_max - std::lrint(std::pow(window_max, rnd));
return window;
}
static unsigned edge2count(uint64_t edge, unsigned count_max) {
const double rnd = u64_to_double1(prng64_map1_white(edge));
const unsigned count = std::lrint(std::pow(count_max, rnd));
return count;
}
void testcase_nested::push_txn() {
MDBX_txn *txn;
int err = mdbx_txn_begin(
db_guard.get(), txn_guard.get(),
prng32() & (MDBX_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC), &txn);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_txn_begin(nested)", err);
#if __cplusplus >= 201703L
stack.emplace(txn, serial, fifo, speculum);
#else
stack.push(std::make_tuple(scoped_txn_guard(txn), serial, fifo, speculum));
#endif
std::swap(txn_guard, std::get<0>(stack.top()));
log_verbose("begin level#%zu txn, serial %" PRIu64, stack.size(), serial);
}
bool testcase_nested::pop_txn(bool abort) {
assert(txn_guard && !stack.empty());
bool should_continue = true;
MDBX_txn *txn = txn_guard.release();
bool commited = false;
if (abort) {
log_verbose("abort level#%zu txn, undo serial %" PRIu64 " <- %" PRIu64,
stack.size(), serial, std::get<1>(stack.top()));
int err = mdbx_txn_abort(txn);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_txn_abort()", err);
} else {
log_verbose("commit level#%zu txn, nested serial %" PRIu64 " -> %" PRIu64,
stack.size(), serial, std::get<1>(stack.top()));
int err = mdbx_txn_commit(txn);
if (likely(err == MDBX_SUCCESS))
commited = true;
else {
should_continue = false;
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
err = mdbx_txn_abort(txn);
if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH &&
err != MDBX_BAD_TXN))
failure_perror("mdbx_txn_abort()", err);
} else
failure_perror("mdbx_txn_commit()", err);
}
}
std::swap(txn_guard, std::get<0>(stack.top()));
if (!commited) {
serial = std::get<1>(stack.top());
std::swap(fifo, std::get<2>(stack.top()));
std::swap(speculum, std::get<3>(stack.top()));
}
stack.pop();
return should_continue;
}
bool testcase_nested::stochastic_breakable_restart_with_nested(
bool force_restart) {
log_trace(">> stochastic_breakable_restart_with_nested%s",
force_restart ? ": force_restart" : "");
if (force_restart)
while (txn_guard)
pop_txn(true);
bool should_continue = true;
while (!stack.empty() &&
(flipcoin() || txn_underutilization_x256(txn_guard.get()) < 42))
should_continue &= pop_txn();
if (should_continue)
while (stack.empty() ||
(is_nested_txn_available() && flipcoin() && stack.size() < 5))
push_txn();
log_trace("<< stochastic_breakable_restart_with_nested: should_continue=%s",
should_continue ? "yes" : "no");
return should_continue;
}
bool testcase_nested::trim_tail(unsigned window_width) {
if (window_width) {
while (fifo.size() > window_width) {
uint64_t tail_serial = fifo.back().first;
const unsigned tail_count = fifo.back().second;
log_trace("nested: pop-tail (serial %" PRIu64 ", count %u)", tail_serial,
tail_count);
fifo.pop_back();
for (unsigned n = 0; n < tail_count; ++n) {
log_trace("nested: remove-tail %" PRIu64, tail_serial);
generate_pair(tail_serial);
int err = remove(key, data);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("nested: tail-bailout due '%s'", mdbx_strerror(err));
return false;
}
failure_perror("mdbx_del(tail)", err);
}
if (unlikely(!keyvalue_maker.increment(tail_serial, 1)))
failure("nested: unexpected key-space overflow on the tail");
}
}
} else {
log_trace("nested: purge state");
db_table_clear(dbi, txn_guard.get());
fifo.clear();
speculum.clear();
}
return true;
}
bool testcase_nested::grow_head(unsigned head_count) {
const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_NODUPDATA
: MDBX_NODUPDATA | MDBX_NOOVERWRITE;
retry:
fifo.push_front(std::make_pair(serial, head_count));
for (unsigned n = 0; n < head_count; ++n) {
log_trace("nested: insert-head %" PRIu64, serial);
generate_pair(serial);
int err = insert(key, data, insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("nested: head-insert skip due '%s'", mdbx_strerror(err));
head_count = n;
stochastic_breakable_restart_with_nested(true);
goto retry;
}
failure_perror("mdbx_put(head)", err);
}
if (unlikely(!keyvalue_maker.increment(serial, 1))) {
log_notice("nested: unexpected key-space overflow");
return false;
}
}
return true;
}
bool testcase_nested::run() {
/* LY: тест "эмуляцией time-to-live" с вложенными транзакциями:
* - организуется "скользящее окно", которое каждую транзакцию сдвигается
* вперед вдоль числовой оси.
* - по переднему краю "скользящего окна" записи добавляются в таблицу,
* а по заднему удаляются.
* - количество добавляемых/удаляемых записей псевдослучайно зависит
* от номера транзакции, но с экспоненциальным распределением.
* - размер "скользящего окна" также псевдослучайно зависит от номера
* транзакции с "отрицательным" экспоненциальным распределением
* MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний
* край и удаляются записи позади него.
* - групповое добавление данных в начало окна и групповое уделение в конце,
* в половине случаев выполняются во вложенных транзакциях.
* - половина запускаемых вложенных транзакций отменяется, последуюим
* повтором групповой операции.
*
* Таким образом имитируется поведение таблицы с TTL: записи стохастически
* добавляются и удаляются, но изредка происходят массивные удаления. */
/* LY: для параметризации используем подходящие параметры, которые не имеют
* здесь смысла в первоначальном значении. */
const unsigned window_max_lower = 333;
const unsigned count_max_lower = 333;
const unsigned window_max = (config.params.batch_read > window_max_lower)
? config.params.batch_read
: window_max_lower;
const unsigned count_max = (config.params.batch_write > count_max_lower)
? config.params.batch_write
: count_max_lower;
log_verbose("nested: using `batch_read` value %u for window_max", window_max);
log_verbose("nested: using `batch_write` value %u for count_max", count_max);
uint64_t seed =
prng64_map2_white(config.params.keygen.seed) + config.actor_id;
while (should_continue()) {
const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
const unsigned window_width =
flipcoin_x4() ? 0 : edge2window(salt, window_max);
const unsigned head_count = edge2count(salt, count_max);
log_debug("nested: step #%zu (serial %" PRIu64
", window %u, count %u) salt %" PRIu64,
nops_completed, serial, window_width, head_count, salt);
if (!trim_tail(window_width))
return false;
if (!stochastic_breakable_restart_with_nested()) {
log_notice("nested: bailout at commit/restart after tail-trim");
return false;
}
if (!speculum_verify()) {
log_notice("nested: bailout after tail-trim");
return false;
}
if (!grow_head(head_count))
return false;
if (!stochastic_breakable_restart_with_nested())
log_notice("nested: skip commit/restart after head-grow");
if (!speculum_verify()) {
log_notice("nested: bailout after head-grow");
return false;
}
report(1);
}
while (!stack.empty())
pop_txn(false);
return speculum_verify();
}

368
contrib/db/libmdbx/test/osal-unix.cc Normal file
View file

@ -0,0 +1,368 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <pthread.h>
#include <signal.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#ifdef __APPLE__
#include "darwin/pthread_barrier.c"
#endif
struct shared_t {
pthread_barrier_t barrier;
pthread_mutex_t mutex;
size_t conds_size;
pthread_cond_t conds[1];
};
static shared_t *shared;
void osal_wait4barrier(void) {
assert(shared != nullptr && shared != MAP_FAILED);
int rc = pthread_barrier_wait(&shared->barrier);
if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) {
failure_perror("pthread_barrier_wait(shared)", rc);
}
}
void osal_setup(const std::vector<actor_config> &actors) {
assert(shared == nullptr);
pthread_mutexattr_t mutexattr;
int rc = pthread_mutexattr_init(&mutexattr);
if (rc)
failure_perror("pthread_mutexattr_init()", rc);
rc = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_mutexattr_setpshared()", rc);
pthread_barrierattr_t barrierattr;
rc = pthread_barrierattr_init(&barrierattr);
if (rc)
failure_perror("pthread_barrierattr_init()", rc);
rc = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_barrierattr_setpshared()", rc);
pthread_condattr_t condattr;
rc = pthread_condattr_init(&condattr);
if (rc)
failure_perror("pthread_condattr_init()", rc);
rc = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_condattr_setpshared()", rc);
shared = (shared_t *)mmap(
nullptr, sizeof(shared_t) + actors.size() * sizeof(pthread_cond_t),
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (MAP_FAILED == (void *)shared)
failure_perror("mmap(shared_conds)", errno);
rc = pthread_mutex_init(&shared->mutex, &mutexattr);
if (rc)
failure_perror("pthread_mutex_init(shared)", rc);
rc = pthread_barrier_init(&shared->barrier, &barrierattr, actors.size() + 1);
if (rc)
failure_perror("pthread_barrier_init(shared)", rc);
const size_t n = actors.size() + 1;
for (size_t i = 0; i < n; ++i) {
pthread_cond_t *event = &shared->conds[i];
rc = pthread_cond_init(event, &condattr);
if (rc)
failure_perror("pthread_cond_init(shared)", rc);
log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i,
__Wpedantic_format_voidptr(event));
}
shared->conds_size = actors.size() + 1;
pthread_barrierattr_destroy(&barrierattr);
pthread_condattr_destroy(&condattr);
pthread_mutexattr_destroy(&mutexattr);
}
void osal_broadcast(unsigned id) {
assert(shared != nullptr && shared != MAP_FAILED);
log_trace("osal_broadcast: event %u", id);
if (id >= shared->conds_size)
failure("osal_broadcast: id > limit");
int rc = pthread_cond_broadcast(shared->conds + id);
if (rc)
failure_perror("sem_post(shared)", rc);
}
int osal_waitfor(unsigned id) {
assert(shared != nullptr && shared != MAP_FAILED);
log_trace("osal_waitfor: event %u", id);
if (id >= shared->conds_size)
failure("osal_waitfor: id > limit");
int rc = pthread_mutex_lock(&shared->mutex);
if (rc != 0)
failure_perror("pthread_mutex_lock(shared)", rc);
rc = pthread_cond_wait(shared->conds + id, &shared->mutex);
if (rc && rc != EINTR)
failure_perror("pthread_cond_wait(shared)", rc);
rc = pthread_mutex_unlock(&shared->mutex);
if (rc != 0)
failure_perror("pthread_mutex_unlock(shared)", rc);
return (rc == 0) ? true : false;
}
//-----------------------------------------------------------------------------
const std::string
actor_config::osal_serialize(simple_checksum &checksum) const {
(void)checksum;
/* not used in workload, but just for testing */
return "unix.fork";
}
bool actor_config::osal_deserialize(const char *str, const char *end,
simple_checksum &checksum) {
(void)checksum;
/* not used in workload, but just for testing */
return strncmp(str, "unix.fork", 9) == 0 && str + 9 == end;
}
//-----------------------------------------------------------------------------
static pid_t overlord_pid;
static volatile sig_atomic_t sigusr1_head, sigusr2_head;
static void handler_SIGUSR(int signum) {
switch (signum) {
case SIGUSR1:
sigusr1_head += 1;
return;
case SIGUSR2:
sigusr2_head += 1;
return;
default:
abort();
}
}
bool osal_progress_push(bool active) {
if (overlord_pid) {
if (kill(overlord_pid, active ? SIGUSR1 : SIGUSR2))
failure_perror("osal_progress_push: kill(overload)", errno);
return true;
}
return false;
}
//-----------------------------------------------------------------------------
static std::unordered_map<pid_t, actor_status> childs;
static volatile sig_atomic_t sigalarm_head;
static void handler_SIGCHLD(int signum) {
if (signum == SIGALRM)
sigalarm_head += 1;
}
mdbx_pid_t osal_getpid(void) { return getpid(); }
int osal_delay(unsigned seconds) { return sleep(seconds) ? errno : 0; }
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
if (childs.empty()) {
struct sigaction act;
memset(&act, 0, sizeof(act));
act.sa_handler = handler_SIGCHLD;
sigaction(SIGCHLD, &act, nullptr);
sigaction(SIGALRM, &act, nullptr);
act.sa_handler = handler_SIGUSR;
sigaction(SIGUSR1, &act, nullptr);
sigaction(SIGUSR2, &act, nullptr);
sigset_t mask;
sigemptyset(&mask);
sigaddset(&mask, SIGCHLD);
sigaddset(&mask, SIGUSR1);
sigaddset(&mask, SIGUSR2);
sigprocmask(SIG_UNBLOCK, &mask, nullptr);
}
pid = fork();
if (pid == 0) {
overlord_pid = getppid();
const bool result = test_execute(config);
exit(result ? EXIT_SUCCESS : EXIT_FAILURE);
}
if (pid < 0)
return errno;
log_trace("osal_actor_start: fork pid %i for %u", pid, config.actor_id);
childs[pid] = as_running;
return 0;
}
actor_status osal_actor_info(const mdbx_pid_t pid) { return childs.at(pid); }
void osal_killall_actors(void) {
for (auto &pair : childs) {
kill(pair.first, SIGKILL);
pair.second = as_killed;
}
}
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) {
static sig_atomic_t sigalarm_tail;
alarm(0) /* cancel prev timeout */;
sigalarm_tail = sigalarm_head /* reset timeout flag */;
int options = WNOHANG;
if (timeout) {
alarm((timeout > INT_MAX) ? INT_MAX : timeout);
options = 0;
}
#ifdef WUNTRACED
options |= WUNTRACED;
#endif
#ifdef WCONTINUED
options |= WCONTINUED;
#endif
while (sigalarm_tail == sigalarm_head) {
int status;
pid = waitpid(0, &status, options);
if (pid > 0) {
if (WIFEXITED(status))
childs[pid] =
(WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed;
else if (WCOREDUMP(status))
childs[pid] = as_coredump;
else if (WIFSIGNALED(status))
childs[pid] = as_killed;
else if (WIFSTOPPED(status))
childs[pid] = as_debuging;
else if (WIFCONTINUED(status))
childs[pid] = as_running;
else {
assert(false);
}
return 0;
}
static sig_atomic_t sigusr1_tail, sigusr2_tail;
if (sigusr1_tail != sigusr1_head) {
sigusr1_tail = sigusr1_head;
logging::progress_canary(true);
if (pid < 0 && errno == EINTR)
continue;
}
if (sigusr2_tail != sigusr2_head) {
sigusr2_tail = sigusr2_head;
logging::progress_canary(false);
if (pid < 0 && errno == EINTR)
continue;
}
if (pid == 0)
break;
int err = errno;
if (err != EINTR)
return err;
}
return 0 /* timeout */;
}
void osal_yield(void) {
if (sched_yield())
failure_perror("sched_yield()", errno);
}
void osal_udelay(unsigned us) {
chrono::time until, now = chrono::now_motonic();
until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint;
struct timespec ts;
static unsigned threshold_us;
if (threshold_us == 0) {
if (clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts)) {
int rc = errno;
failure_perror("clock_getres(CLOCK_PROCESS_CPUTIME_ID)", rc);
}
chrono::time threshold = chrono::from_timespec(ts);
assert(threshold.seconds() == 0);
threshold_us = chrono::fractional2us(threshold.fractional);
if (threshold_us < 1000)
threshold_us = 1000;
}
ts.tv_sec = ts.tv_nsec = 0;
if (us > threshold_us) {
ts.tv_sec = us / 1000000u;
ts.tv_nsec = (us % 1000000u) * 1000u;
}
do {
if (us > threshold_us) {
if (nanosleep(&ts, &ts)) {
int rc = errno;
/* if (rc == EINTR) { ... } ? */
failure_perror("usleep()", rc);
}
us = ts.tv_sec * 1000000u + ts.tv_nsec / 1000u;
}
cpu_relax();
now = chrono::now_motonic();
} while (until.fixedpoint > now.fixedpoint);
}
bool osal_istty(int fd) { return isatty(fd) == 1; }
std::string osal_tempdir(void) {
const char *tempdir = getenv("TMPDIR");
if (!tempdir)
tempdir = getenv("TMP");
if (!tempdir)
tempdir = getenv("TEMPDIR");
if (!tempdir)
tempdir = getenv("TEMP");
if (tempdir) {
std::string dir(tempdir);
if (!dir.empty() && dir.at(dir.length() - 1) != '/')
dir.append("/");
return dir;
}
if (access("/dev/shm/", R_OK | W_OK | X_OK) == 0)
return "/dev/shm/";
return "";
}
int osal_removefile(const std::string &pathname) {
return unlink(pathname.c_str()) ? errno : MDBX_SUCCESS;
}

461
contrib/db/libmdbx/test/osal-windows.cc Normal file
View file

@ -0,0 +1,461 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
static std::unordered_map<unsigned, HANDLE> events;
static HANDLE hBarrierSemaphore, hBarrierEvent;
static HANDLE hProgressActiveEvent, hProgressPassiveEvent;
static int waitstatus2errcode(DWORD result) {
switch (result) {
case WAIT_OBJECT_0:
return MDBX_SUCCESS;
case WAIT_FAILED:
return GetLastError();
case WAIT_ABANDONED:
return ERROR_ABANDONED_WAIT_0;
case WAIT_IO_COMPLETION:
return ERROR_USER_APC;
case WAIT_TIMEOUT:
return ERROR_TIMEOUT;
default:
return ERROR_UNHANDLED_ERROR;
}
}
void osal_wait4barrier(void) {
DWORD rc = WaitForSingleObject(hBarrierSemaphore, 0);
switch (rc) {
default:
failure_perror("WaitForSingleObject(BarrierSemaphore)",
waitstatus2errcode(rc));
case WAIT_OBJECT_0:
rc = WaitForSingleObject(hBarrierEvent, INFINITE);
if (rc != WAIT_OBJECT_0)
failure_perror("WaitForSingleObject(BarrierEvent)",
waitstatus2errcode(rc));
break;
case WAIT_TIMEOUT:
if (!SetEvent(hBarrierEvent))
failure_perror("SetEvent(BarrierEvent)", GetLastError());
break;
}
}
static HANDLE make_inheritable(HANDLE hHandle) {
assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE);
if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(),
&hHandle, 0, TRUE,
DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS))
failure_perror("DuplicateHandle()", GetLastError());
return hHandle;
}
void osal_setup(const std::vector<actor_config> &actors) {
assert(events.empty());
const size_t n = actors.size() + 1;
events.reserve(n);
for (unsigned i = 0; i < n; ++i) {
HANDLE hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (!hEvent)
failure_perror("CreateEvent()", GetLastError());
hEvent = make_inheritable(hEvent);
log_trace("osal_setup: event %" PRIuPTR " -> %p", i, hEvent);
events[i] = hEvent;
}
hBarrierSemaphore = CreateSemaphore(NULL, 0, (LONG)actors.size(), NULL);
if (!hBarrierSemaphore)
failure_perror("CreateSemaphore(BarrierSemaphore)", GetLastError());
hBarrierSemaphore = make_inheritable(hBarrierSemaphore);
hBarrierEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (!hBarrierEvent)
failure_perror("CreateEvent(BarrierEvent)", GetLastError());
hBarrierEvent = make_inheritable(hBarrierEvent);
hProgressActiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
if (!hProgressActiveEvent)
failure_perror("CreateEvent(ProgressActiveEvent)", GetLastError());
hProgressActiveEvent = make_inheritable(hProgressActiveEvent);
hProgressPassiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
if (!hProgressPassiveEvent)
failure_perror("CreateEvent(ProgressPassiveEvent)", GetLastError());
hProgressPassiveEvent = make_inheritable(hProgressPassiveEvent);
}
void osal_broadcast(unsigned id) {
log_trace("osal_broadcast: event %u", id);
if (!SetEvent(events.at(id)))
failure_perror("SetEvent()", GetLastError());
}
int osal_waitfor(unsigned id) {
log_trace("osal_waitfor: event %u", id);
DWORD rc = WaitForSingleObject(events.at(id), INFINITE);
return waitstatus2errcode(rc);
}
mdbx_pid_t osal_getpid(void) { return GetCurrentProcessId(); }
int osal_delay(unsigned seconds) {
Sleep(seconds * 1000u);
return 0;
}
//-----------------------------------------------------------------------------
const std::string
actor_config::osal_serialize(simple_checksum &checksum) const {
checksum.push(hBarrierSemaphore);
checksum.push(hBarrierEvent);
checksum.push(hProgressActiveEvent);
checksum.push(hProgressPassiveEvent);
HANDLE hWait = INVALID_HANDLE_VALUE;
if (wait4id) {
hWait = events.at(wait4id);
checksum.push(hWait);
}
HANDLE hSignal = INVALID_HANDLE_VALUE;
if (wanna_event4signalling()) {
hSignal = events.at(actor_id);
checksum.push(hSignal);
}
return format("%p.%p.%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait,
hSignal, hProgressActiveEvent, hProgressPassiveEvent);
}
bool actor_config::osal_deserialize(const char *str, const char *end,
simple_checksum &checksum) {
std::string copy(str, end - str);
TRACE(">> osal_deserialize(%s)\n", copy.c_str());
assert(hBarrierSemaphore == 0);
assert(hBarrierEvent == 0);
assert(hProgressActiveEvent == 0);
assert(hProgressPassiveEvent == 0);
assert(events.empty());
HANDLE hWait, hSignal;
if (sscanf_s(copy.c_str(), "%p.%p.%p.%p.%p.%p", &hBarrierSemaphore,
&hBarrierEvent, &hWait, &hSignal, &hProgressActiveEvent,
&hProgressPassiveEvent) != 6) {
TRACE("<< osal_deserialize: failed\n");
return false;
}
checksum.push(hBarrierSemaphore);
checksum.push(hBarrierEvent);
checksum.push(hProgressActiveEvent);
checksum.push(hProgressPassiveEvent);
if (wait4id) {
checksum.push(hWait);
events[wait4id] = hWait;
}
if (wanna_event4signalling()) {
checksum.push(hSignal);
events[actor_id] = hSignal;
}
TRACE("<< osal_deserialize: OK\n");
return true;
}
//-----------------------------------------------------------------------------
typedef std::pair<HANDLE, actor_status> child;
static std::unordered_map<mdbx_pid_t, child> childs;
bool osal_progress_push(bool active) {
if (childs.empty()) {
if (!SetEvent(active ? hProgressActiveEvent : hProgressPassiveEvent))
failure_perror("osal_progress_push: SetEvent(overlord.progress)",
GetLastError());
return true;
}
return false;
}
static void ArgvQuote(std::string &CommandLine, const std::string &Argument,
bool Force = false)
/*++
https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
Routine Description:
This routine appends the given argument to a command line such
that CommandLineToArgvW will return the argument string unchanged.
Arguments in a command line should be separated by spaces; this
function does not add these spaces.
Arguments:
Argument - Supplies the argument to encode.
CommandLine - Supplies the command line to which we append the encoded
argument string.
Force - Supplies an indication of whether we should quote
the argument even if it does not contain any characters that would
ordinarily require quoting.
Return Value:
None.
Environment:
Arbitrary.
--*/
{
//
// Unless we're told otherwise, don't quote unless we actually
// need to do so --- hopefully avoid problems if programs won't
// parse quotes properly
//
if (Force == false && Argument.empty() == false &&
Argument.find_first_of(" \t\n\v\"") == Argument.npos) {
CommandLine.append(Argument);
} else {
CommandLine.push_back('"');
for (auto It = Argument.begin();; ++It) {
unsigned NumberBackslashes = 0;
while (It != Argument.end() && *It == '\\') {
++It;
++NumberBackslashes;
}
if (It == Argument.end()) {
//
// Escape all backslashes, but let the terminating
// double quotation mark we add below be interpreted
// as a metacharacter.
//
CommandLine.append(NumberBackslashes * 2, '\\');
break;
} else if (*It == L'"') {
//
// Escape all backslashes and the following
// double quotation mark.
//
CommandLine.append(NumberBackslashes * 2 + 1, '\\');
CommandLine.push_back(*It);
} else {
//
// Backslashes aren't special here.
//
CommandLine.append(NumberBackslashes, '\\');
CommandLine.push_back(*It);
}
}
CommandLine.push_back('"');
}
}
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
if (childs.size() == MAXIMUM_WAIT_OBJECTS)
failure("Could't manage more that %u actors on Windows\n",
MAXIMUM_WAIT_OBJECTS);
_flushall();
STARTUPINFOA StartupInfo;
GetStartupInfoA(&StartupInfo);
char exename[_MAX_PATH + 1];
DWORD exename_size = sizeof(exename);
if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename,
&exename_size))
failure_perror("QueryFullProcessImageName()", GetLastError());
if (exename[1] != ':') {
exename_size = GetModuleFileName(NULL, exename, sizeof(exename));
if (exename_size >= sizeof(exename))
return ERROR_BAD_LENGTH;
}
std::string cmdline = "$ ";
ArgvQuote(cmdline, thunk_param(config));
if (cmdline.size() >= 32767)
return ERROR_BAD_LENGTH;
PROCESS_INFORMATION ProcessInformation;
if (!CreateProcessA(exename, const_cast<char *>(cmdline.c_str()),
NULL, // Retuned process handle is not inheritable.
NULL, // Retuned thread handle is not inheritable.
TRUE, // Child inherits all inheritable handles.
NORMAL_PRIORITY_CLASS | INHERIT_PARENT_AFFINITY,
NULL, // Inherit the parent's environment.
NULL, // Inherit the parent's current directory.
&StartupInfo, &ProcessInformation))
failure_perror(exename, GetLastError());
CloseHandle(ProcessInformation.hThread);
pid = ProcessInformation.dwProcessId;
childs[pid] = std::make_pair(ProcessInformation.hProcess, as_running);
return 0;
}
actor_status osal_actor_info(const mdbx_pid_t pid) {
actor_status status = childs.at(pid).second;
if (status > as_running)
return status;
DWORD ExitCode;
if (!GetExitCodeProcess(childs.at(pid).first, &ExitCode))
failure_perror("GetExitCodeProcess()", GetLastError());
switch (ExitCode) {
case STILL_ACTIVE:
return as_running;
case EXIT_SUCCESS:
status = as_successful;
break;
case EXCEPTION_BREAKPOINT:
case EXCEPTION_SINGLE_STEP:
status = as_debuging;
break;
case STATUS_CONTROL_C_EXIT:
status = as_killed;
break;
case EXCEPTION_ACCESS_VIOLATION:
case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
case EXCEPTION_DATATYPE_MISALIGNMENT:
case EXCEPTION_STACK_OVERFLOW:
case EXCEPTION_INVALID_DISPOSITION:
case EXCEPTION_ILLEGAL_INSTRUCTION:
case EXCEPTION_NONCONTINUABLE_EXCEPTION:
status = as_coredump;
break;
default:
status = as_failed;
break;
}
childs.at(pid).second = status;
return status;
}
void osal_killall_actors(void) {
for (auto &pair : childs)
TerminateProcess(pair.second.first, STATUS_CONTROL_C_EXIT);
}
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) {
std::vector<HANDLE> handles;
handles.reserve(childs.size() + 2);
handles.push_back(hProgressActiveEvent);
handles.push_back(hProgressPassiveEvent);
for (const auto &pair : childs)
if (pair.second.second <= as_running)
handles.push_back(pair.second.first);
while (true) {
DWORD rc =
MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0],
(timeout > 60) ? 60 * 1000 : timeout * 1000,
QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0);
if (rc == WAIT_OBJECT_0) {
logging::progress_canary(true);
continue;
}
if (rc == WAIT_OBJECT_0 + 1) {
logging::progress_canary(false);
continue;
}
if (rc >= WAIT_OBJECT_0 + 2 && rc < WAIT_OBJECT_0 + handles.size()) {
pid = 0;
for (const auto &pair : childs)
if (pair.second.first == handles[rc - WAIT_OBJECT_0]) {
pid = pair.first;
break;
}
return 0;
}
if (rc == WAIT_TIMEOUT) {
pid = 0;
return 0;
}
return waitstatus2errcode(rc);
}
}
void osal_yield(void) { SwitchToThread(); }
void osal_udelay(unsigned us) {
chrono::time until, now = chrono::now_motonic();
until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint;
static unsigned threshold_us;
if (threshold_us == 0) {
#if 1
unsigned timeslice_ms = 1;
while (timeBeginPeriod(timeslice_ms) == TIMERR_NOCANDO)
++timeslice_ms;
threshold_us = timeslice_ms * 1500u;
#else
ULONGLONG InterruptTimePrecise_100ns;
QueryInterruptTimePrecise(&InterruptTimePrecise_100ns);
threshold_us = InterruptTimePrecise_100ns / 5;
#endif
assert(threshold_us > 0);
}
do {
if (us > threshold_us && us > 1000) {
DWORD rc = SleepEx(us / 1000, TRUE);
if (rc)
failure_perror("SleepEx()", waitstatus2errcode(rc));
us = 0;
}
YieldProcessor();
now = chrono::now_motonic();
} while (now.fixedpoint < until.fixedpoint);
}
bool osal_istty(int fd) { return _isatty(fd) != 0; }
std::string osal_tempdir(void) {
char buf[MAX_PATH + 1];
DWORD len = GetTempPathA(sizeof(buf), buf);
return std::string(buf, len);
}
int osal_removefile(const std::string &pathname) {
return DeleteFileA(pathname.c_str()) ? MDBX_SUCCESS : GetLastError();
}

49
contrib/db/libmdbx/test/osal.h Normal file
View file

@ -0,0 +1,49 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
void osal_setup(const std::vector<actor_config> &actors);
void osal_broadcast(unsigned id);
int osal_waitfor(unsigned id);
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid);
actor_status osal_actor_info(const mdbx_pid_t pid);
void osal_killall_actors(void);
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout);
void osal_wait4barrier(void);
bool osal_progress_push(bool active);
mdbx_pid_t osal_getpid(void);
int osal_delay(unsigned seconds);
void osal_udelay(unsigned us);
void osal_yield(void);
bool osal_istty(int fd);
std::string osal_tempdir(void);
int osal_removefile(const std::string &pathname);
#ifdef _MSC_VER
#ifndef STDIN_FILENO
#define STDIN_FILENO _fileno(stdin)
#endif
#ifndef STDOUT_FILENO
#define STDOUT_FILENO _fileno(stdout)
#endif
#ifndef STDERR_FILENO
#define STDERR_FILENO _fileno(stderr)
#endif
#endif /* _MSC_VER */

5
contrib/db/libmdbx/test/pcrf/CMakeLists.txt Normal file
View file

@ -0,0 +1,5 @@
set(TARGET pcrf_test)
add_executable(${TARGET} pcrf_test.c)
target_include_directories(${TARGET} PRIVATE "${PROJECT_SOURCE_DIR}")
target_link_libraries(${TARGET} mdbx)

2
contrib/db/libmdbx/test/pcrf/README.md Normal file
View file

@ -0,0 +1,2 @@
PCRF Session DB emulation test

413
contrib/db/libmdbx/test/pcrf/pcrf_test.c Normal file
View file

@ -0,0 +1,413 @@
/*
* Copyright 2016-2019 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015 Vladimir Romanov
* <https://www.linkedin.com/in/vladimirromanov>, Yota Lab.
*
* This file is part of libmdbx.
*
* ReOpenMDBX is free software; you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* ReOpenMDBX is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/stat.h>
#include <sys/time.h>
#include "mdbx.h"
#include <assert.h>
#include <inttypes.h>
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define IP_PRINTF_ARG_HOST(addr) \
(int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \
(int)((addr)&0xff)
char opt_db_path[PATH_MAX] = "./mdbx_bench2";
static MDBX_env *env;
#define REC_COUNT 10240000
int64_t ids[REC_COUNT * 10];
int32_t ids_count = 0;
int64_t mdbx_add_count = 0;
int64_t mdbx_del_count = 0;
uint64_t mdbx_add_time = 0;
uint64_t mdbx_del_time = 0;
int64_t obj_id = 0;
int64_t mdbx_data_size = 0;
int64_t mdbx_key_size = 0;
typedef struct {
char session_id1[100];
char session_id2[100];
char ip[20];
uint8_t fill[100];
} session_data_t;
typedef struct {
int64_t obj_id;
int8_t event_type;
} __attribute__((__packed__)) event_data_t;
static void add_id_to_pool(int64_t id) {
ids[ids_count] = id;
ids_count++;
}
static inline int64_t getClockUs(void) {
struct timespec val;
#ifdef CYGWIN
clock_gettime(CLOCK_REALTIME, &val);
#else
clock_gettime(CLOCK_MONOTONIC, &val);
#endif
return val.tv_sec * ((int64_t)1000000) + val.tv_nsec / 1000;
}
static int64_t get_id_from_pool() {
if (ids_count == 0) {
return -1;
}
int32_t index = rand() % ids_count;
int64_t id = ids[index];
ids[index] = ids[ids_count - 1];
ids_count--;
return id;
}
#define MDBX_CHECK(x) \
do { \
const int rc = (x); \
if (rc != MDBX_SUCCESS) { \
printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, \
__FILE__, __LINE__); \
exit(EXIT_FAILURE); \
} \
} while (0)
static void db_connect() {
MDBX_dbi dbi_session;
MDBX_dbi dbi_session_id;
MDBX_dbi dbi_event;
MDBX_dbi dbi_ip;
MDBX_CHECK(mdbx_env_create(&env));
MDBX_CHECK(
mdbx_env_set_mapsize(env, REC_COUNT * sizeof(session_data_t) * 10));
MDBX_CHECK(mdbx_env_set_maxdbs(env, 30));
MDBX_CHECK(mdbx_env_open(env, opt_db_path,
MDBX_CREATE | MDBX_WRITEMAP | MDBX_MAPASYNC |
MDBX_NOSYNC | MDBX_LIFORECLAIM,
0664));
MDBX_txn *txn;
// transaction init
MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
// transaction commit
MDBX_CHECK(mdbx_txn_commit(txn));
printf("Connection open\n");
}
static void create_record(uint64_t record_id) {
MDBX_dbi dbi_session;
MDBX_dbi dbi_session_id;
MDBX_dbi dbi_event;
MDBX_dbi dbi_ip;
event_data_t event;
MDBX_txn *txn;
session_data_t data;
// transaction init
snprintf(data.session_id1, sizeof(data.session_id1),
"prefix%02u_%02u.fill.fill.fill.fill.fill.fill;%" PRIu64,
(unsigned)(record_id % 3) + 1, (unsigned)(record_id % 9) + 1,
record_id);
snprintf(data.session_id2, sizeof(data.session_id2),
"dprefix%" PRIu64 ";%" PRIu64 ".fill.fill.;suffix", record_id,
(record_id + UINT64_C(1442695040888963407)) %
UINT64_C(6364136223846793005));
snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d",
IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF));
event.obj_id = record_id;
event.event_type = 1;
MDBX_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)};
MDBX_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)};
MDBX_val _ip_rec = {data.ip, strlen(data.ip)};
MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDBX_val _data_rec = {&data, offsetof(session_data_t, fill) +
(rand() % sizeof(data.fill))};
MDBX_val _event_rec = {&event, sizeof(event)};
uint64_t start = getClockUs();
MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
MDBX_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec,
MDBX_NOOVERWRITE | MDBX_NODUPDATA));
MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec,
MDBX_NOOVERWRITE | MDBX_NODUPDATA));
MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec,
MDBX_NOOVERWRITE | MDBX_NODUPDATA));
MDBX_CHECK(mdbx_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0));
MDBX_CHECK(mdbx_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0));
MDBX_CHECK(mdbx_txn_commit(txn));
mdbx_data_size += (_data_rec.iov_len + _obj_id_rec.iov_len * 4);
mdbx_key_size +=
(_obj_id_rec.iov_len + _session_id1_rec.iov_len +
_session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len);
// transaction commit
mdbx_add_count++;
mdbx_add_time += (getClockUs() - start);
}
static void delete_record(int64_t record_id) {
MDBX_dbi dbi_session;
MDBX_dbi dbi_session_id;
MDBX_dbi dbi_event;
MDBX_dbi dbi_ip;
event_data_t event;
MDBX_txn *txn;
// transaction init
uint64_t start = getClockUs();
MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
// put data
MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDBX_val _data_rec;
// get data
MDBX_CHECK(mdbx_get(txn, dbi_session, &_obj_id_rec, &_data_rec));
session_data_t *data = (session_data_t *)_data_rec.iov_base;
MDBX_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)};
MDBX_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)};
MDBX_val _ip_rec = {data->ip, strlen(data->ip)};
MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id1_rec, NULL));
MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id2_rec, NULL));
MDBX_CHECK(mdbx_del(txn, dbi_ip, &_ip_rec, NULL));
event.obj_id = record_id;
event.event_type = 1;
MDBX_val _event_rec = {&event, sizeof(event)};
MDBX_CHECK(mdbx_del(txn, dbi_event, &_event_rec, NULL));
MDBX_CHECK(mdbx_del(txn, dbi_session, &_obj_id_rec, NULL));
mdbx_data_size -= (_data_rec.iov_len + _obj_id_rec.iov_len * 4);
mdbx_key_size -=
(_obj_id_rec.iov_len + _session_id1_rec.iov_len +
_session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len);
// transaction commit
MDBX_CHECK(mdbx_txn_commit(txn));
mdbx_del_count++;
mdbx_del_time += (getClockUs() - start);
}
static void db_disconnect() {
mdbx_env_close(env);
printf("Connection closed\n");
}
static void get_db_stat(const char *db, int64_t *ms_branch_pages,
int64_t *ms_leaf_pages) {
MDBX_txn *txn;
MDBX_stat stat;
MDBX_dbi dbi;
MDBX_CHECK(mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn));
MDBX_CHECK(mdbx_dbi_open(txn, db, MDBX_CREATE, &dbi));
MDBX_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat)));
mdbx_txn_abort(txn);
printf("%15s | %15" PRIu64 " | %5u | %10" PRIu64 " | %10" PRIu64
" | %11" PRIu64 " |\n",
db, stat.ms_branch_pages, stat.ms_depth, stat.ms_entries,
stat.ms_leaf_pages, stat.ms_overflow_pages);
(*ms_branch_pages) += stat.ms_branch_pages;
(*ms_leaf_pages) += stat.ms_leaf_pages;
}
static void periodic_stat(void) {
int64_t ms_branch_pages = 0;
int64_t ms_leaf_pages = 0;
MDBX_stat mst;
MDBX_envinfo mei;
MDBX_CHECK(mdbx_env_stat(env, &mst, sizeof(mst)));
MDBX_CHECK(mdbx_env_info(env, &mei, sizeof(mei)));
printf("Environment Info\n");
printf(" Pagesize: %u\n", mst.ms_psize);
if (mei.mi_geo.lower != mei.mi_geo.upper) {
printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
"/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
"/-%" PRIu64 ")\n",
mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize,
mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize,
mei.mi_geo.shrink / mst.ms_psize);
printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
} else {
printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
}
printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize);
printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1);
printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
mei.mi_latter_reader_txnid,
mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
printf(" Max readers: %u\n", mei.mi_maxreaders);
printf(" Number of readers used: %u\n", mei.mi_numreaders);
printf(" Name | ms_branch_pages | depth | entries | leaf_pages "
"| overf_pages |\n");
get_db_stat("session", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("event", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages);
printf("%15s | %15" PRIu64 " | %5s | %10s | %10" PRIu64 " | %11s |\n", "",
ms_branch_pages, "", "", ms_leaf_pages, "");
static int64_t prev_add_count;
static int64_t prev_del_count;
static uint64_t prev_add_time;
static uint64_t prev_del_time;
static int64_t t = -1;
if (t > 0) {
int64_t delta = (getClockUs() - t);
printf("CPS: add %" PRIu64 ", delete %" PRIu64
", items processed - %" PRIu64 "K data=%" PRIu64 "K key=%" PRIu64
"K\n",
(mdbx_add_count - prev_add_count) * 1000000 / delta,
(mdbx_del_count - prev_del_count) * 1000000 / delta, obj_id / 1024,
mdbx_data_size / 1024, mdbx_key_size / 1024);
printf("usage data=%" PRIu64 "%%",
((mdbx_data_size + mdbx_key_size) * 100) /
((ms_leaf_pages + ms_branch_pages) * 4096));
if (prev_add_time != mdbx_add_time) {
printf(" Add : %" PRIu64 " c/s", (mdbx_add_count - prev_add_count) *
1000000 /
(mdbx_add_time - prev_add_time));
}
if (prev_del_time != mdbx_del_time) {
printf(" Del : %" PRIu64 " c/s", (mdbx_del_count - prev_del_count) *
1000000 /
(mdbx_del_time - prev_del_time));
}
if (mdbx_add_time) {
printf(" tAdd : %" PRIu64 " c/s",
mdbx_add_count * 1000000 / mdbx_add_time);
}
if (mdbx_del_time) {
printf(" tDel : %" PRIu64 " c/s",
mdbx_del_count * 1000000 / mdbx_del_time);
}
puts("");
}
t = getClockUs();
prev_add_count = mdbx_add_count;
prev_del_count = mdbx_del_count;
prev_add_time = mdbx_add_time;
prev_del_time = mdbx_del_time;
}
// static void periodic_add_rec() {
// for (int i = 0; i < 10240; i++) {
// if (ids_count <= REC_COUNT) {
// int64_t id = obj_id++;
// create_record(id);
// add_id_to_pool(id);
// }
// if (ids_count > REC_COUNT) {
// int64_t id = get_id_from_pool();
// delete_record(id);
// }
// }
// periodic_stat();
//}
int main(int argc, char **argv) {
(void)argc;
(void)argv;
char filename[PATH_MAX];
int i;
mkdir(opt_db_path, 0775);
strcpy(filename, opt_db_path);
strcat(filename, "/mdbx.dat");
remove(filename);
strcpy(filename, opt_db_path);
strcat(filename, "/mdbx.lck");
remove(filename);
puts("Open DB...");
db_connect();
puts("Create data...");
int64_t t = getClockUs();
for (i = 0; i < REC_COUNT; i++) {
int64_t id = obj_id++;
create_record(id);
add_id_to_pool(id);
if (i % 1000 == 0) {
int64_t now = getClockUs();
if ((now - t) > 1000000L) {
periodic_stat();
t = now;
}
}
}
periodic_stat();
while (1) {
int i;
for (i = 0; i < 1000; i++) {
int64_t id = obj_id++;
create_record(id);
add_id_to_pool(id);
id = get_id_from_pool();
delete_record(id);
}
// for (i = 0; i < 50; i++) {
// int64_t id = obj_id++;
// create_record(id);
// add_id_to_pool(id);
// }
// int64_t id = obj_id++;
// create_record(id);
// add_id_to_pool(id);
int64_t now = getClockUs();
if ((now - t) > 10000000L) {
periodic_stat();
t = now;
}
}
db_disconnect();
return 0;
}

736
contrib/db/libmdbx/test/test.cc Normal file
View file

@ -0,0 +1,736 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
const char *testcase2str(const actor_testcase testcase) {
switch (testcase) {
default:
assert(false);
return "?!";
case ac_none:
return "none";
case ac_hill:
return "hill";
case ac_deadread:
return "deadread";
case ac_deadwrite:
return "deadwrite";
case ac_jitter:
return "jitter";
case ac_try:
return "try";
case ac_copy:
return "copy";
case ac_append:
return "append";
case ac_ttl:
return "ttl";
case ac_nested:
return "nested";
}
}
const char *status2str(actor_status status) {
switch (status) {
default:
assert(false);
return "?!";
case as_debuging:
return "debuging";
case as_running:
return "running";
case as_successful:
return "successful";
case as_killed:
return "killed";
case as_failed:
return "failed";
case as_coredump:
return "coredump";
}
}
const char *keygencase2str(const keygen_case keycase) {
switch (keycase) {
default:
assert(false);
return "?!";
case kc_random:
return "random";
case kc_dashes:
return "dashes";
case kc_custom:
return "custom";
}
}
//-----------------------------------------------------------------------------
int testcase::oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid,
uint64_t txn, unsigned gap, size_t space,
int retry) {
testcase *self = (testcase *)mdbx_env_get_userctx(env);
if (retry == 0)
log_notice("oom_callback: waitfor pid %u, thread %" PRIuPTR
", txn #%" PRIu64 ", gap %d, scape %zu",
pid, (size_t)tid, txn, gap, space);
if (self->should_continue(true)) {
osal_yield();
if (retry > 0)
osal_udelay(retry * 100);
return 0 /* always retry */;
}
return -1;
}
void testcase::db_prepare() {
log_trace(">> db_prepare");
assert(!db_guard);
MDBX_env *env = nullptr;
int rc = mdbx_env_create(&env);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_create()", rc);
assert(env != nullptr);
db_guard.reset(env);
rc = mdbx_env_set_userctx(env, this);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_userctx()", rc);
rc = mdbx_env_set_maxreaders(env, config.params.max_readers);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_maxreaders()", rc);
rc = mdbx_env_set_maxdbs(env, config.params.max_tables);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_maxdbs()", rc);
rc = mdbx_env_set_oomfunc(env, testcase::oom_callback);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_oomfunc()", rc);
rc = mdbx_env_set_geometry(
env, config.params.size_lower, config.params.size_now,
config.params.size_upper, config.params.growth_step,
config.params.shrink_threshold, config.params.pagesize);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_mapsize()", rc);
log_trace("<< db_prepare");
}
void testcase::db_open() {
log_trace(">> db_open");
if (!db_guard)
db_prepare();
jitter_delay(true);
int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(),
(unsigned)config.params.mode_flags, 0640);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_open()", rc);
log_trace("<< db_open");
}
void testcase::db_close() {
log_trace(">> db_close");
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
log_trace("<< db_close");
}
void testcase::txn_begin(bool readonly, unsigned flags) {
assert((flags & MDBX_RDONLY) == 0);
log_trace(">> txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write",
flags);
assert(!txn_guard);
MDBX_txn *txn = nullptr;
int rc = mdbx_txn_begin(db_guard.get(), nullptr,
readonly ? flags | MDBX_RDONLY : flags, &txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_begin()", rc);
txn_guard.reset(txn);
log_trace("<< txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write",
flags);
}
int testcase::breakable_commit() {
int rc = MDBX_SUCCESS;
log_trace(">> txn_commit");
assert(txn_guard);
MDBX_txn *txn = txn_guard.release();
txn_inject_writefault(txn);
int err = mdbx_txn_commit(txn);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
rc = err;
err = mdbx_txn_abort(txn);
if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH &&
err != MDBX_BAD_TXN))
failure_perror("mdbx_txn_abort()", err);
} else
failure_perror("mdbx_txn_commit()", err);
}
log_trace("<< txn_commit: %s", rc ? "failed" : "Ok");
return rc;
}
unsigned testcase::txn_underutilization_x256(MDBX_txn *txn) const {
if (txn) {
MDBX_txn_info info;
int err = mdbx_txn_info(txn, &info, false);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_txn_info()", err);
const size_t left = size_t(info.txn_space_leftover);
const size_t total =
size_t(info.txn_space_leftover) + size_t(info.txn_space_dirty);
return (unsigned)(left / (total >> 8));
}
return 0;
}
void testcase::txn_end(bool abort) {
log_trace(">> txn_end(%s)", abort ? "abort" : "commit");
assert(txn_guard);
MDBX_txn *txn = txn_guard.release();
if (abort) {
int err = mdbx_txn_abort(txn);
if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH &&
err != MDBX_BAD_TXN))
failure_perror("mdbx_txn_abort()", err);
} else {
txn_inject_writefault(txn);
int err = mdbx_txn_commit(txn);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_txn_commit()", err);
}
log_trace("<< txn_end(%s)", abort ? "abort" : "commit");
}
void testcase::cursor_open(MDBX_dbi handle) {
log_trace(">> cursor_open(%u)", handle);
assert(!cursor_guard);
assert(txn_guard);
MDBX_cursor *cursor = nullptr;
int rc = mdbx_cursor_open(txn_guard.get(), handle, &cursor);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_cursor_open()", rc);
cursor_guard.reset(cursor);
log_trace("<< cursor_open(%u)", handle);
}
void testcase::cursor_close() {
log_trace(">> cursor_close()");
assert(cursor_guard);
MDBX_cursor *cursor = cursor_guard.release();
mdbx_cursor_close(cursor);
log_trace("<< cursor_close()");
}
int testcase::breakable_restart() {
int rc = MDBX_SUCCESS;
if (txn_guard)
rc = breakable_commit();
if (cursor_guard)
cursor_close();
txn_begin(false, 0);
return rc;
}
void testcase::txn_restart(bool abort, bool readonly, unsigned flags) {
if (txn_guard)
txn_end(abort);
if (cursor_guard)
cursor_close();
txn_begin(readonly, flags);
}
void testcase::txn_inject_writefault(void) {
if (txn_guard)
txn_inject_writefault(txn_guard.get());
}
void testcase::txn_inject_writefault(MDBX_txn *txn) {
if (config.params.inject_writefaultn && txn) {
if (config.params.inject_writefaultn <= nops_completed &&
(mdbx_txn_flags(txn) & MDBX_RDONLY) == 0) {
log_verbose(
"== txn_inject_writefault(): got %u nops or more, inject FAULT",
config.params.inject_writefaultn);
log_flush();
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
TerminateProcess(GetCurrentProcess(), 42);
#else
raise(SIGKILL);
#endif
}
}
}
bool testcase::wait4start() {
if (config.wait4id) {
log_trace(">> wait4start(%u)", config.wait4id);
assert(!global::singlemode);
int rc = osal_waitfor(config.wait4id);
if (rc) {
log_trace("<< wait4start(%u), failed %s", config.wait4id,
test_strerror(rc));
return false;
}
} else {
log_trace("== skip wait4start: not needed");
}
if (config.params.delaystart) {
int rc = osal_delay(config.params.delaystart);
if (rc) {
log_trace("<< delay(%u), failed %s", config.params.delaystart,
test_strerror(rc));
return false;
}
} else {
log_trace("== skip delay: not needed");
}
return true;
}
void testcase::kick_progress(bool active) const {
if (!global::config::progress_indicator)
return;
logging::progress_canary(active);
}
void testcase::report(size_t nops_done) {
assert(nops_done > 0);
if (!nops_done)
return;
nops_completed += nops_done;
log_debug("== complete +%" PRIuPTR " iteration, total %" PRIuPTR " done",
nops_done, nops_completed);
kick_progress(true);
if (config.signal_nops && !signalled &&
config.signal_nops <= nops_completed) {
log_trace(">> signal(n-ops %" PRIuPTR ")", nops_completed);
if (!global::singlemode)
osal_broadcast(config.actor_id);
signalled = true;
log_trace("<< signal(n-ops %" PRIuPTR ")", nops_completed);
}
}
void testcase::signal() {
if (!signalled) {
log_trace(">> signal(forced)");
if (!global::singlemode)
osal_broadcast(config.actor_id);
signalled = true;
log_trace("<< signal(forced)");
}
}
bool testcase::setup() {
db_prepare();
if (!wait4start())
return false;
start_timestamp = chrono::now_motonic();
nops_completed = 0;
return true;
}
bool testcase::teardown() {
log_trace(">> testcase::teardown");
signal();
db_close();
log_trace("<< testcase::teardown");
return true;
}
bool testcase::should_continue(bool check_timeout_only) const {
bool result = true;
if (config.params.test_duration) {
chrono::time since;
since.fixedpoint =
chrono::now_motonic().fixedpoint - start_timestamp.fixedpoint;
if (since.seconds() >= config.params.test_duration)
result = false;
}
if (!check_timeout_only && config.params.test_nops &&
nops_completed >= config.params.test_nops)
result = false;
if (result)
kick_progress(false);
return result;
}
void testcase::fetch_canary() {
mdbx_canary canary_now;
log_trace(">> fetch_canary");
int rc = mdbx_canary_get(txn_guard.get(), &canary_now);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_canary_get()", rc);
if (canary_now.v < last.canary.v)
failure("fetch_canary: %" PRIu64 "(canary-now.v) < %" PRIu64
"(canary-last.v)",
canary_now.v, last.canary.v);
if (canary_now.y < last.canary.y)
failure("fetch_canary: %" PRIu64 "(canary-now.y) < %" PRIu64
"(canary-last.y)",
canary_now.y, last.canary.y);
last.canary = canary_now;
log_trace("<< fetch_canary: db-sequence %" PRIu64
", db-sequence.txnid %" PRIu64,
last.canary.y, last.canary.v);
}
void testcase::update_canary(uint64_t increment) {
mdbx_canary canary_now = last.canary;
log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y,
increment);
canary_now.y += increment;
int rc = mdbx_canary_put(txn_guard.get(), &canary_now);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_canary_put()", rc);
log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y);
}
int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &handle) {
db_open();
int err, retry_left = 42;
for (;;) {
txn_begin(false);
handle = db_table_open(true);
db_table_clear(handle);
err = breakable_commit();
if (likely(err == MDBX_SUCCESS)) {
txn_begin(false);
return MDBX_SUCCESS;
}
if (--retry_left == 0)
break;
jitter_delay(true);
}
log_notice("db_begin_table_create_open_clean: bailout due '%s'",
mdbx_strerror(err));
return err;
}
MDBX_dbi testcase::db_table_open(bool create) {
log_trace(">> testcase::db_table_create");
char tablename_buf[16];
const char *tablename = nullptr;
if (config.space_id) {
int rc = snprintf(tablename_buf, sizeof(tablename_buf), "TBL%04u",
config.space_id);
if (rc < 4 || rc >= (int)sizeof(tablename_buf) - 1)
failure("snprintf(tablename): %d", rc);
tablename = tablename_buf;
}
log_debug("use %s table", tablename ? tablename : "MAINDB");
MDBX_dbi handle = 0;
int rc = mdbx_dbi_open(txn_guard.get(), tablename,
(create ? MDBX_CREATE : 0) | config.params.table_flags,
&handle);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_dbi_open()", rc);
log_trace("<< testcase::db_table_create, handle %u", handle);
return handle;
}
void testcase::db_table_drop(MDBX_dbi handle) {
log_trace(">> testcase::db_table_drop, handle %u", handle);
if (config.params.drop_table) {
int rc = mdbx_drop(txn_guard.get(), handle, true);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_drop(delete=true)", rc);
log_trace("<< testcase::db_table_drop");
} else {
log_trace("<< testcase::db_table_drop: not needed");
}
}
void testcase::db_table_clear(MDBX_dbi handle, MDBX_txn *txn) {
log_trace(">> testcase::db_table_clear, handle %u", handle);
int rc = mdbx_drop(txn ? txn : txn_guard.get(), handle, false);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_drop(delete=false)", rc);
log_trace("<< testcase::db_table_clear");
}
void testcase::db_table_close(MDBX_dbi handle) {
log_trace(">> testcase::db_table_close, handle %u", handle);
assert(!txn_guard);
int rc = mdbx_dbi_close(db_guard.get(), handle);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_dbi_close()", rc);
log_trace("<< testcase::db_table_close");
}
void testcase::checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check,
MDBX_val expected_valued) {
MDBX_val actual_value = expected_valued;
int rc = mdbx_get_nearest(txn_guard.get(), handle, &key2check, &actual_value);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror(step, rc);
if (!is_samedata(&actual_value, &expected_valued))
failure("%s data mismatch", step);
}
//-----------------------------------------------------------------------------
bool test_execute(const actor_config &config_const) {
const mdbx_pid_t pid = osal_getpid();
actor_config config = config_const;
if (global::singlemode) {
logging::setup(format("single_%s", testcase2str(config.testcase)));
} else {
logging::setup((logging::loglevel)config.params.loglevel,
format("child_%u.%u", config.actor_id, config.space_id));
log_trace(">> wait4barrier");
osal_wait4barrier();
log_trace("<< wait4barrier");
}
try {
std::unique_ptr<testcase> test;
switch (config.testcase) {
case ac_hill:
test.reset(new testcase_hill(config, pid));
break;
case ac_deadread:
test.reset(new testcase_deadread(config, pid));
break;
case ac_deadwrite:
test.reset(new testcase_deadwrite(config, pid));
break;
case ac_jitter:
test.reset(new testcase_jitter(config, pid));
break;
case ac_try:
test.reset(new testcase_try(config, pid));
break;
case ac_copy:
test.reset(new testcase_copy(config, pid));
break;
case ac_append:
test.reset(new testcase_append(config, pid));
break;
case ac_ttl:
test.reset(new testcase_ttl(config, pid));
break;
case ac_nested:
test.reset(new testcase_nested(config, pid));
break;
default:
test.reset(new testcase(config, pid));
break;
}
size_t iter = 0;
do {
iter++;
if (!test->setup()) {
log_notice("test setup failed");
return false;
}
if (!test->run()) {
log_notice("test failed");
return false;
}
if (!test->teardown()) {
log_notice("test teardown failed");
return false;
}
if (config.params.nrepeat == 1)
log_verbose("test successed");
else {
if (config.params.nrepeat)
log_verbose("test successed (iteration %zi of %zi)", iter,
size_t(config.params.nrepeat));
else
log_verbose("test successed (iteration %zi)", iter);
config.params.keygen.seed += INT32_C(0xA4F4D37B);
}
} while (config.params.nrepeat == 0 || iter < config.params.nrepeat);
return true;
} catch (const std::exception &pipets) {
failure("***** Exception: %s *****", pipets.what());
return false;
}
}
//-----------------------------------------------------------------------------
int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata,
unsigned flags) {
int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags);
if (err == MDBX_SUCCESS && config.params.speculum) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const bool inserted = speculum.emplace(S_key, S_data).second;
assert(inserted);
(void)inserted;
}
return err;
}
int testcase::replace(const keygen::buffer &akey,
const keygen::buffer &new_data,
const keygen::buffer &old_data, unsigned flags) {
if (config.params.speculum) {
const auto S_key = S(akey);
const auto S_old = S(old_data);
const auto S_new = S(new_data);
const auto removed = speculum.erase(SET::key_type(S_key, S_old));
assert(removed == 1);
(void)removed;
const bool inserted = speculum.emplace(S_key, S_new).second;
assert(inserted);
(void)inserted;
}
return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value,
&old_data->value, flags);
}
int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) {
if (config.params.speculum) {
const auto S_key = S(akey);
const auto S_data = S(adata);
const auto removed = speculum.erase(SET::key_type(S_key, S_data));
assert(removed == 1);
(void)removed;
}
return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value);
}
bool testcase::speculum_verify() const {
if (!config.params.speculum)
return true;
char dump_key[128], dump_value[128];
char dump_mkey[128], dump_mvalue[128];
MDBX_cursor *cursor;
int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
if (err != MDBX_SUCCESS)
failure_perror("mdbx_cursor_open()", err);
bool rc = true;
MDBX_val akey, avalue;
MDBX_val mkey, mvalue;
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST);
assert(std::is_sorted(speculum.cbegin(), speculum.cend(), ItemCompare(this)));
auto it = speculum.cbegin();
while (true) {
if (err != MDBX_SUCCESS) {
akey.iov_len = avalue.iov_len = 0;
akey.iov_base = avalue.iov_base = nullptr;
}
const auto S_key = S(akey);
const auto S_data = S(avalue);
if (it != speculum.cend()) {
mkey.iov_base = (void *)it->first.c_str();
mkey.iov_len = it->first.size();
mvalue.iov_base = (void *)it->second.c_str();
mvalue.iov_len = it->second.size();
}
if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first &&
S_data == it->second) {
++it;
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
} else if (err == MDBX_SUCCESS &&
(it == speculum.cend() || S_key < it->first ||
(S_key == it->first && S_data < it->second))) {
if (it != speculum.cend()) {
log_error("extra pair: db{%s, %s} < mi{%s, %s}",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
} else {
log_error("extra pair: db{%s, %s} < mi.END",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)));
}
err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT);
rc = false;
} else if (it != speculum.cend() &&
(err == MDBX_NOTFOUND || S_key > it->first ||
(S_key == it->first && S_data > it->second))) {
if (err == MDBX_NOTFOUND) {
log_error("lost pair: db.END > mi{%s, %s}",
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
} else {
log_error("lost pair: db{%s, %s} > mi{%s, %s}",
mdbx_dump_val(&akey, dump_key, sizeof(dump_key)),
mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)),
mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)),
mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue)));
}
++it;
rc = false;
} else if (err == MDBX_NOTFOUND && it == speculum.cend()) {
break;
} else if (err != MDBX_SUCCESS) {
failure_perror("mdbx_cursor_get()", err);
} else {
assert(!"WTF?");
}
}
mdbx_cursor_close(cursor);
return rc;
}

314
contrib/db/libmdbx/test/test.h Normal file
View file

@ -0,0 +1,314 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "chrono.h"
#include "config.h"
#include "keygen.h"
#include "log.h"
#include "osal.h"
#include "utils.h"
#include <deque>
#include <set>
#include <stack>
#include <tuple>
#ifndef HAVE_cxx17_std_string_view
#if __cplusplus >= 201703L && __has_include(<string_view>)
#include <string_view>
#define HAVE_cxx17_std_string_view 1
#else
#define HAVE_cxx17_std_string_view 0
#endif
#endif /* HAVE_cxx17_std_string_view */
#if HAVE_cxx17_std_string_view
#include <string_view>
#endif
bool test_execute(const actor_config &config);
std::string thunk_param(const actor_config &config);
void testcase_setup(const char *casename, actor_params &params,
unsigned &last_space_id);
void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
const char *space_id_cstr, const actor_params &params);
void keycase_setup(const char *casename, actor_params &params);
namespace global {
extern const char thunk_param_prefix[];
extern std::vector<actor_config> actors;
extern std::unordered_map<unsigned, actor_config *> events;
extern std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
extern std::set<std::string> databases;
extern unsigned nactors;
extern chrono::time start_motonic;
extern chrono::time deadline_motonic;
extern bool singlemode;
namespace config {
extern unsigned timeout_duration_seconds;
extern bool dump_config;
extern bool cleanup_before;
extern bool cleanup_after;
extern bool failfast;
extern bool progress_indicator;
extern bool console_mode;
} /* namespace config */
} /* namespace global */
//-----------------------------------------------------------------------------
struct db_deleter /* : public std::unary_function<void, MDBX_env *> */ {
void operator()(MDBX_env *env) const { mdbx_env_close(env); }
};
struct txn_deleter /* : public std::unary_function<void, MDBX_txn *> */ {
void operator()(MDBX_txn *txn) const {
int rc = mdbx_txn_abort(txn);
if (rc)
log_trouble(__func__, "mdbx_txn_abort()", rc);
}
};
struct cursor_deleter /* : public std::unary_function<void, MDBX_cursor *> */ {
void operator()(MDBX_cursor *cursor) const { mdbx_cursor_close(cursor); }
};
typedef std::unique_ptr<MDBX_env, db_deleter> scoped_db_guard;
typedef std::unique_ptr<MDBX_txn, txn_deleter> scoped_txn_guard;
typedef std::unique_ptr<MDBX_cursor, cursor_deleter> scoped_cursor_guard;
//-----------------------------------------------------------------------------
class testcase {
protected:
#if HAVE_cxx17_std_string_view
using data_view = std::string_view;
#else
using data_view = std::string;
#endif
static inline data_view S(const MDBX_val &v) {
return data_view(static_cast<const char *>(v.iov_base), v.iov_len);
}
static inline data_view S(const keygen::buffer &b) { return S(b->value); }
using Item = std::pair<std::string, std::string>;
struct ItemCompare {
const testcase *context;
ItemCompare(const testcase *owner) : context(owner) {}
bool operator()(const Item &a, const Item &b) const {
MDBX_val va, vb;
va.iov_base = (void *)a.first.data();
va.iov_len = a.first.size();
vb.iov_base = (void *)b.first.data();
vb.iov_len = b.first.size();
int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb);
if (cmp == 0 &&
(context->config.params.table_flags & MDBX_DUPSORT) != 0) {
va.iov_base = (void *)a.second.data();
va.iov_len = a.second.size();
vb.iov_base = (void *)b.second.data();
vb.iov_len = b.second.size();
cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb);
}
return cmp < 0;
}
};
using SET = std::set<Item, ItemCompare>;
const actor_config &config;
const mdbx_pid_t pid;
MDBX_dbi dbi;
scoped_db_guard db_guard;
scoped_txn_guard txn_guard;
scoped_cursor_guard cursor_guard;
bool signalled;
size_t nops_completed;
chrono::time start_timestamp;
keygen::buffer key;
keygen::buffer data;
keygen::maker keyvalue_maker;
struct {
mdbx_canary canary;
} last;
SET speculum;
bool speculum_verify() const;
int insert(const keygen::buffer &akey, const keygen::buffer &adata,
unsigned flags);
int replace(const keygen::buffer &akey, const keygen::buffer &new_value,
const keygen::buffer &old_value, unsigned flags);
int remove(const keygen::buffer &akey, const keygen::buffer &adata);
static int oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid,
uint64_t txn, unsigned gap, size_t space, int retry);
bool is_nested_txn_available() const {
return (config.params.mode_flags & MDBX_WRITEMAP) == 0;
}
void kick_progress(bool active) const;
void db_prepare();
void db_open();
void db_close();
void txn_begin(bool readonly, unsigned flags = 0);
int breakable_commit();
void txn_end(bool abort);
int breakable_restart();
void txn_restart(bool abort, bool readonly, unsigned flags = 0);
void cursor_open(MDBX_dbi handle);
void cursor_close();
void txn_inject_writefault(void);
void txn_inject_writefault(MDBX_txn *txn);
void fetch_canary();
void update_canary(uint64_t increment);
void checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check,
MDBX_val expected_valued);
unsigned txn_underutilization_x256(MDBX_txn *txn) const;
MDBX_dbi db_table_open(bool create);
void db_table_drop(MDBX_dbi handle);
void db_table_clear(MDBX_dbi handle, MDBX_txn *txn = nullptr);
void db_table_close(MDBX_dbi handle);
int db_open__begin__table_create_open_clean(MDBX_dbi &handle);
bool wait4start();
void report(size_t nops_done);
void signal();
bool should_continue(bool check_timeout_only = false) const;
void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key,
keygen::buffer &out_value, keygen::serial_t data_age = 0) {
keyvalue_maker.pair(serial, out_key, out_value, data_age);
}
void generate_pair(const keygen::serial_t serial,
keygen::serial_t data_age = 0) {
generate_pair(serial, key, data, data_age);
}
bool mode_readonly() const {
return (config.params.mode_flags & MDBX_RDONLY) ? true : false;
}
public:
testcase(const actor_config &config, const mdbx_pid_t pid)
: config(config), pid(pid), signalled(false), nops_completed(0),
speculum(ItemCompare(this)) {
start_timestamp.reset();
memset(&last, 0, sizeof(last));
}
virtual bool setup();
virtual bool run() { return true; }
virtual bool teardown();
virtual ~testcase() {}
};
class testcase_ttl : public testcase {
public:
testcase_ttl(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run() override;
};
class testcase_hill : public testcase {
using inherited = testcase;
SET speculum_commited;
public:
testcase_hill(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid), speculum_commited(ItemCompare(this)) {}
bool run() override;
};
class testcase_append : public testcase {
public:
testcase_append(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run() override;
};
class testcase_deadread : public testcase {
public:
testcase_deadread(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run() override;
};
class testcase_deadwrite : public testcase {
public:
testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run() override;
};
class testcase_jitter : public testcase {
public:
testcase_jitter(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run() override;
};
class testcase_try : public testcase {
public:
testcase_try(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run() override;
};
class testcase_copy : public testcase {
const std::string copy_pathname;
void copy_db(const bool with_compaction);
public:
testcase_copy(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid),
copy_pathname(config.params.pathname_db + "-copy") {}
bool run() override;
};
class testcase_nested : public testcase {
using inherited = testcase;
using FIFO = std::deque<std::pair<uint64_t, unsigned>>;
uint64_t serial;
FIFO fifo;
std::stack<std::tuple<scoped_txn_guard, uint64_t, FIFO, SET>> stack;
bool trim_tail(unsigned window_width);
bool grow_head(unsigned head_count);
bool pop_txn(bool abort);
bool pop_txn() {
return pop_txn(inherited::is_nested_txn_available() ? flipcoin_x3()
: flipcoin_x2());
}
void push_txn();
bool stochastic_breakable_restart_with_nested(bool force_restart = false);
public:
testcase_nested(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool setup() override;
bool run() override;
bool teardown() override;
};

20
contrib/db/libmdbx/test/try.cc Normal file
View file

@ -0,0 +1,20 @@
#include "test.h"
bool testcase_try::run() {
db_open();
assert(!txn_guard);
MDBX_txn *txn = nullptr;
MDBX_txn *txn2 = nullptr;
int rc = mdbx_txn_begin(db_guard.get(), nullptr, 0, &txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_begin(MDBX_TRYTXN)", rc);
else {
rc = mdbx_txn_begin(db_guard.get(), nullptr, MDBX_TRYTXN, &txn2);
if (unlikely(rc != MDBX_BUSY))
failure_perror("mdbx_txn_begin(MDBX_TRYTXN)", rc);
}
txn_guard.reset(txn);
return true;
}

172
contrib/db/libmdbx/test/ttl.cc Normal file
View file

@ -0,0 +1,172 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <cmath>
#include <deque>
static unsigned edge2window(uint64_t edge, unsigned window_max) {
const double rnd = u64_to_double1(bleach64(edge));
const unsigned window = window_max - std::lrint(std::pow(window_max, rnd));
return window;
}
static unsigned edge2count(uint64_t edge, unsigned count_max) {
const double rnd = u64_to_double1(prng64_map1_white(edge));
const unsigned count = std::lrint(std::pow(count_max, rnd));
return count;
}
bool testcase_ttl::run() {
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err));
return false;
}
/* LY: тест "эмуляцией time-to-live":
* - организуется "скользящее окно", которое двигается вперед вдоль
* числовой оси каждую транзакцию.
* - по переднему краю "скользящего окна" записи добавляются в таблицу,
* а по заднему удаляются.
* - количество добавляемых/удаляемых записей псевдослучайно зависит
* от номера транзакции, но с экспоненциальным распределением.
* - размер "скользящего окна" также псевдослучайно зависит от номера
* транзакции с "отрицательным" экспоненциальным распределением
* MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний
* край и удаляются записи позади него.
*
* Таким образом имитируется поведение таблицы с TTL: записи стохастически
* добавляются и удаляются, но изредка происходят массивные удаления.
*/
/* LY: для параметризации используем подходящие параметры, которые не имеют
* здесь смысла в первоначальном значении. */
const unsigned window_max_lower = 333;
const unsigned count_max_lower = 333;
const unsigned window_max = (config.params.batch_read > window_max_lower)
? config.params.batch_read
: window_max_lower;
const unsigned count_max = (config.params.batch_write > count_max_lower)
? config.params.batch_write
: count_max_lower;
log_verbose("ttl: using `batch_read` value %u for window_max", window_max);
log_verbose("ttl: using `batch_write` value %u for count_max", count_max);
uint64_t seed =
prng64_map2_white(config.params.keygen.seed) + config.actor_id;
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
key = keygen::alloc(config.params.keylen_max);
data = keygen::alloc(config.params.datalen_max);
const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_NODUPDATA
: MDBX_NODUPDATA | MDBX_NOOVERWRITE;
std::deque<std::pair<uint64_t, unsigned>> fifo;
uint64_t serial = 0;
bool rc = false;
while (should_continue()) {
const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
const unsigned window_width =
flipcoin_x4() ? 0 : edge2window(salt, window_max);
unsigned head_count = edge2count(salt, count_max);
log_debug("ttl: step #%zu (serial %" PRIu64
", window %u, count %u) salt %" PRIu64,
nops_completed, serial, window_width, head_count, salt);
if (window_width) {
while (fifo.size() > window_width) {
uint64_t tail_serial = fifo.back().first;
const unsigned tail_count = fifo.back().second;
log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial,
tail_count);
fifo.pop_back();
for (unsigned n = 0; n < tail_count; ++n) {
log_trace("ttl: remove-tail %" PRIu64, tail_serial);
generate_pair(tail_serial);
err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("ttl: tail-bailout due '%s'", mdbx_strerror(err));
goto bailout;
}
failure_perror("mdbx_del(tail)", err);
}
if (unlikely(!keyvalue_maker.increment(tail_serial, 1)))
failure("ttl: unexpected key-space overflow on the tail");
}
}
} else {
log_trace("ttl: purge state");
db_table_clear(dbi);
fifo.clear();
}
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
fifo.push_front(std::make_pair(serial, head_count));
retry:
for (unsigned n = 0; n < head_count; ++n) {
log_trace("ttl: insert-head %" PRIu64, serial);
generate_pair(serial);
err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial = fifo.front().first;
fifo.front().second = head_count = n;
goto retry;
}
failure_perror("mdbx_put(head)", err);
}
if (unlikely(!keyvalue_maker.increment(serial, 1))) {
log_notice("ttl: unexpected key-space overflow");
goto bailout;
}
}
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err));
serial = fifo.front().first;
fifo.pop_front();
}
report(1);
rc = true;
}
bailout:
txn_end(true);
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err));
return false;
}
} else
db_table_close(dbi);
}
return rc;
}

370
contrib/db/libmdbx/test/utils.cc Normal file
View file

@ -0,0 +1,370 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <float.h>
#if defined(HAVE_IEEE754_H) || __has_include(<ieee754.h>)
#include <ieee754.h>
#endif
#if defined(__APPLE__) || defined(__MACH__)
#include <mach/mach_time.h>
#endif /* defined(__APPLE__) || defined(__MACH__) */
std::string format(const char *fmt, ...) {
va_list ap, ones;
va_start(ap, fmt);
va_copy(ones, ap);
#ifdef _MSC_VER
int needed = _vscprintf(fmt, ap);
#else
int needed = vsnprintf(nullptr, 0, fmt, ap);
#endif
assert(needed >= 0);
va_end(ap);
std::string result;
result.reserve((size_t)needed + 1);
result.resize((size_t)needed, '\0');
int actual = vsnprintf((char *)result.data(), result.capacity(), fmt, ones);
assert(actual == needed);
(void)actual;
va_end(ones);
return result;
}
std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum) {
std::string result;
if (bytes > 0) {
const uint8_t *data = (const uint8_t *)ptr;
checksum.push(data, bytes);
result.reserve(bytes * 2);
const uint8_t *const end = data + bytes;
do {
char h = *data >> 4;
char l = *data & 15;
result.push_back((l < 10) ? l + '0' : l - 10 + 'a');
result.push_back((h < 10) ? h + '0' : h - 10 + 'a');
} while (++data < end);
}
assert(result.size() == bytes * 2);
return result;
}
bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
size_t bytes, simple_checksum &checksum) {
if (bytes * 2 != (size_t)(hex_end - hex_begin))
return false;
uint8_t *data = (uint8_t *)ptr;
for (const char *hex = hex_begin; hex != hex_end; hex += 2, ++data) {
unsigned l = hex[0], h = hex[1];
if (l >= '0' && l <= '9')
l = l - '0';
else if (l >= 'A' && l <= 'F')
l = l - 'A' + 10;
else if (l >= 'a' && l <= 'f')
l = l - 'a' + 10;
else
return false;
if (h >= '0' && h <= '9')
h = h - '0';
else if (h >= 'A' && h <= 'F')
h = h - 'A' + 10;
else if (h >= 'a' && h <= 'f')
h = h - 'a' + 10;
else
return false;
uint32_t c = l + (h << 4);
checksum.push(c);
*data = (uint8_t)c;
}
return true;
}
bool is_samedata(const MDBX_val *a, const MDBX_val *b) {
return a->iov_len == b->iov_len &&
memcmp(a->iov_base, b->iov_base, a->iov_len) == 0;
}
//-----------------------------------------------------------------------------
/* TODO: replace my 'libmera' from t1ha. */
uint64_t entropy_ticks(void) {
#if defined(EMSCRIPTEN)
return (uint64_t)emscripten_get_now();
#endif /* EMSCRIPTEN */
#if defined(__APPLE__) || defined(__MACH__)
return mach_absolute_time();
#endif /* defined(__APPLE__) || defined(__MACH__) */
#if defined(__sun__) || defined(__sun)
return gethrtime();
#endif /* __sun__ */
#if defined(__GNUC__) || defined(__clang__)
#if defined(__ia64__)
uint64_t ticks;
__asm __volatile("mov %0=ar.itc" : "=r"(ticks));
return ticks;
#elif defined(__hppa__)
uint64_t ticks;
__asm __volatile("mfctl 16, %0" : "=r"(ticks));
return ticks;
#elif defined(__s390__)
uint64_t ticks;
__asm __volatile("stck 0(%0)" : : "a"(&(ticks)) : "memory", "cc");
return ticks;
#elif defined(__alpha__)
uint64_t ticks;
__asm __volatile("rpcc %0" : "=r"(ticks));
return ticks;
#elif defined(__sparc__) || defined(__sparc) || defined(__sparc64__) || \
defined(__sparc64) || defined(__sparc_v8plus__) || \
defined(__sparc_v8plus) || defined(__sparc_v8plusa__) || \
defined(__sparc_v8plusa) || defined(__sparc_v9__) || defined(__sparc_v9)
union {
uint64_t u64;
struct {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
uint32_t h, l;
#else
uint32_t l, h;
#endif
} u32;
} cycles;
#if defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || \
defined(__sparc_v9__) || defined(__sparc_v8plus) || \
defined(__sparc_v8plusa) || defined(__sparc_v9)
#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || \
defined(__sparc64__) || defined(__sparc64)
__asm __volatile("rd %%tick, %0" : "=r"(cycles.u64));
#else
__asm __volatile("rd %%tick, %1; srlx %1, 32, %0"
: "=r"(cycles.u32.h), "=r"(cycles.u32.l));
#endif /* __sparc64__ */
#else
__asm __volatile(".byte 0x83, 0x41, 0x00, 0x00; mov %%g1, %0"
: "=r"(cycles.u64)
:
: "%g1");
#endif /* __sparc8plus__ || __sparc_v9__ */
return cycles.u64;
#elif (defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) || \
defined(__powerpc64))
uint64_t ticks;
__asm __volatile("mfspr %0, 268" : "=r"(ticks));
return ticks;
#elif (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) || \
defined(__ppc))
#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
uint64_t ticks;
__asm __volatile("mftb %0" : "=r"(ticks));
*now = ticks;
#else
uint64_t ticks;
uint32_t low, high_before, high_after;
__asm __volatile("mftbu %0; mftb %1; mftbu %2"
: "=r"(high_before), "=r"(low), "=r"(high_after));
ticks = (uint64_t)high_after << 32;
ticks |= low & /* zeroes if high part has changed */
~(high_before - high_after);
#endif
#elif (defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 7)) && \
!defined(MDBX_SAFE4QEMU)
uint64_t virtual_timer;
__asm __volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer));
return virtual_timer;
#elif (defined(__ARM_ARCH) && __ARM_ARCH > 5 && __ARM_ARCH < 8) || \
defined(_M_ARM)
static uint32_t pmcntenset = 0x00425B00;
if (unlikely(pmcntenset == 0x00425B00)) {
uint32_t pmuseren;
#ifdef _M_ARM
pmuseren = _MoveFromCoprocessor(15, 0, 9, 14, 0);
#else
__asm("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren));
#endif
if (1 & pmuseren /* Is it allowed for user mode code? */) {
#ifdef _M_ARM
pmcntenset = _MoveFromCoprocessor(15, 0, 9, 12, 1);
#else
__asm("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset));
#endif
} else
pmcntenset = 0;
}
if (pmcntenset & 0x80000000ul /* Is it counting? */) {
#ifdef _M_ARM
return __rdpmccntr64();
#else
uint32_t pmccntr;
__asm __volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr));
return pmccntr;
#endif
}
#elif defined(__mips__) || defined(__mips) || defined(_R4000)
unsigned count;
__asm __volatile("rdhwr %0, $2" : "=r"(count));
return count;
#endif /* arch selector */
#endif /* __GNUC__ || __clang__ */
#if defined(__e2k__) || defined(__ia32__)
return __rdtsc();
#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
LARGE_INTEGER PerformanceCount;
if (QueryPerformanceCounter(&PerformanceCount))
return PerformanceCount.QuadPart;
return GetTickCount64();
#else
struct timespec ts;
#if defined(CLOCK_MONOTONIC_COARSE)
clockid_t clk_id = CLOCK_MONOTONIC_COARSE;
#elif defined(CLOCK_MONOTONIC_RAW)
clockid_t clk_id = CLOCK_MONOTONIC_RAW;
#else
clockid_t clk_id = CLOCK_MONOTONIC;
#endif
int rc = clock_gettime(clk_id, &ts);
if (unlikely(rc))
failure_perror("clock_gettime()", rc);
return (((uint64_t)ts.tv_sec) << 32) + ts.tv_nsec;
#endif
}
//-----------------------------------------------------------------------------
uint64_t prng64_white(uint64_t &state) {
state = prng64_map2_careless(state);
return bleach64(state);
}
uint32_t prng32(uint64_t &state) {
return (uint32_t)(prng64_careless(state) >> 32);
}
void prng_fill(uint64_t &state, void *ptr, size_t bytes) {
while (bytes >= 4) {
*((uint32_t *)ptr) = prng32(state);
ptr = (uint32_t *)ptr + 1;
bytes -= 4;
}
switch (bytes & 3) {
case 3: {
uint32_t u32 = prng32(state);
memcpy(ptr, &u32, 3);
} break;
case 2:
*((uint16_t *)ptr) = (uint16_t)prng32(state);
break;
case 1:
*((uint8_t *)ptr) = (uint8_t)prng32(state);
break;
case 0:
break;
}
}
static __thread uint64_t prng_state;
void prng_seed(uint64_t seed) { prng_state = bleach64(seed); }
uint32_t prng32(void) { return prng32(prng_state); }
uint64_t prng64(void) { return prng64_white(prng_state); }
void prng_fill(void *ptr, size_t bytes) { prng_fill(prng_state, ptr, bytes); }
uint64_t entropy_white() { return bleach64(entropy_ticks()); }
double double_from_lower(uint64_t salt) {
#ifdef IEEE754_DOUBLE_BIAS
ieee754_double r;
r.ieee.negative = 0;
r.ieee.exponent = IEEE754_DOUBLE_BIAS;
r.ieee.mantissa0 = (unsigned)(salt >> 32);
r.ieee.mantissa1 = (unsigned)salt;
return r.d;
#else
const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1;
const double scale = 1.0 / (double)top;
return (salt & top) * scale;
#endif
}
double double_from_upper(uint64_t salt) {
#ifdef IEEE754_DOUBLE_BIAS
ieee754_double r;
r.ieee.negative = 0;
r.ieee.exponent = IEEE754_DOUBLE_BIAS;
salt >>= 64 - DBL_MANT_DIG;
r.ieee.mantissa0 = (unsigned)(salt >> 32);
r.ieee.mantissa1 = (unsigned)salt;
return r.d;
#else
const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1;
const double scale = 1.0 / (double)top;
return (salt >> (64 - DBL_MANT_DIG)) * scale;
#endif
}
bool flipcoin() { return bleach32((uint32_t)entropy_ticks()) & 1; }
bool flipcoin_x2() { return (bleach32((uint32_t)entropy_ticks()) & 3) == 0; }
bool flipcoin_x3() { return (bleach32((uint32_t)entropy_ticks()) & 7) == 0; }
bool flipcoin_x4() { return (bleach32((uint32_t)entropy_ticks()) & 15) == 0; }
bool jitter(unsigned probability_percent) {
const uint32_t top = UINT32_MAX - UINT32_MAX % 100;
uint32_t dice, edge = (top) / 100 * probability_percent;
do
dice = bleach32((uint32_t)entropy_ticks());
while (dice >= top);
return dice < edge;
}
void jitter_delay(bool extra) {
unsigned dice = entropy_white() & 3;
if (dice == 0) {
log_trace("== jitter.no-delay");
} else {
log_trace(">> jitter.delay: dice %u", dice);
do {
cpu_relax();
memory_barrier();
cpu_relax();
if (dice > 1) {
osal_yield();
cpu_relax();
if (dice > 2) {
unsigned us = entropy_white() &
(extra ? 0xffff /* 656 ms */ : 0x3ff /* 1 ms */);
log_trace("== jitter.delay: %0.6f", us / 1000000.0);
osal_udelay(us);
}
}
} while (flipcoin());
log_trace("<< jitter.delay: dice %u", dice);
}
}

362
contrib/db/libmdbx/test/utils.h Normal file
View file

@ -0,0 +1,362 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
#error __BYTE_ORDER__ should be defined.
#endif
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
#error Unsupported byte order.
#endif
#if __GNUC_PREREQ(4, 4) || defined(__clang__)
#ifndef bswap64
#define bswap64(v) __builtin_bswap64(v)
#endif
#ifndef bswap32
#define bswap32(v) __builtin_bswap32(v)
#endif
#if (__GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)) && \
!defined(bswap16)
#define bswap16(v) __builtin_bswap16(v)
#endif
#elif defined(_MSC_VER)
#if _MSC_FULL_VER < 190024215
#pragma message( \
"It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.")
#endif
#define bswap64(v) _byteswap_uint64(v)
#define bswap32(v) _byteswap_ulong(v)
#define bswap16(v) _byteswap_ushort(v)
#define rot64(v, s) _rotr64(v, s)
#define rot32(v, s) _rotr(v, s)
#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
#pragma intrinsic(_umul128)
#define mul_64x64_128(a, b, ph) _umul128(a, b, ph)
#pragma intrinsic(__umulh)
#define mul_64x64_high(a, b) __umulh(a, b)
#endif
#if defined(_M_IX86)
#pragma intrinsic(__emulu)
#define mul_32x32_64(a, b) __emulu(a, b)
#elif defined(_M_ARM)
#define mul_32x32_64(a, b) _arm_umull(a, b)
#endif
#endif /* compiler */
#ifndef bswap64
#ifdef __bswap_64
#define bswap64(v) __bswap_64(v)
#else
static __inline uint64_t bswap64(uint64_t v) {
return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
((v << 24) & UINT64_C(0x0000ff0000000000)) |
((v << 8) & UINT64_C(0x000000ff00000000)) |
((v >> 8) & UINT64_C(0x00000000ff0000000)) |
((v >> 24) & UINT64_C(0x0000000000ff0000)) |
((v >> 40) & UINT64_C(0x000000000000ff00));
}
#endif
#endif /* bswap64 */
#ifndef bswap32
#ifdef __bswap_32
#define bswap32(v) __bswap_32(v)
#else
static __inline uint32_t bswap32(uint32_t v) {
return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) |
((v >> 8) & UINT32_C(0x0000ff00));
}
#endif
#endif /* bswap32 */
#ifndef bswap16
#ifdef __bswap_16
#define bswap16(v) __bswap_16(v)
#else
static __inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; }
#endif
#endif /* bswap16 */
#define is_byteorder_le() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define is_byteorder_be() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#ifndef htole16
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe16(v) bswap16(v)
#define htole16(v) (v)
#define be16toh(v) bswap16(v)
#define le16toh(v) (v)
#else
#define htobe16(v) (v)
#define htole16(v) bswap16(v)
#define be16toh(v) (v)
#define le16toh(v) bswap16(v)
#endif
#endif /* htole16 */
#ifndef htole32
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe32(v) bswap32(v)
#define htole32(v) (v)
#define be32toh(v) bswap32(v)
#define le32toh(v) (v)
#else
#define htobe32(v) (v)
#define htole32(v) bswap32(v)
#define be32toh(v) (v)
#define le32toh(v) bswap32(v)
#endif
#endif /* htole32 */
#ifndef htole64
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe64(v) bswap64(v)
#define htole64(v) (v)
#define be64toh(v) bswap64(v)
#define le64toh(v) (v)
#else
#define htobe64(v) (v)
#define htole64(v) bswap_64(v)
#define be64toh(v) (v)
#define le64toh(v) bswap_64(v)
#endif
#endif /* htole64 */
namespace unaligned {
template <typename T> static __inline T load(const void *ptr) {
#if defined(_MSC_VER) && \
(defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64))
return *(const T __unaligned *)ptr;
#elif MDBX_UNALIGNED_OK
return *(const T *)ptr;
#else
T local;
#if defined(__GNUC__) || defined(__clang__)
__builtin_memcpy(&local, (const T *)ptr, sizeof(T));
#else
memcpy(&local, (const T *)ptr, sizeof(T));
#endif /* __GNUC__ || __clang__ */
return local;
#endif /* MDBX_UNALIGNED_OK */
}
template <typename T> static __inline void store(void *ptr, const T &value) {
#if defined(_MSC_VER) && \
(defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64))
*((T __unaligned *)ptr) = value;
#elif MDBX_UNALIGNED_OK
*(volatile T *)ptr = value;
#else
#if defined(__GNUC__) || defined(__clang__)
__builtin_memcpy(ptr, &value, sizeof(T));
#else
memcpy(ptr, &value, sizeof(T));
#endif /* __GNUC__ || __clang__ */
#endif /* MDBX_UNALIGNED_OK */
}
} /* namespace unaligned */
//-----------------------------------------------------------------------------
#ifndef rot64
static __inline uint64_t rot64(uint64_t v, unsigned s) {
return (v >> s) | (v << (64 - s));
}
#endif /* rot64 */
static __inline bool is_power2(size_t x) { return (x & (x - 1)) == 0; }
#undef roundup2
static __inline size_t roundup2(size_t value, size_t granularity) {
assert(is_power2(granularity));
return (value + granularity - 1) & ~(granularity - 1);
}
//-----------------------------------------------------------------------------
static __inline void memory_barrier(void) {
#if __has_extension(c_atomic) || __has_extension(cxx_atomic)
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__ATOMIC_SEQ_CST)
__atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__clang__) || defined(__GNUC__)
__sync_synchronize();
#elif defined(_MSC_VER)
MemoryBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__ia32__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__lwsync();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
static __inline void cpu_relax() {
#if defined(__ia32__)
_mm_pause();
#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || \
defined(YieldProcessor)
YieldProcessor();
#else
/* nope */
#endif
}
//-----------------------------------------------------------------------------
struct simple_checksum {
uint64_t value;
simple_checksum() : value(0) {}
void push(const uint32_t &data) {
value += data * UINT64_C(9386433910765580089) + 1;
value ^= value >> 41;
value *= UINT64_C(0xBD9CACC22C6E9571);
}
void push(const uint64_t &data) {
push((uint32_t)data);
push((uint32_t)(data >> 32));
}
void push(const bool data) {
push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E));
}
void push(const void *ptr, size_t bytes) {
const uint8_t *data = (const uint8_t *)ptr;
for (size_t i = 0; i < bytes; ++i)
push((uint32_t)data[i]);
}
void push(const double &data) { push(&data, sizeof(double)); }
void push(const char *cstr) { push(cstr, strlen(cstr)); }
void push(const std::string &str) { push(str.data(), str.size()); }
void push(unsigned salt, const MDBX_val &val) {
push(unsigned(val.iov_len));
push(salt);
push(val.iov_base, val.iov_len);
}
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
void push(const HANDLE &handle) { push(&handle, sizeof(handle)); }
#endif /* _WINDOWS */
};
std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum);
bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
size_t bytes, simple_checksum &checksum);
bool is_samedata(const MDBX_val *a, const MDBX_val *b);
std::string format(const char *fmt, ...);
uint64_t entropy_ticks(void);
uint64_t entropy_white(void);
static inline uint64_t bleach64(uint64_t v) {
// Tommy Ettinger, https://www.blogger.com/profile/04953541827437796598
// http://mostlymangling.blogspot.com/2019/01/better-stronger-mixer-and-test-procedure.html
v ^= rot64(v, 25) ^ rot64(v, 50);
v *= UINT64_C(0xA24BAED4963EE407);
v ^= rot64(v, 24) ^ rot64(v, 49);
v *= UINT64_C(0x9FB21C651E98DF25);
return v ^ v >> 28;
}
static inline uint32_t bleach32(uint32_t x) {
// https://github.com/skeeto/hash-prospector
// exact bias: 0.17353355999581582
x ^= x >> 16;
x *= UINT32_C(0x7feb352d);
x ^= 0x3027C563 ^ (x >> 15);
x *= UINT32_C(0x846ca68b);
x ^= x >> 16;
return x;
}
static inline uint64_t prng64_map1_careless(uint64_t state) {
return state * UINT64_C(6364136223846793005) + 1;
}
static inline uint64_t prng64_map2_careless(uint64_t state) {
return (state + UINT64_C(1442695040888963407)) *
UINT64_C(6364136223846793005);
}
static inline uint64_t prng64_map1_white(uint64_t state) {
return bleach64(prng64_map1_careless(state));
}
static inline uint64_t prng64_map2_white(uint64_t state) {
return bleach64(prng64_map2_careless(state));
}
static inline uint64_t prng64_careless(uint64_t &state) {
state = prng64_map1_careless(state);
return state;
}
static inline double u64_to_double1(uint64_t v) {
union {
uint64_t u64;
double d;
} casting;
casting.u64 = UINT64_C(0x3ff) << 52 | (v >> 12);
assert(casting.d >= 1.0 && casting.d < 2.0);
return casting.d - 1.0;
}
uint64_t prng64_white(uint64_t &state);
uint32_t prng32(uint64_t &state);
void prng_fill(uint64_t &state, void *ptr, size_t bytes);
void prng_seed(uint64_t seed);
uint32_t prng32(void);
uint64_t prng64(void);
void prng_fill(void *ptr, size_t bytes);
bool flipcoin();
bool flipcoin_x2();
bool flipcoin_x3();
bool flipcoin_x4();
bool jitter(unsigned probability_percent);
void jitter_delay(bool extra = false);

24
contrib/db/libmdbx/test/valgrind_suppress.txt Normal file
View file

@ -0,0 +1,24 @@
{
msync-whole-mmap-1
Memcheck:Param
msync(start)
fun:msync
...
fun:mdbx_sync_locked
}
{
msync-whole-mmap-2
Memcheck:Param
msync(start)
fun:msync
...
fun:mdbx_env_sync_ex
}
{
pwrite-page-flush
Memcheck:Param
pwritev(vector[...])
fun:pwritev
...
fun:mdbx_page_flush
}