Charon/blockchain
1
0
Fork 0
forked from lthn/blockchain
Code Pull requests Activity

added copy of libmdbx into folder tree(ref to git submodule will be added later)

Browse source
This commit is contained in:
cryptozoidberg 2019-08-28 23:25:45 +02:00
parent 4cb6a6a51f
commit c62cb18c48
No known key found for this signature in database
GPG key ID: 22DEB97A54C6FDEC
89 changed files with 36940 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
contrib/db/libmdbx/.circleci/config.yml Normal file
View file

@ -0,0 +1,20 @@
version: 2
jobs:
build:
docker:
- image: circleci/buildpack-deps:artful
environment:
- TESTDB: /tmp/test.db
- TESTLOG: /tmp/test.log
steps:
- checkout
- run: make all
- run: ulimit -c unlimited && make check
- run:
command: |
mkdir -p /tmp/artifacts
mv -t /tmp/artifacts $TESTLOG $TESTDB core.*
when: on_fail
- store_artifacts:
path: /tmp/artifacts
destination: test-artifacts

3
contrib/db/libmdbx/.clang-format Normal file
View file

@ -0,0 +1,3 @@
BasedOnStyle: LLVM
Standard: Cpp11
ReflowComments: true

38
contrib/db/libmdbx/.gitignore vendored Normal file
View file

@ -0,0 +1,38 @@
*.[ao]
*.bak
*.exe
*.gcda
*.gcno
*.gcov
*.lo
*.orig
*.rej
*.so
*[~#]
.idea
.le.ini
.vs/
Win32/
build-*
cmake-build-*
core
example
libmdbx.creator.user
mdbx-dll.VC.VC.opendb
mdbx-dll.VC.db
mdbx-dll.vcxproj.filters
mdbx_chk
mdbx_copy
mdbx_dump
mdbx_load
mdbx_stat
mdbx_test
test.log
test/test.vcxproj.user
test/tmp.db
test/tmp.db-lck
tmp.db
tmp.db-lck
valgrind.*
x64/
x86/

30
contrib/db/libmdbx/.travis.yml Normal file
View file

@ -0,0 +1,30 @@
language: c
dist: xenial
compiler:
- gcc
- clang
os:
- linux
- osx
script: if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make all check; fi
env:
global:
- secure: "M+W+heGGyRQJoBq2W0uqWVrpL4KBXmL0MFL7FSs7f9vmAaDyEgziUXeZRj3GOKzW4kTef3LpIeiu9SmvqSMoQivGGiomZShqPVl045o/OUgRCAT7Al1RLzEZ0efSHpIPf0PZ6byEf6GR2ML76OfuL6JxTVdnz8iVyO2sgLE1HbX1VeB+wgd/jfMeOBhCCXskfK6MLyZihfMYsiYZYSaV98ZDhDLSlzuuRIgzb0bMi8aL6AErs0WLW0NelRBeHkKPYfAUc85pdQHscgrJw6Rh/zT6+8BQ/q5f4IgWhiu4xoRg3Ngl7SNoedRQh93ADM3UG2iGl6HDFpVORaXcFWKAtuYY+kHQ0HB84BRYpQmeBuXNpltsfxQ3d1Q3u0RlE45zRvmr2+X1mFnkcNUAWISLPbsOUlriDQM8irGwRpho77/uYnRC00bJsHW//s6+uPf9zrAw1nI4f0y3PAWukGF/xs6HAI3FZPsuSSnx18Tj3Opgbc9Spop+V3hkhdiJoPGpNKTkFX4ZRXfkPgoRVJmtp4PpbpH0Ps/mCriKjMEfGGi0HcVCi0pEGLXiecdqJ5KPg5+22zNycEujQBJcNTKd9shN+R3glrbmhAxTEzGdGwxXXJ2ybwJ2PWJLMYZ7g98nLyX+uQPaA3BlsbYJHNeS5283/9pJsd9DzfHKsN2nFSc="
before_install:
- echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-
addons:
coverity_scan:
project:
name: "ReOpen/libmdbx"
version: 0.1
description: "Build submitted via Travis CI"
notification_email: leo@yuriev.ru
build_command_prepend: "make clean"
build_command: "make all -j 2"
branch_pattern: coverity_scan

31
contrib/db/libmdbx/AUTHORS Normal file
View file

@ -0,0 +1,31 @@
Contributors
============
Alexey Naumov <alexey.naumov@gmail.com>
Chris Mikkelson <cmikk@qwest.net>
Claude Brisson <claude.brisson@gmail.com>
David Barbour <dmbarbour@gmail.com>
David Wilson <dw@botanicus.net>
dreamsxin <dreamsxin@126.com>
Hallvard Furuseth <hallvard@openldap.org>, <h.b.furuseth@usit.uio.no>
Heiko Becker <heirecka@exherbo.org>
Howard Chu <hyc@openldap.org>, <hyc@symas.com>
Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
James Rouzier <rouzier@gmail.com>
Jean-Christophe DUBOIS <jcd@tribudubois.net>
John Hewson <john@jahewson.com>
Klaus Malorny <klaus.malorny@knipp.de>
Kurt Zeilenga <kurt.zeilenga@isode.com>
Leonid Yuriev <leo@yuriev.ru>, <lyuryev@ptsecurity.com>
Lorenz Bauer <lmb@cloudflare.com>
Luke Yeager <lyeager@nvidia.com>
Martin Hedenfalk <martin@bzero.se>
Ondrej Kuznik <ondrej.kuznik@acision.com>
Orivej Desh <orivej@gmx.fr>
Oskari Timperi <oskari.timperi@iki.fi>
Pavel Medvedev <pmedvedev@gmail.com>
Philipp Storz <philipp.storz@bareos.com>
Quanah Gibson-Mount <quanah@openldap.org>
Salvador Ortiz <sog@msg.com.mx>
Sebastien Launay <sebastien@slaunay.fr>
Vladimir Romanov <vromanov@gmail.com>

22
contrib/db/libmdbx/COPYRIGHT Normal file
View file

@ -0,0 +1,22 @@
Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
Copyright 2011-2015 Howard Chu, Symas Corp.
Copyright 2015,2016 Peter-Service R&D LLC.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted only as authorized by the OpenLDAP
Public License.
A copy of this license is available in the file LICENSE in the
top-level directory of the distribution or, alternatively, at
<http://www.OpenLDAP.org/license.html>.
OpenLDAP is a registered trademark of the OpenLDAP Foundation.
Individual files and/or contributed packages may be copyright by
other parties and/or subject to additional restrictions.
This work also contains materials derived from public sources.
Additional information about OpenLDAP can be obtained at
<http://www.openldap.org/>.

47
contrib/db/libmdbx/LICENSE Normal file
View file

@ -0,0 +1,47 @@
The OpenLDAP Public License
Version 2.8, 17 August 2003
Redistribution and use of this software and associated documentation
("Software"), with or without modification, are permitted provided
that the following conditions are met:
1. Redistributions in source form must retain copyright statements
and notices,
2. Redistributions in binary form must reproduce applicable copyright
statements and notices, this list of conditions, and the following
disclaimer in the documentation and/or other materials provided
with the distribution, and
3. Redistributions must contain a verbatim copy of this document.
The OpenLDAP Foundation may revise this license from time to time.
Each revision is distinguished by a version number. You may use
this Software under terms of this license revision or under the
terms of any subsequent revision of the license.
THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS
CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S)
OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
The names of the authors and copyright holders must not be used in
advertising or otherwise to promote the sale, use or other dealing
in this Software without specific, written prior permission. Title
to copyright in this Software shall at all times remain with copyright
holders.
OpenLDAP is a registered trademark of the OpenLDAP Foundation.
Copyright 1999-2003 The OpenLDAP Foundation, Redwood City,
California, USA. All Rights Reserved. Permission to copy and
distribute verbatim copies of this document is granted.

270
contrib/db/libmdbx/Makefile Normal file
View file

@ -0,0 +1,270 @@
# GNU Makefile for libmdbx, https://abf.io/erthink/libmdbx
########################################################################
# Configuration. The compiler options must enable threaded compilation.
#
# Preprocessor macros (for XCFLAGS) of interest...
# Note that the defaults should already be correct for most
# platforms; you should not need to change any of these.
# Read their descriptions in mdb.c if you do. There may be
# other macros of interest. You should read mdb.c
# before changing any of them.
#
# install sandbox
SANDBOX ?=
# install prefixes (inside sandbox)
prefix ?= /usr/local
mandir ?= $(prefix)/man
# lib/bin suffix for multiarch/biarch, e.g. '.x86_64'
suffix ?=
CC ?= gcc
CXX ?= g++
LD ?= ld
CFLAGS ?= -O2 -g3 -Wall -Werror -Wextra -ffunction-sections -fPIC -fvisibility=hidden
XCFLAGS ?= -DNDEBUG=1 -DLIBMDBX_EXPORTS=1
CFLAGS += -D_GNU_SOURCE=1 -std=gnu11 -pthread $(XCFLAGS)
CXXFLAGS = -std=c++11 $(filter-out -std=gnu11,$(CFLAGS))
TESTDB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.db
TESTLOG ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.log
# LY: '--no-as-needed,-lrt' for ability to built with modern glibc, but then run with the old
LDFLAGS ?= $(shell $(LD) --help 2>/dev/null | grep -q -- --gc-sections && echo '-Wl,--gc-sections,-z,relro,-O1')$(shell $(LD) --help 2>/dev/null | grep -q -- -dead_strip && echo '-Wl,-dead_strip')
EXE_LDFLAGS ?= -pthread
# LY: just for benchmarking
IOARENA ?= $(shell \
(test -x ../ioarena/@BUILD/src/ioarena && echo ../ioarena/@BUILD/src/ioarena) || \
(test -x ../../@BUILD/src/ioarena && echo ../../@BUILD/src/ioarena) || \
(test -x ../../src/ioarena && echo ../../src/ioarena) || which ioarena)
NN ?= 25000000
########################################################################
ifdef MSVC
UNAME := Windows
LCK_IMPL := windows
TEST_OSAL := windows
TEST_ITER := 42
else
UNAME := $(shell uname -s 2>/dev/null || echo Unknown)
define uname2lck
case "$(UNAME)" in
Linux) echo linux;;
CYGWIN*|MINGW*|MSYS*|Windows*) echo windows;;
*) echo posix;;
esac
endef
define uname2osal
case "$(UNAME)" in
CYGWIN*|MINGW*|MSYS*|Windows*) echo windows;;
*) echo unix;;
esac
endef
define uname2titer
case "$(UNAME)" in
Darwin*|Mach*) echo 3;;
*) echo 42;;
esac
endef
define uname2suffix
case "$(UNAME)" in
Darwin*|Mach*) echo dylib;;
CYGWIN*|MINGW*|MSYS*|Windows*) echo dll;;
*) echo so;;
esac
endef
LCK_IMPL := $(shell $(uname2lck))
TEST_OSAL := $(shell $(uname2osal))
TEST_ITER := $(shell $(uname2titer))
SO_SUFFIX := $(shell $(uname2suffix))
endif
HEADERS := mdbx.h
LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX)
TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk
MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1
SHELL := /bin/bash
CORE_SRC := src/lck-$(LCK_IMPL).c $(filter-out $(wildcard src/lck-*.c), $(wildcard src/*.c))
CORE_INC := $(wildcard src/*.h)
CORE_OBJ := $(patsubst %.c,%.o,$(CORE_SRC))
TEST_SRC := test/osal-$(TEST_OSAL).cc $(filter-out $(wildcard test/osal-*.cc), $(wildcard test/*.cc))
TEST_INC := $(wildcard test/*.h)
TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC))
.PHONY: mdbx all install clean check coverage
all: $(LIBRARIES) $(TOOLS) mdbx_test example
mdbx: libmdbx.a libmdbx.$(SO_SUFFIX)
example: mdbx.h tutorial/sample-mdbx.c libmdbx.$(SO_SUFFIX)
$(CC) $(CFLAGS) -I. tutorial/sample-mdbx.c ./libmdbx.$(SO_SUFFIX) -o example
tools: $(TOOLS)
install: $(LIBRARIES) $(TOOLS) $(HEADERS)
mkdir -p $(SANDBOX)$(prefix)/bin$(suffix) \
&& cp -t $(SANDBOX)$(prefix)/bin$(suffix) $(TOOLS) && \
mkdir -p $(SANDBOX)$(prefix)/lib$(suffix) \
&& cp -t $(SANDBOX)$(prefix)/lib$(suffix) $(LIBRARIES) && \
mkdir -p $(SANDBOX)$(prefix)/include \
&& cp -t $(SANDBOX)$(prefix)/include $(HEADERS) && \
mkdir -p $(SANDBOX)$(mandir)/man1 \
&& cp -t $(SANDBOX)$(mandir)/man1 $(MANPAGES)
clean:
rm -rf $(TOOLS) mdbx_test @* *.[ao] *.[ls]o *~ tmp.db/* *.gcov *.log *.err src/*.o test/*.o
check: all
rm -f $(TESTDB) $(TESTLOG) && (set -o pipefail; ./mdbx_test --repeat=$(TEST_ITER) --pathname=$(TESTDB) --dont-cleanup-after basic | tee -a $(TESTLOG) | tail -n 42) \
&& ./mdbx_chk -vvn $(TESTDB) && ./mdbx_chk -vvn $(TESTDB)-copy
check-singleprocess: all
rm -f $(TESTDB) $(TESTLOG) && (set -o pipefail; \
./mdbx_test --repeat=4 --pathname=$(TESTDB) --dont-cleanup-after --hill && \
./mdbx_test --repeat=2 --pathname=$(TESTDB) --dont-cleanup-before --dont-cleanup-after --copy \
| tee -a $(TESTLOG) | tail -n 42) \
&& ./mdbx_chk -vvn $(TESTDB) && ./mdbx_chk -vvn $(TESTDB)-copy
check-fault: all
rm -f $(TESTDB) $(TESTLOG) && (set -o pipefail; ./mdbx_test --pathname=$(TESTDB) --inject-writefault=42 --dump-config --dont-cleanup-after basic | tee -a $(TESTLOG) | tail -n 42) \
; ./mdbx_chk -vvnw $(TESTDB) && ([ ! -e $(TESTDB)-copy ] || ./mdbx_chk -vvn $(TESTDB)-copy)
define core-rule
$(patsubst %.c,%.o,$(1)): $(1) $(CORE_INC) mdbx.h Makefile
$(CC) $(CFLAGS) -c $(1) -o $$@
endef
$(foreach file,$(CORE_SRC),$(eval $(call core-rule,$(file))))
define test-rule
$(patsubst %.cc,%.o,$(1)): $(1) $(TEST_INC) mdbx.h Makefile
$(CXX) $(CXXFLAGS) -c $(1) -o $$@
endef
$(foreach file,$(TEST_SRC),$(eval $(call test-rule,$(file))))
libmdbx.a: $(CORE_OBJ)
$(AR) rs $@ $?
libmdbx.$(SO_SUFFIX): $(CORE_OBJ)
$(CC) $(CFLAGS) -save-temps $^ -pthread -shared $(LDFLAGS) -o $@
mdbx_%: src/tools/mdbx_%.c libmdbx.a
$(CC) $(CFLAGS) $^ $(EXE_LDFLAGS) -o $@
mdbx_test: $(TEST_OBJ) libmdbx.$(SO_SUFFIX)
$(CXX) $(CXXFLAGS) $(TEST_OBJ) -Wl,-rpath . -L . -l mdbx $(EXE_LDFLAGS) -o $@
###############################################################################
ifneq ($(wildcard $(IOARENA)),)
.PHONY: bench clean-bench re-bench
clean-bench:
rm -rf bench-*.txt _ioarena/*
re-bench: clean-bench bench
define bench-rule
bench-$(1)_$(2).txt: $(3) $(IOARENA) Makefile
LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \
$(IOARENA) -D $(1) -B crud -m nosync -n $(2) \
| tee $$@ | grep throughput && \
LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \
$(IOARENA) -D $(1) -B get,iterate -m sync -r 4 -n $(2) \
| tee -a $$@ | grep throughput \
|| mv -f $$@ $$@.error
endef
$(eval $(call bench-rule,mdbx,$(NN),libmdbx.$(SO_SUFFIX)))
$(eval $(call bench-rule,sophia,$(NN)))
$(eval $(call bench-rule,leveldb,$(NN)))
$(eval $(call bench-rule,rocksdb,$(NN)))
$(eval $(call bench-rule,wiredtiger,$(NN)))
$(eval $(call bench-rule,forestdb,$(NN)))
$(eval $(call bench-rule,lmdb,$(NN)))
$(eval $(call bench-rule,nessdb,$(NN)))
$(eval $(call bench-rule,sqlite3,$(NN)))
$(eval $(call bench-rule,ejdb,$(NN)))
$(eval $(call bench-rule,vedisdb,$(NN)))
$(eval $(call bench-rule,dummy,$(NN)))
$(eval $(call bench-rule,debug,10))
bench: bench-mdbx_$(NN).txt
.PHONY: bench-debug
bench-debug: bench-debug_10.txt
bench-quartet: bench-mdbx_$(NN).txt bench-lmdb_$(NN).txt bench-rocksdb_$(NN).txt bench-wiredtiger_$(NN).txt
endif
###############################################################################
ci-rule = ( CC=$$(which $1); if [ -n "$$CC" ]; then \
echo -n "probe by $2 ($$(readlink -f $$(which $$CC))): " && \
$(MAKE) clean >$1.log 2>$1.err && \
$(MAKE) CC=$$(readlink -f $$CC) XCFLAGS="-UNDEBUG -DMDBX_DEBUG=2 -DLIBMDBX_EXPORTS=1" check 1>$1.log 2>$1.err && echo "OK" \
|| ( echo "FAILED"; cat $1.err >&2; exit 1 ); \
else echo "no $2 ($1) for probe"; fi; )
ci:
@if [ "$$(readlink -f $$(which $(CC)))" != "$$(readlink -f $$(which gcc || echo /bin/false))" -a \
"$$(readlink -f $$(which $(CC)))" != "$$(readlink -f $$(which clang || echo /bin/false))" -a \
"$$(readlink -f $$(which $(CC)))" != "$$(readlink -f $$(which icc || echo /bin/false))" ]; then \
$(call ci-rule,$(CC),default C compiler); \
fi
@$(call ci-rule,gcc,GCC)
@$(call ci-rule,clang,clang LLVM)
@$(call ci-rule,icc,Intel C)
###############################################################################
CROSS_LIST = mips-linux-gnu-gcc \
powerpc64-linux-gnu-gcc powerpc-linux-gnu-gcc \
arm-linux-gnueabihf-gcc aarch64-linux-gnu-gcc
# hppa-linux-gnu-gcc - don't supported by current qemu release
# s390x-linux-gnu-gcc - qemu troubles (hang/abort)
# sh4-linux-gnu-gcc - qemu troubles (pread syscall, etc)
# mips64-linux-gnuabi64-gcc - qemu troubles (pread syscall, etc)
# sparc64-linux-gnu-gcc - qemu troubles (fcntl for F_SETLK/F_GETLK)
# alpha-linux-gnu-gcc - qemu (or gcc) troubles (coredump)
CROSS_LIST_NOQEMU = hppa-linux-gnu-gcc s390x-linux-gnu-gcc \
sh4-linux-gnu-gcc mips64-linux-gnuabi64-gcc \
sparc64-linux-gnu-gcc alpha-linux-gnu-gcc
cross-gcc:
@echo "CORRESPONDING CROSS-COMPILERs ARE REQUIRED."
@echo "FOR INSTANCE: apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu"
@for CC in $(CROSS_LIST_NOQEMU) $(CROSS_LIST); do \
echo "===================== $$CC"; \
$(MAKE) clean && CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static $(MAKE) all || exit $$?; \
done
#
# Unfortunately qemu don't provide robust support for futexes.
# Therefore it is impossible to run full multi-process tests.
cross-qemu:
@echo "CORRESPONDING CROSS-COMPILERs AND QEMUs ARE REQUIRED."
@echo "FOR INSTANCE: "
@echo " 1) apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu"
@echo " 2) apt install binfmt-support qemu-user-static qemu-user qemu-system-arm qemu-system-mips qemu-system-misc qemu-system-ppc qemu-system-sparc"
@for CC in $(CROSS_LIST); do \
echo "===================== $$CC + qemu"; \
$(MAKE) clean && \
CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static XCFLAGS="-DMDBX_SAFE4QEMU $(XCFLAGS)" \
$(MAKE) check-singleprocess || exit $$?; \
done

830
contrib/db/libmdbx/README-RU.md Normal file
View file

@ -0,0 +1,830 @@
### The [repository now only mirrored on the Github](https://abf.io/erthink/libmdbx) due to illegal discriminatory restrictions for Russian Crimea and for sovereign crimeans.
<!-- Required extensions: pymdownx.betterem, pymdownx.tilde, pymdownx.emoji, pymdownx.tasklist, pymdownx.superfences -->
-----
libmdbx
======================================
Доработанный и расширенный потомок [Lightning Memory-Mapped Database](https://ru.bmstu.wiki/LMDB_(Lightning_Memory-Mapped_Database)) (aka _LMDB_).
English version is [here](README.md).
_libmdbx_ превосходит LMDB по возможностям и надежности, не уступая в
производительности. _libmdbx_ работает на Linux, FreeBSD, MacOS X и
других ОС соответствующих POSIX.1-2008, а также поддерживает Windows в
качестве дополнительной платформы.
Отдельно ведётся не-публичная разработка следующей версии, в которой
будет кардинальное изменение как API, так и формата базы данных. Цель
этой революции - обеспечение более четкого и надежного API, добавление
новых функций, а также наделение базы данных новыми свойствами.
*Всё будет хорошо. The Future will (be) [Positive](https://www.ptsecurity.ru).*
[![Build Status](https://travis-ci.org/leo-yuriev/libmdbx.svg?branch=master)](https://travis-ci.org/leo-yuriev/libmdbx)
[![Build status](https://ci.appveyor.com/api/projects/status/ue94mlopn50dqiqg/branch/master?svg=true)](https://ci.appveyor.com/project/leo-yuriev/libmdbx/branch/master)
[![Coverity Scan Status](https://scan.coverity.com/projects/12915/badge.svg)](https://scan.coverity.com/projects/reopen-libmdbx)
## Содержание
- [Обзор](#Обзор)
- [Сравнение с другими базами данных](#Сравнение-с-другими-базами-данных)
- [История & Выражение признательности](#История)
- [Описание](#Описание)
- [Ключевые свойства](#Ключевые-свойства)
- [Доработки и усовершенствования относительно LMDB](#Доработки-и-усовершенствования-относительно-lmdb)
- [Недостатки и Компромиссы](#Недостатки-и-Компромиссы)
- [Проблема долгих чтений](#Проблема-долгих-чтений)
- [Сохранность данных в режиме асинхронной фиксации](#Сохранность-данных-в-режиме-асинхронной-фиксации)
- [Использование](#Использование)
- [Сборка](#Сборка)
- [Привязки к другим языкам](#Привязки-к-другим-языкам)
- [Сравнение производительности](#Сравнение-производительности)
- [Интегральная производительность](#Интегральная-производительность)
- [Масштабируемость чтения](#Масштабируемость-чтения)
- [Синхронная фиксация](#Синхронная-фиксация)
- [Отложенная фиксация](#Отложенная-фиксация)
- [Асинхронная фиксация](#Асинхронная-фиксация)
- [Потребление ресурсов](#Потребление-ресурсов)
-----
## Обзор
_libmdbx_ - это встраиваемый key-value движок хранения со специфическим
набором свойств и возможностей, ориентированный на создание уникальных
легковесных решений с предельной производительностью.
_libmdbx_ позволяет множеству процессов совместно читать и обновлять
несколько key-value таблиц с соблюдением
[ACID](https://ru.wikipedia.org/wiki/ACID), при минимальных накладных
расходах и амортизационной стоимости любых операций Olog(N).
_libmdbx_ обеспечивает
[serializability](https://en.wikipedia.org/wiki/Serializability)
изменений и согласованность данных после аварий. При этом транзакции,
изменяющие данные, никак не мешают операциям чтения и выполняются строго
последовательно с использованием единственного
[мьютекса](https://en.wikipedia.org/wiki/Mutual_exclusion).
_libmdbx_ позволяет выполнять операции чтения с гарантиями
[wait-free](https://en.wikipedia.org/wiki/Non-blocking_algorithm#Wait-freedom),
параллельно на каждом ядре CPU, без использования атомарных операций
и/или примитивов синхронизации.
_libmdbx_ не использует
[LSM](https://en.wikipedia.org/wiki/Log-structured_merge-tree), а
основан на [B+Tree](https://en.wikipedia.org/wiki/B%2B_tree) с
[отображением](https://en.wikipedia.org/wiki/Memory-mapped_file) всех
данных в память, при этом текущая версия не использует
[WAL](https://en.wikipedia.org/wiki/Write-ahead_logging). Это
предопределяет многие свойства, в том числе удачные и противопоказанные
сценарии использования.
### Сравнение с другими базами данных
На данный момент, пожалуйста, обратитесь к [главе "сравнение BoltDB с
другими базами
данных"](https://github.com/coreos/bbolt#comparison-with-other-databases),
которая также (в основном) применима к MDBX.
### История
_libmdbx_ является результатом переработки и развития "Lightning
Memory-Mapped Database", известной под аббревиатурой
[LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
Изначально доработка производилась в составе проекта
[ReOpenLDAP](https://github.com/leo-yuriev/ReOpenLDAP). Примерно за год
работы внесенные изменения приобрели самостоятельную ценность. Осенью
2015 доработанный движок был выделен в отдельный проект, который был
[представлен на конференции Highload++
2015](http://www.highload.ru/2015/abstracts/1831.html).
В начале 2017 года движок _libmdbx_ получил новый импульс развития,
благодаря использованию в [Fast Positive
Tables](https://github.com/leo-yuriev/libfpta), aka ["Позитивные
Таблицы"](https://github.com/leo-yuriev/libfpta) by [Positive
Technologies](https://www.ptsecurity.ru).
### Выражение признательности
Говард Чу (Howard Chu) <hyc@openldap.org> является автором движка LMDB, от
которого в 2015 году произошел MDBX.
Мартин Хеденфальк (Martin Hedenfalk) <martin@bzero.se> является автором кода
`btree.c`, который использовался для начала разработки LMDB.
-----
Описание
========
## Ключевые свойства
_libmdbx_ наследует все ключевые возможности и особенности своего
прародителя
[LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database),
но с устранением ряда описываемых далее проблем и архитектурных
недочетов.
1. Данные хранятся в упорядоченном отображении (ordered map), ключи
всегда отсортированы, поддерживается выборка диапазонов (range lookups).
2. Данные отображается в память каждого работающего с БД процесса. К
данным и ключам обеспечивается прямой доступ в памяти без необходимости
их копирования.
3. Транзакции согласно [ACID](https://ru.wikipedia.org/wiki/ACID),
посредством [MVCC](https://ru.wikipedia.org/wiki/MVCC) и
[COW](https://ru.wikipedia.org/wiki/%D0%9A%D0%BE%D0%BF%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%BF%D1%80%D0%B8_%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8).
Изменения строго последовательны и не блокируются чтением, конфликты
между транзакциями невозможны. При этом гарантируется чтение только
зафиксированных данных, см [relaxing
serializability](https://en.wikipedia.org/wiki/Serializability).
4. Чтение и поиск [без
блокировок](https://ru.wikipedia.org/wiki/%D0%9D%D0%B5%D0%B1%D0%BB%D0%BE%D0%BA%D0%B8%D1%80%D1%83%D1%8E%D1%89%D0%B0%D1%8F_%D1%81%D0%B8%D0%BD%D1%85%D1%80%D0%BE%D0%BD%D0%B8%D0%B7%D0%B0%D1%86%D0%B8%D1%8F),
без [атомарных
операций](https://ru.wikipedia.org/wiki/%D0%90%D1%82%D0%BE%D0%BC%D0%B0%D1%80%D0%BD%D0%B0%D1%8F_%D0%BE%D0%BF%D0%B5%D1%80%D0%B0%D1%86%D0%B8%D1%8F).
Читатели не блокируются операциями записи и не конкурируют между собой,
чтение масштабируется линейно по ядрам CPU.
> Для точности следует отметить, что "подключение к БД" (старт первой
> читающей транзакции в потоке) и "отключение от БД" (закрытие БД или
> завершение потока) требуют краткосрочного захвата блокировки для
> регистрации/дерегистрации текущего потока в "таблице читателей".
5. Эффективное хранение дубликатов (ключей с несколькими значениями),
без дублирования ключей, с сортировкой значений, в том числе
целочисленных (для вторичных индексов).
6. Эффективная поддержка коротких ключей фиксированной длины, в том
числе целочисленных.
7. Амортизационная стоимость любой операции Olog(N),
[WAF](https://en.wikipedia.org/wiki/Write_amplification) (Write
Amplification Factor) и RAF (Read Amplification Factor) также Olog(N).
8. Нет [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) и
журнала транзакций, после сбоев не требуется восстановление. Не
требуется компактификация или какое-либо периодическое обслуживание.
Поддерживается резервное копирование "по горячему", на работающей БД без
приостановки изменения данных.
9. Отсутствует какое-либо внутреннее управление памятью или
кэшированием. Всё необходимое штатно выполняет ядро ОС.
## Доработки и усовершенствования относительно LMDB
1. Автоматическое динамическое управление размером БД согласно
параметрам задаваемым функцией `mdbx_env_set_geometry()`, включая шаг
приращения и порог уменьшения размера БД, а также выбор размера
страницы. Соответственно, это позволяет снизить фрагментированность
файла БД на диске и освободить место, в том числе в **Windows**.
2. Автоматическая без-затратная компактификация БД путем возврата
освобождающихся страниц в область нераспределенного резерва в конце
файла данных. При этом уменьшается количество страниц находящихся в
памяти и участвующих в в обмене с диском.
3. Режим `LIFO RECLAIM`.
Для повторного использования выбираются не самые старые, а
самые новые страницы из доступных. За счет этого цикл
использования страниц всегда имеет минимальную длину и не
зависит от общего числа выделенных страниц.
В результате механизмы кэширования и обратной записи работают с
максимально возможной эффективностью. В случае использования
контроллера дисков или системы хранения с
[BBWC](https://en.wikipedia.org/wiki/BBWC) возможно
многократное увеличение производительности по записи
(обновлению данных).
4. Быстрая оценка количества элементов попадающих в запрашиваемый
диапазон значений ключа посредством функций `mdbx_estimate_range()`,
`mdbx_estimate_move()` и `mdbx_estimate_distance()` для выбора
оптимального плана выполнения запроса.
5. Утилита `mdbx_chk` для проверки целостности структуры БД.
6. Поддержка ключей и значений нулевой длины, включая сортированные
дубликаты.
7. Возможность связать с каждой завершаемой транзакцией до 3
дополнительных маркеров посредством `mdbx_canary_put()`, и прочитать их
в транзакции чтения посредством `mdbx_canary_get()`.
8. Возможность посредством `mdbx_replace()` обновить или удалить запись
с получением предыдущего значения данных, а также адресно изменить
конкретное multi-значение.
9. Генерация последовательностей посредством `mdbx_dbi_sequence()`.
10. Обработчик `OOM-KICK`.
Посредством `mdbx_env_set_oomfunc()` может быть установлен
внешний обработчик (callback), который будет вызван при
исчерпании свободных страниц по причине долгой операцией чтения
на фоне интенсивного изменения данных.
Обработчику будет передан PID и pthread_id виновника.
В свою очередь обработчик может предпринять одно из действий:
* нейтрализовать виновника (отправить сигнал kill #9), если
долгое чтение выполняется сторонним процессом;
* отменить или перезапустить проблемную операцию чтения, если
операция выполняется одним из потоков текущего процесса;
* подождать некоторое время, в расчете на то, что проблемная операция
чтения будет штатно завершена;
* прервать текущую операцию изменения данных с возвратом кода
ошибки.
11. Возможность открыть БД в эксклюзивном режиме посредством флага
`MDBX_EXCLUSIVE`, в том числе на сетевом носителе.
12. Возможность получить отставание текущей транзакции чтения от
последней версии данных в БД посредством `mdbx_txn_straggler()`.
13. Возможность явно запросить обновление существующей записи, без
создания новой посредством флажка `MDBX_CURRENT` для `mdbx_put()`.
14. Исправленный вариант `mdbx_cursor_count()`, возвращающий корректное
количество дубликатов для всех типов таблиц и любого положения курсора.
15. Возможность получить посредством `mdbx_env_info()` дополнительную
информацию, включая номер самой старой версии БД (снимка данных),
который используется одним из читателей.
16. Функция `mdbx_del()` не игнорирует дополнительный (уточняющий)
аргумент `data` для таблиц без дубликатов (без флажка `MDBX_DUPSORT`), а
при его ненулевом значении всегда использует его для сверки с удаляемой
записью.
17. Возможность открыть dbi-таблицу, одновременно с установкой
компараторов для ключей и данных, посредством `mdbx_dbi_open_ex()`.
18. Возможность посредством `mdbx_is_dirty()` определить находятся ли
некоторый ключ или данные в "грязной" странице БД. Таким образом,
избегая лишнего копирования данных перед выполнением модифицирующих
операций (значения, размещенные в "грязных" страницах, могут быть
перезаписаны при изменениях, иначе они будут неизменны).
19. Корректное обновление текущей записи, в том числе сортированного
дубликата, при использовании режима `MDBX_CURRENT` в
`mdbx_cursor_put()`.
20. Возможность узнать есть ли за текущей позицией курсора строка данных
посредством `mdbx_cursor_eof()`.
21. Дополнительный код ошибки `MDBX_EMULTIVAL`, который возвращается из
`mdbx_put()` и `mdbx_replace()` при попытке выполнить неоднозначное
обновление или удаления одного из нескольких значений с одним ключом.
22. Возможность посредством `mdbx_get_ex()` получить значение по
заданному ключу, одновременно с количеством дубликатов.
23. Наличие функций `mdbx_cursor_on_first()` и `mdbx_cursor_on_last()`,
которые позволяют быстро выяснить стоит ли курсор на первой/последней
позиции.
24. Возможность автоматического формирования контрольных точек (сброса
данных на диск) при накоплении заданного объёма изменений,
устанавливаемого функцией `mdbx_env_set_syncbytes()`.
25. Управление отладкой и получение отладочных сообщений посредством
`mdbx_setup_debug()`.
26. Функция `mdbx_env_pgwalk()` для обхода всех страниц БД.
27. Три мета-страницы вместо двух, что позволяет гарантированно
консистентно обновлять слабые контрольные точки фиксации без риска
повредить крайнюю сильную точку фиксации.
28. Гарантия сохранности БД в режиме `WRITEMAP+MAPSYNC`.
> В текущей версии _libmdbx_ вам предоставляется выбор между безопасным
> режимом (по умолчанию) асинхронной фиксации, и режимом `UTTERLY_NOSYNC`
> когда при системной аварии есть шанс полного разрушения БД как в LMDB.
> Для подробностей смотрите раздел
> [Сохранность данных в режиме асинхронной фиксации](#Сохранность-данных-в-режиме-асинхронной-фиксации).
29. Возможность закрыть БД в "грязном" состоянии (без сброса данных и
формирования сильной точки фиксации) посредством `mdbx_env_close_ex()`.
30. При завершении читающих транзакций, открытые в них DBI-хендлы не
закрываются и не теряются при завершении таких транзакций посредством
`mdbx_txn_abort()` или `mdbx_txn_reset()`. Что позволяет избавится от ряда
сложно обнаруживаемых ошибок.
31. Все курсоры, как в транзакциях только для чтения, так и в пишущих,
могут быть переиспользованы посредством `mdbx_cursor_renew()` и ДОЛЖНЫ
ОСВОБОЖДАТЬСЯ ЯВНО.
>
> ## _ВАЖНО_, Обратите внимание!
>
> Это единственное изменение в API, которое значимо меняет
> семантику управления курсорами и может приводить к утечкам
> памяти. Следует отметить, что это изменение вынужденно.
> Так устраняется неоднозначность с массой тяжких последствий:
>
> - обращение к уже освобожденной памяти;
> - попытки повторного освобождения памяти;
> - повреждение памяти и ошибки сегментации.
32. На **MacOS X** для синхронизации данных с диском _по-умолчанию_
используется системная функция `fcntl(F_FULLFSYNC)`, так как [только
этим гарантируется сохранность
данных](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html)
при сбое электропитания. К сожалению, в сценариях с высокой
интенсивностью пишущих транзакций, использование `F_FULLFSYNC` приводит
к существенной деградации производительности в сравнении с LMDB, где
используется системная функция `fsync()`. Поэтому _libmdbx_ позволяет
переопределить это поведение определением опции
`MDBX_OSX_SPEED_INSTEADOF_DURABILITY=1` при сборке библиотеки.
33. На **Windows** _libmdbx_ использует файловые блокировки
`LockFileEx()`, так как это позволяет размещать БД на сетевых дисках, а
также обеспечивает защиту от некомпетентных действий пользователя
([защиту от
дурака](https://ru.wikipedia.org/wiki/%D0%97%D0%B0%D1%89%D0%B8%D1%82%D0%B0_%D0%BE%D1%82_%D0%B4%D1%83%D1%80%D0%B0%D0%BA%D0%B0)).
Поэтому _libmdbx_ может немного отставать в тестах производительность от
LMDB, где используются именованные мьютексы.
## Недостатки и Компромиссы
1. Единовременно может выполняться не более одной транзакция изменения данных
(один писатель). Зато все изменения всегда последовательны, не может быть
конфликтов или логических ошибок при откате транзакций.
2. Отсутствие [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging)
обуславливает относительно большой
[WAF](https://en.wikipedia.org/wiki/Write_amplification) (Write
Amplification Factor). Поэтому фиксация изменений на диске может быть
достаточно дорогой и являться главным ограничением производительности
при интенсивном изменении данных.
> В качестве компромисса _libmdbx_ предлагает несколько режимов ленивой
> и/или периодической фиксации. В том числе режим `MAPASYNC`, при котором
> изменения происходят только в памяти и асинхронно фиксируются на диске
> ядром ОС.
>
> Однако, следует воспринимать это свойство аккуратно и взвешенно.
> Например, полная фиксация транзакции в БД с журналом потребует минимум 2
> IOPS (скорее всего 3-4) из-за накладных расходов в файловой системе. В
> _libmdbx_ фиксация транзакции также требует от 2 IOPS. Однако, в БД с
> журналом кол-во IOPS будет меняться в зависимости от файловой системы,
> но не от кол-ва записей или их объема. Тогда как в _libmdbx_ кол-во
> будет расти логарифмически от кол-ва записей/строк в БД (по высоте
> b+tree).
3. [COW](https://ru.wikipedia.org/wiki/%D0%9A%D0%BE%D0%BF%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5_%D0%BF%D1%80%D0%B8_%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8)
для реализации [MVCC](https://ru.wikipedia.org/wiki/MVCC) выполняется на
уровне страниц в [B+
дереве](https://ru.wikipedia.org/wiki/B-%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE).
Поэтому изменение данных амортизационно требует копирования Olog(N)
страниц, что расходует [пропускную способность оперативной
памяти](https://en.wikipedia.org/wiki/Memory_bandwidth) и является
основным ограничителем производительности в режиме `MAPASYNC`.
> Этот недостаток неустраним, тем не менее следует дать некоторые пояснения.
> Дело в том, что фиксация изменений на диске потребует гораздо более
> значительного копирования данных в памяти и массы других затратных операций.
> Поэтому обусловленное этим недостатком падение производительности становится
> заметным только при отказе от фиксации изменений на диске.
> Соответственно, корректнее сказать, что _libmdbx_ позволяет
> получить персистентность ценой минимального падения производительности.
> Если же нет необходимости оперативно сохранять данные, то логичнее
> использовать `std::map`.
4. В _LMDB_ существует проблема долгих чтений (приостановленных читателей),
которая приводит к деградации производительности и переполнению БД.
> В _libmdbx_ предложены средства для предотвращения, быстрого выхода из
> некомфортной ситуации и устранения её последствий. Подробности ниже.
5. В _LMDB_ есть вероятность разрушения БД в режиме `WRITEMAP+MAPASYNC`.
В _libmdbx_ для `WRITEMAP+MAPASYNC` гарантируется как сохранность базы,
так и согласованность данных.
> Дополнительно, в качестве альтернативы, предложен режим `UTTERLY_NOSYNC`.
> Подробности ниже.
### Проблема долгих чтений
*Следует отметить*, что проблема "сборки мусора" так или иначе
существует во всех СУБД (Vacuum в PostgreSQL). Однако в случае _libmdbx_
и LMDB она проявляется более остро, прежде всего из-за высокой
производительности, а также из-за намеренного упрощения внутренних
механизмов ради производительности.
Понимание проблемы требует некоторых пояснений, которые
изложены ниже, но могут быть сложны для быстрого восприятия.
Поэтому, тезисно:
* Изменение данных на фоне долгой операции чтения может
приводить к исчерпанию места в БД.
* После чего любая попытка обновить данные будет приводить к
ошибке `MAP_FULL` до завершения долгой операции чтения.
* Характерными примерами долгих чтений являются горячее
резервное копирования и отладка клиентского приложения при
активной транзакции чтения.
* В оригинальной _LMDB_ после этого будет наблюдаться
устойчивая деградация производительности всех механизмов
обратной записи на диск (в I/O контроллере, в гипервизоре,
в ядре ОС).
* В _libmdbx_ предусмотрен механизм аварийного прерывания таких
операций, а также режим `LIFO RECLAIM` устраняющий последующую
деградацию производительности.
Операции чтения выполняются в контексте снимка данных (версии
БД), который был актуальным на момент старта транзакции чтения. Такой
читаемый снимок поддерживается неизменным до завершения операции. В свою
очередь, это не позволяет повторно использовать страницы БД в
последующих версиях (снимках БД).
Другими словами, если обновление данных выполняется на фоне долгой
операции чтения, то вместо повторного использования "старых" ненужных
страниц будут выделяться новые, так как "старые" страницы составляют
снимок БД, который еще используется долгой операцией чтения.
В результате, при интенсивном изменении данных и достаточно длительной
операции чтения, в БД могут быть исчерпаны свободные страницы, что не
позволит создавать новые снимки/версии БД. Такая ситуация будет
сохраняться до завершения операции чтения, которая использует старый
снимок данных и препятствует повторному использованию страниц БД.
Однако, на этом проблемы не заканчиваются. После описанной ситуации, все
дополнительные страницы, которые были выделены пока переработка старых
была невозможна, будут участвовать в цикле выделения/освобождения до
конца жизни экземпляра БД. В оригинальной _LMDB_ этот цикл использования
страниц работает по принципу [FIFO](https://ru.wikipedia.org/wiki/FIFO).
Поэтому увеличение количества циркулирующий страниц, с точки зрения
механизмов кэширования и/или обратной записи, выглядит как увеличение
рабочего набор данных. Проще говоря, однократное попадание в ситуацию
"уснувшего читателя" приводит к устойчивому эффекту вымывания I/O кэша
при всех последующих изменениях данных.
Для устранения описанных проблемы в _libmdbx_ сделаны существенные
доработки, подробности ниже. Иллюстрации к проблеме "долгих чтений"
можно найти в [слайдах презентации](http://www.slideshare.net/leoyuriev/lmdb).
Там же приведен пример количественной оценки прироста производительности
за счет эффективной работы [BBWC](https://en.wikipedia.org/wiki/BBWC)
при включении `LIFO RECLAIM` в _libmdbx_.
### Сохранность данных в режиме асинхронной фиксации
При работе в режиме `WRITEMAP+MAPSYNC` запись измененных страниц
выполняется ядром ОС, что имеет ряд преимуществ. Так например, при крахе
приложения, ядро ОС сохранит все изменения.
Однако, при аварийном отключении питания или сбое в ядре ОС, на диске
может быть сохранена только часть измененных страниц БД. При этом с
большой вероятностью может оказаться, что будут сохранены мета-страницы
со ссылками на страницы с новыми версиями данных, но не сами новые
данные. В этом случае БД будет безвозвратна разрушена, даже если до
аварии производилась полная синхронизация данных (посредством
`mdbx_env_sync()`).
В _libmdbx_ эта проблема устранена путем полной переработки
пути записи данных:
* В режиме `WRITEMAP+MAPSYNC` _libmdbx_ не обновляет
мета-страницы непосредственно, а поддерживает их теневые копии
с переносом изменений после фиксации данных.
* При завершении транзакций, в зависимости от состояния
синхронности данных между диском и оперативной памятью,
_libmdbx_ помечает точки фиксации либо как сильные (strong),
либо как слабые (weak). Так например, в режиме
`WRITEMAP+MAPSYNC` завершаемые транзакции помечаются как
слабые, а при явной синхронизации данных - как сильные.
* В _libmdbx_ поддерживается не две, а три отдельные мета-страницы.
Это позволяет выполнять фиксацию транзакций с формированием как
сильной, так и слабой точки фиксации, без потери двух предыдущих
точек фиксации (из которых одна может быть сильной, а вторая слабой).
В результате, _libmdbx_ позволяет в произвольном порядке чередовать
сильные и слабые точки фиксации без нарушения соответствующих
гарантий в случае неожиданной системной аварии во время фиксации.
* При открытии БД выполняется автоматический откат к последней
сильной фиксации. Этим обеспечивается гарантия сохранности БД.
Такая гарантия надежности не дается бесплатно. Для сохранности данных,
страницы, формирующие крайний снимок с сильной фиксацией, не должны
повторно использоваться (перезаписываться) до формирования следующей
сильной точки фиксации. Таким образом, крайняя точка фиксации создает
описанный выше эффект "долгого чтения". Разница же здесь в том, что при
исчерпании свободных страниц ситуация будет автоматически исправлена,
посредством записи изменений на диск и формирования новой сильной точки
фиксации.
Таким образом, в режиме безопасной асинхронной фиксации _libmdbx_ будет
всегда использовать новые страницы до исчерпания места в БД или до
явного формирования сильной точки фиксации посредством
`mdbx_env_sync()`. При этом суммарный трафик записи на диск будет
примерно такой же, как если бы отдельно фиксировалась каждая транзакция.
В текущей версии _libmdbx_ вам предоставляется выбор между безопасным
режимом (по умолчанию) асинхронной фиксации, и режимом `UTTERLY_NOSYNC`
когда при системной аварии есть шанс полного разрушения БД как в LMDB.
В последующих версиях _libmdbx_ будут предусмотрены средства для
асинхронной записи данных на диск с автоматическим формированием сильных
точек фиксации.
--------------------------------------------------------------------------------
Использование
=============
## Сборка
Для сборки на всех платформах кроме Windows вам потребуются не-дремучие
версии: GNU Make, [bash](https://ru.wikipedia.org/wiki/Bash), компиляторы C и C++ совместимые с GCC или CLANG.
Исторически сборка _libmdbx_ основывается на одном
[Makefile](https://ru.wikipedia.org/wiki/Makefile), что предполагает
разные рецепты сборки в зависимости от целевой платформы. В следующих
версиях планируется переход на использование
[CMake](https://ru.wikipedia.org/wiki/CMake), с отказом от поддержки
других инструментов.
#### Выгрузка DSO/DLL и деструкторы Thread-Local-Storage объектов
При сборке _libmdbx_ в виде разделяемой библиотеки, либо использовании
статической _libmdbx_ в составе другой динамической библиотеке,
желательно убедиться, что ваша система обеспечивает корректность вызова
деструкторов Thread-Local-Storage объектов при выгрузке динамических
библиотек.
Если это не так, то при выгрузке динамической библиотеки с _libmdbx_
внутри возможна либо утечка ресурсов, либо падения из-за вызова
деструкторов из уже выгруженного DSO/DLL объекта. Проблема может
проявляться только в многопоточном приложении, которое производит
выгрузку разделяемых динамических библиотек с кодом _libmdbx_ внутри,
после использования _libmdbx_. Заведомо известно, что TLS-деструкторы
корректно обслуживаются:
- На всех актуальных версиях Windows (Windows 7 и последующих).
- На системах c функцией
[`__cxa_thread_atexit_impl()`](https://sourceware.org/glibc/wiki/Destructor%20support%20for%20thread_local%20variables)
в стандартной библиотеке C. В том числе на системах с GNU libc версии
2.18 и выше.
- На системах с libpthread/ntpl из состава GNU libc с исправлением
ошибок [#21031](https://sourceware.org/bugzilla/show_bug.cgi?id=21031) и
[#21032](https://sourceware.org/bugzilla/show_bug.cgi?id=21032), либо
где нет подобных ошибок в реализации pthreads.
### Linux и другие платформы с GNU Make
Для сборки библиотеки достаточно выполнить `make all` в директории с
исходными текстами, а для выполнения базовых тестов `make check`.
Если установленный в система `make` не является GNU Make, то при попытке
сборки будет масса ошибок от make. В этом случае, возможно, вместо
`make` вам следует использовать `gmake`, либо даже `gnu-make` и т.п.
### FreeBSD и родственные платформы
Как правило, на таких системах по-умолчанию используется Berkeley Make.
А GNU Make вызывается командой `gmake` или может отсутствовать. Кроме
этого может отсутствовать [`bash`](https://ru.wikipedia.org/wiki/Bash).
Вам необходимо установить требуемые компоненты: GNU Make, bash,
компиляторы C и C++ совместимые с GCC или CLANG. После этого для сборки
библиотеки достаточно выполнить `gmake all` (или `make all`) в
директории с исходными текстами, а для выполнения базовых тестов `gmake
check` (или `make check`).
### Windows
Для сборки libmdbx_ для ОС Windows рекомендуется использовать [Microsoft
Visual Studio](https://ru.wikipedia.org/wiki/Microsoft_Visual_Studio),
но не такие инструменты как MinGW, MSYS или Cygwin. Для этого в набор
исходных кодов _libmdbx_ входят соответствующие файлы проектов
совместимые с Visual Studio 2015, Windows SDK для Windows 8.1 и более
поздними версиями. Достаточно открыть `mdbx.sln` и выполнить сборку
библиотеки.
Для сборки с более новыми версиями SDK или Visual Studio должно быть
достаточно выполнить "Retarget solution". Для сборки под старые версии
Windows (например Windows XP) или более старыми компиляторами вам
потребуется самостоятельно преобразовать или воссоздать файлы проектов.
Сборка посредством MinGW, MSYS или Cygwin потенциально возможна. Однако,
эти сценарии не тестируются и вероятно потребуют от вас доработки
`Makefile`. Следует отметить, что в _libmdbx_ предприняты усилия для
устранения runtime зависимостей от CRT и других библиотек Visual Studio.
Для этого достаточно при сборке определить опцию `MDBX_AVOID_CRT`.
Пример запуска базового сценария тестирования можно найти в
[CI-сценарии](appveyor.yml) для [AppVeyor](https://www.appveyor.com/).
Для выполнения [сценария длительного стохастического
тестирования](test/long_stochastic.sh) потребуется
[`bash`](https://ru.wikipedia.org/wiki/Bash), а само тестирование
рекомендуется выполнять с размещением тестовых данных на
[RAM-диске](https://ru.wikipedia.org/wiki/RAM-%D0%B4%D0%B8%D1%81%D0%BA).
### MacOS X
Актуальные [нативные сборочные
инструменты](https://ru.wikipedia.org/wiki/Xcode) для MacOS X включают
GNU Make, CLANG и устаревшую версию bash. Поэтому для сборки библиотеки
достаточно выполнить `make all` в директории с исходными текстами, а для
выполнения базовых тестов `make check`. Если же что-то пойдет не так, то
рекомендуется установить [Homebrew](https://brew.sh/) и попробовать ещё
раз.
Для выполнения [сценария длительного стохастического
тестирования](test/long_stochastic.sh) потребуется установка актуальной
(не устаревшей) версии [`bash`](https://ru.wikipedia.org/wiki/Bash). Для
этого рекомендуется установить [Homebrew](https://brew.sh/), а затем
выполнить `brew install bash`.
## Привязки к другим языкам
| Runtime | GitHub | Author |
| -------- | ------ | ------ |
| Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) |
| .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) |
--------------------------------------------------------------------------------
Сравнение производительности
============================
Все представленные ниже данные получены многократным прогоном тестов на
ноутбуке Lenovo Carbon-2, i7-4600U 2.1 ГГц, 8 Гб ОЗУ, с SSD-диском
SAMSUNG MZNTD512HAGL-000L1 (DXT23L0Q) 512 Гб.
Исходный код бенчмарка [_IOArena_](https://github.com/pmwkaa/ioarena) и
сценарии тестирования [доступны на
github](https://github.com/pmwkaa/ioarena/tree/HL%2B%2B2015).
## Интегральная производительность
Показана соотнесенная сумма ключевых показателей производительности в трёх
бенчмарках:
- Чтение/Поиск на машине с 4-мя процессорами;
- Транзакции с [CRUD](https://ru.wikipedia.org/wiki/CRUD)-операциями
(вставка, чтение, обновление, удаление) в режиме **синхронной фиксации**
данных (fdatasync при завершении каждой транзакции или аналог);
- Транзакции с [CRUD](https://ru.wikipedia.org/wiki/CRUD)-операциями
(вставка, чтение, обновление, удаление) в режиме **отложенной фиксации**
данных (отложенная запись посредством файловой систем или аналог);
*Бенчмарк в режиме асинхронной записи не включен по двум причинам:*
1. Такое сравнение не совсем правомочно, его следует делать с движками
ориентированными на хранение данных в памяти ([Tarantool](https://tarantool.io/), [Redis](https://redis.io/)).
2. Превосходство libmdbx становится еще более подавляющим, что мешает
восприятию информации.
![Comparison #1: Integral Performance](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-1.png)
--------------------------------------------------------------------------------
## Масштабируемость чтения
Для каждого движка показана суммарная производительность при
одновременном выполнении запросов чтения/поиска в 1-2-4-8 потоков на
машине с 4-мя физическими процессорами.
![Comparison #2: Read Scalability](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-2.png)
--------------------------------------------------------------------------------
## Синхронная фиксация
- Линейная шкала слева и темные прямоугольники соответствуют количеству
транзакций в секунду, усредненному за всё время теста.
- Логарифмическая шкала справа и желтые интервальные отрезки
соответствуют времени выполнения транзакций. При этом каждый отрезок
показывает минимальное и максимальное время, затраченное на выполнение
транзакций, а крестиком отмечено среднеквадратичное значение.
Выполняется **10.000 транзакций в режиме синхронной фиксации данных** на
диске. При этом требуется гарантия, что при аварийном выключении питания
(или другом подобном сбое) все данные будут консистентны и полностью
соответствовать последней завершенной транзакции. В _libmdbx_ в этом
режиме при фиксации каждой транзакции выполняется системный вызов
[fdatasync](https://linux.die.net/man/2/fdatasync).
В каждой транзакции выполняется комбинированная CRUD-операция (две
вставки, одно чтение, одно обновление, одно удаление). Бенчмарк стартует
на пустой базе, а при завершении, в результате выполняемых действий, в
базе насчитывается 10.000 небольших key-value записей.
![Comparison #3: Sync-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-3.png)
--------------------------------------------------------------------------------
## Отложенная фиксация
- Линейная шкала слева и темные прямоугольники соответствуют количеству
транзакций в секунду, усредненному за всё время теста.
- Логарифмическая шкала справа и желтые интервальные отрезки
соответствуют времени выполнения транзакций. При этом каждый отрезок
показывает минимальное и максимальное время, затраченное на выполнение
транзакций, а крестиком отмечено среднеквадратичное значение.
Выполняется **100.000 транзакций в режиме отложенной фиксации данных**
на диске. При этом требуется гарантия, что при аварийном выключении
питания (или другом подобном сбое) все данные будут консистентны на
момент завершения одной из транзакций, но допускается потеря изменений
из некоторого количества последних транзакций, что для многих движков
предполагает включение
[WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) (write-ahead
logging) либо журнала транзакций, который в свою очередь опирается на
гарантию упорядоченности данных в журналируемой файловой системе.
_libmdbx_ при этом не ведет WAL, а передает весь контроль файловой
системе и ядру ОС.
В каждой транзакции выполняется комбинированная CRUD-операция (две
вставки, одно чтение, одно обновление, одно удаление). Бенчмарк стартует
на пустой базе, а при завершении, в результате выполняемых действий, в
базе насчитывается 100.000 небольших key-value записей.
![Comparison #4: Lazy-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-4.png)
--------------------------------------------------------------------------------
## Асинхронная фиксация
- Линейная шкала слева и темные прямоугольники соответствуют количеству
транзакций в секунду, усредненному за всё время теста.
- Логарифмическая шкала справа и желтые интервальные отрезки
соответствуют времени выполнения транзакций. При этом каждый отрезок
показывает минимальное и максимальное время, затраченное на выполнение
транзакций, а крестиком отмечено среднеквадратичное значение.
Выполняется **1.000.000 транзакций в режиме асинхронной фиксации
данных** на диске. При этом требуется гарантия, что при аварийном
выключении питания (или другом подобном сбое) все данные будут
консистентны на момент завершения одной из транзакций, но допускается
потеря изменений из значительного количества последних транзакций. Во
всех движках при этом включался режим предполагающий минимальную
нагрузку на диск по записи, и соответственно минимальную гарантию
сохранности данных. В _libmdbx_ при этом используется режим асинхронной
записи измененных страниц на диск посредством ядра ОС и системного
вызова [msync(MS_ASYNC)](https://linux.die.net/man/2/msync).
В каждой транзакции выполняется комбинированная CRUD-операция (две
вставки, одно чтение, одно обновление, одно удаление). Бенчмарк стартует
на пустой базе, а при завершении, в результате выполняемых действий, в
базе насчитывается 10.000 небольших key-value записей.
![Comparison #5: Async-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-5.png)
--------------------------------------------------------------------------------
## Потребление ресурсов
Показана соотнесенная сумма использованных ресурсов в ходе бенчмарка в
режиме отложенной фиксации:
- суммарное количество операций ввода-вывода (IOPS), как записи, так и
чтения.
- суммарное затраченное время процессора, как в режиме пользовательских
процессов, так и в режиме ядра ОС.
- использованное место на диске при завершении теста, после закрытия БД
из тестирующего процесса, но без ожидания всех внутренних операций
обслуживания (компактификации LSM и т.п.).
Движок _ForestDB_ был исключен при оформлении результатов, так как
относительно конкурентов многократно превысил потребление каждого из
ресурсов (потратил процессорное время на генерацию IOPS для заполнения
диска), что не позволяло наглядно сравнить показатели остальных движков
на одной диаграмме.
Все данные собирались посредством системного вызова
[getrusage()](http://man7.org/linux/man-pages/man2/getrusage.2.html) и
сканированием директорий с данными.
![Comparison #6: Cost comparison](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-6.png)
--------------------------------------------------------------------------------
```
$ objdump -f -h -j .text libmdbx.so
libmdbx.so: file format elf64-x86-64
architecture: i386:x86-64, flags 0x00000150:
HAS_SYMS, DYNAMIC, D_PAGED
start address 0x0000000000003870
Sections:
Idx Name Size VMA LMA File off Algn
11 .text 000173d4 0000000000003870 0000000000003870 00003870 2**4
CONTENTS, ALLOC, LOAD, READONLY, CODE
```

701
contrib/db/libmdbx/README.md Normal file
View file

@ -0,0 +1,701 @@
### The [repository now only mirrored on the Github](https://abf.io/erthink/libmdbx) due to illegal discriminatory restrictions for Russian Crimea and for sovereign crimeans.
<!-- Required extensions: pymdownx.betterem, pymdownx.tilde, pymdownx.emoji, pymdownx.tasklist, pymdownx.superfences -->
-----
libmdbx
======================================
Revised and extended descendant of [Lightning Memory-Mapped Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) (aka _LMDB_).
Русскоязычная версия [здесь](README-RU.md).
_libmdbx_ is superior to LMDB in terms of features and reliability, not
inferior in performance. _libmdbx_ works on Linux, FreeBSD, MacOS X and
other systems compliant with POSIX.1-2008, but also support Windows as a
complementary platform.
The next version is under active non-public development, which will
radically change both the API and the database format. The goal of this
revolution is to provide a clearer and more reliable API, add more
features and new database properties.
*The Future will (be) [Positive](https://www.ptsecurity.com). Всё будет хорошо.*
[![Build Status](https://travis-ci.org/leo-yuriev/libmdbx.svg?branch=master)](https://travis-ci.org/leo-yuriev/libmdbx)
[![Build status](https://ci.appveyor.com/api/projects/status/ue94mlopn50dqiqg/branch/master?svg=true)](https://ci.appveyor.com/project/leo-yuriev/libmdbx/branch/master)
[![Coverity Scan Status](https://scan.coverity.com/projects/12915/badge.svg)](https://scan.coverity.com/projects/reopen-libmdbx)
## Table of Contents
- [Overview](#overview)
- [Comparison with other databases](#comparison-with-other-databases)
- [History & Acknowledgments](#history)
- [Description](#description)
- [Key features](#key-features)
- [Improvements over LMDB](#improvements-over-lmdb)
- [Gotchas](#gotchas)
- [Problem of long-time reading](#problem-of-long-time-reading)
- [Durability in asynchronous writing mode](#durability-in-asynchronous-writing-mode)
- [Usage](#usage)
- [Building](#building)
- [Bindings](#bindings)
- [Performance comparison](#performance-comparison)
- [Integral performance](#integral-performance)
- [Read scalability](#read-scalability)
- [Sync-write mode](#sync-write-mode)
- [Lazy-write mode](#lazy-write-mode)
- [Async-write mode](#async-write-mode)
- [Cost comparison](#cost-comparison)
-----
## Overview
_libmdbx_ is an embedded lightweight key-value database engine oriented
for performance.
_libmdbx_ allows multiple processes to read and update several key-value
tables concurrently, while being
[ACID](https://en.wikipedia.org/wiki/ACID)-compliant, with minimal
overhead and Olog(N) operation cost.
_libmdbx_ enforce
[serializability](https://en.wikipedia.org/wiki/Serializability) for
writers by single
[mutex](https://en.wikipedia.org/wiki/Mutual_exclusion) and affords
[wait-free](https://en.wikipedia.org/wiki/Non-blocking_algorithm#Wait-freedom)
for parallel readers without atomic/interlocked operations, while
writing and reading transactions do not block each other.
_libmdbx_ can guarantee consistency after crash depending of operation
mode.
_libmdbx_ uses [B+Trees](https://en.wikipedia.org/wiki/B%2B_tree) and
[Memory-Mapping](https://en.wikipedia.org/wiki/Memory-mapped_file),
doesn't use [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging)
which might be a caveat for some workloads.
### Comparison with other databases
For now please refer to [chapter of "BoltDB comparison with other
databases"](https://github.com/coreos/bbolt#comparison-with-other-databases)
which is also (mostly) applicable to MDBX.
### History
The _libmdbx_ design is based on [Lightning Memory-Mapped
Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database).
Initial development was going in
[ReOpenLDAP](https://github.com/leo-yuriev/ReOpenLDAP) project. About a
year later libmdbx was isolated to separate project, which was
[presented at Highload++ 2015
conference](http://www.highload.ru/2015/abstracts/1831.html).
Since early 2017 _libmdbx_ is used in [Fast Positive Tables](https://github.com/leo-yuriev/libfpta),
and development is funded by [Positive Technologies](https://www.ptsecurity.com).
### Acknowledgments
Howard Chu <hyc@openldap.org> is the author of LMDB, from which
originated the MDBX in 2015.
Martin Hedenfalk <martin@bzero.se> is the author of `btree.c` code, which
was used for begin development of LMDB.
-----
Description
===========
## Key features
_libmdbx_ inherits all features and characteristics from
[LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database):
1. Key-value pairs are stored in ordered map(s), keys are always sorted,
range lookups are supported.
2. Data is [memory-mapped](https://en.wikipedia.org/wiki/Memory-mapped_file)
into each worker DB process, and could be accessed zero-copy from transactions.
3. Transactions are
[ACID](https://en.wikipedia.org/wiki/ACID)-compliant, through to
[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
and [CoW](https://en.wikipedia.org/wiki/Copy-on-write). Writes are
strongly serialized and aren't blocked by reads, transactions can't
conflict with each other. Reads are guaranteed to get only commited data
([relaxing serializability](https://en.wikipedia.org/wiki/Serializability#Relaxing_serializability)).
4. Read transactions are
[non-blocking](https://en.wikipedia.org/wiki/Non-blocking_algorithm),
don't use [atomic operations](https://en.wikipedia.org/wiki/Linearizability#High-level_atomic_operations).
Readers don't block each other and aren't blocked by writers. Read
performance scales linearly with CPU core count.
> Nonetheless, "connect to DB" (starting the first read transaction in a thread) and
> "disconnect from DB" (closing DB or thread termination) requires a lock
> acquisition to register/unregister at the "readers table".
5. Keys with multiple values are stored efficiently without key
duplication, sorted by value, including integers (valuable for
secondary indexes).
6. Efficient operation on short fixed length keys,
including 32/64-bit integer types.
7. [WAF](https://en.wikipedia.org/wiki/Write_amplification) (Write
Amplification Factor) и RAF (Read Amplification Factor) are Olog(N).
8. No [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) and
transaction journal. In case of a crash no recovery needed. No need for
regular maintenance. Backups can be made on the fly on working DB
without freezing writers.
9. No additional memory management, all done by basic OS services.
## Improvements over LMDB
1. Automatic dynamic DB size management according to the parameters
specified by `mdbx_env_set_geometry()` function. Including
growth step and truncation threshold, as well as the choice of page
size.
2. Automatic returning of freed pages into unallocated space at the end
of database file, with optionally automatic shrinking it. This reduces
amount of pages resides in RAM and circulated in disk I/O. In fact
_libmdbx_ constantly performs DB compactification, without spending
additional resources for that.
3. `LIFO RECLAIM` mode:
The newest pages are picked for reuse instead of the oldest. This allows
to minimize reclaim loop and make it execution time independent of total
page count.
This results in OS kernel cache mechanisms working with maximum
efficiency. In case of using disk controllers or storages with
[BBWC](https://en.wikipedia.org/wiki/Disk_buffer#Write_acceleration)
this may greatly improve write performance.
4. Fast estimation of range query result size via functions
`mdbx_estimate_range()`, `mdbx_estimate_move()` and
`mdbx_estimate_distance()`. E.g. for selection the optimal query
execution plan.
5. `mdbx_chk` tool for DB integrity check.
6. Support for keys and values of zero length, including multi-values
(aka sorted duplicates).
7. Ability to assign up to 3 persistent 64-bit markers to commiting
transaction with `mdbx_canary_put()` and then get them in read
transaction by `mdbx_canary_get()`.
8. Ability to update or delete record and get previous value via
`mdbx_replace()`. Also allows update the specific item from multi-value
with the same key.
9. Sequence generation via `mdbx_dbi_sequence()`.
10. `OOM-KICK` callback.
`mdbx_env_set_oomfunc()` allows to set a callback, which will be called
in the event of DB space exhausting during long-time read transaction in
parallel with extensive updating. Callback will be invoked with PID and
pthread_id of offending thread as parameters. Callback can do any of
these things to remedy the problem:
* wait for read transaction to finish normally;
* kill the offending process (signal 9), if separate process is doing
long-time read;
* abort or restart offending read transaction if it's running in sibling
thread;
* abort current write transaction with returning error code.
11. Ability to open DB in exclusive mode by `MDBX_EXCLUSIVE` flag.
12. Ability to get how far current read-transaction snapshot lags
from the latest version of the DB by `mdbx_txn_straggler()`.
13. Ability to explicitly update the existing record, not insertion
a new one. Implemented as `MDBX_CURRENT` flag for `mdbx_put()`.
14. Fixed `mdbx_cursor_count()`, which returns correct count of
duplicated (aka multi-value) for all cases and any cursor position.
15. `mdbx_env_info()` to getting additional info, including number of
the oldest snapshot of DB, which is used by someone of the readers.
16. `mdbx_del()` doesn't ignore additional argument (specifier) `data`
for tables without duplicates (without flag `MDBX_DUPSORT`), if `data`
is not null then always uses it to verify record, which is being
deleted.
17. Ability to open dbi-table with simultaneous with race-free setup
of comparators for keys and values, via `mdbx_dbi_open_ex()`.
18. `mdbx_is_dirty()`to find out if given key or value is on dirty page, that
useful to avoid copy-out before updates.
19. Correct update of current record in `MDBX_CURRENT` mode of
`mdbx_cursor_put()`, including sorted duplicated.
20. Check if there is a row with data after current cursor position via
`mdbx_cursor_eof()`.
21. Additional error code `MDBX_EMULTIVAL`, which is returned by
`mdbx_put()` and `mdbx_replace()` in case is ambiguous update or delete.
22. Ability to get value by key and duplicates count by `mdbx_get_ex()`.
23. Functions `mdbx_cursor_on_first()` and `mdbx_cursor_on_last()`,
which allows to check cursor is currently on first or last position
respectively.
24. Automatic creation of steady commit-points (flushing data to the
disk) when the volume of changes reaches a threshold, which can be
set by `mdbx_env_set_syncbytes()`.
25. Control over debugging and receiving of debugging messages via
`mdbx_setup_debug()`.
26. Function `mdbx_env_pgwalk()` for page-walking the DB.
27. Three meta-pages instead of two, that allows to guarantee
consistency of data when updating weak commit-points without the
risk of damaging the last steady commit-point.
28. Guarantee of DB integrity in `WRITEMAP+MAPSYNC` mode:
> Current _libmdbx_ gives a choice of safe async-write mode (default)
> and `UTTERLY_NOSYNC` mode which may result in full
> DB corruption during system crash as with LMDB. For details see
> [Data safety in async-write mode](#data-safety-in-async-write-mode).
29. Ability to close DB in "dirty" state (without data flush and
creation of steady synchronization point) via `mdbx_env_close_ex()`.
30. If read transaction is aborted via `mdbx_txn_abort()` or
`mdbx_txn_reset()` then DBI-handles, which were opened during it,
will not be closed or deleted. In several cases this allows
to avoid hard-to-debug errors.
31. All cursors in all read and write transactions can be reused by
`mdbx_cursor_renew()` and MUST be freed explicitly.
> ## Caution, please pay attention!
>
> This is the only change of API, which changes semantics of cursor management
> and can lead to memory leaks on misuse. This is a needed change as it eliminates ambiguity
> which helps to avoid such errors as:
> - use-after-free;
> - double-free;
> - memory corruption and segfaults.
32. On **Mac OS X** the `fcntl(F_FULLFSYNC)` syscall is used _by
default_ to synchronize data with the disk, as this is [the only way to
guarantee data
durability](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html)
in case of power failure. Unfortunately, in scenarios with high write
intensity, the use of `F_FULLFSYNC` significant degrades performance
compared to LMDB, where the `fsync()` syscall is used. Therefore,
_libmdbx_ allows you to override this behavior by defining the
`MDBX_OSX_SPEED_INSTEADOF_DURABILITY=1` option while build the library.
33. On **Windows** the `LockFileEx()` syscall is used for locking, since
it allows place the database on network drives, and provides protection
against incompetent user actions (aka
[poka-yoke](https://en.wikipedia.org/wiki/Poka-yoke)). Therefore
_libmdbx_ may be a little lag in performance tests from LMDB where a
named mutexes are used.
## Gotchas
1. There cannot be more than one writer at a time. This allows serialize an
updates and eliminate any possibility of conflicts, deadlocks or logical errors.
2. No [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) means
relatively big [WAF](https://en.wikipedia.org/wiki/Write_amplification)
(Write Amplification Factor). Because of this syncing data to disk might
be quite resource intensive and be main performance bottleneck during
intensive write workload.
> As compromise _libmdbx_ allows several modes of lazy and/or periodic
> syncing, including `MAPASYNC` mode, which modificate data in memory and
> asynchronously syncs data to disk, moment to sync is picked by OS.
>
> Although this should be used with care, synchronous transactions in a DB
> with transaction journal will require 2 IOPS minimum (probably 3-4 in
> practice) because of filesystem overhead, overhead depends on
> filesystem, not on record count or record size. In _libmdbx_ IOPS count
> will grow logarithmically depending on record count in DB (height of B+
> tree) and will require at least 2 IOPS per transaction too.
3. [CoW](https://en.wikipedia.org/wiki/Copy-on-write) for
[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
is done on memory page level with
[B+trees](https://ru.wikipedia.org/wiki/B-%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE).
Therefore altering data requires to copy about Olog(N) memory pages,
which uses [memory bandwidth](https://en.wikipedia.org/wiki/Memory_bandwidth) and is main
performance bottleneck in `MAPASYNC` mode.
> This is unavoidable, but isn't that bad. Syncing data to disk requires
> much more similar operations which will be done by OS, therefore this is
> noticeable only if data sync to persistent storage is fully disabled.
> _libmdbx_ allows to safely save data to persistent storage with minimal
> performance overhead. If there is no need to save data to persistent
> storage then it's much more preferable to use `std::map`.
4. _LMDB_ has a problem of long-time readers which degrades performance
and bloats DB.
> _libmdbx_ addresses that, details below.
5. _LMDB_ is susceptible to DB corruption in `WRITEMAP+MAPASYNC` mode.
_libmdbx_ in `WRITEMAP+MAPASYNC` guarantees DB integrity and consistency
of data.
> Additionally there is an alternative: `UTTERLY_NOSYNC` mode.
> Details below.
### Problem of long-time reading
Garbage collection problem exists in all databases one way or another
(e.g. VACUUM in PostgreSQL). But in _libmdbx_ and LMDB it's even more
discernible because of high transaction rate and intentional internals
simplification in favor of performance.
Understanding the problem requires some explanation, but can be
difficult for quick perception. So is is reasonable
to simplify this as follows:
* Massive altering of data during a parallel long read operation may
exhaust the free DB space.
* If the available space is exhausted, any attempt to update the data
will cause a "MAP_FULL" error until a long read transaction is
completed.
* A good example of long readers is a hot backup or debugging of
a client application while retaining an active read transaction.
* In _LMDB_ this results in degraded performance of all operations of
writing data to persistent storage.
* _libmdbx_ has the `OOM-KICK` mechanism which allow to abort such
operations and the `LIFO RECLAIM` mode which addresses performance
degradation.
### Durability in asynchronous writing mode
In `WRITEMAP+MAPSYNC` mode updated (aka dirty) pages are written to
persistent storage by the OS kernel. This means that if the application
fails, the OS kernel will finish writing all updated data to disk and
nothing will be lost. However, in the case of hardware malfunction or OS
kernel fatal error, only some updated data can be written to disk and
the database structure is likely to be destroyed. In such situation, DB
is completely corrupted and can't be repaired.
_libmdbx_ addresses this by fully reimplementing write path of data:
* In `WRITEMAP+MAPSYNC` mode meta-data pages aren't updated in place,
instead their shadow copies are used and their updates are synced after
data is flushed to disk.
* During transaction commit _libmdbx_ marks it as a steady or weak
depending on synchronization status between RAM and persistent storage.
For instance, in the `WRITEMAP+MAPSYNC` mode committed transactions
are marked as weak by default, but as steady after explicit data flushes.
* _libmdbx_ maintains three separate meta-pages instead of two. This
allows to commit transaction as steady or weak without losing two
previous commit points (one of them can be steady, and another
weak). Thus, after a fatal system failure, it will be possible to
rollback to the last steady commit point.
* During DB open _libmdbx_ rollbacks to the last steady commit point,
this guarantees database integrity after a crash. However, if the
database opening in read-only mode, such rollback cannot be performed
which will cause returning the MDBX_WANNA_RECOVERY error.
For data integrity a pages which form database snapshot with steady
commit point, must not be updated until next steady commit point.
Therefore the last steady commit point creates an effect analogues to
"long-time read". The only difference that now in case of space
exhaustion the problem will be immediately addressed by writing changes
to disk and forming the new steady commit point.
So in async-write mode _libmdbx_ will always use new pages until the
free DB space will be exhausted or `mdbx_env_sync()` will be invoked,
and the total write traffic to the disk will be the same as in
sync-write mode.
Currently libmdbx gives a choice between a safe async-write mode
(default) and `UTTERLY_NOSYNC` mode which may lead to DB corruption
after a system crash, i.e. like the LMDB.
Next version of _libmdbx_ will be automatically create steady commit
points in async-write mode upon completion transfer data to the disk.
--------------------------------------------------------------------------------
Usage
=====
## Building
To build on all platforms except Windows the prerequirements are the
same: non-obsolete versions of GNU Make,
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)), C and C++
compilers compatible with GCC or CLANG. On Windows you will need only :
Microsoft Visual Studio 2015 or later, Windows SDK for Windows 8 or
later.
Historically, the libmdbx builing is based on single
[Makefile](https://en.wikipedia.org/wiki/Makefile) which assumes
different recipes depending on target platform. In the next versions, it
is planned to switch to [CMake](https://en.wikipedia.org/wiki/CMake),
with the refusal to support other tools.
#### DSO/DLL unloading and destructors of Thread-Local-Storage objects
When building _libmdbx_ as a shared library or use static _libmdbx_ as a
part of another dynamic library, it is advisable to make sure that your
system ensures the correctness of the call destructors of
Thread-Local-Storage objects when unloading dynamic libraries'.
If this is not the case, then unloading a dynamic-link library with
_libmdbx_ code inside, can result in either a resource leak or a crash
due to calling destructors from an already unloaded DSO/DLL object. The
problem can only manifest in a multithreaded application, which makes
the unloading of shared dynamic libraries with _libmdbx_ code inside,
after using _libmdbx_. It is known that TLS-destructors are properly
maintained in the following cases:
- On all modern versions of Windows (Windows 7 and later).
- On systems with the
[`__cxa_thread_atexit_impl()`](https://sourceware.org/glibc/wiki/Destructor%20support%20for%20thread_local%20variables)
function in the standard C library, including systems with GNU libc
version 2.18 and later.
- On systems with libpthread/ntpl from GNU libc with bug fixes
[#21031](https://sourceware.org/bugzilla/show_bug.cgi?id=21031) and
[#21032](https://sourceware.org/bugzilla/show_bug.cgi?id=21032), or
where there are no similar bugs in the pthreads implementation.
### Linux and other platforms with GNU Make
To build the library it is enough to execute `make all` in the directory
of source code, and `make check` for execute the basic tests.
If the `make` installed on the system is not GNU Make, there will be a
lot of errors from make when trying to build. In this case, perhaps you
should use `gmake` instead of `make`, or even `gnu-make`, etc.
### FreeBSD and related platforms
As a rule, in such systems, the default is to use Berkeley Make. And GNU
Make is called by the gmake command or may be missing. In addition,
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) may be absent.
You need to install the required components: GNU Make, bash, C and C++
compilers compatible with GCC or CLANG. After that, to build the
library, it is enough execute `gmake all` (or `make all`) in the
directory with source code, and `gmake check` (or `make check`) to run
the basic tests.
### Windows
For building _libmdbx_ on Windows the [Microsoft Visual
Studio](https://en.wikipedia.org/wiki/Microsoft_Visual_Studio) is
recommended, but not tools such as MinGW, MSYS, or Cygwin. To do this,
the libmdbx source code includes the set of appropriate project files
that are compatible with Visual Studio 2015, the Windows SDK for Windows
8.1, and later. Just open `mdbx.sln` in Visual Studio and build the
library.
To build with newer versions of the SDK or Visual Studio, it should be
sufficient to execute "Retarget solution". To build for older versions
of Windows (such as Windows XP) or by older compilers, you will need to
convert or recreate the corresponding project files yourself.
Building by MinGW, MSYS or Cygwin is potentially possible. However,
these scripts are not tested and will probably require you to modify the
Makefile. It should be noted that in _libmdbx_ was efforts to resolve
runtime dependencies from CRT and other libraries Visual Studio.
For this is enough define the `MDBX_AVOID_CRT` during build.
An example of running a basic test script can be found in the
[CI-script](appveyor.yml) for [AppVeyor](https://www.appveyor.com/). To
run the [long stochastic test scenario](test/long_stochastic.sh),
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) is required, and
the such testing is recommended with place the test data on the
[RAM-disk](https://en.wikipedia.org/wiki/RAM_drive).
### MacOS X
Current [native build tools](https://en.wikipedia.org/wiki/Xcode) for
MacOS X include GNU Make, CLANG and an outdated version of bash.
Therefore, to build the library, it is enough to run `make all` in the
directory with source code, and run `make check` to execute the base
tests. If something goes wrong, it is recommended to install
[Homebrew](https://brew.sh/) and try again.
To run the [long stochastic test scenario](test/long_stochastic.sh), you
will need to install the current (not outdated) version of
[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)). To do this, we
recommend that you install [Homebrew](https://brew.sh/) and then execute
`brew install bash`.
## Bindings
| Runtime | GitHub | Author |
| -------- | ------ | ------ |
| Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) |
| .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) |
--------------------------------------------------------------------------------
Performance comparison
======================
All benchmarks were done by [IOArena](https://github.com/pmwkaa/ioarena)
and multiple [scripts](https://github.com/pmwkaa/ioarena/tree/HL%2B%2B2015)
runs on Lenovo Carbon-2 laptop, i7-4600U 2.1 GHz, 8 Gb RAM,
SSD SAMSUNG MZNTD512HAGL-000L1 (DXT23L0Q) 512 Gb.
## Integral performance
Here showed sum of performance metrics in 3 benchmarks:
- Read/Search on 4 CPU cores machine;
- Transactions with [CRUD](https://en.wikipedia.org/wiki/CRUD)
operations in sync-write mode (fdatasync is called after each
transaction);
- Transactions with [CRUD](https://en.wikipedia.org/wiki/CRUD)
operations in lazy-write mode (moment to sync data to persistent storage
is decided by OS).
*Reasons why asynchronous mode isn't benchmarked here:*
1. It doesn't make sense as it has to be done with DB engines, oriented
for keeping data in memory e.g. [Tarantool](https://tarantool.io/),
[Redis](https://redis.io/)), etc.
2. Performance gap is too high to compare in any meaningful way.
![Comparison #1: Integral Performance](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-1.png)
--------------------------------------------------------------------------------
## Read Scalability
Summary performance with concurrent read/search queries in 1-2-4-8
threads on 4 CPU cores machine.
![Comparison #2: Read Scalability](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-2.png)
--------------------------------------------------------------------------------
## Sync-write mode
- Linear scale on left and dark rectangles mean arithmetic mean
transactions per second;
- Logarithmic scale on right is in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**10,000 transactions in sync-write mode**. In case of a crash all data
is consistent and state is right after last successful transaction.
[fdatasync](https://linux.die.net/man/2/fdatasync) syscall is used after
each write transaction in this mode.
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 10,000 small key-value records.
![Comparison #3: Sync-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-3.png)
--------------------------------------------------------------------------------
## Lazy-write mode
- Linear scale on left and dark rectangles mean arithmetic mean of
thousands transactions per second;
- Logarithmic scale on right in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**100,000 transactions in lazy-write mode**. In case of a crash all data
is consistent and state is right after one of last transactions, but
transactions after it will be lost. Other DB engines use
[WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) or transaction
journal for that, which in turn depends on order of operations in
journaled filesystem. _libmdbx_ doesn't use WAL and hands I/O operations
to filesystem and OS kernel (mmap).
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 100,000 small key-value
records.
![Comparison #4: Lazy-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-4.png)
--------------------------------------------------------------------------------
## Async-write mode
- Linear scale on left and dark rectangles mean arithmetic mean of
thousands transactions per second;
- Logarithmic scale on right in seconds and yellow intervals mean
execution time of transactions. Each interval shows minimal and maximum
execution time, cross marks standard deviation.
**1,000,000 transactions in async-write mode**. In case of a crash all
data will be consistent and state will be right after one of last
transactions, but lost transaction count is much higher than in
lazy-write mode. All DB engines in this mode do as little writes as
possible on persistent storage. _libmdbx_ uses
[msync(MS_ASYNC)](https://linux.die.net/man/2/msync) in this mode.
In the benchmark each transaction contains combined CRUD operations (2
inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database
and after full run the database contains 10,000 small key-value records.
![Comparison #5: Async-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-5.png)
--------------------------------------------------------------------------------
## Cost comparison
Summary of used resources during lazy-write mode benchmarks:
- Read and write IOPS;
- Sum of user CPU time and sys CPU time;
- Used space on persistent storage after the test and closed DB, but not
waiting for the end of all internal housekeeping operations (LSM
compactification, etc).
_ForestDB_ is excluded because benchmark showed it's resource
consumption for each resource (CPU, IOPS) much higher than other engines
which prevents to meaningfully compare it with them.
All benchmark data is gathered by
[getrusage()](http://man7.org/linux/man-pages/man2/getrusage.2.html)
syscall and by scanning data directory.
![Comparison #6: Cost comparison](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-6.png)
--------------------------------------------------------------------------------
```
$ objdump -f -h -j .text libmdbx.so
libmdbx.so: file format elf64-x86-64
architecture: i386:x86-64, flags 0x00000150:
HAS_SYMS, DYNAMIC, D_PAGED
start address 0x0000000000003870
Sections:
Idx Name Size VMA LMA File off Algn
11 .text 000173d4 0000000000003870 0000000000003870 00003870 2**4
CONTENTS, ALLOC, LOAD, READONLY, CODE
```

54
contrib/db/libmdbx/appveyor.yml Normal file
View file

@ -0,0 +1,54 @@
version: 0.3.2.{build}
environment:
matrix:
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
TOOLSET: v141
# - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
# TOOLSET: v140
# - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013
# TOOLSET: v120
branches:
except:
- coverity_scan
configuration:
- Debug
- Release
platform:
- x86
- x64
#- ARM
build_script:
- ps: >
msbuild "C:\projects\libmdbx\mdbx.sln" /verbosity:minimal
/logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll"
/property:PlatformToolset=$env:TOOLSET
/property:Configuration=$env:CONFIGURATION
/property:Platform=$env:PLATFORM
test_script:
- ps: |
if (($env:PLATFORM -eq "x86") -and (Test-Path "C:\projects\libmdbx\Win32\$env:CONFIGURATION\mdbx_test.exe" -PathType Leaf)) {
$mdbx_test = "C:\projects\libmdbx\Win32\$env:CONFIGURATION\mdbx_test.exe"
$mdbx_chk = "C:\projects\libmdbx\Win32\$env:CONFIGURATION\mdbx_chk.exe"
} elseif (($env:PLATFORM -ne "ARM") -and ($env:PLATFORM -ne "ARM64")) {
$mdbx_test = "C:\projects\libmdbx\$env:PLATFORM\$env:CONFIGURATION\mdbx_test.exe"
$mdbx_chk = "C:\projects\libmdbx\$env:PLATFORM\$env:CONFIGURATION\mdbx_chk.exe"
} else {
$mdbx_test = ""
$mdbx_chk = ""
}
if ($mdbx_test -ne "") {
& "$mdbx_test" --pathname=test.db --dont-cleanup-after basic | Tee-Object -file test.log | Select-Object -last 42
& "$mdbx_chk" -nvv test.db | Tee-Object -file chk.log | Select-Object -last 42
}
on_failure:
- ps: Push-AppveyorArtifact test.log
- ps: Push-AppveyorArtifact test.db
- ps: Push-AppveyorArtifact chk.log

223
contrib/db/libmdbx/dll.vcxproj Normal file
View file

@ -0,0 +1,223 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{6D19209B-ECE7-4B9C-941C-0AA2B484F199}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<TargetName>mdbx</TargetName>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
<CustomBuildBeforeTargets>PreLinkEvent</CustomBuildBeforeTargets>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<TargetName>mdbx</TargetName>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
<CustomBuildBeforeTargets>PreLinkEvent</CustomBuildBeforeTargets>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<TargetName>mdbx</TargetName>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
<CustomBuildBeforeTargets>PreLinkEvent</CustomBuildBeforeTargets>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<TargetName>mdbx</TargetName>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
<CustomBuildBeforeTargets>PreLinkEvent</CustomBuildBeforeTargets>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PreprocessorDefinitions>WIN32;_DEBUG;_WINDOWS;_USRDLL;LIBMDBX_EXPORTS;MDBX_BUILD_DLL;MDBX_AVOID_CRT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<WarningLevel>EnableAllWarnings</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<Optimization>Disabled</Optimization>
<StringPooling>true</StringPooling>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<TargetMachine>MachineX86</TargetMachine>
<GenerateDebugInformation>true</GenerateDebugInformation>
<SubSystem>Windows</SubSystem>
<AdditionalDependencies>ntdll.lib;$(IntermediateOutputPath)mdbx_ntdll_extra.lib;kernel32.lib;advapi32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
<CustomBuildStep>
<Message>Generate fake-library mdbx_ntdll_extra.lib for $(PlatformTarget)</Message>
<Outputs>$(IntermediateOutputPath)mdbx_ntdll_extra.lib</Outputs>
<Inputs>$(ProjectDir)src/ntdll.def</Inputs>
<Command>lib.exe /def:%(Inputs) /out:%(Outputs) /machine:$(PlatformTarget)</Command>
</CustomBuildStep>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;LIBMDBX_EXPORTS;MDBX_BUILD_DLL;MDBX_AVOID_CRT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<WarningLevel>EnableAllWarnings</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<StringPooling>true</StringPooling>
<Optimization>Full</Optimization>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<IntrinsicFunctions>true</IntrinsicFunctions>
<FavorSizeOrSpeed>Size</FavorSizeOrSpeed>
<OmitFramePointers>true</OmitFramePointers>
<WholeProgramOptimization>true</WholeProgramOptimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<BufferSecurityCheck>false</BufferSecurityCheck>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile>
<Link>
<TargetMachine>MachineX86</TargetMachine>
<GenerateDebugInformation>true</GenerateDebugInformation>
<SubSystem>Windows</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
<AdditionalDependencies>ntdll.lib;$(IntermediateOutputPath)mdbx_ntdll_extra.lib;kernel32.lib;advapi32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<IgnoreAllDefaultLibraries>true</IgnoreAllDefaultLibraries>
</Link>
<CustomBuildStep>
<Message>Generate fake-library mdbx_ntdll_extra.lib for $(PlatformTarget)</Message>
<Outputs>$(IntermediateOutputPath)mdbx_ntdll_extra.lib</Outputs>
<Inputs>$(ProjectDir)src/ntdll.def</Inputs>
<Command>lib.exe /def:%(Inputs) /out:%(Outputs) /machine:$(PlatformTarget)</Command>
</CustomBuildStep>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
</ClCompile>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PreprocessorDefinitions>WIN64;_DEBUG;_WINDOWS;_USRDLL;LIBMDBX_EXPORTS;MDBX_BUILD_DLL;MDBX_AVOID_CRT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<StringPooling>true</StringPooling>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<AdditionalDependencies>ntdll.lib;$(IntermediateOutputPath)mdbx_ntdll_extra.lib;kernel32.lib;advapi32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
<CustomBuildStep>
<Message>Generate fake-library mdbx_ntdll_extra.lib for $(PlatformTarget)</Message>
<Outputs>$(IntermediateOutputPath)mdbx_ntdll_extra.lib</Outputs>
<Inputs>$(ProjectDir)src/ntdll.def</Inputs>
<Command>lib.exe /def:%(Inputs) /out:%(Outputs) /machine:$(PlatformTarget)</Command>
</CustomBuildStep>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<PreprocessorDefinitions>WIN64;NDEBUG;_WINDOWS;_USRDLL;LIBMDBX_EXPORTS;MDBX_BUILD_DLL;MDBX_AVOID_CRT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<StringPooling>true</StringPooling>
<Optimization>Full</Optimization>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
<IntrinsicFunctions>true</IntrinsicFunctions>
<FavorSizeOrSpeed>Size</FavorSizeOrSpeed>
<OmitFramePointers>true</OmitFramePointers>
<WholeProgramOptimization>true</WholeProgramOptimization>
<WarningLevel>EnableAllWarnings</WarningLevel>
<FunctionLevelLinking>true</FunctionLevelLinking>
<BufferSecurityCheck>false</BufferSecurityCheck>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile>
<Link>
<LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
<AdditionalDependencies>ntdll.lib;$(IntermediateOutputPath)mdbx_ntdll_extra.lib;kernel32.lib;advapi32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<IgnoreAllDefaultLibraries>true</IgnoreAllDefaultLibraries>
</Link>
<CustomBuildStep>
<Message>Generate fake-library mdbx_ntdll_extra.lib for $(PlatformTarget)</Message>
<Outputs>$(IntermediateOutputPath)mdbx_ntdll_extra.lib</Outputs>
<Inputs>$(ProjectDir)src/ntdll.def</Inputs>
<Command>lib.exe /def:%(Inputs) /out:%(Outputs) /machine:$(PlatformTarget)</Command>
</CustomBuildStep>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="src\lck-windows.c" />
<ClCompile Include="src\mdbx.c" />
<ClCompile Include="src\osal.c" />
<ClCompile Include="src\version.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="mdbx.h" />
<ClInclude Include="src\bits.h" />
<ClInclude Include="src\defs.h" />
<ClInclude Include="src\osal.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

1
contrib/db/libmdbx/libmdbx.cflags Normal file
View file

@ -0,0 +1 @@
-std=c11

2
contrib/db/libmdbx/libmdbx.config Normal file
View file

@ -0,0 +1,2 @@
// Add predefined macros for your project here. For example:
// #define THE_ANSWER 42

1
contrib/db/libmdbx/libmdbx.creator Normal file
View file

@ -0,0 +1 @@
[General]

1
contrib/db/libmdbx/libmdbx.cxxflags Normal file
View file

@ -0,0 +1 @@
-std=c++17

62
contrib/db/libmdbx/libmdbx.files Normal file
View file

@ -0,0 +1,62 @@
AUTHORS
CMakeLists.txt
LICENSE
Makefile
README-RU.md
README.md
TODO.md
mdbx.h
src/bits.h
src/defs.h
src/lck-linux.c
src/lck-posix.c
src/lck-windows.c
src/mdbx.c
src/osal.c
src/osal.h
src/tools/CMakeLists.txt
src/tools/mdbx_chk.c
src/tools/mdbx_copy.1
src/tools/mdbx_copy.c
src/tools/mdbx_dump.1
src/tools/mdbx_dump.c
src/tools/mdbx_load.1
src/tools/mdbx_load.c
src/tools/mdbx_stat.1
src/tools/mdbx_stat.c
src/tools/wingetopt.c
src/tools/wingetopt.h
src/version.c
test/CMakeLists.txt
test/actor.cc
test/append.cc
test/base.h
test/cases.cc
test/chrono.cc
test/chrono.h
test/config.cc
test/config.h
test/copy.cc
test/dead.cc
test/hill.cc
test/jitter.cc
test/keygen.cc
test/keygen.h
test/log.cc
test/log.h
test/main.cc
test/osal-unix.cc
test/osal-windows.cc
test/osal.h
test/pcrf/CMakeLists.txt
test/test.cc
test/test.h
test/try.cc
test/ttl.cc
test/utils.cc
test/utils.h
tutorial/CMakeLists.txt
tutorial/README.md
tutorial/sample-bdb.txt
tutorial/sample-mdb.txt
tutorial/sample-mdbx.c

4
contrib/db/libmdbx/libmdbx.includes Normal file
View file

@ -0,0 +1,4 @@
.
src
src/tools
test

2005
contrib/db/libmdbx/mdbx.h Normal file
View file

File diff suppressed because it is too large Load diff

97
contrib/db/libmdbx/mdbx.sln Normal file
View file

@ -0,0 +1,97 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 14
VisualStudioVersion = 14.0.25420.1
MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test", "test\test.vcxproj", "{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dll", "dll.vcxproj", "{6D19209B-ECE7-4B9C-941C-0AA2B484F199}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{0A147F9F-22D5-44E6-B389-218CFFB0C524}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_load", "src\tools\mdbx_load.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_dump", "src\tools\mdbx_dump.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_copy", "src\tools\mdbx_copy.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_chk", "src\tools\mdbx_chk.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mdbx_stat", "src\tools\mdbx_stat.vcxproj", "{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x64.ActiveCfg = Debug|x64
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x64.Build.0 = Debug|x64
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x86.ActiveCfg = Debug|Win32
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Debug|x86.Build.0 = Debug|Win32
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x64.ActiveCfg = Release|x64
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x64.Build.0 = Release|x64
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x86.ActiveCfg = Release|Win32
{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}.Release|x86.Build.0 = Release|Win32
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Debug|x64.ActiveCfg = Debug|x64
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Debug|x64.Build.0 = Debug|x64
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Debug|x86.ActiveCfg = Debug|Win32
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Debug|x86.Build.0 = Debug|Win32
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x64.ActiveCfg = Release|x64
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x64.Build.0 = Release|x64
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x86.ActiveCfg = Release|Win32
{6D19209B-ECE7-4B9C-941C-0AA2B484F199}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}.Release|x86.Build.0 = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Debug|x64.ActiveCfg = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Debug|x64.Build.0 = Debug|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Debug|x86.ActiveCfg = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Debug|x86.Build.0 = Debug|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Release|x64.ActiveCfg = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Release|x64.Build.0 = Release|x64
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Release|x86.ActiveCfg = Release|Win32
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}.Release|x86.Build.0 = Release|Win32
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{15030120-5F7F-48F9-ABE5-DFC814F2A4BB} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
{15030120-5F7F-48F9-ABE5-DFC814F2A4BC} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
{15030120-5F7F-48F9-ABE5-DFC814F2A4BD} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
{15030120-5F7F-48F9-ABE5-DFC814F2A4BE} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
{15030120-5F7F-48F9-ABE5-DFC814F2A4BF} = {0A147F9F-22D5-44E6-B389-218CFFB0C524}
EndGlobalSection
EndGlobal

184
contrib/db/libmdbx/packages/rpm/CMakeLists.txt Normal file
View file

@ -0,0 +1,184 @@
cmake_minimum_required(VERSION 2.8.7)
set(TARGET mdbx)
project(${TARGET})
set(MDBX_VERSION_MAJOR 0)
set(MDBX_VERSION_MINOR 3)
set(MDBX_VERSION_RELEASE 1)
set(MDBX_VERSION_REVISION 0)
set(MDBX_VERSION_STRING ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}.${MDBX_VERSION_RELEASE})
enable_language(C)
enable_language(CXX)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED on)
add_definitions(-DNDEBUG=1 -DMDBX_DEBUG=0 -DLIBMDBX_EXPORTS=1 -D_GNU_SOURCE=1)
find_package(Threads REQUIRED)
get_directory_property(hasParent PARENT_DIRECTORY)
if(hasParent)
set(STANDALONE_BUILD 0)
else()
set(STANDALONE_BUILD 1)
enable_testing()
if (CMAKE_C_COMPILER_ID MATCHES GNU)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g3")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wextra")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread")
endif()
if (CMAKE_CXX_COMPILER_ID MATCHES GNU)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpointer-arith")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat-security")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Woverloaded-virtual")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wwrite-strings")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=20")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wunused-function -Wunused-variable -Wunused-value -Wmissing-declarations")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wcast-qual")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finline-functions-called-once")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-packed-bitfield-compat")
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g3")
endif()
if (COVERAGE)
if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
message(FATAL_ERROR "Coverage requires -DCMAKE_BUILD_TYPE=Debug Current value=${CMAKE_BUILD_TYPE}")
endif()
message(STATUS "Setting coverage compiler flags")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage")
add_definitions(-DCOVERAGE_TEST)
endif()
if (NOT TRAVIS)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fstack-protector-strong -static-libasan")
endif()
endif()
set(${TARGET}_SRC
mdbx.h
src/bits.h
src/defs.h
src/lck-linux.c
src/mdbx.c
src/osal.c
src/osal.h
src/version.c
)
add_library(${TARGET}_STATIC STATIC
${${TARGET}_SRC}
)
add_library(${TARGET} ALIAS ${TARGET}_STATIC)
add_library(${TARGET}_SHARED SHARED
${${TARGET}_SRC}
)
set_target_properties(${TARGET}_SHARED PROPERTIES
VERSION ${MDBX_VERSION_STRING}
SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}
OUTPUT_NAME ${TARGET}
CLEAN_DIRECT_OUTPUT 1
)
set_target_properties(${TARGET}_STATIC PROPERTIES
VERSION ${MDBX_VERSION_STRING}
SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}
OUTPUT_NAME ${TARGET}
CLEAN_DIRECT_OUTPUT 1
)
target_include_directories(${TARGET}_STATIC PUBLIC
${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(${TARGET}_SHARED PUBLIC
${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(${TARGET}_STATIC ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(${TARGET}_SHARED ${CMAKE_THREAD_LIBS_INIT})
if(UNIX AND NOT APPLE)
target_link_libraries(${TARGET}_STATIC rt)
target_link_libraries(${TARGET}_SHARED rt)
endif()
install(TARGETS ${TARGET}_STATIC DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx)
install(TARGETS ${TARGET}_SHARED DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx)
install(FILES mdbx.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT mdbx-devel)
add_subdirectory(src/tools)
add_subdirectory(test)
add_subdirectory(test/pcrf)
add_subdirectory(tutorial)
##############################################################################
set(CPACK_GENERATOR "RPM")
set(CPACK_RPM_COMPONENT_INSTALL ON)
# Version
if (NOT "$ENV{BUILD_NUMBER}" STREQUAL "")
set(CPACK_PACKAGE_RELEASE $ENV{BUILD_NUMBER})
else()
if (NOT "$ENV{CI_PIPELINE_ID}" STREQUAL "")
set(CPACK_PACKAGE_RELEASE $ENV{CI_PIPELINE_ID})
else()
set(CPACK_PACKAGE_RELEASE 1)
endif()
endif()
set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE})
set(CPACK_PACKAGE_VERSION ${MDBX_VERSION_STRING})
set(CPACK_PACKAGE_VERSION_FULL ${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE})
set(CPACK_RPM_mdbx-devel_PACKAGE_REQUIRES "mdbx = ${CPACK_PACKAGE_VERSION}")
set(CPACK_RPM_SPEC_INSTALL_POST "/bin/true")
set(CPACK_RPM_mdbx_PACKAGE_NAME mdbx)
set(CPACK_RPM_mdbx-devel_PACKAGE_NAME mdbx-devel)
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "The revised and extended descendant of Symas LMDB")
set(CPACK_PACKAGE_VENDOR "???")
set(CPACK_PACKAGE_CONTACT "Vladimir Romanov")
set(CPACK_PACKAGE_RELOCATABLE false)
set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
set(CPACK_RPM_PACKAGE_REQUIRES "")
set(CPACK_RPM_PACKAGE_GROUP "Applications/Database")
set(CPACK_RPM_mdbx_FILE_NAME "${CPACK_RPM_mdbx_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm")
set(CPACK_RPM_mdbx-devel_FILE_NAME "${CPACK_RPM_mdbx-devel_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm")
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/usr/local
/usr/local/bin
/usr/local/lib64
/usr/local/include
/usr/local/man
/usr/local/man/man1
)
include(CPack)

18
contrib/db/libmdbx/packages/rpm/build.sh Normal file
View file

@ -0,0 +1,18 @@
#!/bin/bash
set -e
CONFIG=$1
if [[ -z "${CONFIG}" ]]; then
CONFIG=Debug
fi
if [[ -r /opt/rh/devtoolset-6/enable ]]; then
source /opt/rh/devtoolset-6/enable
fi
#rm -f -r build || true
mkdir -p cmake-build-${CONFIG}
pushd cmake-build-${CONFIG} &> /dev/null
if [[ ! -r Makefile ]]; then
cmake .. -DCMAKE_BUILD_TYPE=${CONFIG}
fi
make -j8 || exit 1
popd &> /dev/null

25
contrib/db/libmdbx/packages/rpm/package.sh Normal file
View file

@ -0,0 +1,25 @@
#!/bin/bash
set -e
CONFIG=$1
if [[ -z "${CONFIG}" ]]; then
CONFIG=Debug
fi
DIRNAME=`dirname ${BASH_SOURCE[0]}`
DIRNAME=`readlink --canonicalize ${DIRNAME}`
if [[ -r /opt/rh/devtoolset-6/enable ]]; then
source /opt/rh/devtoolset-6/enable
fi
mkdir -p cmake-build-${CONFIG}
pushd cmake-build-${CONFIG} &> /dev/null
if [[ ! -r Makefile ]]; then
cmake .. -DCMAKE_BUILD_TYPE=${CONFIG}
fi
rm -f *.rpm
make -j8 package || exit 1
rm -f *-Unspecified.rpm
popd &> /dev/null

1252
contrib/db/libmdbx/src/bits.h Normal file
View file

File diff suppressed because it is too large Load diff

447
contrib/db/libmdbx/src/defs.h Normal file
View file

@ -0,0 +1,447 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
/* *INDENT-OFF* */
/* clang-format off */
#ifndef __GNUC_PREREQ
# if defined(__GNUC__) && defined(__GNUC_MINOR__)
# define __GNUC_PREREQ(maj, min) \
((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GNUC_PREREQ(maj, min) (0)
# endif
#endif /* __GNUC_PREREQ */
#ifndef __CLANG_PREREQ
# ifdef __clang__
# define __CLANG_PREREQ(maj,min) \
((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))
# else
# define __CLANG_PREREQ(maj,min) (0)
# endif
#endif /* __CLANG_PREREQ */
#ifndef __GLIBC_PREREQ
# if defined(__GLIBC__) && defined(__GLIBC_MINOR__)
# define __GLIBC_PREREQ(maj, min) \
((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min))
# else
# define __GLIBC_PREREQ(maj, min) (0)
# endif
#endif /* __GLIBC_PREREQ */
#ifndef __has_attribute
# define __has_attribute(x) (0)
#endif
#ifndef __has_feature
# define __has_feature(x) (0)
#endif
#ifndef __has_extension
# define __has_extension(x) (0)
#endif
#ifndef __has_builtin
# define __has_builtin(x) (0)
#endif
#ifndef __has_warning
# define __has_warning(x) (0)
#endif
#ifndef __has_include
# define __has_include(x) (0)
#endif
#if __has_feature(thread_sanitizer)
# define __SANITIZE_THREAD__ 1
#endif
#if __has_feature(address_sanitizer)
# define __SANITIZE_ADDRESS__ 1
#endif
/*----------------------------------------------------------------------------*/
#ifndef __extern_C
# ifdef __cplusplus
# define __extern_C extern "C"
# else
# define __extern_C
# endif
#endif /* __extern_C */
#ifndef __cplusplus
# ifndef bool
# define bool _Bool
# endif
# ifndef true
# define true (1)
# endif
# ifndef false
# define false (0)
# endif
#endif
#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER))
# define nullptr NULL
#endif
/*----------------------------------------------------------------------------*/
#ifndef __always_inline
# if defined(__GNUC__) || __has_attribute(__always_inline__)
# define __always_inline __inline __attribute__((__always_inline__))
# elif defined(_MSC_VER)
# define __always_inline __forceinline
# else
# define __always_inline
# endif
#endif /* __always_inline */
#ifndef __noinline
# if defined(__GNUC__) || __has_attribute(__noinline__)
# define __noinline __attribute__((__noinline__))
# elif defined(_MSC_VER)
# define __noinline __declspec(noinline)
# elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
# define __noinline inline
# elif !defined(__INTEL_COMPILER)
# define __noinline /* FIXME ? */
# endif
#endif /* __noinline */
#ifndef __must_check_result
# if defined(__GNUC__) || __has_attribute(__warn_unused_result__)
# define __must_check_result __attribute__((__warn_unused_result__))
# else
# define __must_check_result
# endif
#endif /* __must_check_result */
#ifndef __deprecated
# if defined(__GNUC__) || __has_attribute(__deprecated__)
# define __deprecated __attribute__((__deprecated__))
# elif defined(_MSC_VER)
# define __deprecated __declspec(deprecated)
# else
# define __deprecated
# endif
#endif /* __deprecated */
#if !defined(__noop) && !defined(_MSC_VER)
# ifdef __cplusplus
static inline void __noop_consume_args() {}
template <typename First, typename... Rest>
static inline void
__noop_consume_args(const First &first, const Rest &... rest) {
(void) first; __noop_consume_args(rest...);
}
# define __noop(...) __noop_consume_args(__VA_ARGS__)
# elif defined(__GNUC__) && (!defined(__STRICT_ANSI__) || !__STRICT_ANSI__)
static __inline void __noop_consume_args(void* anchor, ...) {
(void) anchor;
}
# define __noop(...) __noop_consume_args(0, ##__VA_ARGS__)
# else
# define __noop(...) do {} while(0)
# endif
#endif /* __noop */
#ifndef __fallthrough
# if __GNUC_PREREQ(7, 0) || __has_attribute(__fallthrough__)
# define __fallthrough __attribute__((__fallthrough__))
# else
# define __fallthrough __noop()
# endif
#endif /* __fallthrough */
#ifndef __unreachable
# if __GNUC_PREREQ(4,5)
# define __unreachable() __builtin_unreachable()
# elif defined(_MSC_VER)
# define __unreachable() __assume(0)
# else
# define __unreachable() __noop()
# endif
#endif /* __unreachable */
#ifndef __prefetch
# if defined(__GNUC__) || defined(__clang__)
# define __prefetch(ptr) __builtin_prefetch(ptr)
# else
# define __prefetch(ptr) __noop(ptr)
# endif
#endif /* __prefetch */
#ifndef __noreturn
# if defined(__GNUC__) || __has_attribute(__noreturn__)
# define __noreturn __attribute__((__noreturn__))
# elif defined(_MSC_VER)
# define __noreturn __declspec(noreturn)
# else
# define __noreturn
# endif
#endif /* __noreturn */
#ifndef __nothrow
# if defined(__cplusplus)
# if __cplusplus < 201703L
# define __nothrow throw()
# else
# define __nothrow noexcept(true)
# endif /* __cplusplus */
# elif defined(__GNUC__) || __has_attribute(__nothrow__)
# define __nothrow __attribute__((__nothrow__))
# elif defined(_MSC_VER) && defined(__cplusplus)
# define __nothrow __declspec(nothrow)
# else
# define __nothrow
# endif
#endif /* __nothrow */
#ifndef __pure_function
/* Many functions have no effects except the return value and their
* return value depends only on the parameters and/or global variables.
* Such a function can be subject to common subexpression elimination
* and loop optimization just as an arithmetic operator would be.
* These functions should be declared with the attribute pure. */
# if defined(__GNUC__) || __has_attribute(__pure__)
# define __pure_function __attribute__((__pure__))
# else
# define __pure_function
# endif
#endif /* __pure_function */
#ifndef __const_function
/* Many functions do not examine any values except their arguments,
* and have no effects except the return value. Basically this is just
* slightly more strict class than the PURE attribute, since function
* is not allowed to read global memory.
*
* Note that a function that has pointer arguments and examines the
* data pointed to must not be declared const. Likewise, a function
* that calls a non-const function usually must not be const.
* It does not make sense for a const function to return void. */
# if defined(__GNUC__) || __has_attribute(__const__)
# define __const_function __attribute__((__const__))
# else
# define __const_function
# endif
#endif /* __const_function */
#ifndef __hidden
# if defined(__GNUC__) || __has_attribute(__visibility__)
# define __hidden __attribute__((__visibility__("hidden")))
# else
# define __hidden
# endif
#endif /* __hidden */
#ifndef __optimize
# if defined(__OPTIMIZE__)
# if defined(__clang__) && !__has_attribute(__optimize__)
# define __optimize(ops)
# elif defined(__GNUC__) || __has_attribute(__optimize__)
# define __optimize(ops) __attribute__((__optimize__(ops)))
# else
# define __optimize(ops)
# endif
# else
# define __optimize(ops)
# endif
#endif /* __optimize */
#ifndef __hot
# if defined(__OPTIMIZE__)
# if defined(__e2k__)
# define __hot __attribute__((__hot__)) __optimize(3)
# elif defined(__clang__) && !__has_attribute(__hot_) \
&& __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
/* just put frequently used functions in separate section */
# define __hot __attribute__((__section__("text.hot"))) __optimize("O3")
# elif defined(__GNUC__) || __has_attribute(__hot__)
# define __hot __attribute__((__hot__)) __optimize("O3")
# else
# define __hot __optimize("O3")
# endif
# else
# define __hot
# endif
#endif /* __hot */
#ifndef __cold
# if defined(__OPTIMIZE__)
# if defined(__e2k__)
# define __cold __attribute__((__cold__)) __optimize(1)
# elif defined(__clang__) && !__has_attribute(cold) \
&& __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__))
/* just put infrequently used functions in separate section */
# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os")
# elif defined(__GNUC__) || __has_attribute(cold)
# define __cold __attribute__((__cold__)) __optimize("Os")
# else
# define __cold __optimize("Os")
# endif
# else
# define __cold
# endif
#endif /* __cold */
#ifndef __flatten
# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__))
# define __flatten __attribute__((__flatten__))
# else
# define __flatten
# endif
#endif /* __flatten */
#ifndef likely
# if (defined(__GNUC__) || defined(__clang__)) && !defined(__COVERITY__)
# define likely(cond) __builtin_expect(!!(cond), 1)
# else
# define likely(x) (x)
# endif
#endif /* likely */
#ifndef unlikely
# if (defined(__GNUC__) || defined(__clang__)) && !defined(__COVERITY__)
# define unlikely(cond) __builtin_expect(!!(cond), 0)
# else
# define unlikely(x) (x)
# endif
#endif /* unlikely */
/* Workaround for Coverity Scan */
#if defined(__COVERITY__) && __GNUC_PREREQ(7, 0) && !defined(__cplusplus)
typedef float _Float32;
typedef double _Float32x;
typedef double _Float64;
typedef long double _Float64x;
typedef float _Float128 __attribute__((__mode__(__TF__)));
typedef __complex__ float __cfloat128 __attribute__ ((__mode__ (__TC__)));
typedef _Complex float __cfloat128 __attribute__ ((__mode__ (__TC__)));
#endif /* Workaround for Coverity Scan */
/* Wrapper around __func__, which is a C99 feature */
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
# define mdbx_func_ __func__
#elif (defined(__GNUC__) && __GNUC__ >= 2) || defined(__clang__) || defined(_MSC_VER)
# define mdbx_func_ __FUNCTION__
#else
# define mdbx_func_ "<mdbx_unknown>"
#endif
#if defined(__GNUC__) || __has_attribute(__format__)
#define __printf_args(format_index, first_arg) \
__attribute__((__format__(printf, format_index, first_arg)))
#else
#define __printf_args(format_index, first_arg)
#endif
/*----------------------------------------------------------------------------*/
#if defined(USE_VALGRIND)
# include <valgrind/memcheck.h>
# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE
/* LY: available since Valgrind 3.10 */
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# endif
#elif !defined(RUNNING_ON_VALGRIND)
# define VALGRIND_CREATE_MEMPOOL(h,r,z)
# define VALGRIND_DESTROY_MEMPOOL(h)
# define VALGRIND_MEMPOOL_TRIM(h,a,s)
# define VALGRIND_MEMPOOL_ALLOC(h,a,s)
# define VALGRIND_MEMPOOL_FREE(h,a)
# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s)
# define VALGRIND_MAKE_MEM_NOACCESS(a,s)
# define VALGRIND_MAKE_MEM_DEFINED(a,s)
# define VALGRIND_MAKE_MEM_UNDEFINED(a,s)
# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s)
# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0)
# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0)
# define RUNNING_ON_VALGRIND (0)
#endif /* USE_VALGRIND */
#ifdef __SANITIZE_ADDRESS__
# include <sanitizer/asan_interface.h>
#elif !defined(ASAN_POISON_MEMORY_REGION)
# define ASAN_POISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \
((void)(addr), (void)(size))
#endif /* __SANITIZE_ADDRESS__ */
/*----------------------------------------------------------------------------*/
#ifndef ARRAY_LENGTH
# ifdef __cplusplus
template <typename T, size_t N>
char (&__ArraySizeHelper(T (&array)[N]))[N];
# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array)))
# else
# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
# endif
#endif /* ARRAY_LENGTH */
#ifndef ARRAY_END
# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)])
#endif /* ARRAY_END */
#ifndef STRINGIFY
# define STRINGIFY_HELPER(x) #x
# define STRINGIFY(x) STRINGIFY_HELPER(x)
#endif /* STRINGIFY */
#ifndef offsetof
# define offsetof(type, member) __builtin_offsetof(type, member)
#endif /* offsetof */
#ifndef container_of
# define container_of(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
#endif /* container_of */
#define MDBX_TETRAD(a, b, c, d) \
((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d))
#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3])
#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__)
#ifndef STATIC_ASSERT_MSG
# if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \
|| __has_feature(c_static_assert)
# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg)
# elif defined(static_assert)
# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg)
# elif defined(_MSC_VER)
# include <crtdbg.h>
# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr)
# else
# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;}
# endif
#endif /* STATIC_ASSERT */
#ifndef STATIC_ASSERT
# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr)
#endif
/* *INDENT-ON* */
/* clang-format on */

428
contrib/db/libmdbx/src/lck-linux.c Normal file
View file

@ -0,0 +1,428 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#if !(defined(__linux__) || defined(__gnu_linux__))
#error "This implementation of locking only supports Linux,\
where is no interaction between the types of lock placed\
by flock() and fcntl()."
#endif
#include "./bits.h"
#include <sys/utsname.h>
/* Some platforms define the EOWNERDEAD error code
* even though they don't support Robust Mutexes.
* Compile with -DMDBX_USE_ROBUST=0. */
#ifndef MDBX_USE_ROBUST
/* Howard Chu: Android currently lacks Robust Mutex support */
#if defined(EOWNERDEAD) && \
!defined(__ANDROID__) /* LY: glibc before 2.10 has a troubles \
with Robust Mutex too. */ \
&& (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) || \
_POSIX_C_SOURCE >= 200809L)
#define MDBX_USE_ROBUST 1
#else
#define MDBX_USE_ROBUST 0
#endif
#endif /* MDBX_USE_ROBUST */
/*----------------------------------------------------------------------------*/
/* global constructor/destructor */
uint32_t mdbx_linux_kernel_version;
static __cold __attribute__((__constructor__)) void
mdbx_global_constructor(void) {
struct utsname buffer;
if (uname(&buffer) == 0) {
int i = 0;
char *p = buffer.release;
while (*p && i < 4) {
if (*p >= '0' && *p <= '9') {
long number = strtol(p, &p, 10);
if (number > 0) {
if (number > 255)
number = 255;
mdbx_linux_kernel_version += number << (24 - i * 8);
}
++i;
} else {
++p;
}
}
}
mdbx_rthc_global_init();
}
static __cold __attribute__((__destructor__)) void
mdbx_global_destructor(void) {
mdbx_rthc_global_dtor();
}
/*----------------------------------------------------------------------------*/
/* lck */
/* Описание реализации блокировок для Linux:
*
* lck-файл отображается в память, в нём организуется таблица читателей и
* размещаются совместно используемые posix-мьютексы (futex). Посредством
* этих мьютексов (см struct MDBX_lockinfo) реализуются:
* - Блокировка таблицы читателей для регистрации,
* т.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock().
* - Блокировка БД для пишущих транзакций,
* т.е. функции mdbx_txn_lock() и mdbx_txn_unlock().
*
* Остальной функционал реализуется отдельно посредством файловых блокировок:
* - Первоначальный захват БД в режиме exclusive/shared и последующий перевод
* в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade().
* - Проверка присутствие процессов-читателей,
* т.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check().
*
* Используется два вида файловых блокировок flock() и fcntl(F_SETLK),
* как для lck-файла, так и для основного файла БД:
* - Для контроля процессов-читателей используются однобайтовые
* range-блокировки lck-файла посредством fcntl(F_SETLK). При этом
* в качестве позиции используется pid процесса-читателя.
* - Для первоначального захвата и shared/exclusive блокировок используется
* комбинация flock() и fcntl(F_SETLK) блокировки одного байта lck-файла
* в нулевой позиции (нулевая позиция не используется механизмом контроля
* процессов-читателей, так как pid пользовательского процесса в Linux
* всегда больше 0).
* - Кроме этого, flock() блокировка основного файла БД используется при работе
* в режимах без lck-файла, как в в read-only, так и в эксклюзивном.
* - Блокировки flock() и fcntl(F_SETLK) в Linux работают независимо. Поэтому
* их комбинирование позволяет предотвратить совместное использование БД
* через NFS, что позволяет fcntl(F_SETLK), одновременно защитившись
* от проблем не-аторманости flock() при переходе между эксклюзивным
* и атомарным режимами блокировок.
*/
static int op_setlk, op_setlkw, op_getlk;
static void __cold choice_fcntl() {
assert(!op_setlk && !op_setlkw && !op_getlk);
#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)
if (mdbx_linux_kernel_version >
0x030f0000 /* OFD locks are available since 3.15, but engages here
only for 3.16 and larer kernels (LTS) for reliability reasons */
&& (mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0) {
op_setlk = F_OFD_SETLK;
op_setlkw = F_OFD_SETLKW;
op_getlk = F_OFD_GETLK;
return;
}
#endif /* OFD locks */
op_setlk = F_SETLK;
op_setlkw = F_SETLKW;
op_getlk = F_GETLK;
}
#ifndef OFF_T_MAX
#define OFF_T_MAX \
((sizeof(off_t) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff)
#endif
#define LCK_WHOLE OFF_T_MAX
static int mdbx_lck_op(mdbx_filehandle_t fd, int cmd, short lck, off_t offset,
off_t len) {
for (;;) {
struct flock lock_op;
memset(&lock_op, 0, sizeof(lock_op));
lock_op.l_type = lck;
lock_op.l_whence = SEEK_SET;
lock_op.l_start = offset;
lock_op.l_len = len;
if (fcntl(fd, cmd, &lock_op) == 0) {
if (cmd == op_getlk) {
/* Checks reader by pid. Returns:
* MDBX_RESULT_TRUE - if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE - if pid is dead (lock acquired). */
return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE
: MDBX_RESULT_TRUE;
}
return 0;
}
int rc = errno;
if (rc != EINTR || cmd == op_setlkw)
return rc;
}
}
static __inline int mdbx_lck_exclusive(int lfd, bool fallback2shared) {
assert(lfd != INVALID_HANDLE_VALUE);
if (flock(lfd, LOCK_EX | LOCK_NB))
return errno;
int rc = mdbx_lck_op(lfd, op_setlk, F_WRLCK, 0, 1);
if (rc != 0 && fallback2shared) {
while (flock(lfd, LOCK_SH)) {
int rc = errno;
if (rc != EINTR)
return rc;
}
}
return rc;
}
static __inline int mdbx_lck_shared(int lfd) {
assert(lfd != INVALID_HANDLE_VALUE);
while (flock(lfd, LOCK_SH)) {
int rc = errno;
if (rc != EINTR)
return rc;
}
return mdbx_lck_op(lfd, op_setlkw, F_RDLCK, 0, 1);
}
int mdbx_lck_downgrade(MDBX_env *env, bool complete) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
return complete ? mdbx_lck_shared(env->me_lfd) : MDBX_SUCCESS;
}
int mdbx_rpid_set(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0);
return mdbx_lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1);
}
int mdbx_rpid_clear(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0);
return mdbx_lck_op(env->me_lfd, op_setlkw, F_UNLCK, env->me_pid, 1);
}
int mdbx_rpid_check(MDBX_env *env, mdbx_pid_t pid) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(pid > 0);
return mdbx_lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1);
}
/*---------------------------------------------------------------------------*/
static int mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
const int rc);
int __cold mdbx_lck_init(MDBX_env *env) {
pthread_mutexattr_t ma;
int rc = pthread_mutexattr_init(&ma);
if (rc)
return rc;
rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED);
if (rc)
goto bailout;
#if MDBX_USE_ROBUST
#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) && \
!defined(pthread_mutex_consistent) && _POSIX_C_SOURCE < 200809L
rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP);
#else
rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
#endif
if (rc)
goto bailout;
#endif /* MDBX_USE_ROBUST */
#if _POSIX_C_SOURCE >= 199506L && !defined(MDBX_SAFE4QEMU)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
if (rc == ENOTSUP)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
if (rc)
goto bailout;
#endif /* PTHREAD_PRIO_INHERIT */
rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_rmutex, &ma);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_wmutex, &ma);
bailout:
pthread_mutexattr_destroy(&ma);
return rc;
}
void __cold mdbx_lck_destroy(MDBX_env *env) {
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* try get exclusive access */
if (env->me_lck && mdbx_lck_exclusive(env->me_lfd, false) == 0) {
mdbx_info("%s: got exclusive, drown mutexes", mdbx_func_);
int rc = pthread_mutex_destroy(&env->me_lck->mti_rmutex);
if (rc == 0)
rc = pthread_mutex_destroy(&env->me_lck->mti_wmutex);
assert(rc == 0);
(void)rc;
/* file locks would be released (by kernel)
* while the me_lfd will be closed */
}
}
}
static int mdbx_robust_lock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_lock(mutex);
if (unlikely(rc != 0))
rc = mdbx_mutex_failed(env, mutex, rc);
return rc;
}
static int mdbx_robust_trylock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_trylock(mutex);
if (unlikely(rc != 0 && rc != EBUSY))
rc = mdbx_mutex_failed(env, mutex, rc);
return (rc != EBUSY) ? rc : MDBX_BUSY;
}
static int mdbx_robust_unlock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_unlock(mutex);
if (unlikely(rc != 0))
rc = mdbx_mutex_failed(env, mutex, rc);
return rc;
}
int mdbx_rdt_lock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_lock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
return rc;
}
void mdbx_rdt_unlock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_unlock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", mdbx_func_, rc);
}
int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
mdbx_trace(">>");
int rc = dontwait ? mdbx_robust_trylock(env, env->me_wmutex)
: mdbx_robust_lock(env, env->me_wmutex);
mdbx_trace("<< rc %d", rc);
return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS;
}
void mdbx_txn_unlock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_unlock(env, env->me_wmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", mdbx_func_, rc);
}
static int __cold internal_seize_lck(int lfd) {
assert(lfd != INVALID_HANDLE_VALUE);
/* try exclusive access */
int rc = mdbx_lck_exclusive(lfd, false);
if (rc == 0)
/* got exclusive */
return MDBX_RESULT_TRUE;
if (rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK) {
/* get shared access */
rc = mdbx_lck_shared(lfd);
if (rc == 0) {
/* got shared, try exclusive again */
rc = mdbx_lck_exclusive(lfd, true);
if (rc == 0)
/* now got exclusive */
return MDBX_RESULT_TRUE;
if (rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK)
/* unable exclusive, but stay shared */
return MDBX_RESULT_FALSE;
}
}
assert(MDBX_IS_ERROR(rc));
return rc;
}
int __cold mdbx_lck_seize(MDBX_env *env) {
assert(env->me_fd != INVALID_HANDLE_VALUE);
if (unlikely(op_setlk == 0))
choice_fcntl();
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
int rc = mdbx_lck_op(env->me_fd, op_setlk,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0,
LCK_WHOLE);
if (rc != 0) {
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "without-lck", rc);
return rc;
}
return MDBX_RESULT_TRUE;
}
if ((env->me_flags & MDBX_RDONLY) == 0) {
/* Check that another process don't operates in without-lck mode. */
int rc = mdbx_lck_op(env->me_fd, op_setlk, F_WRLCK, env->me_pid, 1);
if (rc != 0) {
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"lock-against-without-lck", rc);
return rc;
}
}
return internal_seize_lck(env->me_lfd);
}
static int __cold mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
const int err) {
int rc = err;
#if MDBX_USE_ROBUST
if (err == EOWNERDEAD) {
/* We own the mutex. Clean up after dead previous owner. */
int rlocked = (env->me_lck && mutex == &env->me_lck->mti_rmutex);
rc = MDBX_SUCCESS;
if (!rlocked) {
if (unlikely(env->me_txn)) {
/* env is hosed if the dead thread was ours */
env->me_flags |= MDBX_FATAL_ERROR;
env->me_txn = NULL;
rc = MDBX_PANIC;
}
}
mdbx_notice("%cmutex owner died, %s", (rlocked ? 'r' : 'w'),
(rc ? "this process' env is hosed" : "recovering"));
int check_rc = mdbx_reader_check0(env, rlocked, NULL);
check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) && \
!defined(pthread_mutex_consistent) && _POSIX_C_SOURCE < 200809L
int mreco_rc = pthread_mutex_consistent_np(mutex);
#else
int mreco_rc = pthread_mutex_consistent(mutex);
#endif
check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;
if (unlikely(mreco_rc))
mdbx_error("mutex recovery failed, %s", mdbx_strerror(mreco_rc));
rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
if (MDBX_IS_ERROR(rc))
pthread_mutex_unlock(mutex);
return rc;
}
#else
(void)mutex;
#endif /* MDBX_USE_ROBUST */
mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err));
if (rc != EDEADLK)
env->me_flags |= MDBX_FATAL_ERROR;
return rc;
}

389
contrib/db/libmdbx/src/lck-posix.c Normal file
View file

@ -0,0 +1,389 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "./bits.h"
/* Some platforms define the EOWNERDEAD error code
* even though they don't support Robust Mutexes.
* Compile with -DMDBX_USE_ROBUST=0. */
#ifndef MDBX_USE_ROBUST
#if (defined(EOWNERDEAD) || _POSIX_C_SOURCE >= 200809L) && !defined(__APPLE__)
#define MDBX_USE_ROBUST 1
#else
#define MDBX_USE_ROBUST 0
#endif
#endif /* MDBX_USE_ROBUST */
/*----------------------------------------------------------------------------*/
/* rthc */
static __cold __attribute__((__constructor__)) void
mdbx_global_constructor(void) {
mdbx_rthc_global_init();
}
static __cold __attribute__((__destructor__)) void
mdbx_global_destructor(void) {
mdbx_rthc_global_dtor();
}
/*----------------------------------------------------------------------------*/
/* lck */
/* Описание реализации блокировок для POSIX:
*
* lck-файл отображается в память, в нём организуется таблица читателей и
* размещаются совместно используемые posix-мьютексы (futex). Посредством
* этих мьютексов (см struct MDBX_lockinfo) реализуются:
* - Блокировка таблицы читателей для регистрации,
* т.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock().
* - Блокировка БД для пишущих транзакций,
* т.е. функции mdbx_txn_lock() и mdbx_txn_unlock().
*
* Остальной функционал реализуется отдельно посредством файловых блокировок:
* - Первоначальный захват БД в режиме exclusive/shared и последующий перевод
* в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade().
* - Проверка присутствие процессов-читателей,
* т.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check().
*
* Для блокировки файлов Используется только fcntl(F_SETLK), так как:
* - lockf() оперирует только эксклюзивной блокировкой и требует
* открытия файла в RW-режиме.
* - flock() не гарантирует атомарности при смене блокировок
* и оперирует только всем файлом целиком.
* - Для контроля процессов-читателей используются однобайтовые
* range-блокировки lck-файла посредством fcntl(F_SETLK). При этом
* в качестве позиции используется pid процесса-читателя.
* - Для первоначального захвата и shared/exclusive выполняется блокировка
* основного файла БД и при успехе lck-файла.
*/
#ifndef OFF_T_MAX
#define OFF_T_MAX \
((sizeof(off_t) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff)
#endif
#ifndef PID_T_MAX
#define PID_T_MAX INT_MAX
#endif
#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)
#define OP_SETLK F_OFD_SETLK
#define OP_SETLKW F_OFD_SETLKW
#define OP_GETLK F_OFD_GETLK
#else
#define OP_SETLK F_SETLK
#define OP_SETLKW F_SETLKW
#define OP_GETLK F_GETLK
#endif /* OFD locks */
static int mdbx_lck_op(mdbx_filehandle_t fd, int cmd, short lck, off_t offset,
off_t len) {
for (;;) {
struct flock lock_op;
memset(&lock_op, 0, sizeof(lock_op));
lock_op.l_type = lck;
lock_op.l_whence = SEEK_SET;
lock_op.l_start = offset;
lock_op.l_len = len;
if (fcntl(fd, cmd, &lock_op) == 0) {
if (cmd == OP_GETLK) {
/* Checks reader by pid. Returns:
* MDBX_RESULT_TRUE - if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE - if pid is dead (lock acquired). */
return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE
: MDBX_RESULT_TRUE;
}
return 0;
}
int rc = errno;
if (rc != EINTR || cmd == F_SETLKW)
return rc;
}
}
int mdbx_rpid_set(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0 && env->me_pid <= PID_T_MAX);
return mdbx_lck_op(env->me_lfd, OP_SETLK, F_WRLCK, env->me_pid, 1);
}
int mdbx_rpid_clear(MDBX_env *env) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0 && env->me_pid <= PID_T_MAX);
return mdbx_lck_op(env->me_lfd, OP_SETLKW, F_UNLCK, env->me_pid, 1);
}
int mdbx_rpid_check(MDBX_env *env, mdbx_pid_t pid) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
assert(pid > 0 && pid <= PID_T_MAX);
assert(PID_T_MAX < OFF_T_MAX);
return mdbx_lck_op(env->me_lfd, OP_GETLK, F_WRLCK, pid, 1);
}
int __cold mdbx_lck_seize(MDBX_env *env) {
assert(env->me_fd != INVALID_HANDLE_VALUE);
assert(env->me_pid > 0 && env->me_pid <= PID_T_MAX);
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */
int rc = mdbx_lck_op(env->me_fd, OP_SETLK,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0,
OFF_T_MAX);
if (rc != 0) {
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "without-lck", rc);
return rc;
}
return MDBX_RESULT_TRUE;
}
/* try exclusive access */
int rc = mdbx_lck_op(env->me_fd, OP_SETLK,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0,
OFF_T_MAX);
if (rc == 0) {
continue_exclusive:
/* got dxb-exclusive, continue lck-exclusive */
rc = mdbx_lck_op(env->me_lfd, OP_SETLKW, F_WRLCK, 0, OFF_T_MAX);
if (rc == 0) {
/* got both exclusive */
return MDBX_RESULT_TRUE;
}
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"lck-after-dxb-exclusive", rc);
assert(MDBX_IS_ERROR(rc));
goto bailout;
}
if (rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK) {
rc = mdbx_lck_op(env->me_fd, OP_SETLKW,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
env->me_pid, 1);
if (rc == 0) {
/* got dxb-shared, try again dxb-exclusive */
rc = mdbx_lck_op(env->me_fd, OP_SETLK,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0,
OFF_T_MAX);
if (rc == 0)
goto continue_exclusive;
/* continue lck-shared */
rc = mdbx_lck_op(env->me_lfd, OP_SETLKW, F_RDLCK, 0, 1);
if (rc == 0) {
/* got both dxb and lck shared lock */
return MDBX_RESULT_FALSE;
}
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "lck-shared", rc);
} else {
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "dxb-shared", rc);
}
assert(MDBX_IS_ERROR(rc));
}
bailout:
(void)mdbx_lck_op(env->me_lfd, OP_SETLK, F_UNLCK, 0, OFF_T_MAX);
(void)mdbx_lck_op(env->me_fd, OP_SETLK, F_UNLCK, 0, OFF_T_MAX);
assert(MDBX_IS_ERROR(rc));
return rc;
}
int mdbx_lck_downgrade(MDBX_env *env, bool complete) {
assert(env->me_lfd != INVALID_HANDLE_VALUE);
int rc = mdbx_lck_op(env->me_lfd, OP_SETLK, F_UNLCK, 1, OFF_T_MAX - 1);
if (rc == 0)
rc = mdbx_lck_op(env->me_lfd, OP_SETLKW, F_RDLCK, 0, 1);
if (unlikely(rc != 0)) {
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "lck", rc);
goto bailout;
}
if (complete) {
rc = mdbx_lck_op(env->me_fd, OP_SETLK,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK,
env->me_pid, 1);
if (unlikely(rc != 0)) {
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "dxb", rc);
goto bailout;
}
}
return MDBX_SUCCESS;
bailout:
(void)mdbx_lck_op(env->me_lfd, OP_SETLK, F_UNLCK, 0, OFF_T_MAX);
(void)mdbx_lck_op(env->me_fd, OP_SETLK, F_UNLCK, 0, OFF_T_MAX);
assert(MDBX_IS_ERROR(rc));
return rc;
}
/*---------------------------------------------------------------------------*/
static int mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
const int rc);
int __cold mdbx_lck_init(MDBX_env *env) {
pthread_mutexattr_t ma;
int rc = pthread_mutexattr_init(&ma);
if (rc)
return rc;
rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED);
if (rc)
goto bailout;
#if MDBX_USE_ROBUST
rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST);
if (rc)
goto bailout;
#endif /* MDBX_USE_ROBUST */
#if _POSIX_C_SOURCE >= 199506L && !defined(MDBX_SAFE4QEMU)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT);
if (rc == ENOTSUP)
rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE);
if (rc)
goto bailout;
#endif /* PTHREAD_PRIO_INHERIT */
rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_rmutex, &ma);
if (rc)
goto bailout;
rc = pthread_mutex_init(&env->me_lck->mti_wmutex, &ma);
bailout:
pthread_mutexattr_destroy(&ma);
return rc;
}
void __cold mdbx_lck_destroy(MDBX_env *env) {
/* File locks would be released (by kernel) while the file-descriptors
* will be closed. But to avoid false-positive EDEADLK from the kernel,
* locks should be released here explicitly with properly order. */
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* try get exclusive access */
if (env->me_lck &&
mdbx_lck_op(env->me_fd, OP_SETLK,
(env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0,
OFF_T_MAX) == 0 &&
mdbx_lck_op(env->me_lfd, OP_SETLK, F_WRLCK, 0, OFF_T_MAX) == 0) {
mdbx_info("%s: got exclusive, drown mutexes", mdbx_func_);
int rc = pthread_mutex_destroy(&env->me_lck->mti_rmutex);
if (rc == 0)
rc = pthread_mutex_destroy(&env->me_lck->mti_wmutex);
assert(rc == 0);
(void)rc;
msync(env->me_lck, env->me_os_psize, MS_ASYNC);
}
(void)mdbx_lck_op(env->me_lfd, OP_SETLK, F_UNLCK, 0, OFF_T_MAX);
}
if (env->me_fd != INVALID_HANDLE_VALUE)
(void)mdbx_lck_op(env->me_fd, OP_SETLK, F_UNLCK, 0, OFF_T_MAX);
}
static int mdbx_robust_lock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_lock(mutex);
if (unlikely(rc != 0))
rc = mdbx_mutex_failed(env, mutex, rc);
return rc;
}
static int mdbx_robust_trylock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_trylock(mutex);
if (unlikely(rc != 0 && rc != EBUSY))
rc = mdbx_mutex_failed(env, mutex, rc);
return (rc != EBUSY) ? rc : MDBX_BUSY;
}
static int mdbx_robust_unlock(MDBX_env *env, pthread_mutex_t *mutex) {
int rc = pthread_mutex_unlock(mutex);
if (unlikely(rc != 0))
rc = mdbx_mutex_failed(env, mutex, rc);
return rc;
}
int mdbx_rdt_lock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_lock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
return rc;
}
void mdbx_rdt_unlock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_unlock(env, &env->me_lck->mti_rmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", mdbx_func_, rc);
}
int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
mdbx_trace(">>");
int rc = dontwait ? mdbx_robust_trylock(env, env->me_wmutex)
: mdbx_robust_lock(env, env->me_wmutex);
mdbx_trace("<< rc %d", rc);
return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS;
}
void mdbx_txn_unlock(MDBX_env *env) {
mdbx_trace(">>");
int rc = mdbx_robust_unlock(env, env->me_wmutex);
mdbx_trace("<< rc %d", rc);
if (unlikely(MDBX_IS_ERROR(rc)))
mdbx_panic("%s() failed: errcode %d\n", mdbx_func_, rc);
}
static int __cold mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex,
const int err) {
int rc = err;
#if MDBX_USE_ROBUST
if (err == EOWNERDEAD) {
/* We own the mutex. Clean up after dead previous owner. */
int rlocked = (env->me_lck && mutex == &env->me_lck->mti_rmutex);
rc = MDBX_SUCCESS;
if (!rlocked) {
if (unlikely(env->me_txn)) {
/* env is hosed if the dead thread was ours */
env->me_flags |= MDBX_FATAL_ERROR;
env->me_txn = NULL;
rc = MDBX_PANIC;
}
}
mdbx_notice("%cmutex owner died, %s", (rlocked ? 'r' : 'w'),
(rc ? "this process' env is hosed" : "recovering"));
int check_rc = mdbx_reader_check0(env, rlocked, NULL);
check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc;
int mreco_rc = pthread_mutex_consistent(mutex);
check_rc = (mreco_rc == 0) ? check_rc : mreco_rc;
if (unlikely(mreco_rc))
mdbx_error("mutex recovery failed, %s", mdbx_strerror(mreco_rc));
rc = (rc == MDBX_SUCCESS) ? check_rc : rc;
if (MDBX_IS_ERROR(rc))
pthread_mutex_unlock(mutex);
return rc;
}
#else
(void)mutex;
#endif /* MDBX_USE_ROBUST */
mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err));
if (rc != EDEADLK)
env->me_flags |= MDBX_FATAL_ERROR;
return rc;
}

707
contrib/db/libmdbx/src/lck-windows.c Normal file
View file

@ -0,0 +1,707 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "./bits.h"
/* PREAMBLE FOR WINDOWS:
*
* We are not concerned for performance here.
* If you are running Windows a performance could NOT be the goal.
* Otherwise please use Linux.
*
* Regards,
* LY
*/
static void mdbx_winnt_import(void);
#ifdef MDBX_BUILD_DLL
/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks.
*
* Therefore we don't define dll's entry point for debug/checked builds by MSVC.
* In this case MSVC's will automatically use DllMainCRTStartup() from CRT
* library, which also automatically call DllMain() from our mdbx.dll
*
* On the other side, for RELEASE builds
* we explicitly define DllMain() as the entry point and don't linking with
* any CRT libraries (IgnoreAllDefaultLibraries = Yes). */
#if !defined(_MSC_VER) || defined(NDEBUG)
#pragma comment(linker, "/ENTRY:DllMain")
#endif
BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved)
#else
#if !MDBX_CONFIG_MANUAL_TLS_CALLBACK
static
#endif /* !MDBX_CONFIG_MANUAL_TLS_CALLBACK */
void NTAPI
mdbx_dll_callback(PVOID module, DWORD reason, PVOID reserved)
#endif /* MDBX_BUILD_DLL */
{
(void)reserved;
switch (reason) {
case DLL_PROCESS_ATTACH:
mdbx_winnt_import();
mdbx_rthc_global_init();
break;
case DLL_PROCESS_DETACH:
mdbx_rthc_global_dtor();
break;
case DLL_THREAD_ATTACH:
break;
case DLL_THREAD_DETACH:
mdbx_rthc_thread_dtor(module);
break;
}
#ifdef MDBX_BUILD_DLL
return TRUE;
#endif
}
#if !defined(MDBX_BUILD_DLL) && !MDBX_CONFIG_MANUAL_TLS_CALLBACK
/* *INDENT-OFF* */
/* clang-format off */
#if defined(_MSC_VER)
# pragma const_seg(push)
# pragma data_seg(push)
# ifdef _WIN64
/* kick a linker to create the TLS directory if not already done */
# pragma comment(linker, "/INCLUDE:_tls_used")
/* Force some symbol references. */
# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor")
/* specific const-segment for WIN64 */
# pragma const_seg(".CRT$XLB")
const
# else
/* kick a linker to create the TLS directory if not already done */
# pragma comment(linker, "/INCLUDE:__tls_used")
/* Force some symbol references. */
# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor")
/* specific data-segment for WIN32 */
# pragma data_seg(".CRT$XLB")
# endif
__declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_dll_callback;
# pragma data_seg(pop)
# pragma const_seg(pop)
#elif defined(__GNUC__)
# ifdef _WIN64
const
# endif
PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_dll_callback;
#else
# error FIXME
#endif
/* *INDENT-ON* */
/* clang-format on */
#endif /* !defined(MDBX_BUILD_DLL) && !MDBX_CONFIG_MANUAL_TLS_CALLBACK */
/*----------------------------------------------------------------------------*/
#define LCK_SHARED 0
#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK
#define LCK_WAITFOR 0
#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY
static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset,
size_t bytes) {
OVERLAPPED ov;
ov.hEvent = 0;
ov.Offset = (DWORD)offset;
ov.OffsetHigh = HIGH_DWORD(offset);
return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov);
}
static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset,
size_t bytes) {
return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes,
HIGH_DWORD(bytes));
}
/*----------------------------------------------------------------------------*/
/* global `write` lock for write-txt processing,
* exclusive locking both meta-pages) */
#define LCK_MAXLEN (1u + (size_t)(MAXSSIZE_T))
#define LCK_META_OFFSET 0
#define LCK_META_LEN 0x10000u
#define LCK_BODY_OFFSET LCK_META_LEN
#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET)
#define LCK_META LCK_META_OFFSET, LCK_META_LEN
#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN
#define LCK_WHOLE 0, LCK_MAXLEN
int mdbx_txn_lock(MDBX_env *env, bool dontwait) {
if (dontwait) {
if (!TryEnterCriticalSection(&env->me_windowsbug_lock))
return MDBX_BUSY;
} else {
EnterCriticalSection(&env->me_windowsbug_lock);
}
if ((env->me_flags & MDBX_EXCLUSIVE) ||
flock(env->me_fd,
dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT)
: (LCK_EXCLUSIVE | LCK_WAITFOR),
LCK_BODY))
return MDBX_SUCCESS;
int rc = GetLastError();
LeaveCriticalSection(&env->me_windowsbug_lock);
return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY;
}
void mdbx_txn_unlock(MDBX_env *env) {
int rc =
(env->me_flags & MDBX_EXCLUSIVE) ? TRUE : funlock(env->me_fd, LCK_BODY);
LeaveCriticalSection(&env->me_windowsbug_lock);
if (!rc)
mdbx_panic("%s failed: errcode %u", mdbx_func_, GetLastError());
}
/*----------------------------------------------------------------------------*/
/* global `read` lock for readers registration,
* exclusive locking `mti_numreaders` (second) cacheline */
#define LCK_LO_OFFSET 0
#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders)
#define LCK_UP_OFFSET LCK_LO_LEN
#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET)
#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN
#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN
int mdbx_rdt_lock(MDBX_env *env) {
mdbx_srwlock_AcquireShared(&env->me_remap_guard);
if (env->me_lfd == INVALID_HANDLE_VALUE)
return MDBX_SUCCESS; /* readonly database in readonly filesystem */
/* transite from S-? (used) to S-E (locked), e.g. exclusive lock upper-part */
if ((env->me_flags & MDBX_EXCLUSIVE) ||
flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER))
return MDBX_SUCCESS;
int rc = GetLastError();
mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
return rc;
}
void mdbx_rdt_unlock(MDBX_env *env) {
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* transite from S-E (locked) to S-? (used), e.g. unlock upper-part */
if ((env->me_flags & MDBX_EXCLUSIVE) == 0 &&
!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s failed: errcode %u", mdbx_func_, GetLastError());
}
mdbx_srwlock_ReleaseShared(&env->me_remap_guard);
}
static int suspend_and_append(mdbx_handle_array_t **array,
const DWORD ThreadId) {
const unsigned limit = (*array)->limit;
if ((*array)->count == limit) {
void *ptr = mdbx_realloc(
(limit > ARRAY_LENGTH((*array)->handles))
? *array
: /* don't free initial array on the stack */ NULL,
sizeof(mdbx_handle_array_t) +
sizeof(HANDLE) * (limit * 2 - ARRAY_LENGTH((*array)->handles)));
if (!ptr)
return MDBX_ENOMEM;
if (limit == ARRAY_LENGTH((*array)->handles))
memcpy(ptr, *array, sizeof(mdbx_handle_array_t));
*array = (mdbx_handle_array_t *)ptr;
(*array)->limit = limit * 2;
}
HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION,
FALSE, ThreadId);
if (hThread == NULL)
return GetLastError();
if (SuspendThread(hThread) == -1) {
int err = GetLastError();
DWORD ExitCode;
if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED ||
!GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE)
err = MDBX_SUCCESS;
CloseHandle(hThread);
return err;
}
(*array)->handles[(*array)->count++] = hThread;
return MDBX_SUCCESS;
}
int mdbx_suspend_threads_before_remap(MDBX_env *env,
mdbx_handle_array_t **array) {
const mdbx_pid_t CurrentTid = GetCurrentThreadId();
int rc;
if (env->me_lck) {
/* Scan LCK for threads of the current process */
const MDBX_reader *const begin = env->me_lck->mti_readers;
const MDBX_reader *const end = begin + env->me_lck->mti_numreaders;
const mdbx_tid_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0;
for (const MDBX_reader *reader = begin; reader < end; ++reader) {
if (reader->mr_pid != env->me_pid || !reader->mr_tid) {
skip_lck:
continue;
}
if (reader->mr_tid == CurrentTid || reader->mr_tid == WriteTxnOwner)
goto skip_lck;
if (env->me_flags & MDBX_NOTLS) {
/* Skip duplicates in no-tls mode */
for (const MDBX_reader *scan = reader; --scan >= begin;)
if (scan->mr_tid == reader->mr_tid)
goto skip_lck;
}
rc = suspend_and_append(array, reader->mr_tid);
if (rc != MDBX_SUCCESS) {
bailout_lck:
(void)mdbx_resume_threads_after_remap(*array);
return rc;
}
}
if (WriteTxnOwner && WriteTxnOwner != CurrentTid) {
rc = suspend_and_append(array, WriteTxnOwner);
if (rc != MDBX_SUCCESS)
goto bailout_lck;
}
} else {
/* Without LCK (i.e. read-only mode).
* Walk thougth a snapshot of all running threads */
mdbx_assert(env,
env->me_txn0 == NULL || (env->me_flags & MDBX_EXCLUSIVE) != 0);
const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
if (hSnapshot == INVALID_HANDLE_VALUE)
return GetLastError();
THREADENTRY32 entry;
entry.dwSize = sizeof(THREADENTRY32);
if (!Thread32First(hSnapshot, &entry)) {
rc = GetLastError();
bailout_toolhelp:
CloseHandle(hSnapshot);
(void)mdbx_resume_threads_after_remap(*array);
return rc;
}
do {
if (entry.th32OwnerProcessID != env->me_pid ||
entry.th32ThreadID == CurrentTid)
continue;
rc = suspend_and_append(array, entry.th32ThreadID);
if (rc != MDBX_SUCCESS)
goto bailout_toolhelp;
} while (Thread32Next(hSnapshot, &entry));
rc = GetLastError();
if (rc != ERROR_NO_MORE_FILES)
goto bailout_toolhelp;
CloseHandle(hSnapshot);
}
return MDBX_SUCCESS;
}
int mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) {
int rc = MDBX_SUCCESS;
for (unsigned i = 0; i < array->count; ++i) {
const HANDLE hThread = array->handles[i];
if (ResumeThread(hThread) == -1) {
const int err = GetLastError();
DWORD ExitCode;
if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED &&
GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE)
rc = err;
}
CloseHandle(hThread);
}
return rc;
}
/*----------------------------------------------------------------------------*/
/* global `initial` lock for lockfile initialization,
* exclusive/shared locking first cacheline */
/* FIXME: locking schema/algo descritpion.
?-? = free
S-? = used
E-? = exclusive-read
?-S
?-E = middle
S-S
S-E = locked
E-S
E-E = exclusive-write
*/
int mdbx_lck_init(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE)
* or as 'used' (S-? and returns MDBX_RESULT_FALSE), otherwise returns an error
*/
static int internal_seize_lck(HANDLE lfd) {
int rc;
assert(lfd != INVALID_HANDLE_VALUE);
/* 1) now on ?-? (free), get ?-E (middle) */
mdbx_jitter4testing(false);
if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) {
rc = GetLastError() /* 2) something went wrong, give up */;
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"?-?(free) >> ?-E(middle)", rc);
return rc;
}
/* 3) now on ?-E (middle), try E-E (exclusive-write) */
mdbx_jitter4testing(false);
if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER))
return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */;
/* 5) still on ?-E (middle) */
rc = GetLastError();
mdbx_jitter4testing(false);
if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) {
/* 6) something went wrong, give up */
if (!funlock(lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"?-E(middle) >> ?-?(free)", GetLastError());
return rc;
}
/* 7) still on ?-E (middle), try S-E (locked) */
mdbx_jitter4testing(false);
rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE
: GetLastError();
mdbx_jitter4testing(false);
if (rc != MDBX_RESULT_FALSE)
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"?-E(middle) >> S-E(locked)", rc);
/* 8) now on S-E (locked) or still on ?-E (middle),
* transite to S-? (used) or ?-? (free) */
if (!funlock(lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"X-E(locked/middle) >> X-?(used/free)", GetLastError());
/* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */
return rc;
}
int mdbx_lck_seize(MDBX_env *env) {
int rc;
assert(env->me_fd != INVALID_HANDLE_VALUE);
if (env->me_flags & MDBX_EXCLUSIVE)
return MDBX_RESULT_TRUE /* nope since files were must be opened
non-shareable */
;
if (env->me_lfd == INVALID_HANDLE_VALUE) {
/* LY: without-lck mode (e.g. on read-only filesystem) */
mdbx_jitter4testing(false);
if (!flock(env->me_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) {
rc = GetLastError();
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_, "without-lck", rc);
return rc;
}
return MDBX_RESULT_FALSE;
}
rc = internal_seize_lck(env->me_lfd);
mdbx_jitter4testing(false);
if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) {
/* Check that another process don't operates in without-lck mode.
* Doing such check by exclusive locking the body-part of db. Should be
* noted:
* - we need an exclusive lock for do so;
* - we can't lock meta-pages, otherwise other process could get an error
* while opening db in valid (non-conflict) mode. */
if (!flock(env->me_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) {
rc = GetLastError();
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"lock-against-without-lck", rc);
mdbx_jitter4testing(false);
mdbx_lck_destroy(env);
} else {
mdbx_jitter4testing(false);
if (!funlock(env->me_fd, LCK_BODY))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"unlock-against-without-lck", GetLastError());
}
}
return rc;
}
int mdbx_lck_downgrade(MDBX_env *env, bool complete) {
/* Transite from exclusive state (E-?) to used (S-?) */
assert(env->me_fd != INVALID_HANDLE_VALUE);
assert(env->me_lfd != INVALID_HANDLE_VALUE);
if (env->me_flags & MDBX_EXCLUSIVE)
return MDBX_SUCCESS /* nope since files were must be opened non-shareable */
;
/* 1) must be at E-E (exclusive-write) */
if (!complete) {
/* transite from E-E to E_? (exclusive-read) */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"E-E(exclusive-write) >> E-?(exclusive-read)", GetLastError());
return MDBX_SUCCESS /* 2) now at E-? (exclusive-read), done */;
}
/* 3) now at E-E (exclusive-write), transite to ?_E (middle) */
if (!funlock(env->me_lfd, LCK_LOWER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"E-E(exclusive-write) >> ?-E(middle)", GetLastError());
/* 4) now at ?-E (middle), transite to S-E (locked) */
if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) {
int rc = GetLastError() /* 5) something went wrong, give up */;
mdbx_error("%s(%s) failed: errcode %u", mdbx_func_,
"?-E(middle) >> S-E(locked)", rc);
return rc;
}
/* 6) got S-E (locked), continue transition to S-? (used) */
if (!funlock(env->me_lfd, LCK_UPPER))
mdbx_panic("%s(%s) failed: errcode %u", mdbx_func_,
"S-E(locked) >> S-?(used)", GetLastError());
return MDBX_SUCCESS /* 7) now at S-? (used), done */;
}
void mdbx_lck_destroy(MDBX_env *env) {
int rc;
if (env->me_lfd != INVALID_HANDLE_VALUE) {
/* double `unlock` for robustly remove overlapped shared/exclusive locks */
while (funlock(env->me_lfd, LCK_LOWER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_lfd, LCK_UPPER))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
}
if (env->me_fd != INVALID_HANDLE_VALUE) {
/* explicitly unlock to avoid latency for other processes (windows kernel
* releases such locks via deferred queues) */
while (funlock(env->me_fd, LCK_BODY))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_fd, LCK_META))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
while (funlock(env->me_fd, LCK_WHOLE))
;
rc = GetLastError();
assert(rc == ERROR_NOT_LOCKED);
(void)rc;
SetLastError(ERROR_SUCCESS);
}
}
/*----------------------------------------------------------------------------*/
/* reader checking (by pid) */
int mdbx_rpid_set(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
int mdbx_rpid_clear(MDBX_env *env) {
(void)env;
return MDBX_SUCCESS;
}
/* Checks reader by pid.
*
* Returns:
* MDBX_RESULT_TRUE, if pid is live (unable to acquire lock)
* MDBX_RESULT_FALSE, if pid is dead (lock acquired)
* or otherwise the errcode. */
int mdbx_rpid_check(MDBX_env *env, mdbx_pid_t pid) {
(void)env;
HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid);
int rc;
if (likely(hProcess)) {
rc = WaitForSingleObject(hProcess, 0);
if (unlikely(rc == WAIT_FAILED))
rc = GetLastError();
CloseHandle(hProcess);
} else {
rc = GetLastError();
}
switch (rc) {
case ERROR_INVALID_PARAMETER:
/* pid seem invalid */
return MDBX_RESULT_FALSE;
case WAIT_OBJECT_0:
/* process just exited */
return MDBX_RESULT_FALSE;
case WAIT_TIMEOUT:
/* pid running */
return MDBX_RESULT_TRUE;
default:
/* failure */
return rc;
}
}
//----------------------------------------------------------------------------
// Stub for slim read-write lock
// Copyright (C) 1995-2002 Brad Wilson
static void WINAPI stub_srwlock_Init(MDBX_srwlock *srwl) {
srwl->readerCount = srwl->writerCount = 0;
}
static void WINAPI stub_srwlock_AcquireShared(MDBX_srwlock *srwl) {
while (true) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
// If there's a writer already, spin without unnecessarily
// interlocking the CPUs
if (srwl->writerCount != 0) {
YieldProcessor();
continue;
}
// Add to the readers list
_InterlockedIncrement(&srwl->readerCount);
// Check for writers again (we may have been pre-empted). If
// there are no writers writing or waiting, then we're done.
if (srwl->writerCount == 0)
break;
// Remove from the readers list, spin, try again
_InterlockedDecrement(&srwl->readerCount);
YieldProcessor();
}
}
static void WINAPI stub_srwlock_ReleaseShared(MDBX_srwlock *srwl) {
assert(srwl->readerCount > 0);
_InterlockedDecrement(&srwl->readerCount);
}
static void WINAPI stub_srwlock_AcquireExclusive(MDBX_srwlock *srwl) {
while (true) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
// If there's a writer already, spin without unnecessarily
// interlocking the CPUs
if (srwl->writerCount != 0) {
YieldProcessor();
continue;
}
// See if we can become the writer (expensive, because it inter-
// locks the CPUs, so writing should be an infrequent process)
if (_InterlockedExchange(&srwl->writerCount, 1) == 0)
break;
}
// Now we're the writer, but there may be outstanding readers.
// Spin until there aren't any more; new readers will wait now
// that we're the writer.
while (srwl->readerCount != 0) {
assert(srwl->writerCount >= 0 && srwl->readerCount >= 0);
YieldProcessor();
}
}
static void WINAPI stub_srwlock_ReleaseExclusive(MDBX_srwlock *srwl) {
assert(srwl->writerCount == 1 && srwl->readerCount >= 0);
srwl->writerCount = 0;
}
MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared,
mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive,
mdbx_srwlock_ReleaseExclusive;
/*----------------------------------------------------------------------------*/
MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx;
MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW;
MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle;
MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
MDBX_NtFsControlFile mdbx_NtFsControlFile;
static void mdbx_winnt_import(void) {
const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll");
const MDBX_srwlock_function init =
(MDBX_srwlock_function)GetProcAddress(hKernel32dll, "InitializeSRWLock");
if (init != NULL) {
mdbx_srwlock_Init = init;
mdbx_srwlock_AcquireShared = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "AcquireSRWLockShared");
mdbx_srwlock_ReleaseShared = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "ReleaseSRWLockShared");
mdbx_srwlock_AcquireExclusive = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "AcquireSRWLockExclusive");
mdbx_srwlock_ReleaseExclusive = (MDBX_srwlock_function)GetProcAddress(
hKernel32dll, "ReleaseSRWLockExclusive");
} else {
mdbx_srwlock_Init = stub_srwlock_Init;
mdbx_srwlock_AcquireShared = stub_srwlock_AcquireShared;
mdbx_srwlock_ReleaseShared = stub_srwlock_ReleaseShared;
mdbx_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive;
mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive;
}
#define GET_KERNEL32_PROC(ENTRY) \
mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(hKernel32dll, #ENTRY)
GET_KERNEL32_PROC(GetFileInformationByHandleEx);
GET_KERNEL32_PROC(GetVolumeInformationByHandleW);
GET_KERNEL32_PROC(GetFinalPathNameByHandleW);
GET_KERNEL32_PROC(SetFileInformationByHandle);
GET_KERNEL32_PROC(PrefetchVirtualMemory);
const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll");
mdbx_NtFsControlFile =
(MDBX_NtFsControlFile)GetProcAddress(hNtdll, "NtFsControlFile");
}

14492
contrib/db/libmdbx/src/mdbx.c Normal file
View file

File diff suppressed because it is too large Load diff

1244
contrib/db/libmdbx/src/ntdll.def Normal file
View file

File diff suppressed because it is too large Load diff

1319
contrib/db/libmdbx/src/osal.c Normal file
View file

File diff suppressed because it is too large Load diff

899
contrib/db/libmdbx/src/osal.h Normal file
View file

@ -0,0 +1,899 @@
/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
/*----------------------------------------------------------------------------*/
/* Microsoft compiler generates a lot of warning for self includes... */
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
#if defined(_WIN32) || defined(_WIN64)
#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
#endif
#if !defined(_NO_CRT_STDIO_INLINE) && defined(MDBX_BUILD_DLL)
#define _NO_CRT_STDIO_INLINE
#endif
#endif /* Windows */
/*----------------------------------------------------------------------------*/
/* C99 includes */
#include <inttypes.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#include <fcntl.h>
#include <limits.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
/* C11 stdalign.h */
#if __has_include(<stdalign.h>)
#include <stdalign.h>
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
#define alignas(N) _Alignas(N)
#elif defined(_MSC_VER)
#define alignas(N) __declspec(align(N))
#elif __has_attribute(__aligned__) || defined(__GNUC__)
#define alignas(N) __attribute__((__aligned__(N)))
#else
#error "FIXME: Required _alignas() or equivalent."
#endif
/*----------------------------------------------------------------------------*/
/* Systems includes */
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
defined(__BSD__) || defined(__NETBSD__) || defined(__bsdi__) || \
defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__)
#include <sys/cdefs.h>
#else
#include <malloc.h>
#ifndef _POSIX_C_SOURCE
#ifdef _POSIX_SOURCE
#define _POSIX_C_SOURCE 1
#else
#define _POSIX_C_SOURCE 0
#endif
#endif
#endif /* !xBSD */
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 0
#endif
#if defined(_WIN32) || defined(_WIN64)
#define WIN32_LEAN_AND_MEAN
#include <tlhelp32.h>
#include <windows.h>
#include <winnt.h>
#include <winternl.h>
#define HAVE_SYS_STAT_H
#define HAVE_SYS_TYPES_H
typedef HANDLE mdbx_thread_t;
typedef unsigned mdbx_thread_key_t;
#define MDBX_OSAL_SECTION HANDLE
#define MAP_FAILED NULL
#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0))
#define THREAD_CALL WINAPI
#define THREAD_RESULT DWORD
typedef struct {
HANDLE mutex;
HANDLE event;
} mdbx_condmutex_t;
typedef CRITICAL_SECTION mdbx_fastmutex_t;
#ifdef MDBX_AVOID_CRT
#ifndef mdbx_malloc
static inline void *mdbx_malloc(size_t bytes) {
return LocalAlloc(LMEM_FIXED, bytes);
}
#endif /* mdbx_malloc */
#ifndef mdbx_calloc
static inline void *mdbx_calloc(size_t nelem, size_t size) {
return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size);
}
#endif /* mdbx_calloc */
#ifndef mdbx_realloc
static inline void *mdbx_realloc(void *ptr, size_t bytes) {
return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE);
}
#endif /* mdbx_realloc */
#ifndef mdbx_free
#define mdbx_free LocalFree
#endif /* mdbx_free */
#else
#define mdbx_malloc malloc
#define mdbx_calloc calloc
#define mdbx_realloc realloc
#define mdbx_free free
#define mdbx_strdup _strdup
#endif /* MDBX_AVOID_CRT */
#ifndef snprintf
#define snprintf _snprintf /* ntdll */
#endif
#ifndef vsnprintf
#define vsnprintf _vsnprintf /* ntdll */
#endif
#else /*----------------------------------------------------------------------*/
#include <pthread.h>
#include <signal.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/uio.h>
#include <unistd.h>
typedef pthread_t mdbx_thread_t;
typedef pthread_key_t mdbx_thread_key_t;
#define INVALID_HANDLE_VALUE (-1)
#define THREAD_CALL
#define THREAD_RESULT void *
typedef struct {
pthread_mutex_t mutex;
pthread_cond_t cond;
} mdbx_condmutex_t;
typedef pthread_mutex_t mdbx_fastmutex_t;
#define mdbx_malloc malloc
#define mdbx_calloc calloc
#define mdbx_realloc realloc
#define mdbx_free free
#define mdbx_strdup strdup
#endif /* Platform */
/* *INDENT-OFF* */
/* clang-format off */
#if defined(HAVE_SYS_STAT_H) || __has_include(<sys/stat.h>)
#include <sys/stat.h>
#endif
#if defined(HAVE_SYS_TYPES_H) || __has_include(<sys/types.h>)
#include <sys/types.h>
#endif
#if defined(HAVE_SYS_FILE_H) || __has_include(<sys/file.h>)
#include <sys/file.h>
#endif
/* *INDENT-ON* */
/* clang-format on */
#ifndef SSIZE_MAX
#define SSIZE_MAX INTPTR_MAX
#endif
#if !defined(MADV_DODUMP) && defined(MADV_CORE)
#define MADV_DODUMP MADV_CORE
#endif /* MADV_CORE -> MADV_DODUMP */
#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE)
#define MADV_DONTDUMP MADV_NOCORE
#endif /* MADV_NOCORE -> MADV_DONTDUMP */
#ifndef MADV_REMOVE_OR_FREE
#ifdef MADV_REMOVE
#define MADV_REMOVE_OR_FREE MADV_REMOVE
#elif defined(MADV_FREE)
#define MADV_REMOVE_OR_FREE MADV_FREE
#endif
#endif /* MADV_REMOVE_OR_FREE */
#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \
defined(i486) || defined(__i486) || defined(__i486__) || \
defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \
defined(__i686) || defined(__i686__) || defined(_M_IX86) || \
defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \
defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \
defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__)
#ifndef __ia32__
/* LY: define neutral __ia32__ for x86 and x86-64 archs */
#define __ia32__ 1
#endif /* __ia32__ */
#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64) || defined(_M_X64))
/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */
#define __amd64__ 1
#endif /* __amd64__ */
#endif /* all x86 */
#if !defined(UNALIGNED_OK)
#if (defined(__ia32__) || defined(__e2k__) || \
defined(__ARM_FEATURE_UNALIGNED)) && \
!defined(__ALIGNED__)
#define UNALIGNED_OK 1
#else
#define UNALIGNED_OK 0
#endif
#endif /* UNALIGNED_OK */
#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
#error \
"Sanity checking failed: Two's complement, reasonably sized integer types"
#endif
/*----------------------------------------------------------------------------*/
/* Compiler's includes for builtins/intrinsics */
#if defined(_MSC_VER) || defined(__INTEL_COMPILER)
#include <intrin.h>
#elif __GNUC_PREREQ(4, 4) || defined(__clang__)
#if defined(__ia32__) || defined(__e2k__)
#include <x86intrin.h>
#endif /* __ia32__ */
#if defined(__ia32__)
#include <cpuid.h>
#endif /* __ia32__ */
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
#include <mbarrier.h>
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
#include <machine/sys/inline.h>
#elif defined(__IBMC__) && defined(__powerpc)
#include <atomic.h>
#elif defined(_AIX)
#include <builtins.h>
#include <sys/atomic_op.h>
#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha)
#include <c_asm.h>
#include <machine/builtins.h>
#elif defined(__MWERKS__)
/* CodeWarrior - troubles ? */
#pragma gcc_extensions
#elif defined(__SNC__)
/* Sony PS3 - troubles ? */
#elif defined(__hppa__) || defined(__hppa)
#include <machine/inline.h>
#else
#error Unsupported C compiler, please use GNU C 4.4 or newer
#endif /* Compiler */
/*----------------------------------------------------------------------------*/
/* Byteorder */
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
/* *INDENT-OFF* */
/* clang-format off */
#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \
defined(HAVE_ENDIAN_H) || __has_include(<endian.h>)
#include <endian.h>
#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \
defined(HAVE_MACHINE_ENDIAN_H) || __has_include(<machine/endian.h>)
#include <machine/endian.h>
#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include(<sys/isa_defs.h>)
#include <sys/isa_defs.h>
#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \
(__has_include(<sys/types.h>) && __has_include(<sys/endian.h>))
#include <sys/endian.h>
#include <sys/types.h>
#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \
defined(__NETBSD__) || defined(__NetBSD__) || \
defined(HAVE_SYS_PARAM_H) || __has_include(<sys/param.h>)
#include <sys/param.h>
#endif /* OS */
/* *INDENT-ON* */
/* clang-format on */
#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN
#define __BYTE_ORDER__ __BYTE_ORDER
#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)
#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN
#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN
#define __BYTE_ORDER__ _BYTE_ORDER
#else
#define __ORDER_LITTLE_ENDIAN__ 1234
#define __ORDER_BIG_ENDIAN__ 4321
#if defined(__LITTLE_ENDIAN__) || \
(defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \
defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \
defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \
defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \
defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \
defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \
defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \
defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \
defined(__WINDOWS__)
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
#elif defined(__BIG_ENDIAN__) || \
(defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \
defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \
defined(__m68k__) || defined(M68000) || defined(__hppa__) || \
defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \
defined(__sparc) || defined(__370__) || defined(__THW_370__) || \
defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__)
#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__
#else
#error __BYTE_ORDER__ should be defined.
#endif /* Arch */
#endif
#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */
/*----------------------------------------------------------------------------*/
/* Memory/Compiler barriers, cache coherence */
static __inline void mdbx_compiler_barrier(void) {
#if defined(__clang__) || defined(__GNUC__)
__asm__ __volatile__("" ::: "memory");
#elif defined(_MSC_VER)
_ReadWriteBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
__memory_barrier();
if (type > MDBX_BARRIER_COMPILER)
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__compiler_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */);
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__fence();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
static __inline void mdbx_memory_barrier(void) {
#if __has_extension(c_atomic) || __has_extension(cxx_atomic)
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__ATOMIC_SEQ_CST)
__atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__clang__) || defined(__GNUC__)
__sync_synchronize();
#elif defined(_MSC_VER)
MemoryBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__i386__) || defined(__x86_64__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__lwsync();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
/*----------------------------------------------------------------------------*/
/* Cache coherence and invalidation */
#ifndef MDBX_CPU_WRITEBACK_IS_COHERENT
#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \
defined(__hppa__)
#define MDBX_CPU_WRITEBACK_IS_COHERENT 1
#else
#define MDBX_CPU_WRITEBACK_IS_COHERENT 0
#endif
#endif /* MDBX_CPU_WRITEBACK_IS_COHERENT */
#ifndef MDBX_CACHELINE_SIZE
#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE)
#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE
#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
#define MDBX_CACHELINE_SIZE 128
#else
#define MDBX_CACHELINE_SIZE 64
#endif
#endif /* MDBX_CACHELINE_SIZE */
#if MDBX_CPU_WRITEBACK_IS_COHERENT
#define mdbx_flush_noncoherent_cpu_writeback() mdbx_compiler_barrier()
#else
#define mdbx_flush_noncoherent_cpu_writeback() mdbx_memory_barrier()
#endif
#if __has_include(<sys/cachectl.h>)
#include <sys/cachectl.h>
#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
/* MIPS should have explicit cache control */
#include <sys/cachectl.h>
#endif
#ifndef MDBX_CPU_CACHE_MMAP_NONCOHERENT
#if defined(__mips) || defined(__mips__) || defined(__mips64) || \
defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \
defined(__MWERKS__) || defined(__sgi)
/* MIPS has cache coherency issues. */
#define MDBX_CPU_CACHE_MMAP_NONCOHERENT 1
#else
/* LY: assume no relevant mmap/dcache issues. */
#define MDBX_CPU_CACHE_MMAP_NONCOHERENT 0
#endif
#endif /* ndef MDBX_CPU_CACHE_MMAP_NONCOHERENT */
static __inline void mdbx_invalidate_mmap_noncoherent_cache(void *addr,
size_t nbytes) {
#if MDBX_CPU_CACHE_MMAP_NONCOHERENT
#ifdef DCACHE
/* MIPS has cache coherency issues.
* Note: for any nbytes >= on-chip cache size, entire is flushed. */
cacheflush(addr, nbytes, DCACHE);
#else
#error "Oops, cacheflush() not available"
#endif /* DCACHE */
#else /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */
(void)addr;
(void)nbytes;
#endif /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */
}
/*----------------------------------------------------------------------------*/
/* libc compatibility stuff */
#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \
(defined(_GNU_SOURCE) || defined(_BSD_SOURCE))
#define mdbx_asprintf asprintf
#define mdbx_vasprintf vasprintf
#else
__printf_args(2, 3) int mdbx_asprintf(char **strp, const char *fmt, ...);
int mdbx_vasprintf(char **strp, const char *fmt, va_list ap);
#endif
/*----------------------------------------------------------------------------*/
/* OS abstraction layer stuff */
/* max bytes to write in one call */
#define MAX_WRITE UINT32_C(0x3fff0000)
#if defined(__linux__) || defined(__gnu_linux__)
extern uint32_t mdbx_linux_kernel_version;
#endif /* Linux */
/* Get the size of a memory page for the system.
* This is the basic size that the platform's memory manager uses, and is
* fundamental to the use of memory-mapped files. */
static __inline size_t mdbx_syspagesize(void) {
#if defined(_WIN32) || defined(_WIN64)
SYSTEM_INFO si;
GetSystemInfo(&si);
return si.dwPageSize;
#else
return sysconf(_SC_PAGE_SIZE);
#endif
}
#ifndef mdbx_strdup
LIBMDBX_API char *mdbx_strdup(const char *str);
#endif
static __inline int mdbx_get_errno(void) {
#if defined(_WIN32) || defined(_WIN64)
DWORD rc = GetLastError();
#else
int rc = errno;
#endif
return rc;
}
#ifndef mdbx_memalign_alloc
int mdbx_memalign_alloc(size_t alignment, size_t bytes, void **result);
#endif
#ifndef mdbx_memalign_free
void mdbx_memalign_free(void *ptr);
#endif
int mdbx_condmutex_init(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_lock(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_unlock(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_signal(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_wait(mdbx_condmutex_t *condmutex);
int mdbx_condmutex_destroy(mdbx_condmutex_t *condmutex);
int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex);
int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex);
int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex);
int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex);
int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, int iovcnt,
uint64_t offset, size_t expected_written);
int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset);
int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count,
uint64_t offset);
int mdbx_thread_create(mdbx_thread_t *thread,
THREAD_RESULT(THREAD_CALL *start_routine)(void *),
void *arg);
int mdbx_thread_join(mdbx_thread_t thread);
enum mdbx_syncmode_bits {
MDBX_SYNC_DATA = 1,
MDBX_SYNC_SIZE = 2,
MDBX_SYNC_IODQ = 4
};
int mdbx_filesync(mdbx_filehandle_t fd, enum mdbx_syncmode_bits mode_bits);
int mdbx_filesize_sync(mdbx_filehandle_t fd);
int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length);
int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos);
int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length);
int mdbx_openfile(const char *pathname, int flags, mode_t mode,
mdbx_filehandle_t *fd, bool exclusive);
int mdbx_closefile(mdbx_filehandle_t fd);
int mdbx_removefile(const char *pathname);
typedef struct mdbx_mmap_param {
union {
void *address;
uint8_t *dxb;
struct MDBX_lockinfo *lck;
};
mdbx_filehandle_t fd;
size_t length; /* mapping length, but NOT a size of file or DB */
#if defined(_WIN32) || defined(_WIN64)
size_t current; /* mapped region size, e.g. file and DB */
uint64_t filesize;
#endif
#ifdef MDBX_OSAL_SECTION
MDBX_OSAL_SECTION section;
#endif
} mdbx_mmap_t;
int mdbx_mmap(int flags, mdbx_mmap_t *map, size_t must, size_t limit);
int mdbx_munmap(mdbx_mmap_t *map);
int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, size_t wanna);
#if defined(_WIN32) || defined(_WIN64)
typedef struct {
unsigned limit, count;
HANDLE handles[31];
} mdbx_handle_array_t;
int mdbx_suspend_threads_before_remap(MDBX_env *env,
mdbx_handle_array_t **array);
int mdbx_resume_threads_after_remap(mdbx_handle_array_t *array);
#endif /* Windows */
int mdbx_msync(mdbx_mmap_t *map, size_t offset, size_t length, int async);
int mdbx_check4nonlocal(mdbx_filehandle_t handle, int flags);
static __inline mdbx_pid_t mdbx_getpid(void) {
#if defined(_WIN32) || defined(_WIN64)
return GetCurrentProcessId();
#else
return getpid();
#endif
}
static __inline mdbx_tid_t mdbx_thread_self(void) {
#if defined(_WIN32) || defined(_WIN64)
return GetCurrentThreadId();
#else
return pthread_self();
#endif
}
void mdbx_osal_jitter(bool tiny);
uint64_t mdbx_osal_monotime(void);
uint64_t mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16);
/*----------------------------------------------------------------------------*/
/* lck stuff */
#if defined(_WIN32) || defined(_WIN64)
#undef MDBX_OSAL_LOCK
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0xF10C)
#else
#define MDBX_OSAL_LOCK pthread_mutex_t
#define MDBX_OSAL_LOCK_SIGN UINT32_C(0x8017)
#endif /* MDBX_OSAL_LOCK */
/// \brief Инициализация объектов синхронизации внутри текущего процесса
/// связанных с экземпляром MDBX_env.
/// \return Код ошибки или 0 в случае успеха.
int mdbx_lck_init(MDBX_env *env);
/// \brief Отключение от общих межпроцесных объектов и разрушение объектов
/// синхронизации внутри текущего процесса связанных с экземпляром MDBX_env.
void mdbx_lck_destroy(MDBX_env *env);
/// \brief Подключение к общим межпроцесным объектам блокировки с попыткой
/// захвата блокировки максимального уровня (разделяемой при недоступности
/// эксклюзивной).
/// В зависимости от реализации и/или платформы (Windows) может
/// захватывать блокировку не-операционного супер-уровня (например, для
/// инициализации разделяемых объектов синхронизации), которая затем будет
/// понижена до операционно-эксклюзивной или разделяемой посредством
/// явного вызова mdbx_lck_downgrade().
/// \return
/// MDBX_RESULT_TRUE (-1) - если удалось захватить эксклюзивную блокировку и,
/// следовательно, текущий процесс является первым и единственным
/// после предыдущего использования БД.
/// MDBX_RESULT_FALSE (0) - если удалось захватить только разделяемую
/// блокировку и, следовательно, БД уже открыта и используется другими
/// процессами.
/// Иначе (не 0 и не -1) - код ошибки.
int mdbx_lck_seize(MDBX_env *env);
/// \brief Снижает уровень первоначальной захваченной блокировки до
/// операционного уровня определяемого аргументом.
/// \param
/// complete = TRUE - понижение до разделяемой блокировки.
/// complete = FALSE - понижение до эксклюзивной операционной блокировки.
/// \return Код ошибки или 0 в случае успеха.
int mdbx_lck_downgrade(MDBX_env *env, bool complete);
/// \brief Блокирует lck-файл и/или таблицу читателей для (де)регистрации.
/// \return Код ошибки или 0 в случае успеха.
int mdbx_rdt_lock(MDBX_env *env);
/// \brief Разблокирует lck-файл и/или таблицу читателей после (де)регистрации.
void mdbx_rdt_unlock(MDBX_env *env);
/// \brief Захватывает блокировку для изменения БД (при старте пишущей
/// транзакции). Транзакции чтения при этом никак не блокируются.
/// \return Код ошибки или 0 в случае успеха.
LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dontwait);
/// \brief Освобождает блокировку по окончанию изменения БД (после завершения
/// пишущей транзакции).
LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env);
/// \brief Устанавливает alive-флажок присутствия (индицирующую блокировку)
/// читателя для pid текущего процесса. Функции может выполнить не более
/// необходимого минимума для корректной работы mdbx_rpid_check() в других
/// процессах.
/// \return Код ошибки или 0 в случае успеха.
int mdbx_rpid_set(MDBX_env *env);
/// \brief Снимает alive-флажок присутствия (индицирующую блокировку)
/// читателя для pid текущего процесса. Функции может выполнить не более
/// необходимого минимума для корректной работы mdbx_rpid_check() в других
/// процессах.
/// \return Код ошибки или 0 в случае успеха.
int mdbx_rpid_clear(MDBX_env *env);
/// \brief Проверяет жив ли процесс-читатель с заданным pid
/// по alive-флажку присутствия (индицирующей блокировку),
/// либо любым другим способом.
/// \return
/// MDBX_RESULT_TRUE (-1) - если процесс-читатель с соответствующим pid жив
/// и работает с БД (индицирующая блокировка присутствует).
/// MDBX_RESULT_FALSE (0) - если процесс-читатель с соответствующим pid
/// отсутствует или не работает с БД (индицирующая блокировка отсутствует).
/// Иначе (не 0 и не -1) - код ошибки.
int mdbx_rpid_check(MDBX_env *env, mdbx_pid_t pid);
#if defined(_WIN32) || defined(_WIN64)
typedef union MDBX_srwlock {
struct {
long volatile readerCount;
long volatile writerCount;
};
RTL_SRWLOCK native;
} MDBX_srwlock;
typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *);
extern MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared,
mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive,
mdbx_srwlock_ReleaseExclusive;
typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(
_In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
extern MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx;
typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)(
_In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer,
_In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber,
_Out_opt_ LPDWORD lpMaximumComponentLength,
_Out_opt_ LPDWORD lpFileSystemFlags,
_Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize);
extern MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW;
typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile,
_Out_ LPWSTR lpszFilePath,
_In_ DWORD cchFilePath,
_In_ DWORD dwFlags);
extern MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW;
typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(
_In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
_Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize);
extern MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle;
typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(
IN HANDLE FileHandle, IN OUT HANDLE Event,
IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext,
OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode,
IN OUT PVOID InputBuffer, IN ULONG InputBufferLength,
OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength);
extern MDBX_NtFsControlFile mdbx_NtFsControlFile;
#ifndef _WIN32_WINNT_WIN8
typedef struct _WIN32_MEMORY_RANGE_ENTRY {
PVOID VirtualAddress;
SIZE_T NumberOfBytes;
} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY;
#endif
typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(
HANDLE hProcess, ULONG_PTR NumberOfEntries,
PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags);
extern MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory;
#endif /* Windows */
/*----------------------------------------------------------------------------*/
/* Atomics */
#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \
!defined(__STDC_NO_ATOMICS__) && \
(__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \
!(defined(__GNUC__) || defined(__clang__)))
#include <stdatomic.h>
#elif defined(__GNUC__) || defined(__clang__)
/* LY: nothing required */
#elif defined(_MSC_VER)
#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */
#pragma warning(disable : 4133) /* 'function': incompatible types - from \
'size_t' to 'LONGLONG' */
#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \
'std::size_t', possible loss of data */
#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \
'long', possible loss of data */
#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange)
#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64)
#elif defined(__APPLE__)
#include <libkern/OSAtomic.h>
#else
#error FIXME atomic-ops
#endif
static __inline uint32_t mdbx_atomic_add32(volatile uint32_t *p, uint32_t v) {
#if !defined(__cplusplus) && defined(ATOMIC_VAR_INIT)
assert(atomic_is_lock_free(p));
return atomic_fetch_add((_Atomic uint32_t *)p, v);
#elif defined(__GNUC__) || defined(__clang__)
return __sync_fetch_and_add(p, v);
#else
#ifdef _MSC_VER
STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
return _InterlockedExchangeAdd((volatile long *)p, v);
#endif
#ifdef __APPLE__
return OSAtomicAdd32(v, (volatile int32_t *)p);
#endif
#endif
}
static __inline uint64_t mdbx_atomic_add64(volatile uint64_t *p, uint64_t v) {
#if !defined(__cplusplus) && defined(ATOMIC_VAR_INIT)
assert(atomic_is_lock_free(p));
return atomic_fetch_add((_Atomic uint64_t *)p, v);
#elif defined(__GNUC__) || defined(__clang__)
return __sync_fetch_and_add(p, v);
#else
#ifdef _MSC_VER
#ifdef _WIN64
return _InterlockedExchangeAdd64((volatile int64_t *)p, v);
#else
return InterlockedExchangeAdd64((volatile int64_t *)p, v);
#endif
#endif /* _MSC_VER */
#ifdef __APPLE__
return OSAtomicAdd64(v, (volatile int64_t *)p);
#endif
#endif
}
#define mdbx_atomic_sub32(p, v) mdbx_atomic_add32(p, 0 - (v))
#define mdbx_atomic_sub64(p, v) mdbx_atomic_add64(p, 0 - (v))
static __inline bool mdbx_atomic_compare_and_swap32(volatile uint32_t *p,
uint32_t c, uint32_t v) {
#if !defined(__cplusplus) && defined(ATOMIC_VAR_INIT)
assert(atomic_is_lock_free(p));
return atomic_compare_exchange_strong((_Atomic uint32_t *)p, &c, v);
#elif defined(__GNUC__) || defined(__clang__)
return __sync_bool_compare_and_swap(p, c, v);
#else
#ifdef _MSC_VER
STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t));
return c == _InterlockedCompareExchange((volatile long *)p, v, c);
#endif
#ifdef __APPLE__
return c == OSAtomicCompareAndSwap32Barrier(c, v, (volatile int32_t *)p);
#endif
#endif
}
static __inline bool mdbx_atomic_compare_and_swap64(volatile uint64_t *p,
uint64_t c, uint64_t v) {
#if !defined(__cplusplus) && defined(ATOMIC_VAR_INIT)
assert(atomic_is_lock_free(p));
return atomic_compare_exchange_strong((_Atomic uint64_t *)p, &c, v);
#elif defined(__GNUC__) || defined(__clang__)
return __sync_bool_compare_and_swap(p, c, v);
#else
#ifdef _MSC_VER
return c == _InterlockedCompareExchange64((volatile int64_t *)p, v, c);
#endif
#ifdef __APPLE__
return c == OSAtomicCompareAndSwap64Barrier(c, v, (volatile uint64_t *)p);
#endif
#endif
}
/*----------------------------------------------------------------------------*/
#if defined(_MSC_VER) && _MSC_VER >= 1900
/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros
* for internal format-args checker. */
#undef PRIuPTR
#undef PRIiPTR
#undef PRIdPTR
#undef PRIxPTR
#define PRIuPTR "Iu"
#define PRIiPTR "Ii"
#define PRIdPTR "Id"
#define PRIxPTR "Ix"
#define PRIuSIZE "zu"
#define PRIiSIZE "zi"
#define PRIdSIZE "zd"
#define PRIxSIZE "zx"
#endif /* fix PRI*PTR for _MSC_VER */
#ifndef PRIuSIZE
#define PRIuSIZE PRIuPTR
#define PRIiSIZE PRIiPTR
#define PRIdSIZE PRIdPTR
#define PRIxSIZE PRIxPTR
#endif /* PRI*SIZE macros for MSVC */
#ifdef _MSC_VER
#pragma warning(pop)
#endif

19
contrib/db/libmdbx/src/tools/CMakeLists.txt Normal file
View file

@ -0,0 +1,19 @@
project(mdbx_tools)
set(MDBX_TOOLS
mdbx_chk
mdbx_copy
mdbx_dump
mdbx_load
mdbx_stat
)
foreach (TOOL ${MDBX_TOOLS})
add_executable(${TOOL} ${TOOL}.c)
target_link_libraries(${TOOL} mdbx)
install(TARGETS ${TOOL} DESTINATION ${CMAKE_INSTALL_PREFIX}/bin COMPONENT mdbx)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TOOL}.1)
install(FILES ${TOOL}.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/man/man1 COMPONENT mdbx)
endif()
endforeach()

1417
contrib/db/libmdbx/src/tools/mdbx_chk.c Normal file
View file

File diff suppressed because it is too large Load diff

166
contrib/db/libmdbx/src/tools/mdbx_chk.vcxproj Normal file
View file

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BE}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_chk</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_chk.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

57
contrib/db/libmdbx/src/tools/mdbx_copy.1 Normal file
View file

@ -0,0 +1,57 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_COPY 1 "2014/06/20" "LMDB 0.9.14"
.SH NAME
mdbx_copy \- MDBX environment copy tool
.SH SYNOPSIS
.B mdbx_copy
[\c
.BR \-V ]
[\c
.BR \-c ]
[\c
.BR \-n ]
.B srcpath
[\c
.BR dstpath ]
.SH DESCRIPTION
The
.B mdbx_copy
utility copies an MDBX environment. The environment can
be copied regardless of whether it is currently in use.
No lockfile is created, since it gets recreated at need.
If
.I dstpath
is specified it must be the path of an empty directory
for storing the backup. Otherwise, the backup will be
written to stdout.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-c
Compact while copying. Only current data pages will be copied; freed
or unused pages will be omitted from the copy. This option will
slow down the backup process as it is more CPU-intensive.
Currently it fails if the environment has suffered a page leak.
.TP
.BR \-n
Open LDMB environment(s) which do not use subdirectories.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH CAVEATS
This utility can trigger significant file size growth if run
in parallel with write transactions, because pages which they
free during copying cannot be reused until the copy is done.
.SH "SEE ALSO"
.BR mdbx_stat (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

109
contrib/db/libmdbx/src/tools/mdbx_copy.c Normal file
View file

@ -0,0 +1,109 @@
/* mdbx_copy.c - memory-mapped database backup tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
/* Avoid reference to mdbx_runtime_flags from assert() */
#define mdbx_runtime_flags (~0u)
#include "../bits.h"
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
int main(int argc, char *argv[]) {
int rc;
MDBX_env *env = NULL;
const char *progname = argv[0], *act;
unsigned flags = MDBX_RDONLY;
unsigned cpflags = 0;
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
if (argv[1][1] == 'n' && argv[1][2] == '\0')
flags |= MDBX_NOSUBDIR;
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
cpflags |= MDBX_CP_COMPACT;
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
printf("%s (%s, build %s)\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
} else
argc = 0;
}
if (argc < 2 || argc > 3) {
fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname);
exit(EXIT_FAILURE);
}
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
act = "opening environment";
rc = mdbx_env_create(&env);
if (rc == MDBX_SUCCESS) {
rc = mdbx_env_open(env, argv[1], flags, 0640);
}
if (rc == MDBX_SUCCESS) {
act = "copying";
if (argc == 2) {
mdbx_filehandle_t fd;
#if defined(_WIN32) || defined(_WIN64)
fd = GetStdHandle(STD_OUTPUT_HANDLE);
#else
fd = fileno(stdout);
#endif
rc = mdbx_env_copy2fd(env, fd, cpflags);
} else
rc = mdbx_env_copy(env, argv[2], cpflags);
}
if (rc)
fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc,
mdbx_strerror(rc));
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

166
contrib/db/libmdbx/src/tools/mdbx_copy.vcxproj Normal file
View file

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BD}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_copy</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_copy.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

77
contrib/db/libmdbx/src/tools/mdbx_dump.1 Normal file
View file

@ -0,0 +1,77 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_DUMP 1 "2014/06/20" "LMDB 0.9.14"
.SH NAME
mdbx_dump \- MDBX environment export tool
.SH SYNOPSIS
.B mdbx_dump
[\c
.BR \-V ]
[\c
.BI \-f \ file\fR]
[\c
.BR \-l ]
[\c
.BR \-n ]
[\c
.BR \-p ]
[\c
.BR \-a \ |
.BI \-s \ subdb\fR]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_dump
utility reads a database and writes its contents to the
standard output using a portable flat-text format
understood by the
.BR mdbx_load (1)
utility.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-f \ file
Write to the specified file instead of to the standard output.
.TP
.BR \-l
List the databases stored in the environment. Just the
names will be listed, no data will be output.
.TP
.BR \-n
Dump an MDBX database which does not use subdirectories.
.TP
.BR \-p
If characters in either the key or data items are printing characters (as
defined by isprint(3)), output them directly. This option permits users to
use standard text editors and tools to modify the contents of databases.
Note: different systems may have different notions about what characters
are considered printing characters, and databases dumped in this manner may
be less portable to external systems.
.TP
.BR \-a
Dump all of the subdatabases in the environment.
.TP
.BR \-s \ subdb
Dump a specific subdatabase. If no database is specified, only the main database is dumped.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
Dumping and reloading databases that use user-defined comparison functions
will result in new databases that use the default comparison functions.
\fBIn this case it is quite likely that the reloaded database will be
damaged beyond repair permitting neither record storage nor retrieval.\fP
The only available workaround is to modify the source for the
.BR mdbx_load (1)
utility to load the database using the correct comparison functions.
.SH "SEE ALSO"
.BR mdbx_load (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

340
contrib/db/libmdbx/src/tools/mdbx_dump.c Normal file
View file

@ -0,0 +1,340 @@
/* mdbx_dump.c - memory-mapped database dump tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
/* Avoid reference to mdbx_runtime_flags from assert() */
#define mdbx_runtime_flags (~0u)
#include "../bits.h"
#include <ctype.h>
#define PRINT 1
static int mode;
typedef struct flagbit {
int bit;
char *name;
} flagbit;
flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"},
{MDBX_DUPSORT, "dupsort"},
{MDBX_INTEGERKEY, "integerkey"},
{MDBX_DUPFIXED, "dupfixed"},
{MDBX_INTEGERDUP, "integerdup"},
{MDBX_REVERSEDUP, "reversedup"},
{0, NULL}};
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
static const char hexc[] = "0123456789abcdef";
static void dumpbyte(unsigned char c) {
putchar(hexc[c >> 4]);
putchar(hexc[c & 0xf]);
}
static void text(MDBX_val *v) {
unsigned char *c, *end;
putchar(' ');
c = v->iov_base;
end = c + v->iov_len;
while (c < end) {
if (isprint(*c) && *c != '\\') {
putchar(*c);
} else {
putchar('\\');
dumpbyte(*c);
}
c++;
}
putchar('\n');
}
static void dumpval(MDBX_val *v) {
unsigned char *c, *end;
putchar(' ');
c = v->iov_base;
end = c + v->iov_len;
while (c < end) {
dumpbyte(*c++);
}
putchar('\n');
}
/* Dump in BDB-compatible format */
static int dumpit(MDBX_txn *txn, MDBX_dbi dbi, char *name) {
MDBX_cursor *mc;
MDBX_stat ms;
MDBX_val key, data;
MDBX_envinfo info;
unsigned int flags;
int rc, i;
rc = mdbx_dbi_flags(txn, dbi, &flags);
if (rc)
return rc;
rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms));
if (rc)
return rc;
rc = mdbx_env_info(mdbx_txn_env(txn), &info, sizeof(info));
if (rc)
return rc;
printf("VERSION=3\n");
printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
if (name)
printf("database=%s\n", name);
printf("type=btree\n");
printf("mapsize=%" PRIu64 "\n", info.mi_mapsize);
printf("maxreaders=%u\n", info.mi_maxreaders);
for (i = 0; dbflags[i].bit; i++)
if (flags & dbflags[i].bit)
printf("%s=1\n", dbflags[i].name);
printf("db_pagesize=%d\n", ms.ms_psize);
printf("HEADER=END\n");
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc)
return rc;
while ((rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT)) == MDBX_SUCCESS) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
if (mode & PRINT) {
text(&key);
text(&data);
} else {
dumpval(&key);
dumpval(&data);
}
}
printf("DATA=END\n");
if (rc == MDBX_NOTFOUND)
rc = MDBX_SUCCESS;
return rc;
}
static void usage(char *prog) {
fprintf(stderr,
"usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env;
MDBX_txn *txn;
MDBX_dbi dbi;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envflags = 0, list = 0;
if (argc < 2) {
usage(prog);
}
/* -a: dump main DB and all subDBs
* -s: dump only the named subDB
* -n: use NOSUBDIR flag on env_open
* -p: use printable characters
* -f: write to file instead of stdout
* -V: print version and exit
* (default) dump only the main DB
*/
while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) {
switch (i) {
case 'V':
printf("%s (%s, build %s)\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
break;
case 'l':
list = 1;
/*FALLTHROUGH*/;
__fallthrough;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'f':
if (freopen(optarg, "w", stdout) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 'p':
mode |= PRINT;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdbx_env_set_maxdbs(env, 2);
}
rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_dbi_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
if (alldbs) {
MDBX_cursor *cursor;
MDBX_val key;
int count = 0;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
char *str;
MDBX_dbi db2;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;
count++;
str = mdbx_malloc(key.iov_len + 1);
memcpy(str, key.iov_base, key.iov_len);
str[key.iov_len] = '\0';
rc = mdbx_dbi_open(txn, str, 0, &db2);
if (rc == MDBX_SUCCESS) {
if (list) {
printf("%s\n", str);
list++;
} else {
rc = dumpit(txn, db2, str);
if (rc)
break;
}
mdbx_dbi_close(env, db2);
}
mdbx_free(str);
if (rc)
continue;
}
mdbx_cursor_close(cursor);
if (!count) {
fprintf(stderr, "%s: %s does not contain multiple databases\n", prog,
envname);
rc = MDBX_NOTFOUND;
} else if (rc == MDBX_INCOMPATIBLE) {
/* LY: the record it not a named sub-db. */
rc = MDBX_SUCCESS;
}
} else {
rc = dumpit(txn, dbi, subname);
}
if (rc && rc != MDBX_NOTFOUND)
fprintf(stderr, "%s: %s: %s\n", prog, envname, mdbx_strerror(rc));
mdbx_dbi_close(env, dbi);
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

166
contrib/db/libmdbx/src/tools/mdbx_dump.vcxproj Normal file
View file

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BC}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_dump</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_dump.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

86
contrib/db/libmdbx/src/tools/mdbx_load.1 Normal file
View file

@ -0,0 +1,86 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_LOAD 1 "2014/06/20" "LMDB 0.9.14"
.SH NAME
mdbx_load \- MDBX environment import tool
.SH SYNOPSIS
.B mdbx_load
[\c
.BR \-V ]
[\c
.BI \-f \ file\fR]
[\c
.BR \-n ]
[\c
.BI \-s \ subdb\fR]
[\c
.BR \-N ]
[\c
.BR \-T ]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_load
utility reads from the standard input and loads it into the
MDBX environment
.BR envpath .
The input to
.B mdbx_load
must be in the output format specified by the
.BR mdbx_dump (1)
utility or as specified by the
.B -T
option below.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-a
Append all records in the order they appear in the input. The input is assumed to already be
in correctly sorted order and no sorting or checking for redundant values will be performed.
This option must be used to reload data that was produced by running
.B mdbx_dump
on a database that uses custom compare functions.
.TP
.BR \-f \ file
Read from the specified file instead of from the standard input.
.TP
.BR \-n
Load an MDBX database which does not use subdirectories.
.TP
.BR \-s \ subdb
Load a specific subdatabase. If no database is specified, data is loaded into the main database.
.TP
.BR \-N
Don't overwrite existing records when loading into an already existing database; just skip them.
.TP
.BR \-T
Load data from simple text files. The input must be paired lines of text, where the first
line of the pair is the key item, and the second line of the pair is its corresponding
data item.
A simple escape mechanism, where newline and backslash (\\) characters are special, is
applied to the text input. Newline characters are interpreted as record separators.
Backslash characters in the text will be interpreted in one of two ways: If the backslash
character precedes another backslash character, the pair will be interpreted as a literal
backslash. If the backslash character precedes any other character, the two characters
following the backslash will be interpreted as a hexadecimal specification of a single
character; for example, \\0a is a newline character in the ASCII character set.
For this reason, any backslash or newline characters that naturally occur in the text
input must be escaped to avoid misinterpretation by
.BR mdbx_load .
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdbx_dump (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

554
contrib/db/libmdbx/src/tools/mdbx_load.c Normal file
View file

@ -0,0 +1,554 @@
/* mdbx_load.c - memory-mapped database load tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
/* Avoid reference to mdbx_runtime_flags from assert() */
#define mdbx_runtime_flags (~0u)
#include "../bits.h"
#include <ctype.h>
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
#define PRINT 1
#define NOHDR 2
static int mode;
static char *subname = NULL;
static size_t lineno;
static int version;
static int dbi_flags;
static char *prog;
static int Eof;
static MDBX_envinfo envinfo;
static MDBX_val kbuf, dbuf;
static MDBX_val k0buf;
#define STRLENOF(s) (sizeof(s) - 1)
typedef struct flagbit {
int bit;
char *name;
int len;
} flagbit;
#define S(s) s, STRLENOF(s)
flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")},
{MDBX_DUPSORT, S("dupsort")},
{MDBX_INTEGERKEY, S("integerkey")},
{MDBX_DUPFIXED, S("dupfixed")},
{MDBX_INTEGERDUP, S("integerdup")},
{MDBX_REVERSEDUP, S("reversedup")},
{0, NULL, 0}};
static void readhdr(void) {
char *ptr;
dbi_flags = 0;
while (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) {
lineno++;
if (!strncmp(dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) ||
!strncmp(dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) {
/* LY: silently ignore information fields. */
continue;
} else if (!strncmp(dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) {
version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION="));
if (version > 3) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n",
prog, lineno, version);
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) {
break;
} else if (!strncmp(dbuf.iov_base, "format=", STRLENOF("format="))) {
if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print",
STRLENOF("print")))
mode |= PRINT;
else if (strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "bytevalue",
STRLENOF("bytevalue"))) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported FORMAT %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "database=", STRLENOF("database="))) {
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
if (subname)
mdbx_free(subname);
subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database="));
} else if (!strncmp(dbuf.iov_base, "type=", STRLENOF("type="))) {
if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree",
STRLENOF("btree"))) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("type="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
void *unused;
i = sscanf((char *)dbuf.iov_base + STRLENOF("mapaddr="), "%p", &unused);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapaddr %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
i = sscanf((char *)dbuf.iov_base + STRLENOF("mapsize="), "%" PRIu64,
&envinfo.mi_mapsize);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapsize %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("mapsize="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.iov_base,
"maxreaders=", STRLENOF("maxreaders="))) {
int i;
ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len);
if (ptr)
*ptr = '\0';
i = sscanf((char *)dbuf.iov_base + STRLENOF("maxreaders="), "%u",
&envinfo.mi_maxreaders);
if (i != 1) {
fprintf(stderr, "%s: line %" PRIiSIZE ": invalid maxreaders %s\n", prog,
lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders="));
exit(EXIT_FAILURE);
}
} else {
int i;
for (i = 0; dbflags[i].bit; i++) {
if (!strncmp(dbuf.iov_base, dbflags[i].name, dbflags[i].len) &&
((char *)dbuf.iov_base)[dbflags[i].len] == '=') {
if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1')
dbi_flags |= dbflags[i].bit;
break;
}
}
if (!dbflags[i].bit) {
ptr = memchr(dbuf.iov_base, '=', dbuf.iov_len);
if (!ptr) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog,
lineno);
exit(EXIT_FAILURE);
} else {
*ptr = '\0';
fprintf(stderr,
"%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n",
prog, lineno, (char *)dbuf.iov_base);
}
}
}
}
}
static void badend(void) {
fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog,
lineno);
}
static int unhex(unsigned char *c2) {
int x, c;
x = *c2++ & 0x4f;
if (x & 0x40)
x -= 55;
c = x << 4;
x = *c2 & 0x4f;
if (x & 0x40)
x -= 55;
c |= x;
return c;
}
static int readline(MDBX_val *out, MDBX_val *buf) {
unsigned char *c1, *c2, *end;
size_t len, l2;
int c;
if (!(mode & NOHDR)) {
c = fgetc(stdin);
if (c == EOF) {
Eof = 1;
return EOF;
}
if (c != ' ') {
lineno++;
if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
badend:
Eof = 1;
badend();
return EOF;
}
if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END")))
return EOF;
goto badend;
}
}
if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) {
Eof = 1;
return EOF;
}
lineno++;
c1 = buf->iov_base;
len = strlen((char *)c1);
l2 = len;
/* Is buffer too short? */
while (c1[len - 1] != '\n') {
buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2);
if (!buf->iov_base) {
Eof = 1;
fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n",
prog, lineno);
return EOF;
}
c1 = buf->iov_base;
c1 += l2;
if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) {
Eof = 1;
badend();
return EOF;
}
buf->iov_len *= 2;
len = strlen((char *)c1);
l2 += len;
}
c1 = c2 = buf->iov_base;
len = l2;
c1[--len] = '\0';
end = c1 + len;
if (mode & PRINT) {
while (c2 < end) {
if (unlikely(*c2 == '\\')) {
if (c2[1] == '\\') {
*c1++ = '\\';
} else {
if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = (char)unhex(++c2);
}
c2 += 2;
} else {
/* copies are redundant when no escapes were used */
*c1++ = *c2++;
}
}
} else {
/* odd length not allowed */
if (len & 1) {
Eof = 1;
badend();
return EOF;
}
while (c2 < end) {
if (!isxdigit(*c2) || !isxdigit(c2[1])) {
Eof = 1;
badend();
return EOF;
}
*c1++ = (char)unhex(c2);
c2 += 2;
}
}
c2 = out->iov_base = buf->iov_base;
out->iov_len = c1 - c2;
return 0;
}
static void usage(void) {
fprintf(stderr,
"usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
static int anyway_greater(const MDBX_val *a, const MDBX_val *b) {
(void)a;
(void)b;
return 1;
}
int main(int argc, char *argv[]) {
int i, rc;
MDBX_env *env = NULL;
MDBX_txn *txn = NULL;
MDBX_cursor *mc = NULL;
MDBX_dbi dbi;
char *envname = NULL;
int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0;
int append = 0;
MDBX_val prevk;
prog = argv[0];
if (argc < 2)
usage();
/* -a: append records in input order
* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
switch (i) {
case 'V':
printf("%s (%s, build %s)\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
break;
case 'a':
append = 1;
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg,
mdbx_strerror(errno));
exit(EXIT_FAILURE);
}
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 's':
subname = mdbx_strdup(optarg);
break;
case 'N':
putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA;
break;
case 'T':
mode |= NOHDR | PRINT;
break;
default:
usage();
}
}
if (optind != argc - 1)
usage();
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
dbuf.iov_len = 4096;
dbuf.iov_base = mdbx_malloc(dbuf.iov_len);
/* read first header for mapsize= */
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
mdbx_env_set_maxdbs(env, 2);
if (envinfo.mi_maxreaders)
mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders);
if (envinfo.mi_mapsize) {
if (envinfo.mi_mapsize > SIZE_MAX) {
fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc,
mdbx_strerror(MDBX_TOO_LARGE));
return EXIT_FAILURE;
}
mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize);
}
#ifdef MDBX_FIXEDMAP
if (info.mi_mapaddr)
envflags |= MDBX_FIXEDMAP;
#endif
rc = mdbx_env_open(env, envname, envflags, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
kbuf.iov_len = mdbx_env_get_maxkeysize(env);
if (kbuf.iov_len >= SIZE_MAX / 4) {
fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n",
kbuf.iov_len);
goto env_close;
}
kbuf.iov_len = (kbuf.iov_len + 1) * 2;
kbuf.iov_base = malloc(kbuf.iov_len * 2);
k0buf.iov_len = kbuf.iov_len;
k0buf.iov_base = (char *)kbuf.iov_base + kbuf.iov_len;
prevk.iov_base = k0buf.iov_base;
while (!Eof) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi,
append ? anyway_greater : NULL,
append ? anyway_greater : NULL);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
int batch = 0;
prevk.iov_len = 0;
while (1) {
MDBX_val key;
rc = readline(&key, &kbuf);
if (rc) /* rc == EOF */
break;
MDBX_val data;
rc = readline(&data, &dbuf);
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n",
prog, lineno);
goto txn_abort;
}
int appflag = 0;
if (append) {
appflag = MDBX_APPEND;
if (dbi_flags & MDBX_DUPSORT) {
if (prevk.iov_len == key.iov_len &&
memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0)
appflag = MDBX_APPEND | MDBX_APPENDDUP;
else
memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len);
}
}
rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag);
if (rc == MDBX_KEYEXIST && putflags)
continue;
if (rc) {
fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
batch++;
if (batch == 100) {
rc = mdbx_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog,
lineno, mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
rc = mdbx_cursor_open(txn, dbi, &mc);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
batch = 0;
}
}
rc = mdbx_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog, lineno,
mdbx_strerror(rc));
goto env_close;
}
mdbx_dbi_close(env, dbi);
/* try read next header */
if (!(mode & NOHDR))
readhdr();
}
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

166
contrib/db/libmdbx/src/tools/mdbx_load.vcxproj Normal file
View file

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BB}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_load</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_load.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

66
contrib/db/libmdbx/src/tools/mdbx_stat.1 Normal file
View file

@ -0,0 +1,66 @@
.\" Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2015,2016 Peter-Service R&D LLC <http://billing.ru/>.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.TH MDBX_STAT 1 "2014/06/20" "LMDB 0.9.14"
.SH NAME
mdbx_stat \- MDBX environment status tool
.SH SYNOPSIS
.B mdbx_stat
[\c
.BR \-V ]
[\c
.BR \-e ]
[\c
.BR \-f [ f [ f ]]]
[\c
.BR \-n ]
[\c
.BR \-r [ r ]]
[\c
.BR \-a \ |
.BI \-s \ subdb\fR]
.BR \ envpath
.SH DESCRIPTION
The
.B mdbx_stat
utility displays the status of an MDBX environment.
.SH OPTIONS
.TP
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-e
Display information about the database environment.
.TP
.BR \-f
Display information about the environment freelist.
If \fB\-ff\fP is given, summarize each freelist entry.
If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
.TP
.BR \-n
Display the status of an MDBX database which does not use subdirectories.
.TP
.BR \-r
Display information about the environment reader table.
Shows the process ID, thread ID, and transaction ID for each active
reader slot. The process ID and transaction ID are in decimal, the
thread ID is in hexadecimal. The transaction ID is displayed as "-"
if the reader does not currently have a read transaction open.
If \fB\-rr\fP is given, check for stale entries in the reader
table and clear them. The reader table will be printed again
after the check is performed.
.TP
.BR \-a
Display the status of all of the subdatabases in the environment.
.TP
.BR \-s \ subdb
Display the status of a specific subdatabase.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.
Errors result in a non-zero exit status and
a diagnostic message being written to standard error.
.SH "SEE ALSO"
.BR mdbx_copy (1)
.SH AUTHOR
Howard Chu of Symas Corporation <http://www.symas.com>

392
contrib/db/libmdbx/src/tools/mdbx_stat.c Normal file
View file

@ -0,0 +1,392 @@
/* mdbx_stat.c - memory-mapped database status tool */
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>. */
#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif /* _MSC_VER (warnings) */
/* Avoid reference to mdbx_runtime_flags from assert() */
#define mdbx_runtime_flags (~0u)
#include "../bits.h"
#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"
static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
(void)dwCtrlType;
user_break = true;
return true;
}
#else /* WINDOWS */
static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
(void)sig;
user_break = 1;
}
#endif /* !WINDOWS */
static void prstat(MDBX_stat *ms) {
printf(" Pagesize: %u\n", ms->ms_psize);
printf(" Tree depth: %u\n", ms->ms_depth);
printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages);
printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages);
printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages);
printf(" Entries: %" PRIu64 "\n", ms->ms_entries);
}
static void usage(char *prog) {
fprintf(stderr,
"usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n",
prog);
exit(EXIT_FAILURE);
}
int main(int argc, char *argv[]) {
int o, rc;
MDBX_env *env;
MDBX_txn *txn;
MDBX_dbi dbi;
MDBX_stat mst;
MDBX_envinfo mei;
char *prog = argv[0];
char *envname;
char *subname = NULL;
int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
if (argc < 2) {
usage(prog);
}
/* -a: print stat of main DB and all subDBs
* -s: print stat of only the named subDB
* -e: print env info
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -V: print version and exit
* (default) print stat of only the main DB
*/
while ((o = getopt(argc, argv, "Vaefnrs:")) != EOF) {
switch (o) {
case 'V':
printf("%s (%s, build %s)\n", mdbx_version.git.describe,
mdbx_version.git.datetime, mdbx_build.datetime);
exit(EXIT_SUCCESS);
break;
case 'a':
if (subname)
usage(prog);
alldbs++;
break;
case 'e':
envinfo++;
break;
case 'f':
freinfo++;
break;
case 'n':
envflags |= MDBX_NOSUBDIR;
break;
case 'r':
rdrinfo++;
break;
case 's':
if (alldbs)
usage(prog);
subname = optarg;
break;
default:
usage(prog);
}
}
if (optind != argc - 1)
usage(prog);
#if defined(_WIN32) || defined(_WIN64)
SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true);
#else
#ifdef SIGPIPE
signal(SIGPIPE, signal_handler);
#endif
#ifdef SIGHUP
signal(SIGHUP, signal_handler);
#endif
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
#endif /* !WINDOWS */
envname = argv[optind];
rc = mdbx_env_create(&env);
if (rc) {
fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc,
mdbx_strerror(rc));
return EXIT_FAILURE;
}
if (alldbs || subname) {
mdbx_env_set_maxdbs(env, 4);
}
rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664);
if (rc) {
fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (envinfo || freinfo) {
(void)mdbx_env_info(env, &mei, sizeof(mei));
} else {
/* LY: zap warnings from gcc */
memset(&mei, 0, sizeof(mei));
}
if (envinfo) {
(void)mdbx_env_stat(env, &mst, sizeof(mst));
printf("Environment Info\n");
printf(" Pagesize: %u\n", mst.ms_psize);
if (mei.mi_geo.lower != mei.mi_geo.upper) {
printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
"/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
"/-%" PRIu64 ")\n",
mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize,
mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize,
mei.mi_geo.shrink / mst.ms_psize);
printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
} else {
printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
}
printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize);
printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1);
printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
mei.mi_latter_reader_txnid,
mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
printf(" Max readers: %u\n", mei.mi_maxreaders);
printf(" Number of readers used: %u\n", mei.mi_numreaders);
} else {
/* LY: zap warnings from gcc */
memset(&mst, 0, sizeof(mst));
}
if (rdrinfo) {
printf("Reader Table Status\n");
rc = mdbx_reader_list(env, (MDBX_msg_func *)fputs, stdout);
if (rdrinfo > 1) {
int dead;
mdbx_reader_check(env, &dead);
printf(" %d stale readers cleared.\n", dead);
rc = mdbx_reader_list(env, (MDBX_msg_func *)fputs, stdout);
}
if (!(subname || alldbs || freinfo))
goto env_close;
}
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc) {
fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto env_close;
}
if (freinfo) {
MDBX_cursor *cursor;
MDBX_val key, data;
pgno_t pages = 0, *iptr;
pgno_t reclaimable = 0;
printf("Freelist Status\n");
dbi = 0;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
prstat(&mst);
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) ==
MDBX_SUCCESS) {
if (user_break) {
rc = MDBX_EINTR;
break;
}
iptr = data.iov_base;
const pgno_t number = *iptr++;
pages += number;
if (envinfo && mei.mi_latter_reader_txnid > *(size_t *)key.iov_base)
reclaimable += number;
if (freinfo > 1) {
char *bad = "";
pgno_t prev =
MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1;
pgno_t span = 1;
for (unsigned i = 0; i < number; ++i) {
pgno_t pg = iptr[i];
if (MDBX_PNL_DISORDERED(prev, pg))
bad = " [bad sequence]";
prev = pg;
while (i + span < number &&
iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
: pgno_sub(pg, span)))
++span;
}
printf(" Transaction %" PRIaTXN ", %" PRIaPGNO
" pages, maxspan %" PRIaPGNO "%s\n",
*(txnid_t *)key.iov_base, number, span, bad);
if (freinfo > 2) {
for (unsigned i = 0; i < number; i += span) {
const pgno_t pg = iptr[i];
for (span = 1;
i + span < number &&
iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span)
: pgno_sub(pg, span));
++span)
;
if (span > 1)
printf(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pg, span);
else
printf(" %9" PRIaPGNO "\n", pg);
}
}
}
}
mdbx_cursor_close(cursor);
switch (rc) {
case MDBX_SUCCESS:
case MDBX_NOTFOUND:
break;
case MDBX_EINTR:
fprintf(stderr, "Interrupted by signal/user\n");
goto txn_abort;
default:
fprintf(stderr, "mdbx_cursor_get failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
if (envinfo) {
uint64_t value = mei.mi_mapsize / mst.ms_psize;
double percent = value / 100.0;
printf("Page Allocation Info\n");
printf(" Max pages: %" PRIu64 " 100%%\n", value);
value = mei.mi_last_pgno + 1;
printf(" Pages used: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1);
printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent);
value = mei.mi_last_pgno + 1 - pages;
printf(" Used now: %" PRIu64 " %.1f%%\n", value, value / percent);
value = pages;
printf(" Unallocated: %" PRIu64 " %.1f%%\n", value, value / percent);
value = pages - reclaimable;
printf(" Detained: %" PRIu64 " %.1f%%\n", value, value / percent);
value = reclaimable;
printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent);
value =
mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1) + reclaimable;
printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent);
} else
printf(" Free pages: %" PRIaPGNO "\n", pages);
}
rc = mdbx_dbi_open(txn, subname, 0, &dbi);
if (rc) {
fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc));
goto txn_abort;
}
rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
prstat(&mst);
if (alldbs) {
MDBX_cursor *cursor;
MDBX_val key;
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc) {
fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) {
char *str;
MDBX_dbi db2;
if (memchr(key.iov_base, '\0', key.iov_len))
continue;
str = mdbx_malloc(key.iov_len + 1);
memcpy(str, key.iov_base, key.iov_len);
str[key.iov_len] = '\0';
rc = mdbx_dbi_open(txn, str, 0, &db2);
if (rc == MDBX_SUCCESS)
printf("Status of %s\n", str);
mdbx_free(str);
if (rc)
continue;
rc = mdbx_dbi_stat(txn, db2, &mst, sizeof(mst));
if (rc) {
fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc,
mdbx_strerror(rc));
goto txn_abort;
}
prstat(&mst);
mdbx_dbi_close(env, db2);
}
mdbx_cursor_close(cursor);
}
if (rc == MDBX_NOTFOUND)
rc = MDBX_SUCCESS;
mdbx_dbi_close(env, dbi);
txn_abort:
mdbx_txn_abort(txn);
env_close:
mdbx_env_close(env);
return rc ? EXIT_FAILURE : EXIT_SUCCESS;
}

166
contrib/db/libmdbx/src/tools/mdbx_stat.vcxproj Normal file
View file

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{15030120-5F7F-48F9-ABE5-DFC814F2A4BF}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbx_stat</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>EnableAllWarnings</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>EnableAllWarnings</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ProjectReference Include="..\..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClCompile Include="mdbx_stat.c" />
<ClCompile Include="wingetopt.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\mdbx.h" />
<ClInclude Include="wingetopt.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

95
contrib/db/libmdbx/src/tools/wingetopt.c Normal file
View file

@ -0,0 +1,95 @@
/*
* POSIX getopt for Windows
*
* AT&T Public License
*
* Code given out at the 1985 UNIFORUM conference in Dallas.
*/
/*----------------------------------------------------------------------------*/
/* Microsoft compiler generates a lot of warning for self includes... */
#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
* semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
* mode specified; termination on exception is \
* not guaranteed. Specify /EHsc */
#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS
#endif
#endif /* _MSC_VER (warnings) */
#include "wingetopt.h"
#include <stdio.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(pop)
#endif
/*----------------------------------------------------------------------------*/
#ifndef NULL
#define NULL 0
#endif
#ifndef EOF
#define EOF (-1)
#endif
#define ERR(s, c) \
if (opterr) { \
fputs(argv[0], stderr); \
fputs(s, stderr); \
fputc(c, stderr); \
}
int opterr = 1;
int optind = 1;
int optopt;
char *optarg;
int getopt(int argc, char *const argv[], const char *opts) {
static int sp = 1;
int c;
const char *cp;
if (sp == 1) {
if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0')
return EOF;
else if (strcmp(argv[optind], "--") == 0) {
optind++;
return EOF;
}
}
optopt = c = argv[optind][sp];
if (c == ':' || (cp = strchr(opts, c)) == NULL) {
ERR(": illegal option -- ", c);
if (argv[optind][++sp] == '\0') {
optind++;
sp = 1;
}
return '?';
}
if (*++cp == ':') {
if (argv[optind][sp + 1] != '\0')
optarg = &argv[optind++][sp + 1];
else if (++optind >= argc) {
ERR(": option requires an argument -- ", c);
sp = 1;
return '?';
} else
optarg = argv[optind++];
sp = 1;
} else {
if (argv[optind][++sp] == '\0') {
sp = 1;
optind++;
}
optarg = NULL;
}
return c;
}

26
contrib/db/libmdbx/src/tools/wingetopt.h Normal file
View file

@ -0,0 +1,26 @@
/*
* POSIX getopt for Windows
*
* AT&T Public License
*
* Code given out at the 1985 UNIFORUM conference in Dallas.
*/
#ifndef _WINGETOPT_H_
#define _WINGETOPT_H_
#ifdef __cplusplus
extern "C" {
#endif
extern int opterr;
extern int optind;
extern int optopt;
extern char *optarg;
int getopt(int argc, char *const argv[], const char *optstring);
#ifdef __cplusplus
}
#endif
#endif /* _GETOPT_H_ */

34
contrib/db/libmdbx/src/version.c Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "./bits.h"
#if MDBX_VERSION_MAJOR != 0 || MDBX_VERSION_MINOR != 3
#error "API version mismatch!"
#endif
#define MDBX_VERSION_RELEASE 0
#define MDBX_VERSION_REVISION 0
/*LIBMDBX_EXPORTS*/ const mdbx_version_info mdbx_version = {
MDBX_VERSION_MAJOR,
MDBX_VERSION_MINOR,
MDBX_VERSION_RELEASE,
MDBX_VERSION_REVISION,
{"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@",
"@MDBX_GIT_DESCRIBE@"}};
/*LIBMDBX_EXPORTS*/ const mdbx_build_info mdbx_build = {
"@MDBX_BUILD_TIMESTAMP@", "@MDBX_BUILD_TARGET@", "@MDBX_BUILD_OPTIONS@",
"@MDBX_BUILD_COMPILER@", "@MDBX_BUILD_FLAGS@"};

37
contrib/db/libmdbx/test/CMakeLists.txt Normal file
View file

@ -0,0 +1,37 @@
set(TARGET mdbx_test)
project(${TARGET})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-declarations")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-cast-qual")
add_executable(${TARGET}
base.h
cases.cc
chrono.cc
chrono.h
config.cc
config.h
copy.cc
dead.cc
hill.cc
jitter.cc
keygen.cc
keygen.h
log.cc
log.h
main.cc
osal.h
osal-unix.cc
test.cc
test.h
try.cc
utils.cc
utils.h
append.cc
ttl.cc
)
target_link_libraries(${TARGET}
mdbx
)

165
contrib/db/libmdbx/test/append.cc Normal file
View file

@ -0,0 +1,165 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_append::run() {
MDBX_dbi dbi;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
/* LY: тест наполнения таблиц в append-режиме,
* при котором записи добавляются строго в конец (в порядке сортировки) */
const unsigned flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_APPEND | MDBX_APPENDDUP
: MDBX_APPEND;
keyvalue_maker.make_ordered();
key = keygen::alloc(config.params.keylen_max);
data = keygen::alloc(config.params.datalen_max);
keygen::buffer last_key = keygen::alloc(config.params.keylen_max);
keygen::buffer last_data = keygen::alloc(config.params.datalen_max);
last_key->value.iov_base = last_key->bytes;
last_key->value.iov_len = 0;
last_data->value.iov_base = last_data->bytes;
last_data->value.iov_len = 0;
simple_checksum inserted_checksum;
uint64_t inserted_number = 0;
uint64_t serial_count = 0;
unsigned txn_nops = 0;
uint64_t commited_inserted_number = inserted_number;
simple_checksum commited_inserted_checksum = inserted_checksum;
while (should_continue()) {
const keygen::serial_t serial = serial_count;
if (!keyvalue_maker.increment(serial_count, 1)) {
// дошли до границы пространства ключей
break;
}
log_trace("append: append-a %" PRIu64, serial);
generate_pair(serial, key, data);
int cmp = inserted_number ? mdbx_cmp(txn_guard.get(), dbi, &key->value,
&last_key->value)
: 1;
if (cmp == 0 && (config.params.table_flags & MDBX_DUPSORT))
cmp = mdbx_dcmp(txn_guard.get(), dbi, &data->value, &last_data->value);
err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags);
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("append: bailout-insert due '%s'", mdbx_strerror(err));
txn_end(true);
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
break;
}
if (cmp > 0) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_put(appenda-a)", err);
memcpy(last_key->value.iov_base, key->value.iov_base,
last_key->value.iov_len = key->value.iov_len);
memcpy(last_data->value.iov_base, data->value.iov_base,
last_data->value.iov_len = data->value.iov_len);
++inserted_number;
inserted_checksum.push((uint32_t)inserted_number, key->value);
inserted_checksum.push(10639, data->value);
} else {
if (unlikely(err != MDBX_EKEYMISMATCH))
failure_perror("mdbx_put(appenda-a) != MDBX_EKEYMISMATCH", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-commit due '%s'", mdbx_strerror(err));
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
break;
}
commited_inserted_number = inserted_number;
commited_inserted_checksum = inserted_checksum;
txn_nops = 0;
}
report(1);
}
if (txn_guard) {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-commit due '%s'", mdbx_strerror(err));
inserted_number = commited_inserted_number;
inserted_checksum = commited_inserted_checksum;
}
}
//----------------------------------------------------------------------------
txn_begin(true);
cursor_open(dbi);
MDBX_val check_key, check_data;
err =
mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_FIRST);
if (likely(inserted_number)) {
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_cursor_get(MDBX_FIRST)", err);
}
simple_checksum read_checksum;
uint64_t read_count = 0;
while (err == MDBX_SUCCESS) {
++read_count;
read_checksum.push((uint32_t)read_count, check_key);
read_checksum.push(10639, check_data);
err =
mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_NEXT);
}
if (unlikely(err != MDBX_NOTFOUND))
failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", err);
if (unlikely(read_count != inserted_number))
failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")",
read_count, inserted_number);
if (unlikely(read_checksum.value != inserted_checksum.value))
failure("read_checksum(0x%016" PRIu64 ") "
"!= inserted_checksum(0x%016" PRIu64 ")",
read_checksum.value, inserted_checksum.value);
cursor_close();
txn_end(true);
//----------------------------------------------------------------------------
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("append: bailout-clean due '%s'", mdbx_strerror(err));
return true;
}
} else
db_table_close(dbi);
}
return true;
}

113
contrib/db/libmdbx/test/base.h Normal file
View file

@ -0,0 +1,113 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#ifndef NOMINMAX
#define NOMINMAX
#endif
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
#ifdef _MSC_VER
#define _CRT_SECURE_NO_WARNINGS
#pragma warning(push, 1)
#pragma warning(disable : 4548) /* expression before comma has no effect; \
expected expression with side - effect */
#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \
semantics are not enabled. Specify /EHsc */
#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \
mode specified; termination on exception \
is not guaranteed. Specify /EHsc */
#endif /* _MSC_VER (warnings) */
/* If you wish to build your application for a previous Windows platform,
* include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you
* wish to support before including SDKDDKVer.h.
*
* TODO: #define _WIN32_WINNT WIN32_MUSTDIE */
#include <SDKDDKVer.h>
#endif /* WINDOWS */
#ifdef __APPLE__
#define _DARWIN_C_SOURCE
#endif
#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
#include <io.h>
#else
#include <fcntl.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#endif
#ifdef _BSD_SOURCE
#include <endian.h>
#endif
#include <algorithm>
#include <cassert>
#include <cinttypes> // for PRId64, PRIu64
#include <cstdarg>
#include <cstddef>
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "../mdbx.h"
#include "../src/defs.h"
#include "../src/osal.h"
#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__))
#define __thread __declspec(thread)
#endif /* __thread */
#ifdef _MSC_VER
#pragma warning(pop)
#pragma warning(disable : 4201) /* nonstandard extension used : \
nameless struct / union */
#pragma warning(disable : 4127) /* conditional expression is constant */
#if _MSC_VER < 1900
#pragma warning(disable : 4510) /* default constructor could \
not be generated */
#pragma warning(disable : 4512) /* assignment operator could \
not be generated */
#pragma warning(disable : 4610) /* user-defined constructor required */
#ifndef snprintf
#define snprintf(buffer, buffer_size, format, ...) \
_snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__)
#endif
#ifndef vsnprintf
#define vsnprintf(buffer, buffer_size, format, args) \
_vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args)
#endif
#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \
may be unsafe */
#endif
#endif /* _MSC_VER */

99
contrib/db/libmdbx/test/cases.cc Normal file
View file

@ -0,0 +1,99 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
const char *space_id_cstr, const actor_params &params) {
unsigned wait4id = 0;
if (params.waitfor_nops) {
for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) {
if (i->is_waitable(params.waitfor_nops)) {
if (i->signal_nops && i->signal_nops != params.waitfor_nops)
failure("Previous waitable actor (id=%u) already linked on %u-ops\n",
i->actor_id, i->signal_nops);
wait4id = i->actor_id;
i->signal_nops = params.waitfor_nops;
break;
}
}
if (!wait4id)
failure("No previous waitable actor for %u-ops\n", params.waitfor_nops);
}
unsigned space_id = 0;
if (!space_id_cstr || strcmp(space_id_cstr, "auto") == 0)
space_id = last_space_id + 1;
else {
char *end = nullptr;
errno = 0;
space_id = strtoul(space_id_cstr, &end, 0);
if (errno)
failure_perror("Expects an integer value for space-id\n", errno);
if (end && *end)
failure("The '%s' is unexpected for space-id\n", end);
}
if (space_id > ACTOR_ID_MAX)
failure("Invalid space-id %u\n", space_id);
last_space_id = space_id;
log_trace("configure_actor: space %u for %s", space_id,
testcase2str(testcase));
global::actors.emplace_back(
actor_config(testcase, params, space_id, wait4id));
global::databases.insert(params.pathname_db);
}
void testcase_setup(const char *casename, actor_params &params,
unsigned &last_space_id) {
if (strcmp(casename, "basic") == 0) {
log_notice(">>> testcase_setup(%s)", casename);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_hill, nullptr, params);
configure_actor(last_space_id, ac_ttl, nullptr, params);
configure_actor(last_space_id, ac_jitter, nullptr, params);
configure_actor(last_space_id, ac_hill, nullptr, params);
configure_actor(last_space_id, ac_ttl, nullptr, params);
configure_actor(last_space_id, ac_try, nullptr, params);
configure_actor(last_space_id, ac_copy, nullptr, params);
configure_actor(last_space_id, ac_append, nullptr, params);
log_notice("<<< testcase_setup(%s): done", casename);
} else {
failure("unknown testcase `%s`", casename);
}
}
void keycase_setup(const char *casename, actor_params &params) {
if (strcmp(casename, "random") == 0 || strcmp(casename, "prng") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_random;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "dashes") == 0 ||
strcmp(casename, "aside") == 0) {
log_notice(">>> keycase_setup(%s)", casename);
params.keygen.keycase = kc_dashes;
// TODO
log_notice("<<< keycase_setup(%s): done", casename);
} else if (strcmp(casename, "custom") == 0) {
log_notice("=== keycase_setup(%s): skip", casename);
params.keygen.keycase = kc_custom;
} else {
failure("unknown keycase `%s`", casename);
}
}
/* TODO */

129
contrib/db/libmdbx/test/chrono.cc Normal file
View file

@ -0,0 +1,129 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
namespace chrono {
#define NSEC_PER_SEC 1000000000u
uint32_t ns2fractional(uint32_t ns) {
assert(ns < NSEC_PER_SEC);
/* LY: здесь и далее используется "длинное деление", которое
* для ясности кода оставлено как есть (без ручной оптимизации). Так как
* GCC, Clang и даже MSVC сами давно умеют конвертировать деление на
* константу в быструю reciprocal-форму. */
return ((uint64_t)ns << 32) / NSEC_PER_SEC;
}
uint32_t fractional2ns(uint32_t fractional) {
return (fractional * (uint64_t)NSEC_PER_SEC) >> 32;
}
#define USEC_PER_SEC 1000000u
uint32_t us2fractional(uint32_t us) {
assert(us < USEC_PER_SEC);
return ((uint64_t)us << 32) / USEC_PER_SEC;
}
uint32_t fractional2us(uint32_t fractional) {
return (fractional * (uint64_t)USEC_PER_SEC) >> 32;
}
#define MSEC_PER_SEC 1000u
uint32_t ms2fractional(uint32_t ms) {
assert(ms < MSEC_PER_SEC);
return ((uint64_t)ms << 32) / MSEC_PER_SEC;
}
uint32_t fractional2ms(uint32_t fractional) {
return (fractional * (uint64_t)MSEC_PER_SEC) >> 32;
}
time from_ns(uint64_t ns) {
time result;
result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) |
ns2fractional((uint32_t)(ns % NSEC_PER_SEC));
return result;
}
time from_us(uint64_t us) {
time result;
result.fixedpoint = ((us / USEC_PER_SEC) << 32) |
us2fractional((uint32_t)(us % USEC_PER_SEC));
return result;
}
time from_ms(uint64_t ms) {
time result;
result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) |
ms2fractional((uint32_t)(ms % MSEC_PER_SEC));
return result;
}
time now_realtime() {
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
static void(WINAPI * query_time)(LPFILETIME);
if (!query_time) {
query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress(
GetModuleHandle(TEXT("kernel32.dll")),
"GetSystemTimePreciseAsFileTime");
if (!query_time)
query_time = GetSystemTimeAsFileTime;
}
FILETIME filetime;
query_time(&filetime);
uint64_t ns100 =
(uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime;
return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u);
#else
struct timespec ts;
if (unlikely(clock_gettime(CLOCK_REALTIME, &ts)))
failure_perror("clock_gettime(CLOCK_REALTIME", errno);
return from_timespec(ts);
#endif
}
time now_motonic() {
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
static uint64_t reciprocal;
static LARGE_INTEGER Frequency;
if (reciprocal == 0) {
if (!QueryPerformanceFrequency(&Frequency))
failure_perror("QueryPerformanceFrequency()", GetLastError());
reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) /
Frequency.QuadPart);
assert(reciprocal);
}
LARGE_INTEGER Counter;
if (!QueryPerformanceCounter(&Counter))
failure_perror("QueryPerformanceCounter()", GetLastError());
time result;
result.fixedpoint = (Counter.QuadPart / Frequency.QuadPart) << 32;
uint64_t mod = Counter.QuadPart % Frequency.QuadPart;
result.fixedpoint += (mod * reciprocal) >> 16;
return result;
#else
struct timespec ts;
if (unlikely(clock_gettime(CLOCK_MONOTONIC, &ts)))
failure_perror("clock_gettime(CLOCK_MONOTONIC)", errno);
return from_timespec(ts);
#endif
}
} /* namespace chrono */

100
contrib/db/libmdbx/test/chrono.h Normal file
View file

@ -0,0 +1,100 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "log.h"
#include "utils.h"
namespace chrono {
#pragma pack(push, 1)
typedef union time {
uint64_t fixedpoint;
struct {
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
uint32_t fractional;
union {
uint32_t utc;
uint32_t integer;
};
#else
union {
uint32_t utc;
uint32_t integer;
};
uint32_t fractional;
#endif
};
void reset() { fixedpoint = 0; }
uint32_t seconds() const { return utc; }
} time;
#pragma pack(pop)
uint32_t ns2fractional(uint32_t);
uint32_t fractional2ns(uint32_t);
uint32_t us2fractional(uint32_t);
uint32_t fractional2us(uint32_t);
uint32_t ms2fractional(uint32_t);
uint32_t fractional2ms(uint32_t);
time from_ns(uint64_t us);
time from_us(uint64_t ns);
time from_ms(uint64_t ms);
inline time from_seconds(uint64_t seconds) {
assert(seconds < UINT32_MAX);
time result;
result.fixedpoint = seconds << 32;
return result;
}
inline time from_utc(time_t utc) {
assert(utc >= 0);
return from_seconds((uint64_t)utc);
}
inline time infinite() {
time result;
result.fixedpoint = UINT64_MAX;
return result;
}
#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \
defined(CLOCK_REALTIME)
inline time from_timespec(const struct timespec &ts) {
time result;
result.fixedpoint =
((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec);
return result;
}
#endif /* HAVE_TIMESPEC_TV_NSEC */
#if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL)
inline time from_timeval(const struct timeval &tv) {
time result;
result.fixedpoint =
((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec);
return result;
}
#endif /* HAVE_TIMEVAL_TV_USEC */
time now_realtime();
time now_motonic();
} /* namespace chrono */

576
contrib/db/libmdbx/test/config.cc Normal file
View file

@ -0,0 +1,576 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#if defined(_MSC_VER) && !defined(strcasecmp)
#define strcasecmp(str, len) _stricmp(str, len)
#endif /* _MSC_VER && strcasecmp() */
namespace config {
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value) {
assert(narg < argc);
const char *current = argv[narg];
const size_t optlen = strlen(option);
if (strncmp(current, "--", 2) || strncmp(current + 2, option, optlen))
return false;
if (!value) {
if (current[optlen + 2] == '=')
failure("Option '--%s' doen't accept any value\n", option);
return true;
}
*value = nullptr;
if (current[optlen + 2] == '=') {
*value = &current[optlen + 3];
return true;
}
if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2) != 0) {
*value = argv[narg + 1];
if (strcmp(*value, "default") == 0) {
if (!default_value)
failure("Option '--%s' doen't accept default value\n", option);
*value = default_value;
}
++narg;
return true;
}
if (default_value) {
*value = default_value;
return true;
}
failure("No value given for '--%s' option\n", option);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty) {
return parse_option(argc, argv, narg, option, value, allow_empty,
allow_empty ? "" : nullptr);
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty,
const char *default_value) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr, default_value))
return false;
if (!allow_empty && strlen(value_cstr) == 0)
failure("Value for option '--%s' could't be empty\n", option);
value = value_cstr;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs) {
const char *list;
if (!parse_option(argc, argv, narg, option, &list))
return false;
unsigned clear = 0;
while (*list) {
if (*list == ',' || *list == ' ' || *list == '\t') {
++list;
continue;
}
const char *const comma = strchr(list, ',');
const bool strikethrough = *list == '-' || *list == '~';
if (strikethrough || *list == '+')
++list;
else
mask = clear;
const size_t len = (comma) ? comma - list : strlen(list);
const option_verb *scan = verbs;
while (true) {
if (!scan->verb)
failure("Unknown verb '%.*s', for option '==%s'\n", (int)len, list,
option);
if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) {
mask = strikethrough ? mask & ~scan->mask : mask | scan->mask;
clear = strikethrough ? clear & ~scan->mask : clear | scan->mask;
list += len;
break;
}
++scan;
}
}
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval, const uint64_t maxval,
const uint64_t default_value) {
const char *value_cstr;
if (!parse_option(argc, argv, narg, option, &value_cstr))
return false;
if (default_value && strcmp(value_cstr, "default") == 0) {
value = default_value;
return true;
}
if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0) {
value = minval;
return true;
}
if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0) {
value = maxval;
return true;
}
char *suffix = nullptr;
errno = 0;
unsigned long long raw = strtoull(value_cstr, &suffix, 0);
if ((suffix && *suffix) || errno) {
suffix = nullptr;
errno = 0;
raw = strtoull(value_cstr, &suffix, 10);
}
if (errno)
failure("Option '--%s' expects a numeric value (%s)\n", option,
test_strerror(errno));
uint64_t multipler = 1;
if (suffix && *suffix) {
if (scale == no_scale)
failure("Option '--%s' doen't accepts suffixes, so '%s' is unexpected\n",
option, suffix);
if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024);
else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0)
multipler =
(scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024;
else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024;
else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0)
multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000
: UINT64_C(1024) * 1024 * 1024 * 1024;
else if (scale == duration &&
(strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0))
multipler = 1;
else if (scale == duration &&
(strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0))
multipler = 60;
else if (scale == duration &&
(strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0))
multipler = 3600;
else if (scale == duration &&
(strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0))
multipler = 3600 * 24;
else
failure(
"Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s"
"suffixes, but '%s' is unexpected\n",
option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "",
suffix);
}
if (raw >= UINT64_MAX / multipler)
failure("The value for option '--%s' is too huge\n", option);
value = raw * multipler;
if (maxval && value > maxval)
failure("The maximal value for option '--%s' is %" PRIu64 "\n", option,
maxval);
if (value < minval)
failure("The minimal value for option '--%s' is %" PRIu64 "\n", option,
minval);
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval, const unsigned maxval,
const unsigned default_value) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval,
default_value))
return false;
value = (unsigned)huge;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval, const uint8_t maxval,
const uint8_t default_value) {
uint64_t huge;
if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval,
default_value))
return false;
value = (uint8_t)huge;
return true;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value) {
uint64_t proxy = (uint64_t)value;
if (parse_option(argc, argv, narg, option, proxy, config::binary,
(uint64_t)minval, (uint64_t)maxval,
(uint64_t)default_value)) {
value = (int64_t)proxy;
return true;
}
return false;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value) {
uint64_t proxy = (uint64_t)value;
if (parse_option(argc, argv, narg, option, proxy, config::binary,
(uint64_t)minval, (uint64_t)maxval,
(uint64_t)default_value)) {
value = (int32_t)proxy;
return true;
}
return false;
}
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value) {
const char *value_cstr = nullptr;
if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) {
const char *current = argv[narg];
if (strncmp(current, "--no-", 5) == 0 && strcmp(current + 5, option) == 0) {
value = false;
return true;
}
if (strncmp(current, "--dont-", 7) == 0 &&
strcmp(current + 7, option) == 0) {
value = false;
return true;
}
return false;
}
if (!value_cstr) {
value = true;
return true;
}
if (strcasecmp(value_cstr, "yes") == 0 || strcasecmp(value_cstr, "1") == 0) {
value = true;
return true;
}
if (strcasecmp(value_cstr, "no") == 0 || strcasecmp(value_cstr, "0") == 0) {
value = false;
return true;
}
failure(
"Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n",
option, value_cstr);
}
//-----------------------------------------------------------------------------
const struct option_verb mode_bits[] = {
{"rdonly", MDBX_RDONLY}, {"mapasync", MDBX_MAPASYNC},
{"utterly", MDBX_UTTERLY_NOSYNC}, {"nosubdir", MDBX_NOSUBDIR},
{"nosync", MDBX_NOSYNC}, {"nometasync", MDBX_NOMETASYNC},
{"writemap", MDBX_WRITEMAP}, {"notls", MDBX_NOTLS},
{"nordahead", MDBX_NORDAHEAD}, {"nomeminit", MDBX_NOMEMINIT},
{"coalesce", MDBX_COALESCE}, {"lifo", MDBX_LIFORECLAIM},
{"perturb", MDBX_PAGEPERTURB}, {nullptr, 0}};
const struct option_verb table_bits[] = {
{"key.reverse", MDBX_REVERSEKEY},
{"key.integer", MDBX_INTEGERKEY},
{"data.integer", MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.fixed", MDBX_DUPFIXED | MDBX_DUPSORT},
{"data.reverse", MDBX_REVERSEDUP | MDBX_DUPSORT},
{"data.dups", MDBX_DUPSORT},
{nullptr, 0}};
static void dump_verbs(const char *caption, size_t bits,
const struct option_verb *verbs) {
log_info("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits);
const char *comma = "";
while (verbs->mask && bits) {
if ((bits & verbs->mask) == verbs->mask) {
logging::feed("%s%s", comma, verbs->verb);
bits -= verbs->mask;
comma = ", ";
}
++verbs;
}
logging::feed("%s\n", (*comma == '\0') ? "none" : "");
}
static void dump_duration(const char *caption, unsigned duration) {
log_info("%s: ", caption);
if (duration) {
if (duration > 24 * 3600)
logging::feed("%u_", duration / (24 * 3600));
if (duration > 3600)
logging::feed("%02u:", (duration % (24 * 3600)) / 3600);
logging::feed("%02u:%02u", (duration % 3600) / 60, duration % 60);
} else {
logging::feed("INFINITE");
}
logging::feed("\n");
}
void dump(const char *title) {
logging::local_suffix indent(title);
for (auto i = global::actors.begin(); i != global::actors.end(); ++i) {
log_info("#%u, testcase %s, space_id/table %u\n", i->actor_id,
testcase2str(i->testcase), i->space_id);
indent.push();
if (i->params.loglevel) {
log_info("log: level %u, %s\n", i->params.loglevel,
i->params.pathname_log.empty() ? "console"
: i->params.pathname_log.c_str());
}
log_info("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR
", %i %i, %i]\n",
i->params.pathname_db.c_str(), i->params.size_now,
i->params.size_lower, i->params.size_upper,
i->params.shrink_threshold, i->params.growth_step,
i->params.pagesize);
dump_verbs("mode", i->params.mode_flags, mode_bits);
dump_verbs("table", i->params.table_flags, table_bits);
if (i->params.test_nops)
log_info("iterations/records %u\n", i->params.test_nops);
else
dump_duration("duration", i->params.test_duration);
if (i->params.nrepeat)
log_info("repeat %u\n", i->params.nrepeat);
else
log_info("repeat ETERNALLY\n");
log_info("threads %u\n", i->params.nthreads);
log_info(
"keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64
", split %u/%u\n",
keygencase2str(i->params.keygen.keycase), i->params.keygen.width,
i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset,
i->params.keygen.split,
i->params.keygen.width - i->params.keygen.split);
log_info("keygen.seed: %u\n", i->params.keygen.seed);
log_info("key: minlen %u, maxlen %u\n", i->params.keylen_min,
i->params.keylen_max);
log_info("data: minlen %u, maxlen %u\n", i->params.datalen_min,
i->params.datalen_max);
log_info("batch: read %u, write %u\n", i->params.batch_read,
i->params.batch_write);
if (i->params.waitfor_nops)
log_info("wait: actor %u for %u ops\n", i->wait4id,
i->params.waitfor_nops);
else if (i->params.delaystart)
dump_duration("delay", i->params.delaystart);
else
log_info("no-delay\n");
if (i->params.inject_writefaultn)
log_info("inject-writefault on %u ops\n", i->params.inject_writefaultn);
else
log_info("no-inject-writefault\n");
log_info("limits: readers %u, tables %u\n", i->params.max_readers,
i->params.max_tables);
log_info("drop table: %s\n", i->params.drop_table ? "Yes" : "No");
log_info("ignore MDBX_MAP_FULL error: %s\n",
i->params.ignore_dbfull ? "Yes" : "No");
indent.pop();
}
dump_duration("timeout", global::config::timeout_duration_seconds);
log_info("cleanup: before %s, after %s\n",
global::config::cleanup_before ? "Yes" : "No",
global::config::cleanup_after ? "Yes" : "No");
log_info("failfast: %s\n", global::config::failfast ? "Yes" : "No");
log_info("progress indicator: %s\n",
global::config::progress_indicator ? "Yes" : "No");
}
} /* namespace config */
//-----------------------------------------------------------------------------
using namespace config;
actor_config::actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id)
: params(params) {
this->space_id = space_id;
this->actor_id = 1 + (unsigned)global::actors.size();
this->testcase = testcase;
this->wait4id = wait4id;
signal_nops = 0;
}
const std::string actor_config::serialize(const char *prefix) const {
simple_checksum checksum;
std::string result;
if (prefix)
result.append(prefix);
checksum.push(params.pathname_db);
result.append(params.pathname_db);
result.append("|");
checksum.push(params.pathname_log);
result.append(params.pathname_log);
result.append("|");
static_assert(std::is_pod<actor_params_pod>::value,
"actor_params_pod should by POD");
result.append(data2hex(static_cast<const actor_params_pod *>(&params),
sizeof(actor_params_pod), checksum));
result.append("|");
static_assert(std::is_pod<actor_config_pod>::value,
"actor_config_pod should by POD");
result.append(data2hex(static_cast<const actor_config_pod *>(this),
sizeof(actor_config_pod), checksum));
result.append("|");
result.append(osal_serialize(checksum));
result.append("|");
result.append(std::to_string(checksum.value));
return result;
}
bool actor_config::deserialize(const char *str, actor_config &config) {
simple_checksum checksum;
TRACE(">> actor_config::deserialize: %s\n", str);
const char *slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-1\n");
return false;
}
config.params.pathname_db.assign(str, slash - str);
checksum.push(config.params.pathname_db);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-2\n");
return false;
}
config.params.pathname_log.assign(str, slash - str);
checksum.push(config.params.pathname_log);
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-3\n");
return false;
}
static_assert(std::is_pod<actor_params_pod>::value,
"actor_params_pod should by POD");
if (!hex2data(str, slash, static_cast<actor_params_pod *>(&config.params),
sizeof(actor_params_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n",
(int)(slash - str), str);
return false;
}
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-4\n");
return false;
}
static_assert(std::is_pod<actor_config_pod>::value,
"actor_config_pod should by POD");
if (!hex2data(str, slash, static_cast<actor_config_pod *>(&config),
sizeof(actor_config_pod), checksum)) {
TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n",
(int)(slash - str), str);
return false;
}
str = slash + 1;
slash = strchr(str, '|');
if (!slash) {
TRACE("<< actor_config::deserialize: slash-5\n");
return false;
}
if (!config.osal_deserialize(str, slash, checksum)) {
TRACE("<< actor_config::deserialize: osal\n");
return false;
}
str = slash + 1;
uint64_t verify = std::stoull(std::string(str));
if (checksum.value != verify) {
TRACE("<< actor_config::deserialize: checksum mismatch\n");
return false;
}
TRACE("<< actor_config::deserialize: OK\n");
return true;
}
unsigned actor_params::mdbx_keylen_min() const {
return (table_flags & MDBX_INTEGERKEY) ? 4 : 0;
}
unsigned actor_params::mdbx_keylen_max() const {
return (table_flags & MDBX_INTEGERKEY)
? 8
: std::min((unsigned)mdbx_limits_keysize_max(pagesize),
(unsigned)UINT16_MAX);
}
unsigned actor_params::mdbx_datalen_min() const {
return (table_flags & MDBX_INTEGERDUP) ? 4 : 0;
}
unsigned actor_params::mdbx_datalen_max() const {
return (table_flags & MDBX_INTEGERDUP)
? 8
: std::min((table_flags & MDBX_DUPSORT)
? (unsigned)mdbx_limits_keysize_max(pagesize)
: (unsigned)MDBX_MAXDATASIZE,
(unsigned)UINT16_MAX);
}

326
contrib/db/libmdbx/test/config.h Normal file
View file

@ -0,0 +1,326 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "log.h"
#include "utils.h"
#define ACTOR_ID_MAX INT16_MAX
enum actor_testcase {
ac_none,
ac_hill,
ac_deadread,
ac_deadwrite,
ac_jitter,
ac_try,
ac_copy,
ac_append,
ac_ttl
};
enum actor_status {
as_unknown,
as_debuging,
as_running,
as_successful,
as_killed,
as_failed,
as_coredump,
};
const char *testcase2str(const actor_testcase);
const char *status2str(actor_status status);
enum keygen_case {
kc_random, /* [ 6.. 2.. 7.. 4.. 0.. 1.. 5.. 3.. ] */
kc_dashes, /* [ 0123.. 4567.. ] */
kc_custom,
/* TODO: more cases */
};
const char *keygencase2str(const keygen_case);
//-----------------------------------------------------------------------------
namespace config {
enum scale_mode { no_scale, decimal, binary, duration };
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
const char **value, const char *default_value = nullptr);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty = false);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
std::string &value, bool allow_empty,
const char *default_value);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
bool &value);
struct option_verb {
const char *const verb;
unsigned mask;
};
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &mask, const option_verb *verbs);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint64_t &value, const scale_mode scale,
const uint64_t minval = 0, const uint64_t maxval = INT64_MAX,
const uint64_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
unsigned &value, const scale_mode scale,
const unsigned minval = 0, const unsigned maxval = INT32_MAX,
const unsigned default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
uint8_t &value, const uint8_t minval = 0,
const uint8_t maxval = 255, const uint8_t default_value = 0);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int64_t &value, const int64_t minval, const int64_t maxval,
const int64_t default_value = -1);
bool parse_option(int argc, char *const argv[], int &narg, const char *option,
int32_t &value, const int32_t minval, const int32_t maxval,
const int32_t default_value = -1);
inline bool parse_option_intptr(int argc, char *const argv[], int &narg,
const char *option, intptr_t &value,
const intptr_t minval, const intptr_t maxval,
const intptr_t default_value = -1) {
static_assert(sizeof(intptr_t) == 4 || sizeof(intptr_t) == 8, "WTF?");
if (sizeof(intptr_t) == 8)
return parse_option(argc, argv, narg, option,
*reinterpret_cast<int64_t *>(&value), int64_t(minval),
int64_t(maxval), int64_t(default_value));
else
return parse_option(argc, argv, narg, option,
*reinterpret_cast<int32_t *>(&value), int32_t(minval),
int32_t(maxval), int32_t(default_value));
}
//-----------------------------------------------------------------------------
#pragma pack(push, 1)
struct keygen_params_pod {
keygen_case keycase;
/* Параметры генератора пар key-value.
*
* Ключи и значения генерируются по задаваемым параметрам на основе "плоской"
* исходной координаты. При этом, в общем случае, в процессе тестов исходная
* координата последовательно итерируется в заданном диапазоне, а необходимые
* паттерны/последовательности/узоры получаются за счет преобразования
* исходной координаты, согласно описанным ниже параметрам.
*
* Стоит отметить, что порядок описания параметров для удобства совпадает с
* порядком их использования, т.е. с порядком соответствующих преобразований.
*
* Второе важное замечание касается ограничений одновременной координированной
* генерации паттеров как для ключей, так и для значений. Суть в том, что
* такая возможность не нужна по следующим причинам:
* - libmdbx поддерживает два существенно различающихся вида таблиц,
* "уникальные" (без дубликатов и без multi-value), и так называемые
* "с дубликатами" (c multi-value).
* - Для таблиц "без дубликатов" только размер связанных к ключами значений
* (данных) оказывает влияния на работу движка, непосредственно содержимое
* данных не анализируется движком и не оказывает влияния на его работу.
* - Для таблиц "с дубликатами", при наличии более одного значения для
* некоторого ключа, формируется дочернее btree-поддерево. Это дерево
* формируется в отдельном "кусте" страниц и обслуживается независимо
* от окружения родительского ключа.
* - Таким образом, паттерн генерации значений имеет смысл только для
* таблиц "с дубликатами" и только в контексте одного значения ключа.
* Иначе говоря, нет смысла в со-координации генерации паттернов для
* ключей и значений. Более того, генерацию значений всегда необходимо
* рассматривать в контексте связки с одним значением ключа.
* - Тем не менее, во всех случаях достаточно важным является равномерная
* всех возможных сочетаний длин ключей и данных.
*
* width:
* Большинство тестов предполагают создание или итерирование некоторого
* количества записей. При этом требуется итерирование или генерация
* значений и ключей из некоторого ограниченного пространства вариантов.
*
* Параметр width задает такую ширину пространства вариантов в битах.
* Таким образом мощность пространства вариантов (пока) всегда равна
* степени двойки. Это ограничение можно снять, но ценой увеличения
* вычислительной сложности, включая потерю простоты и прозрачности.
*
* С другой стороны, не-битовый width может быть полезен:
* - Позволит генерировать ключи/значения в точно задаваемом диапазоне.
* Например, перебрать в псевдо-случайном порядке 10001 значение.
* - Позволит поровну разделять заданное пространство (диапазон)
* ключей/значений между количеством потоков некратным степени двойки.
*
* mesh и seed:
* Позволяют получить псевдо-случайные последовательности ключей/значений.
* Параметр mesh задает сколько младших бит исходной плоской координаты
* будет "перемешано" (инъективно отображено), а параметр seed позволяет
* выбрать конкретный вариант "перемешивания".
*
* Перемешивание выполняется при ненулевом значении mesh. Перемешивание
* реализуется посредством применения двух инъективных функций для
* заданного количества бит:
* - применяется первая инъективная функция;
* - к результату добавляется salt полученный из seed;
* - применяется вторая инъективная функция;
*
* Следует отметить, что mesh умышленно позволяет перемешать только младшую
* часть, что при ненулевом значении split (см далее) не позволяет получать
* псевдо-случайные значений ключей без псевдо-случайности в значениях.
*
* Такое ограничение соответствуют внутренней алгоритмике libmdbx. Проще
* говоря, мы можем проверить движок псевдо-случайной последовательностью
* ключей на таблицах без дубликатов (без multi-value), а затем проверить
* корректность работу псевдо-случайной последовательностью значений на
* таблицах с дубликатами (с multi-value), опционально добавляя
* псевдо-случайности к последовательности ключей. Однако, нет смысла
* генерировать псевдо-случайные ключи, одновременно с формированием
* какого-либо паттерна в значениях, так как содержимое в данных либо
* не будет иметь значения (для таблиц без дубликатов), либо будет
* обрабатываться в отдельных btree-поддеревьях.
*
* rotate и offset:
* Для проверки слияния и разделения страниц внутри движка требуются
* генерация ключей/значений в виде не-смежных последовательностей, как-бы
* в виде "пунктира", который постепенно заполняет весь заданных диапазон.
*
* Параметры позволяют генерировать такой "пунктир". Соответственно rotate
* задает циклический сдвиг вправо, а offset задает смещение, точнее говоря
* сложение по модулю внутри диапазона заданного посредством width.
*
* Например, при rotate равном 1 (циклический сдвиг вправо на 1 бит),
* четные и нечетные исходные значения сложатся в две линейные
* последовательности, которые постепенно закроют старшую и младшую
* половины диапазона.
*
* split:
* Для таблиц без дубликатов (без multi-value ключей) фактически требуется
* генерация только ключей, а данные могут быть постоянным. Но для таблиц с
* дубликатами (с multi-value ключами) также требуется генерация значений.
*
* Ненулевое значение параметра split фактически включает генерацию значений,
* при этом значение split определяет сколько бит исходного абстрактного
* номера будет отрезано для генерации значения.
*/
uint8_t width;
uint8_t mesh;
uint8_t rotate;
uint8_t split;
uint32_t seed;
uint64_t offset;
};
struct actor_params_pod {
unsigned loglevel;
unsigned mode_flags;
unsigned table_flags;
intptr_t size_lower;
intptr_t size_now;
intptr_t size_upper;
int shrink_threshold;
int growth_step;
int pagesize;
unsigned test_duration;
unsigned test_nops;
unsigned nrepeat;
unsigned nthreads;
unsigned keylen_min, keylen_max;
unsigned datalen_min, datalen_max;
unsigned batch_read;
unsigned batch_write;
unsigned delaystart;
unsigned waitfor_nops;
unsigned inject_writefaultn;
unsigned max_readers;
unsigned max_tables;
keygen_params_pod keygen;
bool drop_table;
bool ignore_dbfull;
};
struct actor_config_pod {
unsigned actor_id, space_id;
actor_testcase testcase;
unsigned wait4id;
unsigned signal_nops;
};
#pragma pack(pop)
extern const struct option_verb mode_bits[];
extern const struct option_verb table_bits[];
void dump(const char *title = "config-dump: ");
} /* namespace config */
struct actor_params : public config::actor_params_pod {
std::string pathname_log;
std::string pathname_db;
void set_defaults(const std::string &tmpdir);
unsigned mdbx_keylen_min() const;
unsigned mdbx_keylen_max() const;
unsigned mdbx_datalen_min() const;
unsigned mdbx_datalen_max() const;
};
struct actor_config : public config::actor_config_pod {
actor_params params;
bool wanna_event4signalling() const { return true /* TODO ? */; }
actor_config(actor_testcase testcase, const actor_params &params,
unsigned space_id, unsigned wait4id);
actor_config(const char *str) {
if (!deserialize(str, *this))
failure("Invalid internal parameter '%s'\n", str);
}
const std::string osal_serialize(simple_checksum &) const;
bool osal_deserialize(const char *str, const char *end, simple_checksum &);
const std::string serialize(const char *prefix) const;
static bool deserialize(const char *str, actor_config &config);
bool is_waitable(size_t nops) const {
switch (testcase) {
case ac_hill:
if (!params.test_nops || params.test_nops >= nops)
return true;
__fallthrough;
default:
return false;
}
}
};

26
contrib/db/libmdbx/test/copy.cc Normal file
View file

@ -0,0 +1,26 @@
#include "test.h"
void testcase_copy::copy_db(const bool with_compaction) {
int err = osal_removefile(copy_pathname);
if (err != MDBX_SUCCESS && err != MDBX_ENOFILE)
failure_perror("mdbx_removefile()", err);
err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(),
with_compaction ? MDBX_CP_COMPACT : 0);
if (unlikely(err != MDBX_SUCCESS))
failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)"
: "mdbx_env_copy(MDBX_CP_ASIS)",
err);
}
bool testcase_copy::run() {
jitter_delay();
db_open();
assert(!txn_guard);
const bool order = flipcoin();
jitter_delay();
copy_db(order);
jitter_delay();
copy_db(!order);
return true;
}

24
contrib/db/libmdbx/test/darwin/LICENSE Normal file
View file

@ -0,0 +1,24 @@
Copyright (c) 2015, Aleksey Demakov
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

8
contrib/db/libmdbx/test/darwin/README.md Normal file
View file

@ -0,0 +1,8 @@
# DarwinPthreadBarrier
A pthread_barrier_t implementation for Mac OS/X
There is no pthread_barrier_t in Mac OS/X pthreads. This project fixes
this omission by providing a simple-minded barrier implementation based
on a pair of pthread_mutex_t and pthread_cond_t.

110
contrib/db/libmdbx/test/darwin/pthread_barrier.c Normal file
View file

@ -0,0 +1,110 @@
/*
* Copyright (c) 2015, Aleksey Demakov
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "pthread_barrier.h"
#include <errno.h>
#ifdef __APPLE__
int pthread_barrierattr_init(pthread_barrierattr_t *attr) {
memset(attr, 0, sizeof(pthread_barrierattr_t));
int m = pthread_mutexattr_init(&attr->mattr);
int c = pthread_condattr_init(&attr->cattr);
return m ? m : c;
}
int pthread_barrierattr_destroy(pthread_barrierattr_t *attr) {
int c = pthread_condattr_destroy(&attr->cattr);
int m = pthread_mutexattr_destroy(&attr->mattr);
return m ? m : c;
}
int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr,
int *__restrict pshared) {
return pthread_condattr_getpshared(&attr->cattr, pshared);
}
int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) {
int m = pthread_mutexattr_setpshared(&attr->mattr, pshared);
int c = pthread_condattr_setpshared(&attr->cattr, pshared);
return m ? m : c;
}
int pthread_barrier_init(pthread_barrier_t *__restrict barrier,
const pthread_barrierattr_t *__restrict attr,
unsigned count) {
if (count == 0)
return errno = EINVAL;
int rc = pthread_mutex_init(&barrier->mutex, attr ? &attr->mattr : 0);
if (rc)
return rc;
rc = pthread_cond_init(&barrier->cond, attr ? &attr->cattr : 0);
if (rc) {
int errno_save = errno;
pthread_mutex_destroy(&barrier->mutex);
errno = errno_save;
return rc;
}
barrier->limit = count;
barrier->count = 0;
barrier->phase = 0;
return 0;
}
int pthread_barrier_destroy(pthread_barrier_t *barrier) {
pthread_mutex_destroy(&barrier->mutex);
pthread_cond_destroy(&barrier->cond);
return 0;
}
int pthread_barrier_wait(pthread_barrier_t *barrier) {
int rc = pthread_mutex_lock(&barrier->mutex);
if (rc)
return rc;
barrier->count++;
if (barrier->count >= barrier->limit) {
barrier->phase++;
barrier->count = 0;
pthread_cond_broadcast(&barrier->cond);
pthread_mutex_unlock(&barrier->mutex);
return PTHREAD_BARRIER_SERIAL_THREAD;
} else {
unsigned phase = barrier->phase;
do
pthread_cond_wait(&barrier->cond, &barrier->mutex);
while (phase == barrier->phase);
pthread_mutex_unlock(&barrier->mutex);
return 0;
}
}
#endif /* __APPLE__ */

83
contrib/db/libmdbx/test/darwin/pthread_barrier.h Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright (c) 2015, Aleksey Demakov
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef PTHREAD_BARRIER_H
#define PTHREAD_BARRIER_H
#include <pthread.h>
#ifdef __APPLE__
#ifdef __cplusplus
extern "C" {
#endif
#if !defined(PTHREAD_BARRIER_SERIAL_THREAD)
#define PTHREAD_BARRIER_SERIAL_THREAD (1)
#endif
#if !defined(PTHREAD_PROCESS_PRIVATE)
#define PTHREAD_PROCESS_PRIVATE (42)
#endif
#if !defined(PTHREAD_PROCESS_SHARED)
#define PTHREAD_PROCESS_SHARED (43)
#endif
typedef struct {
pthread_mutexattr_t mattr;
pthread_condattr_t cattr;
} pthread_barrierattr_t;
typedef struct {
pthread_mutex_t mutex;
pthread_cond_t cond;
unsigned int limit;
unsigned int count;
unsigned int phase;
} pthread_barrier_t;
int pthread_barrierattr_init(pthread_barrierattr_t *attr);
int pthread_barrierattr_destroy(pthread_barrierattr_t *attr);
int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr,
int *__restrict pshared);
int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared);
int pthread_barrier_init(pthread_barrier_t *__restrict barrier,
const pthread_barrierattr_t *__restrict attr,
unsigned int count);
int pthread_barrier_destroy(pthread_barrier_t *barrier);
int pthread_barrier_wait(pthread_barrier_t *barrier);
#ifdef __cplusplus
}
#endif
#endif /* __APPLE__ */
#endif /* PTHREAD_BARRIER_H */

35
contrib/db/libmdbx/test/dead.cc Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_deadread::run() {
db_open();
txn_begin(true);
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
return true;
}
//-----------------------------------------------------------------------------
bool testcase_deadwrite::run() {
db_open();
txn_begin(false);
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
return true;
}

321
contrib/db/libmdbx/test/hill.cc Normal file
View file

@ -0,0 +1,321 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_hill::run() {
MDBX_dbi dbi;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
/* LY: тест "холмиком":
* - сначала наполняем таблицу циклическими CRUD-манипуляциями,
* которые в каждом цикле делают несколько операций, включая удаление,
* но в результате добавляют записи.
* - затем очищаем таблицу также CRUD-манипуляциями, но уже с другой
* пропорцией удалений.
*
* При этом очень многое зависит от порядка перебора ключей:
* - (псевдо)случайное распределение требуется лишь для полноты картины,
* но в целом не покрывает важных кейсов.
* - кроме (псевдо)случайного перебора требуется последовательное
* итерирование ключей интервалами различной ширины, с тем чтобы
* проверить различные варианты как разделения, так и слияния страниц
* внутри движка.
* - при не-уникальных ключах (MDBX_DUPSORT с подвариантами), для каждого
* повтора внутри движка формируется вложенное btree-дерево,
* соответственно требуется соблюдение аналогичных принципов
* итерирования для значений.
*/
/* TODO: работа в несколько потоков */
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
keygen::buffer a_key = keygen::alloc(config.params.keylen_max);
keygen::buffer a_data_0 = keygen::alloc(config.params.datalen_max);
keygen::buffer a_data_1 = keygen::alloc(config.params.datalen_max);
keygen::buffer b_key = keygen::alloc(config.params.keylen_max);
keygen::buffer b_data = keygen::alloc(config.params.datalen_max);
const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_NODUPDATA
: MDBX_NODUPDATA | MDBX_NOOVERWRITE;
const unsigned update_flags =
(config.params.table_flags & MDBX_DUPSORT)
? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE
: MDBX_NODUPDATA;
uint64_t serial_count = 0;
uint64_t commited_serial = serial_count;
unsigned txn_nops = 0;
while (should_continue()) {
const keygen::serial_t a_serial = serial_count;
if (unlikely(!keyvalue_maker.increment(serial_count, 1))) {
log_notice("uphill: unexpected key-space overflow");
break;
}
const keygen::serial_t b_serial = serial_count;
assert(b_serial > a_serial);
// создаем первую запись из пары
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_1, age_shift);
err = mdbx_put(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
break;
}
failure_perror("mdbx_put(insert-a.1)", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
break;
}
commited_serial = a_serial;
txn_nops = 0;
}
// создаем вторую запись из пары
log_trace("uphill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
break;
}
commited_serial = a_serial;
txn_nops = 0;
}
// обновляем данные в первой записи
log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value);
err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_0->value,
&a_data_1->value, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
break;
}
failure_perror("mdbx_replace(update-a: 1->0)", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
break;
}
commited_serial = a_serial;
txn_nops = 0;
}
// удаляем вторую запись
log_trace("uphill: delete-b %" PRIu64, b_serial);
checkdata("uphill: delete-b", dbi, b_key->value, b_data->value);
err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial_count = commited_serial;
break;
}
failure_perror("mdbx_del(b)", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err));
serial_count = commited_serial;
break;
}
commited_serial = a_serial;
txn_nops = 0;
}
report(1);
if (!keyvalue_maker.increment(serial_count, 1)) {
// дошли до границы пространства ключей
serial_count = a_serial;
goto overflow;
}
}
while (serial_count > 1) {
if (unlikely(!keyvalue_maker.increment(serial_count, -2)))
failure("downhill: unexpected key-space underflow");
overflow:
const keygen::serial_t a_serial = serial_count;
const keygen::serial_t b_serial = a_serial + 1;
assert(b_serial > a_serial);
// обновляем первую запись из пары
const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31);
log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift,
a_serial);
generate_pair(a_serial, a_key, a_data_0, 0);
generate_pair(a_serial, a_key, a_data_1, age_shift);
checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value);
err = mdbx_replace(txn_guard.get(), dbi, &a_key->value, &a_data_1->value,
&a_data_0->value, update_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at update-a due '%s'",
mdbx_strerror(err));
txn_end(true);
break;
}
failure_perror("mdbx_put(update-a: 0->1)", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
txn_nops = 0;
}
// создаем вторую запись из пары
log_trace("downhill: insert-b %" PRIu64, b_serial);
generate_pair(b_serial, b_key, b_data, 0);
err = mdbx_put(txn_guard.get(), dbi, &b_key->value, &b_data->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at insert-a due '%s'",
mdbx_strerror(err));
txn_end(true);
break;
}
failure_perror("mdbx_put(insert-b)", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
txn_nops = 0;
}
// удаляем первую запись
log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift,
a_serial);
checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value);
err = mdbx_del(txn_guard.get(), dbi, &a_key->value, &a_data_1->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-a due '%s'",
mdbx_strerror(err));
txn_end(true);
break;
}
failure_perror("mdbx_del(a)", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
txn_nops = 0;
}
// удаляем вторую запись
log_trace("downhill: delete-b %" PRIu64, b_serial);
checkdata("downhill: delete-b", dbi, b_key->value, b_data->value);
err = mdbx_del(txn_guard.get(), dbi, &b_key->value, &b_data->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("downhill: bailout at delete-b due '%s'",
mdbx_strerror(err));
txn_end(true);
break;
}
failure_perror("mdbx_del(b)", err);
}
if (++txn_nops >= config.params.batch_write) {
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
txn_nops = 0;
}
report(1);
}
if (txn_guard) {
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS))
log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err));
}
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err));
return true;
}
} else
db_table_close(dbi);
}
return true;
}

59
contrib/db/libmdbx/test/jitter.cc Normal file
View file

@ -0,0 +1,59 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
bool testcase_jitter::run() {
while (should_continue()) {
jitter_delay();
db_open();
if (flipcoin()) {
jitter_delay();
txn_begin(true);
fetch_canary();
jitter_delay();
txn_end(flipcoin());
}
jitter_delay();
txn_begin(mode_readonly());
jitter_delay();
if (!mode_readonly()) {
fetch_canary();
update_canary(1);
/* TODO:
* - db_setsize()
* ...
*/
}
txn_end(flipcoin());
if (flipcoin()) {
jitter_delay();
txn_begin(true);
jitter_delay();
txn_end(flipcoin());
}
jitter_delay();
db_close();
/* just 'align' nops with other tests with batching */
const auto batching =
std::max(config.params.batch_read, config.params.batch_write);
report(std::max(1u, batching / 2));
}
return true;
}

275
contrib/db/libmdbx/test/keygen.cc Normal file
View file

@ -0,0 +1,275 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
namespace keygen {
static inline __pure_function serial_t mask(unsigned bits) {
assert(bits > 0 && bits <= serial_maxwith);
return serial_allones >> (serial_maxwith - bits);
}
/* LY: https://en.wikipedia.org/wiki/Injective_function */
serial_t injective(const serial_t serial,
const unsigned bits /* at least serial_minwith (8) */,
const serial_t salt) {
assert(bits > serial_minwith && bits <= serial_maxwith);
/* LY: All these "magic" prime numbers were found
* and verified with a bit of brute force. */
static const uint64_t m[64 - serial_minwith + 1] = {
/* 8 - 24 */
113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937,
309277, 834497, 1499933, 4373441, 10184137,
/* 25 - 64 */
10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573,
6924051577, 19204053433, 45840188887, 53625693977, 73447827913,
141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197,
5871903036137, 11238507001417, 45264020802263, 105008404482889,
81921776907059, 199987980256399, 307207457507641, 946769023178273,
2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153,
23171543400565993, 53353226456762893, 155627817337932409,
227827205384840249, 816509268558278821, 576933057762605689,
2623957345935638441, 5048241705479929949, 4634245581946485653,
4613509448041658233, 4952535426879925961};
static const uint8_t s[64 - serial_minwith + 1] = {
/* 8 - 24 */
2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11,
/* 25 - 64 */
11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20,
10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20,
19};
const auto mult = m[bits - 8];
const auto shift = s[bits - 8];
serial_t result = serial * mult;
if (salt) {
const unsigned left = bits / 2;
const unsigned right = bits - left;
result = (result << left) | ((result & mask(bits)) >> right);
result = (result ^ salt) * mult;
}
result ^= result << shift;
result &= mask(bits);
log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64
" => %" PRIu64 "/%u",
serial, bits, mult, shift, salt, result, bits);
return result;
}
void __hot maker::pair(serial_t serial, const buffer &key, buffer &value,
serial_t value_age) {
assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith);
assert(mapping.split <= mapping.width);
assert(mapping.mesh <= mapping.width);
assert(mapping.rotate <= mapping.width);
assert(mapping.offset <= mask(mapping.width));
assert(!(key_essentials.flags &
~(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT)));
assert(!(value_essentials.flags & ~(MDBX_INTEGERDUP | MDBX_REVERSEDUP)));
log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial,
value_age);
if (mapping.mesh >= serial_minwith) {
serial =
(serial & ~mask(mapping.mesh)) | injective(serial, mapping.mesh, salt);
log_trace("keygen-pair: mesh@%u => %" PRIu64, mapping.mesh, serial);
}
if (mapping.rotate) {
const unsigned right = mapping.rotate;
const unsigned left = mapping.width - right;
serial = (serial << left) | ((serial & mask(mapping.width)) >> right);
log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64,
mapping.rotate, serial, serial);
}
if (mapping.offset) {
serial = (serial + mapping.offset) & mask(mapping.width);
log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset,
serial);
}
if (base) {
serial += base;
log_trace("keygen-pair: base@%" PRIu64 " => %" PRIu64, base, serial);
}
serial_t key_serial = serial;
serial_t value_serial = value_age << mapping.split;
if (mapping.split) {
if (key_essentials.flags & MDBX_DUPSORT) {
key_serial >>= mapping.split;
value_serial += serial & mask(mapping.split);
} else {
/* Без MDBX_DUPSORT требуется уникальность ключей, а для этого нельзя
* отбрасывать какие-либо биты serial после инъективного преобразования.
* Поэтому key_serial не трогаем, а в value_serial нелинейно вмешиваем
* запрошенное количество бит из serial */
value_serial +=
(serial ^ (serial >> mapping.split)) & mask(mapping.split);
}
value_serial |= value_age << mapping.split;
log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split,
key_serial, value_serial);
}
log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial,
value_serial);
mk(key_serial, key_essentials, *key);
mk(value_serial, value_essentials, *value);
if (log_enabled(logging::trace)) {
char dump_key[128], dump_value[128];
log_trace("keygen-pair: key %s, value %s",
mdbx_dkey(&key->value, dump_key, sizeof(dump_key)),
mdbx_dkey(&value->value, dump_value, sizeof(dump_value)));
}
}
void maker::setup(const config::actor_params_pod &actor, unsigned actor_id,
unsigned thread_number) {
key_essentials.flags =
actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT);
assert(actor.keylen_min <= UINT8_MAX);
key_essentials.minlen = (uint8_t)actor.keylen_min;
assert(actor.keylen_max <= UINT16_MAX);
key_essentials.maxlen = (uint16_t)actor.keylen_max;
value_essentials.flags =
actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP);
assert(actor.datalen_min <= UINT8_MAX);
value_essentials.minlen = (uint8_t)actor.datalen_min;
assert(actor.datalen_max <= UINT16_MAX);
value_essentials.maxlen = (uint16_t)actor.datalen_max;
assert(thread_number < 2);
(void)thread_number;
mapping = actor.keygen;
salt = (actor.keygen.seed + actor_id) * UINT64_C(14653293970879851569);
// FIXME: TODO
base = 0;
}
void maker::make_ordered() {
mapping.mesh = 0;
mapping.rotate = 0;
}
bool maker::is_unordered() const {
return (mapping.mesh >= serial_minwith || mapping.rotate) != 0;
}
bool maker::increment(serial_t &serial, int delta) const {
if (serial > mask(mapping.width)) {
log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial,
mask(mapping.width));
return false;
}
serial_t target = serial + (int64_t)delta;
if (target > mask(mapping.width) ||
((delta > 0) ? target < serial : target > serial)) {
log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", overflow",
serial, delta, target);
return false;
}
log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", continue", serial,
delta, target);
serial = target;
return true;
}
//-----------------------------------------------------------------------------
static size_t length(serial_t serial) {
size_t n = 0;
if (serial > UINT32_MAX) {
n = 4;
serial >>= 32;
}
if (serial > UINT16_MAX) {
n += 2;
serial >>= 16;
}
if (serial > UINT8_MAX) {
n += 1;
serial >>= 8;
}
return (serial > 0) ? n + 1 : n;
}
buffer alloc(size_t limit) {
result *ptr = (result *)malloc(sizeof(result) + limit);
if (unlikely(ptr == nullptr))
failure_perror("malloc(keyvalue_buffer)", errno);
ptr->value.iov_base = ptr->bytes;
ptr->value.iov_len = 0;
ptr->limit = limit;
return buffer(ptr);
}
void __hot maker::mk(const serial_t serial, const essentials &params,
result &out) {
assert(out.limit >= params.maxlen);
assert(params.maxlen >= params.minlen);
assert(params.maxlen >= length(serial));
out.value.iov_base = out.bytes;
out.value.iov_len =
(params.maxlen > params.minlen)
? params.minlen + serial % (params.maxlen - params.minlen)
: params.minlen;
if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) {
assert(params.maxlen == params.minlen);
assert(params.minlen == 4 || params.minlen == 8);
if (is_byteorder_le() || params.minlen == 8)
out.u64 = serial;
else
out.u32 = (uint32_t)serial;
} else if (params.flags & (MDBX_REVERSEKEY | MDBX_REVERSEDUP)) {
if (out.value.iov_len > 8) {
memset(out.bytes, '\0', out.value.iov_len - 8);
unaligned::store(out.bytes + out.value.iov_len - 8, htobe64(serial));
} else {
out.u64 = htobe64(serial);
if (out.value.iov_len < 8) {
out.value.iov_len = std::max(length(serial), out.value.iov_len);
out.value.iov_base = out.bytes + 8 - out.value.iov_len;
}
}
} else {
out.u64 = htole64(serial);
if (out.value.iov_len > 8)
memset(out.bytes + 8, '\0', out.value.iov_len - 8);
else
out.value.iov_len = std::max(length(serial), out.value.iov_len);
}
assert(out.value.iov_len >= params.minlen);
assert(out.value.iov_len <= params.maxlen);
assert(out.value.iov_len >= length(serial));
assert(out.value.iov_base >= out.bytes);
assert((uint8_t *)out.value.iov_base + out.value.iov_len <=
out.bytes + out.limit);
}
} /* namespace keygen */

130
contrib/db/libmdbx/test/keygen.h Normal file
View file

@ -0,0 +1,130 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "config.h"
#include "log.h"
#include "utils.h"
namespace keygen {
/* Под "генерацией ключей" здесь понимается генерация обоих значений для
* пар key-value, т.е. не только ключей, но и ассоциированных с ними данных.
*/
/* Генерацию ключей нельзя отнести к простым задачам, так как требования
* примерно следующие:
* - генерация разного количества уникальных ключей различной длины
* в задаваемом диапазоне;
* - возможность выбора как псевдо-случайного порядка ключей,
* так и по некоторым специфическим законам (ограниченными упорядоченными
* последовательностями, в шахматном порядке по граница диапазона и т.д.);
* - возможность генерации дубликатов с задаваемым законом распределения;
* - возможность генерации непересекающимися кластерами для параллельного
* использования в нескольких потоках;
* - использовать минимум ресурсов, как CPU, так и RAM, в том числе
* включая cache pollution и ram bandwidth.
*
* При этом заведомо известно, что для MDBX не имеет значения:
* - используемый алфавит (значения байтов);
* - частотное распределение по алфавиту;
* - абсолютное значение ключей или разность между отдельными значениями;
*
* Соответственно, в общих чертах, схема генерации следующая:
* - вводится плоская одномерная "координата" serial (uint64_t);
* - генерация специфических паттернов (последовательностей)
* реализуется посредством соответствующих преобразований "координат", при
* этом все подобные преобразования выполняются только над "координатой";
* - итоговая "координата" преобразуется в 8-байтное суррогатное значение
* ключа;
* - для получения ключей длиной МЕНЕЕ 8 байт суррогат может усекаться
* до ненулевых байт, в том числе до нулевой длины;
* - для получения ключей длиной БОЛЕЕ 8 байт суррогат дополняется
* нулями или псевдослучайной последовательностью;
*
* Механизм генерации паттернов:
* - реализованный механизм является компромиссом между скоростью/простотой
* и гибкостью, необходимой для получения последовательностей, которых
* будет достаточно для проверки сценариев разделения и слияния страниц
* с данными внутри mdbx;
* - псевдо-случайные паттерны реализуются посредством набора инъективных
* отображающих функций;
* - не-псевдо-случайные паттерны реализуются посредством параметризируемого
* трех-этапного преобразования:
* 1) смещение (сложение) по модулю;
* 2) циклический сдвиг;
* 3) добавление абсолютного смещения (базы);
*/
typedef uint64_t serial_t;
enum : serial_t {
serial_minwith = 8,
serial_maxwith = sizeof(serial_t) * 8,
serial_allones = ~(serial_t)0u
};
struct result {
MDBX_val value;
size_t limit;
union {
uint8_t bytes[sizeof(uint64_t)];
uint32_t u32;
uint64_t u64;
};
std::string as_string() const {
return std::string((const char *)value.iov_base, value.iov_len);
}
};
//-----------------------------------------------------------------------------
struct buffer_deleter : public std::unary_function<void, result *> {
void operator()(result *buffer) const { free(buffer); }
};
typedef std::unique_ptr<result, buffer_deleter> buffer;
buffer alloc(size_t limit);
class maker {
config::keygen_params_pod mapping;
serial_t base;
serial_t salt;
struct essentials {
uint8_t minlen;
uint8_t flags;
uint16_t maxlen;
} key_essentials, value_essentials;
static void mk(const serial_t serial, const essentials &params, result &out);
public:
maker() { memset(this, 0, sizeof(*this)); }
void pair(serial_t serial, const buffer &key, buffer &value,
serial_t value_age);
void setup(const config::actor_params_pod &actor, unsigned actor_id,
unsigned thread_number);
void make_ordered();
bool is_unordered() const;
bool increment(serial_t &serial, int delta) const;
};
} /* namespace keygen */

325
contrib/db/libmdbx/test/log.cc Normal file
View file

@ -0,0 +1,325 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
static void fflushall() { fflush(nullptr); }
void failure(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
fflushall();
logging::output(logging::failure, fmt, ap);
va_end(ap);
fflushall();
exit(EXIT_FAILURE);
}
const char *test_strerror(int errnum) {
static __thread char buf[1024];
return mdbx_strerror_r(errnum, buf, sizeof(buf));
}
void __noreturn failure_perror(const char *what, int errnum) {
failure("%s failed: %s (%d)\n", what, test_strerror(errnum), errnum);
}
//-----------------------------------------------------------------------------
static void mdbx_logger(int type, const char *function, int line,
const char *msg, va_list args) {
logging::loglevel level = logging::info;
if (type & MDBX_DBG_EXTRA)
level = logging::extra;
if (type & MDBX_DBG_TRACE)
level = logging::trace;
if (type & MDBX_DBG_PRINT)
level = logging::verbose;
if (!function)
function = "unknown";
if (type & MDBX_DBG_ASSERT) {
log_error("mdbx: assertion failure: %s, %d", function, line);
level = logging::failure;
}
if (logging::output(
level,
strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx: %s: ", function))
logging::feed_ap(msg, args);
if (type & MDBX_DBG_ASSERT)
abort();
}
namespace logging {
static std::string prefix;
static std::string suffix;
static loglevel level;
static FILE *last;
void setlevel(loglevel _level) {
level = (_level > error) ? failure : _level;
int mdbx_dbg_opts = MDBX_DBG_ASSERT | MDBX_DBG_JITTER | MDBX_DBG_DUMP;
if (level <= trace)
mdbx_dbg_opts |= MDBX_DBG_TRACE;
if (level <= verbose)
mdbx_dbg_opts |= MDBX_DBG_PRINT;
int rc = mdbx_setup_debug(mdbx_dbg_opts, mdbx_logger);
log_trace("set mdbx debug-opts: 0x%02x", rc);
}
void setup(loglevel _level, const std::string &_prefix) {
setlevel(_level);
prefix = _prefix;
}
void setup(const std::string &_prefix) { prefix = _prefix; }
const char *level2str(const loglevel alevel) {
switch (alevel) {
default:
return "invalid/unknown";
case extra:
return "extra";
case trace:
return "trace";
case verbose:
return "verbose";
case info:
return "info";
case notice:
return "notice";
case warning:
return "warning";
case error:
return "error";
case failure:
return "failure";
}
}
bool output(const loglevel priority, const char *format, ...) {
if (priority < level)
return false;
va_list ap;
va_start(ap, format);
output(priority, format, ap);
va_end(ap);
return true;
}
bool output(const logging::loglevel priority, const char *format, va_list ap) {
if (last) {
putc('\n', last);
fflush(last);
last = nullptr;
}
if (priority < level)
return false;
chrono::time now = chrono::now_realtime();
struct tm tm;
#ifdef _MSC_VER
int rc = _localtime32_s(&tm, (const __time32_t *)&now.utc);
#else
time_t time = now.utc;
int rc = localtime_r(&time, &tm) ? MDBX_SUCCESS : errno;
#endif
if (rc != MDBX_SUCCESS)
failure_perror("localtime_r()", rc);
last = stdout;
fprintf(last,
"[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05u %-10s %.4s ] %s" /* TODO */,
tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min,
tm.tm_sec, chrono::fractional2us(now.fractional), osal_getpid(),
prefix.c_str(), level2str(priority), suffix.c_str());
va_list ones;
memset(&ones, 0, sizeof(ones)) /* zap MSVC and other stupid compilers */;
if (priority >= error)
va_copy(ones, ap);
vfprintf(last, format, ap);
size_t len = strlen(format);
char end = len ? format[len - 1] : '\0';
switch (end) {
default:
putc('\n', last);
// fall through
case '\n':
fflush(last);
last = nullptr;
// fall through
case ' ':
case '_':
case ':':
case '|':
case ',':
case '\t':
case '\b':
case '\r':
case '\0':
break;
}
if (priority >= error) {
if (last != stderr) {
fprintf(stderr, "[ %05u %-10s %.4s ] %s", osal_getpid(), prefix.c_str(),
level2str(priority), suffix.c_str());
vfprintf(stderr, format, ones);
if (end != '\n')
putc('\n', stderr);
fflush(stderr);
}
va_end(ones);
}
return true;
}
bool feed_ap(const char *format, va_list ap) {
if (!last)
return false;
vfprintf(last, format, ap);
size_t len = strlen(format);
if (len && format[len - 1] == '\n') {
fflush(last);
last = nullptr;
}
return true;
}
bool feed(const char *format, ...) {
if (!last)
return false;
va_list ap;
va_start(ap, format);
feed_ap(format, ap);
va_end(ap);
return true;
}
local_suffix::local_suffix(const char *c_str)
: trim_pos(suffix.size()), indent(0) {
suffix.append(c_str);
}
local_suffix::local_suffix(const std::string &str)
: trim_pos(suffix.size()), indent(0) {
suffix.append(str);
}
void local_suffix::push() {
indent += 1;
suffix.push_back('\t');
}
void local_suffix::pop() {
assert(indent > 0);
if (indent > 0) {
indent -= 1;
suffix.pop_back();
}
}
local_suffix::~local_suffix() { suffix.erase(trim_pos); }
} // namespace logging
void log_extra(const char *msg, ...) {
if (logging::extra >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::extra, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_trace(const char *msg, ...) {
if (logging::trace >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::trace, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_verbose(const char *msg, ...) {
if (logging::verbose >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::verbose, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_info(const char *msg, ...) {
if (logging::info >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::info, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_notice(const char *msg, ...) {
if (logging::notice >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::notice, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_warning(const char *msg, ...) {
if (logging::warning >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::warning, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_error(const char *msg, ...) {
if (logging::error >= logging::level) {
va_list ap;
va_start(ap, msg);
logging::output(logging::error, msg, ap);
va_end(ap);
} else
logging::last = nullptr;
}
void log_trouble(const char *where, const char *what, int errnum) {
log_error("%s: %s %s", where, what, test_strerror(errnum));
}
bool log_enabled(const logging::loglevel priority) {
return (priority >= logging::level);
}
void log_flush(void) { fflushall(); }

83
contrib/db/libmdbx/test/log.h Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
void __noreturn usage(void);
void __noreturn __printf_args(1, 2) failure(const char *fmt, ...);
void __noreturn failure_perror(const char *what, int errnum);
const char *test_strerror(int errnum);
namespace logging {
enum loglevel {
extra,
trace,
verbose,
info,
notice,
warning,
error,
failure,
};
const char *level2str(const loglevel level);
void setup(loglevel level, const std::string &prefix);
void setup(const std::string &prefix);
void setlevel(loglevel level);
bool output(const loglevel priority, const char *format, va_list ap);
bool __printf_args(2, 3)
output(const loglevel priority, const char *format, ...);
bool feed_ap(const char *format, va_list ap);
bool __printf_args(1, 2) feed(const char *format, ...);
class local_suffix {
protected:
size_t trim_pos;
int indent;
public:
local_suffix(const local_suffix &) = delete;
local_suffix(const local_suffix &&) = delete;
const local_suffix &operator=(const local_suffix &) = delete;
local_suffix(const char *c_str);
local_suffix(const std::string &str);
void push();
void pop();
~local_suffix();
};
} // namespace logging
void __printf_args(1, 2) log_extra(const char *msg, ...);
void __printf_args(1, 2) log_trace(const char *msg, ...);
void __printf_args(1, 2) log_verbose(const char *msg, ...);
void __printf_args(1, 2) log_info(const char *msg, ...);
void __printf_args(1, 2) log_notice(const char *msg, ...);
void __printf_args(1, 2) log_warning(const char *msg, ...);
void __printf_args(1, 2) log_error(const char *msg, ...);
void log_trouble(const char *where, const char *what, int errnum);
void log_flush(void);
bool log_enabled(const logging::loglevel priority);
#ifdef _DEBUG
#define TRACE(...) log_trace(__VA_ARGS__)
#else
#define TRACE(...) __noop(__VA_ARGS__)
#endif

138
contrib/db/libmdbx/test/long_stochastic.sh Normal file
View file

@ -0,0 +1,138 @@
#!/bin/bash
if ! which make cc c++ tee lz4 >/dev/null; then
echo "Please install the following prerequisites: make cc c++ tee lz4" >&2
exit 1
fi
set -euo pipefail
UNAME="$(uname -s 2>/dev/null || echo Unknown)"
case ${UNAME} in
Linux)
MAKE=make
if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then
TESTDB_DIR="/dev/shm/mdbx-test.$$"
fi
mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
if LC_ALL=C free | grep -q -i available; then
ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 7) / 1024))
else
ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 4) / 1024))
fi
;;
FreeBSD)
MAKE=gmake
if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then
for old_test_dir in $(ls -d /tmp/mdbx-test.[0-9]*); do
umount $old_test_dir && rm -r $old_test_dir
done
TESTDB_DIR="/tmp/mdbx-test.$$"
rm -rf $TESTDB_DIR && mkdir -p $TESTDB_DIR && mount -t tmpfs tmpfs $TESTDB_DIR
else
mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/*
fi
ram_avail_mb=$(($(LC_ALL=C vmstat -s | grep -ie '[0-9] pages free$' | cut -d p -f 1) * ($(LC_ALL=C vmstat -s | grep -ie '[0-9] bytes per page$' | cut -d b -f 1) / 1024) / 1024))
;;
*)
echo "FIXME: ${UNAME} not supported by this script"
exit 2
;;
esac
echo "=== ${ram_avail_mb}M RAM available"
ram_reserve4logs_mb=1234
if [ $ram_avail_mb -lt $ram_reserve4logs_mb ]; then
echo "=== At least ${ram_reserve4logs_mb}Mb RAM required"
exit 3
fi
#
# В режимах отличных от MDBX_WRITEMAP изменения до записи в файл
# будут накапливаться в памяти, что может потребовать свободной
# памяти размером с БД. Кроме этого, в тест входит сценарий
# создания копия БД на ходу. Поэтому БД не может быть больше 1/3
# от доступной памяти. Однако, следует учесть что malloc() будет
# не сразу возвращать выделенную память системе, а также
# предусмотреть места для логов.
#
# In non-MDBX_WRITEMAP modes, updates (dirty pages) will
# accumulate in memory before writing to the disk, which may
# require a free memory up to the size of a whole database. In
# addition, the test includes a script create a copy of the
# database on the go. Therefore, the database cannot be more 1/3
# of available memory. Moreover, should be taken into account
# that malloc() will not return the allocated memory to the
# system immediately, as well some space is required for logs.
#
db_size_mb=$(((ram_avail_mb - ram_reserve4logs_mb) / 4))
if [ $db_size_mb -gt 3072 ]; then
db_size_mb=3072
fi
echo "=== use ${db_size_mb}M for DB"
${MAKE} TESTDB=${TESTDB_DIR}/smoke.db TESTLOG=${TESTDB_DIR}/smoke.log check
rm -f ${TESTDB_DIR}/*
###############################################################################
function rep9 { printf "%*s" $1 '' | tr ' ' '9'; }
function join { local IFS="$1"; shift; echo "$*"; }
function bit2option { local -n arr=$1; (( ($2&(1<<$3)) != 0 )) && echo -n '+' || echo -n '-'; echo "${arr[$3]}"; }
options=(writemap coalesce lifo)
function bits2list {
local -n arr=$1
local i
local list=()
for ((i=0; i<${#arr[@]}; ++i)) do
list[$i]=$(bit2option $1 $2 $i)
done
join , "${list[@]}"
}
function probe {
echo "=============================================== $(date)"
echo "${caption}: $*"
rm -f ${TESTDB_DIR}/* \
&& ./mdbx_test --ignore-dbfull --repeat=42 --pathname=${TESTDB_DIR}/long.db "$@" | lz4 > ${TESTDB_DIR}/long.log.lz4 \
&& ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db | tee ${TESTDB_DIR}/long-chk.log \
&& ([ ! -e ${TESTDB_DIR}/long.db-copy ] || ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db-copy | tee ${TESTDB_DIR}/long-chk-copy.log) \
|| (echo "FAILED"; exit 1)
}
###############################################################################
count=0
for nops in $(seq 2 6); do
for ((wbatch=nops-1; wbatch > 0; --wbatch)); do
loops=$(((111 >> nops) / nops + 3))
for ((rep=0; rep++ < loops; )); do
for ((bits=2**${#options[@]}; --bits >= 0; )); do
seed=$(($(date +%s) + RANDOM))
caption="Probe #$((++count)) int-key,w/o-dups, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
caption="Probe #$((++count)) int-key,with-dups, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
caption="Probe #$((++count)) int-key,int-data, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.integer --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
caption="Probe #$((++count)) w/o-dups, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
caption="Probe #$((++count)) with-dups, repeat ${rep} of ${loops}" probe \
--pagesize=min --size-upper=${db_size_mb}M --table=+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \
--nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \
--keygen.seed=${seed} basic
done
done
done
done
echo "=== ALL DONE ====================== $(date)"

488
contrib/db/libmdbx/test/main.cc Normal file
View file

@ -0,0 +1,488 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
void __noreturn usage(void) {
printf("usage:\n"
"\tFIXME\n");
exit(EXIT_FAILURE);
}
//-----------------------------------------------------------------------------
void actor_params::set_defaults(const std::string &tmpdir) {
pathname_log = "";
loglevel =
#ifdef NDEBUG
logging::info;
#elif defined(_WIN32) || defined(_WIN64)
logging::verbose;
#else
logging::trace;
#endif
pathname_db = tmpdir + "mdbx-test.db";
mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NORDAHEAD |
MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_LIFORECLAIM;
table_flags = MDBX_DUPSORT;
size_lower = -1;
size_now =
intptr_t(1024) * 1024 * ((table_flags & MDBX_DUPSORT) ? 256 : 1024);
size_upper = -1;
shrink_threshold = -1;
growth_step = -1;
pagesize = -1;
keygen.seed = 1;
keygen.keycase = kc_random;
keygen.width = (table_flags & MDBX_DUPSORT) ? 32 : 64;
keygen.mesh = keygen.width;
keygen.split = keygen.width / 2;
keygen.rotate = 3;
keygen.offset = 41;
test_duration = 0;
test_nops = 1000;
nrepeat = 1;
nthreads = 1;
keylen_min = mdbx_keylen_min();
keylen_max = mdbx_keylen_max();
datalen_min = mdbx_datalen_min();
datalen_max = std::min(mdbx_datalen_max(), 256u * 1024 + 42);
batch_read = 42;
batch_write = 42;
delaystart = 0;
waitfor_nops = 0;
inject_writefaultn = 0;
drop_table = false;
ignore_dbfull = false;
max_readers = 42;
max_tables = 42;
global::config::timeout_duration_seconds = 0 /* infinite */;
global::config::dump_config = true;
global::config::cleanup_before = true;
global::config::cleanup_after = true;
global::config::failfast = true;
global::config::progress_indicator = osal_istty(STDERR_FILENO);
}
namespace global {
std::vector<actor_config> actors;
std::unordered_map<unsigned, actor_config *> events;
std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
std::set<std::string> databases;
unsigned nactors;
chrono::time start_motonic;
chrono::time deadline_motonic;
bool singlemode;
namespace config {
unsigned timeout_duration_seconds;
bool dump_config;
bool cleanup_before;
bool cleanup_after;
bool failfast;
bool progress_indicator;
} /* namespace config */
} /* namespace global */
//-----------------------------------------------------------------------------
const char global::thunk_param_prefix[] = "--execute=";
std::string thunk_param(const actor_config &config) {
return config.serialize(global::thunk_param_prefix);
}
void cleanup() {
log_trace(">> cleanup");
/* TODO: remove each database */
log_trace("<< cleanup");
}
int main(int argc, char *const argv[]) {
#ifdef _DEBUG
log_trace("#argc = %d", argc);
for (int i = 0; i < argc; ++i)
log_trace("#argv[%d] = %s", i, argv[i]);
#endif /* _DEBUG */
if (argc < 2)
failure("No parameters given\n");
if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix,
strlen(global::thunk_param_prefix)) == 0)
return test_execute(
actor_config(argv[1] + strlen(global::thunk_param_prefix)))
? EXIT_SUCCESS
: EXIT_FAILURE;
actor_params params;
params.set_defaults(osal_tempdir());
global::config::dump_config = true;
logging::setup((logging::loglevel)params.loglevel, "main");
unsigned last_space_id = 0;
for (int narg = 1; narg < argc; ++narg) {
const char *value = nullptr;
if (config::parse_option(argc, argv, narg, "case", &value)) {
testcase_setup(value, params, last_space_id);
continue;
}
if (config::parse_option(argc, argv, narg, "pathname", params.pathname_db))
continue;
if (config::parse_option(argc, argv, narg, "mode", params.mode_flags,
config::mode_bits))
continue;
if (config::parse_option(argc, argv, narg, "table", params.table_flags,
config::table_bits)) {
if ((params.table_flags & MDBX_DUPFIXED) == 0)
params.table_flags &= ~MDBX_INTEGERDUP;
if ((params.table_flags & MDBX_DUPSORT) == 0)
params.table_flags &=
~(MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP);
continue;
}
if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize,
mdbx_limits_pgsize_min(),
mdbx_limits_pgsize_max())) {
const unsigned keylen_max = params.mdbx_keylen_max();
if (params.keylen_min > keylen_max)
params.keylen_min = keylen_max;
if (params.keylen_max > keylen_max)
params.keylen_max = keylen_max;
const unsigned datalen_max = params.mdbx_datalen_max();
if (params.datalen_min > datalen_max)
params.datalen_min = datalen_max;
if (params.datalen_max > datalen_max)
params.datalen_max = datalen_max;
continue;
}
if (config::parse_option_intptr(argc, argv, narg, "size-lower",
params.size_lower,
mdbx_limits_dbsize_min(params.pagesize),
mdbx_limits_dbsize_max(params.pagesize)))
continue;
if (config::parse_option_intptr(argc, argv, narg, "size-upper",
params.size_upper,
mdbx_limits_dbsize_min(params.pagesize),
mdbx_limits_dbsize_max(params.pagesize)))
continue;
if (config::parse_option_intptr(argc, argv, narg, "size", params.size_now,
mdbx_limits_dbsize_min(params.pagesize),
mdbx_limits_dbsize_max(params.pagesize)))
continue;
if (config::parse_option(
argc, argv, narg, "shrink-threshold", params.shrink_threshold, 0,
(int)std::min((intptr_t)INT_MAX,
mdbx_limits_dbsize_max(params.pagesize) -
mdbx_limits_dbsize_min(params.pagesize))))
continue;
if (config::parse_option(
argc, argv, narg, "growth-step", params.growth_step, 0,
(int)std::min((intptr_t)INT_MAX,
mdbx_limits_dbsize_max(params.pagesize) -
mdbx_limits_dbsize_min(params.pagesize))))
continue;
if (config::parse_option(argc, argv, narg, "keygen.width",
params.keygen.width, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.mesh",
params.keygen.mesh, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.seed",
params.keygen.seed, config::no_scale))
continue;
if (config::parse_option(argc, argv, narg, "keygen.split",
params.keygen.split, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.rotate",
params.keygen.rotate, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "keygen.offset",
params.keygen.offset, config::binary))
continue;
if (config::parse_option(argc, argv, narg, "keygen.case", &value)) {
keycase_setup(value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat,
config::no_scale))
continue;
if (config::parse_option(argc, argv, narg, "threads", params.nthreads,
config::no_scale, 1, 64))
continue;
if (config::parse_option(argc, argv, narg, "timeout",
global::config::timeout_duration_seconds,
config::duration, 1))
continue;
if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min,
config::no_scale, params.mdbx_keylen_min(),
params.mdbx_keylen_max())) {
if ((params.table_flags & MDBX_INTEGERKEY) ||
params.keylen_max < params.keylen_min)
params.keylen_max = params.keylen_min;
continue;
}
if (config::parse_option(argc, argv, narg, "keylen.max", params.keylen_max,
config::no_scale, params.mdbx_keylen_min(),
params.mdbx_keylen_max())) {
if ((params.table_flags & MDBX_INTEGERKEY) ||
params.keylen_min > params.keylen_max)
params.keylen_min = params.keylen_max;
continue;
}
if (config::parse_option(argc, argv, narg, "datalen.min",
params.datalen_min, config::no_scale,
params.mdbx_datalen_min(),
params.mdbx_datalen_max())) {
if ((params.table_flags & MDBX_DUPFIXED) ||
params.datalen_max < params.datalen_min)
params.datalen_max = params.datalen_min;
continue;
}
if (config::parse_option(argc, argv, narg, "datalen.max",
params.datalen_max, config::no_scale,
params.mdbx_datalen_min(),
params.mdbx_datalen_max())) {
if ((params.table_flags & MDBX_DUPFIXED) ||
params.datalen_min > params.datalen_max)
params.datalen_min = params.datalen_max;
continue;
}
if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read,
config::no_scale, 1))
continue;
if (config::parse_option(argc, argv, narg, "batch.write",
params.batch_write, config::no_scale, 1))
continue;
if (config::parse_option(argc, argv, narg, "delay", params.delaystart,
config::duration))
continue;
if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops,
config::decimal))
continue;
if (config::parse_option(argc, argv, narg, "inject-writefault",
params.inject_writefaultn, config::decimal))
continue;
if (config::parse_option(argc, argv, narg, "drop", params.drop_table))
continue;
if (config::parse_option(argc, argv, narg, "ignore-dbfull",
params.ignore_dbfull))
continue;
if (config::parse_option(argc, argv, narg, "dump-config",
global::config::dump_config))
continue;
if (config::parse_option(argc, argv, narg, "cleanup-before",
global::config::cleanup_before))
continue;
if (config::parse_option(argc, argv, narg, "cleanup-after",
global::config::cleanup_after))
continue;
if (config::parse_option(argc, argv, narg, "max-readers",
params.max_readers, config::no_scale, 1, 255))
continue;
if (config::parse_option(argc, argv, narg, "max-tables", params.max_tables,
config::no_scale, 1, INT16_MAX))
continue;
if (config::parse_option(argc, argv, narg, "no-delay", nullptr)) {
params.delaystart = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "no-wait", nullptr)) {
params.waitfor_nops = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "duration", params.test_duration,
config::duration, 1)) {
params.test_nops = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "nops", params.test_nops,
config::decimal, 1)) {
params.test_duration = 0;
continue;
}
if (config::parse_option(argc, argv, narg, "hill", &value, "auto")) {
configure_actor(last_space_id, ac_hill, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "jitter", nullptr)) {
configure_actor(last_space_id, ac_jitter, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "dead.reader", nullptr)) {
configure_actor(last_space_id, ac_deadread, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "dead.writer", nullptr)) {
configure_actor(last_space_id, ac_deadwrite, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "copy", nullptr)) {
configure_actor(last_space_id, ac_copy, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "append", nullptr)) {
configure_actor(last_space_id, ac_append, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "ttl", nullptr)) {
configure_actor(last_space_id, ac_ttl, value, params);
continue;
}
if (config::parse_option(argc, argv, narg, "failfast",
global::config::failfast))
continue;
if (config::parse_option(argc, argv, narg, "progress",
global::config::progress_indicator))
continue;
if (*argv[narg] != '-')
testcase_setup(argv[narg], params, last_space_id);
else
failure("Unknown option '%s'\n", argv[narg]);
}
if (global::config::dump_config)
config::dump();
//--------------------------------------------------------------------------
if (global::actors.empty()) {
log_notice("no testcase(s) configured, exiting");
return EXIT_SUCCESS;
}
bool failed = false;
global::start_motonic = chrono::now_motonic();
global::deadline_motonic.fixedpoint =
(global::config::timeout_duration_seconds == 0)
? chrono::infinite().fixedpoint
: global::start_motonic.fixedpoint +
chrono::from_seconds(global::config::timeout_duration_seconds)
.fixedpoint;
if (global::config::cleanup_before)
cleanup();
log_trace(">> probe entropy_ticks()");
entropy_ticks();
log_trace("<< probe entropy_ticks()");
if (global::actors.size() == 1) {
logging::setup("main");
global::singlemode = true;
if (!test_execute(global::actors.front()))
failed = true;
} else {
logging::setup("overlord");
log_trace("=== preparing...");
log_trace(">> osal_setup");
osal_setup(global::actors);
log_trace("<< osal_setup");
for (auto &a : global::actors) {
mdbx_pid_t pid;
log_trace(">> actor_start");
int rc = osal_actor_start(a, pid);
log_trace("<< actor_start");
if (rc) {
log_trace(">> killall_actors: (%s)", "start failed");
osal_killall_actors();
log_trace("<< killall_actors");
failure("Failed to start actor #%u (%s)\n", a.actor_id,
test_strerror(rc));
}
global::pid2actor[pid] = &a;
}
log_trace("=== ready to start...");
atexit(osal_killall_actors);
log_trace(">> wait4barrier");
osal_wait4barrier();
log_trace("<< wait4barrier");
size_t left = global::actors.size();
log_trace("=== polling...");
while (left > 0) {
unsigned timeout_seconds_left = INT_MAX;
chrono::time now_motonic = chrono::now_motonic();
if (now_motonic.fixedpoint >= global::deadline_motonic.fixedpoint)
timeout_seconds_left = 0;
else {
chrono::time left_motonic;
left_motonic.fixedpoint =
global::deadline_motonic.fixedpoint - now_motonic.fixedpoint;
timeout_seconds_left = left_motonic.seconds();
}
mdbx_pid_t pid;
int rc = osal_actor_poll(pid, timeout_seconds_left);
if (rc)
failure("Poll error: %s (%d)\n", test_strerror(rc), rc);
if (pid) {
actor_status status = osal_actor_info(pid);
actor_config *actor = global::pid2actor.at(pid);
if (!actor)
continue;
log_info("actor #%u, id %d, pid %u: %s\n", actor->actor_id,
actor->space_id, pid, status2str(status));
if (status > as_running) {
left -= 1;
if (status != as_successful) {
if (global::config::failfast && !failed) {
log_trace(">> killall_actors: (%s)", "failfast");
osal_killall_actors();
log_trace("<< killall_actors");
}
failed = true;
}
}
} else {
if (timeout_seconds_left == 0)
failure("Timeout\n");
}
}
log_trace("=== done...");
}
log_notice("RESULT: %s\n", failed ? "Failed" : "Successful");
if (global::config::cleanup_before) {
if (failed)
log_info("skip cleanup");
else
cleanup();
}
return failed ? EXIT_FAILURE : EXIT_SUCCESS;
}

313
contrib/db/libmdbx/test/osal-unix.cc Normal file
View file

@ -0,0 +1,313 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <pthread.h>
#include <signal.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#ifdef __APPLE__
#include "darwin/pthread_barrier.c"
#endif
struct shared_t {
pthread_barrier_t barrier;
pthread_mutex_t mutex;
size_t conds_size;
pthread_cond_t conds[1];
};
static shared_t *shared;
void osal_wait4barrier(void) {
assert(shared != nullptr && shared != MAP_FAILED);
int rc = pthread_barrier_wait(&shared->barrier);
if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) {
failure_perror("pthread_barrier_wait(shared)", rc);
}
}
void osal_setup(const std::vector<actor_config> &actors) {
assert(shared == nullptr);
pthread_mutexattr_t mutexattr;
int rc = pthread_mutexattr_init(&mutexattr);
if (rc)
failure_perror("pthread_mutexattr_init()", rc);
rc = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_mutexattr_setpshared()", rc);
pthread_barrierattr_t barrierattr;
rc = pthread_barrierattr_init(&barrierattr);
if (rc)
failure_perror("pthread_barrierattr_init()", rc);
rc = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_barrierattr_setpshared()", rc);
pthread_condattr_t condattr;
rc = pthread_condattr_init(&condattr);
if (rc)
failure_perror("pthread_condattr_init()", rc);
rc = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED);
if (rc)
failure_perror("pthread_condattr_setpshared()", rc);
shared = (shared_t *)mmap(
nullptr, sizeof(shared_t) + actors.size() * sizeof(pthread_cond_t),
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (MAP_FAILED == (void *)shared)
failure_perror("mmap(shared_conds)", errno);
rc = pthread_mutex_init(&shared->mutex, &mutexattr);
if (rc)
failure_perror("pthread_mutex_init(shared)", rc);
rc = pthread_barrier_init(&shared->barrier, &barrierattr, actors.size() + 1);
if (rc)
failure_perror("pthread_barrier_init(shared)", rc);
const size_t n = actors.size() + 1;
for (size_t i = 0; i < n; ++i) {
pthread_cond_t *event = &shared->conds[i];
rc = pthread_cond_init(event, &condattr);
if (rc)
failure_perror("pthread_cond_init(shared)", rc);
log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i,
event);
}
shared->conds_size = actors.size() + 1;
pthread_barrierattr_destroy(&barrierattr);
pthread_condattr_destroy(&condattr);
pthread_mutexattr_destroy(&mutexattr);
}
void osal_broadcast(unsigned id) {
assert(shared != nullptr && shared != MAP_FAILED);
log_trace("osal_broadcast: event %u", id);
if (id >= shared->conds_size)
failure("osal_broadcast: id > limit");
int rc = pthread_cond_broadcast(shared->conds + id);
if (rc)
failure_perror("sem_post(shared)", rc);
}
int osal_waitfor(unsigned id) {
assert(shared != nullptr && shared != MAP_FAILED);
log_trace("osal_waitfor: event %u", id);
if (id >= shared->conds_size)
failure("osal_waitfor: id > limit");
int rc = pthread_mutex_lock(&shared->mutex);
if (rc != 0)
failure_perror("pthread_mutex_lock(shared)", rc);
rc = pthread_cond_wait(shared->conds + id, &shared->mutex);
if (rc && rc != EINTR)
failure_perror("pthread_cond_wait(shared)", rc);
rc = pthread_mutex_unlock(&shared->mutex);
if (rc != 0)
failure_perror("pthread_mutex_unlock(shared)", rc);
return (rc == 0) ? true : false;
}
//-----------------------------------------------------------------------------
const std::string
actor_config::osal_serialize(simple_checksum &checksum) const {
(void)checksum;
/* not used in workload, but just for testing */
return "unix.fork";
}
bool actor_config::osal_deserialize(const char *str, const char *end,
simple_checksum &checksum) {
(void)checksum;
/* not used in workload, but just for testing */
return strncmp(str, "unix.fork", 9) == 0 && str + 9 == end;
}
//-----------------------------------------------------------------------------
static std::unordered_map<pid_t, actor_status> childs;
static void handler_SIGCHLD(int unused) { (void)unused; }
mdbx_pid_t osal_getpid(void) { return getpid(); }
int osal_delay(unsigned seconds) { return sleep(seconds) ? errno : 0; }
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
if (childs.empty())
signal(SIGCHLD, handler_SIGCHLD);
pid = fork();
if (pid == 0) {
const bool result = test_execute(config);
exit(result ? EXIT_SUCCESS : EXIT_FAILURE);
}
if (pid < 0)
return errno;
log_trace("osal_actor_start: fork pid %i for %u", pid, config.actor_id);
childs[pid] = as_running;
return 0;
}
actor_status osal_actor_info(const mdbx_pid_t pid) { return childs.at(pid); }
void osal_killall_actors(void) {
for (auto &pair : childs) {
kill(pair.first, SIGKILL);
pair.second = as_killed;
}
}
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) {
struct timespec ts;
ts.tv_nsec = 0;
ts.tv_sec = (timeout > INT_MAX) ? INT_MAX : timeout;
retry:
int status, options = WNOHANG;
#ifdef WUNTRACED
options |= WUNTRACED;
#endif
#ifdef WCONTINUED
options |= WCONTINUED;
#endif
pid = waitpid(0, &status, options);
if (pid > 0) {
if (WIFEXITED(status))
childs[pid] =
(WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed;
else if (WCOREDUMP(status))
childs[pid] = as_coredump;
else if (WIFSIGNALED(status))
childs[pid] = as_killed;
else if (WIFSTOPPED(status))
childs[pid] = as_debuging;
else if (WIFCONTINUED(status))
childs[pid] = as_running;
else {
assert(false);
}
return 0;
}
if (pid == 0) {
/* child still running */
if (ts.tv_sec == 0 && ts.tv_nsec == 0)
ts.tv_nsec = 1;
if (nanosleep(&ts, &ts) == 0) {
/* timeout and no signal from child */
pid = 0;
return 0;
}
if (errno == EINTR)
goto retry;
}
switch (errno) {
case EINTR:
pid = 0;
return 0;
case ECHILD:
default:
pid = 0;
return errno;
}
}
void osal_yield(void) {
if (sched_yield())
failure_perror("sched_yield()", errno);
}
void osal_udelay(unsigned us) {
chrono::time until, now = chrono::now_motonic();
until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint;
struct timespec ts;
static unsigned threshold_us;
if (threshold_us == 0) {
if (clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts)) {
int rc = errno;
failure_perror("clock_getres(CLOCK_PROCESS_CPUTIME_ID)", rc);
}
chrono::time threshold = chrono::from_timespec(ts);
assert(threshold.seconds() == 0);
threshold_us = chrono::fractional2us(threshold.fractional);
if (threshold_us < 1000)
threshold_us = 1000;
}
ts.tv_sec = ts.tv_nsec = 0;
if (us > threshold_us) {
ts.tv_sec = us / 1000000u;
ts.tv_nsec = (us % 1000000u) * 1000u;
}
do {
if (us > threshold_us) {
if (nanosleep(&ts, &ts)) {
int rc = errno;
/* if (rc == EINTR) { ... } ? */
failure_perror("usleep()", rc);
}
us = ts.tv_sec * 1000000u + ts.tv_nsec / 1000u;
}
cpu_relax();
now = chrono::now_motonic();
} while (until.fixedpoint > now.fixedpoint);
}
bool osal_istty(int fd) { return isatty(fd) == 1; }
std::string osal_tempdir(void) {
const char *tempdir = getenv("TMPDIR");
if (!tempdir)
tempdir = getenv("TMP");
if (!tempdir)
tempdir = getenv("TEMPDIR");
if (!tempdir)
tempdir = getenv("TEMP");
if (tempdir) {
std::string dir(tempdir);
if (!dir.empty() && dir.at(dir.length() - 1) != '/')
dir.append("/");
return dir;
}
if (access("/dev/shm/", R_OK | W_OK | X_OK) == 0)
return "/dev/shm/";
return "";
}
int osal_removefile(const std::string &pathname) {
return unlink(pathname.c_str()) ? errno : MDBX_SUCCESS;
}

419
contrib/db/libmdbx/test/osal-windows.cc Normal file
View file

@ -0,0 +1,419 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
static std::unordered_map<unsigned, HANDLE> events;
static HANDLE hBarrierSemaphore, hBarrierEvent;
static int waitstatus2errcode(DWORD result) {
switch (result) {
case WAIT_OBJECT_0:
return MDBX_SUCCESS;
case WAIT_FAILED:
return GetLastError();
case WAIT_ABANDONED:
return ERROR_ABANDONED_WAIT_0;
case WAIT_IO_COMPLETION:
return ERROR_USER_APC;
case WAIT_TIMEOUT:
return ERROR_TIMEOUT;
default:
return ERROR_UNHANDLED_ERROR;
}
}
void osal_wait4barrier(void) {
DWORD rc = WaitForSingleObject(hBarrierSemaphore, 0);
switch (rc) {
default:
failure_perror("WaitForSingleObject(BarrierSemaphore)",
waitstatus2errcode(rc));
case WAIT_OBJECT_0:
rc = WaitForSingleObject(hBarrierEvent, INFINITE);
if (rc != WAIT_OBJECT_0)
failure_perror("WaitForSingleObject(BarrierEvent)",
waitstatus2errcode(rc));
break;
case WAIT_TIMEOUT:
if (!SetEvent(hBarrierEvent))
failure_perror("SetEvent(BarrierEvent)", GetLastError());
break;
}
}
static HANDLE make_inheritable(HANDLE hHandle) {
assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE);
if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(),
&hHandle, 0, TRUE,
DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS))
failure_perror("DuplicateHandle()", GetLastError());
return hHandle;
}
void osal_setup(const std::vector<actor_config> &actors) {
assert(events.empty());
const size_t n = actors.size() + 1;
events.reserve(n);
for (unsigned i = 0; i < n; ++i) {
HANDLE hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (!hEvent)
failure_perror("CreateEvent()", GetLastError());
hEvent = make_inheritable(hEvent);
log_trace("osal_setup: event %" PRIuPTR " -> %p", i, hEvent);
events[i] = hEvent;
}
hBarrierSemaphore = CreateSemaphore(NULL, 0, (LONG)actors.size(), NULL);
if (!hBarrierSemaphore)
failure_perror("CreateSemaphore(BarrierSemaphore)", GetLastError());
hBarrierSemaphore = make_inheritable(hBarrierSemaphore);
hBarrierEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (!hBarrierEvent)
failure_perror("CreateEvent(BarrierEvent)", GetLastError());
hBarrierEvent = make_inheritable(hBarrierEvent);
}
void osal_broadcast(unsigned id) {
log_trace("osal_broadcast: event %u", id);
if (!SetEvent(events.at(id)))
failure_perror("SetEvent()", GetLastError());
}
int osal_waitfor(unsigned id) {
log_trace("osal_waitfor: event %u", id);
DWORD rc = WaitForSingleObject(events.at(id), INFINITE);
return waitstatus2errcode(rc);
}
mdbx_pid_t osal_getpid(void) { return GetCurrentProcessId(); }
int osal_delay(unsigned seconds) {
Sleep(seconds * 1000u);
return 0;
}
//-----------------------------------------------------------------------------
const std::string
actor_config::osal_serialize(simple_checksum &checksum) const {
checksum.push(hBarrierSemaphore);
checksum.push(hBarrierEvent);
HANDLE hWait = INVALID_HANDLE_VALUE;
if (wait4id) {
hWait = events.at(wait4id);
checksum.push(hWait);
}
HANDLE hSignal = INVALID_HANDLE_VALUE;
if (wanna_event4signalling()) {
hSignal = events.at(actor_id);
checksum.push(hSignal);
}
return format("%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait,
hSignal);
}
bool actor_config::osal_deserialize(const char *str, const char *end,
simple_checksum &checksum) {
std::string copy(str, end - str);
TRACE(">> osal_deserialize(%s)\n", copy.c_str());
assert(hBarrierSemaphore == 0);
assert(hBarrierEvent == 0);
assert(events.empty());
HANDLE hWait, hSignal;
if (sscanf_s(copy.c_str(), "%p.%p.%p.%p", &hBarrierSemaphore, &hBarrierEvent,
&hWait, &hSignal) != 4) {
TRACE("<< osal_deserialize: failed\n");
return false;
}
checksum.push(hBarrierSemaphore);
checksum.push(hBarrierEvent);
if (wait4id) {
checksum.push(hWait);
events[wait4id] = hWait;
}
if (wanna_event4signalling()) {
checksum.push(hSignal);
events[actor_id] = hSignal;
}
TRACE("<< osal_deserialize: OK\n");
return true;
}
//-----------------------------------------------------------------------------
typedef std::pair<HANDLE, actor_status> child;
static std::unordered_map<mdbx_pid_t, child> childs;
static void ArgvQuote(std::string &CommandLine, const std::string &Argument,
bool Force = false)
/*++
https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
Routine Description:
This routine appends the given argument to a command line such
that CommandLineToArgvW will return the argument string unchanged.
Arguments in a command line should be separated by spaces; this
function does not add these spaces.
Arguments:
Argument - Supplies the argument to encode.
CommandLine - Supplies the command line to which we append the encoded
argument string.
Force - Supplies an indication of whether we should quote
the argument even if it does not contain any characters that would
ordinarily require quoting.
Return Value:
None.
Environment:
Arbitrary.
--*/
{
//
// Unless we're told otherwise, don't quote unless we actually
// need to do so --- hopefully avoid problems if programs won't
// parse quotes properly
//
if (Force == false && Argument.empty() == false &&
Argument.find_first_of(" \t\n\v\"") == Argument.npos) {
CommandLine.append(Argument);
} else {
CommandLine.push_back('"');
for (auto It = Argument.begin();; ++It) {
unsigned NumberBackslashes = 0;
while (It != Argument.end() && *It == '\\') {
++It;
++NumberBackslashes;
}
if (It == Argument.end()) {
//
// Escape all backslashes, but let the terminating
// double quotation mark we add below be interpreted
// as a metacharacter.
//
CommandLine.append(NumberBackslashes * 2, '\\');
break;
} else if (*It == L'"') {
//
// Escape all backslashes and the following
// double quotation mark.
//
CommandLine.append(NumberBackslashes * 2 + 1, '\\');
CommandLine.push_back(*It);
} else {
//
// Backslashes aren't special here.
//
CommandLine.append(NumberBackslashes, '\\');
CommandLine.push_back(*It);
}
}
CommandLine.push_back('"');
}
}
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) {
if (childs.size() == MAXIMUM_WAIT_OBJECTS)
failure("Could't manage more that %u actors on Windows\n",
MAXIMUM_WAIT_OBJECTS);
_flushall();
STARTUPINFOA StartupInfo;
GetStartupInfoA(&StartupInfo);
char exename[_MAX_PATH + 1];
DWORD exename_size = sizeof(exename);
if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename,
&exename_size))
failure_perror("QueryFullProcessImageName()", GetLastError());
if (exename[1] != ':') {
exename_size = GetModuleFileName(NULL, exename, sizeof(exename));
if (exename_size >= sizeof(exename))
return ERROR_BAD_LENGTH;
}
std::string cmdline = "$ ";
ArgvQuote(cmdline, thunk_param(config));
if (cmdline.size() >= 32767)
return ERROR_BAD_LENGTH;
PROCESS_INFORMATION ProcessInformation;
if (!CreateProcessA(exename, const_cast<char *>(cmdline.c_str()),
NULL, // Retuned process handle is not inheritable.
NULL, // Retuned thread handle is not inheritable.
TRUE, // Child inherits all inheritable handles.
NORMAL_PRIORITY_CLASS | INHERIT_PARENT_AFFINITY,
NULL, // Inherit the parent's environment.
NULL, // Inherit the parent's current directory.
&StartupInfo, &ProcessInformation))
failure_perror(exename, GetLastError());
CloseHandle(ProcessInformation.hThread);
pid = ProcessInformation.dwProcessId;
childs[pid] = std::make_pair(ProcessInformation.hProcess, as_running);
return 0;
}
actor_status osal_actor_info(const mdbx_pid_t pid) {
actor_status status = childs.at(pid).second;
if (status > as_running)
return status;
DWORD ExitCode;
if (!GetExitCodeProcess(childs.at(pid).first, &ExitCode))
failure_perror("GetExitCodeProcess()", GetLastError());
switch (ExitCode) {
case STILL_ACTIVE:
return as_running;
case EXIT_SUCCESS:
status = as_successful;
break;
case EXCEPTION_BREAKPOINT:
case EXCEPTION_SINGLE_STEP:
status = as_debuging;
break;
case STATUS_CONTROL_C_EXIT:
status = as_killed;
break;
case EXCEPTION_ACCESS_VIOLATION:
case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
case EXCEPTION_DATATYPE_MISALIGNMENT:
case EXCEPTION_STACK_OVERFLOW:
case EXCEPTION_INVALID_DISPOSITION:
case EXCEPTION_ILLEGAL_INSTRUCTION:
case EXCEPTION_NONCONTINUABLE_EXCEPTION:
status = as_coredump;
break;
default:
status = as_failed;
break;
}
childs.at(pid).second = status;
return status;
}
void osal_killall_actors(void) {
for (auto &pair : childs)
TerminateProcess(pair.second.first, STATUS_CONTROL_C_EXIT);
}
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) {
std::vector<HANDLE> handles;
handles.reserve(childs.size());
for (const auto &pair : childs)
if (pair.second.second <= as_running)
handles.push_back(pair.second.first);
DWORD rc =
MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0],
(timeout > 60) ? 60 * 1000 : timeout * 1000,
QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0);
if (rc >= WAIT_OBJECT_0 && rc < WAIT_OBJECT_0 + handles.size()) {
pid = 0;
for (const auto &pair : childs)
if (pair.second.first == handles[rc - WAIT_OBJECT_0]) {
pid = pair.first;
break;
}
return 0;
}
if (rc == WAIT_TIMEOUT) {
pid = 0;
return 0;
}
return waitstatus2errcode(rc);
}
void osal_yield(void) { SwitchToThread(); }
void osal_udelay(unsigned us) {
chrono::time until, now = chrono::now_motonic();
until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint;
static unsigned threshold_us;
if (threshold_us == 0) {
#if 1
unsigned timeslice_ms = 1;
while (timeBeginPeriod(timeslice_ms) == TIMERR_NOCANDO)
++timeslice_ms;
threshold_us = timeslice_ms * 1500u;
#else
ULONGLONG InterruptTimePrecise_100ns;
QueryInterruptTimePrecise(&InterruptTimePrecise_100ns);
threshold_us = InterruptTimePrecise_100ns / 5;
#endif
assert(threshold_us > 0);
}
do {
if (us > threshold_us && us > 1000) {
DWORD rc = SleepEx(us / 1000, TRUE);
if (rc)
failure_perror("SleepEx()", waitstatus2errcode(rc));
us = 0;
}
YieldProcessor();
now = chrono::now_motonic();
} while (now.fixedpoint < until.fixedpoint);
}
bool osal_istty(int fd) { return _isatty(fd) != 0; }
std::string osal_tempdir(void) {
char buf[MAX_PATH + 1];
DWORD len = GetTempPathA(sizeof(buf), buf);
return std::string(buf, len);
}
int osal_removefile(const std::string &pathname) {
return DeleteFileA(pathname.c_str()) ? MDBX_SUCCESS : GetLastError();
}

47
contrib/db/libmdbx/test/osal.h Normal file
View file

@ -0,0 +1,47 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
void osal_setup(const std::vector<actor_config> &actors);
void osal_broadcast(unsigned id);
int osal_waitfor(unsigned id);
int osal_actor_start(const actor_config &config, mdbx_pid_t &pid);
actor_status osal_actor_info(const mdbx_pid_t pid);
void osal_killall_actors(void);
int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout);
void osal_wait4barrier(void);
mdbx_pid_t osal_getpid(void);
int osal_delay(unsigned seconds);
void osal_udelay(unsigned us);
void osal_yield(void);
bool osal_istty(int fd);
std::string osal_tempdir(void);
int osal_removefile(const std::string &pathname);
#ifdef _MSC_VER
#ifndef STDIN_FILENO
#define STDIN_FILENO _fileno(stdin)
#endif
#ifndef STDOUT_FILENO
#define STDOUT_FILENO _fileno(stdout)
#endif
#ifndef STDERR_FILENO
#define STDERR_FILENO _fileno(stderr)
#endif
#endif /* _MSC_VER */

7
contrib/db/libmdbx/test/pcrf/CMakeLists.txt Normal file
View file

@ -0,0 +1,7 @@
set(TARGET pcrf_test)
project(${TARGET})
add_executable(${TARGET} pcrf_test.c)
target_link_libraries(${TARGET} mdbx)

2
contrib/db/libmdbx/test/pcrf/README.md Normal file
View file

@ -0,0 +1,2 @@
PCRF Session DB emulation test

404
contrib/db/libmdbx/test/pcrf/pcrf_test.c Normal file
View file

@ -0,0 +1,404 @@
/*
* Copyright 2016-2017 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2015 Vladimir Romanov
* <https://www.linkedin.com/in/vladimirromanov>, Yota Lab.
*
* This file is part of libmdbx.
*
* ReOpenMDBX is free software; you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* ReOpenMDBX is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <sys/stat.h>
#include <sys/time.h>
#include "mdbx.h"
#include <assert.h>
#include <inttypes.h>
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define IP_PRINTF_ARG_HOST(addr) \
(int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \
(int)((addr)&0xff)
char opt_db_path[PATH_MAX] = "/root/lmdbx_bench2";
static MDBX_env *env;
#define REC_COUNT 10240000
int64_t ids[REC_COUNT * 10];
int32_t ids_count = 0;
int64_t mdbx_add_count = 0;
int64_t mdbx_del_count = 0;
uint64_t mdbx_add_time = 0;
uint64_t mdbx_del_time = 0;
int64_t obj_id = 0;
int64_t mdbx_data_size = 0;
int64_t mdbx_key_size = 0;
typedef struct {
char session_id1[100];
char session_id2[100];
char ip[20];
uint8_t fill[100];
} session_data_t;
typedef struct {
int64_t obj_id;
int8_t event_type;
} __attribute__((__packed__)) event_data_t;
static void add_id_to_pool(int64_t id) {
ids[ids_count] = id;
ids_count++;
}
static inline int64_t getClockUs(void) {
struct timespec val;
#ifdef CYGWIN
clock_gettime(CLOCK_REALTIME, &val);
#else
clock_gettime(CLOCK_MONOTONIC, &val);
#endif
return val.tv_sec * ((int64_t)1000000) + val.tv_nsec / 1000;
}
static int64_t get_id_from_pool() {
if (ids_count == 0) {
return -1;
}
int32_t index = rand() % ids_count;
int64_t id = ids[index];
ids[index] = ids[ids_count - 1];
ids_count--;
return id;
}
#define MDBX_CHECK(x) \
do { \
const int rc = (x); \
if (rc != MDBX_SUCCESS) { \
printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, \
__FILE__, __LINE__); \
exit(EXIT_FAILURE); \
} \
} while (0)
static void db_connect() {
MDBX_dbi dbi_session;
MDBX_dbi dbi_session_id;
MDBX_dbi dbi_event;
MDBX_dbi dbi_ip;
MDBX_CHECK(mdbx_env_create(&env));
MDBX_CHECK(
mdbx_env_set_mapsize(env, REC_COUNT * sizeof(session_data_t) * 10));
MDBX_CHECK(mdbx_env_set_maxdbs(env, 30));
MDBX_CHECK(mdbx_env_open(env, opt_db_path,
MDBX_CREATE | MDBX_WRITEMAP | MDBX_MAPASYNC |
MDBX_NOSYNC | MDBX_LIFORECLAIM,
0664));
MDBX_txn *txn;
// transaction init
MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
// transaction commit
MDBX_CHECK(mdbx_txn_commit(txn));
printf("Connection open\n");
}
static void create_record(int64_t record_id) {
MDBX_dbi dbi_session;
MDBX_dbi dbi_session_id;
MDBX_dbi dbi_event;
MDBX_dbi dbi_ip;
event_data_t event;
MDBX_txn *txn;
session_data_t data;
// transaction init
snprintf(data.session_id1, sizeof(data.session_id1),
"prefix%02ld_%02ld.fill.fill.fill.fill.fill.fill;%ld",
record_id % 3 + 1, record_id % 9 + 1, record_id);
snprintf(data.session_id2, sizeof(data.session_id2),
"dprefix%ld;%ld.fill.fill.;suffix", record_id,
record_id % 1000000000 + 99999);
snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d",
IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF));
event.obj_id = record_id;
event.event_type = 1;
MDBX_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)};
MDBX_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)};
MDBX_val _ip_rec = {data.ip, strlen(data.ip)};
MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDBX_val _data_rec = {&data, offsetof(session_data_t, fill) +
(rand() % sizeof(data.fill))};
MDBX_val _event_rec = {&event, sizeof(event)};
uint64_t start = getClockUs();
MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
MDBX_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec,
MDBX_NOOVERWRITE | MDBX_NODUPDATA));
MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec,
MDBX_NOOVERWRITE | MDBX_NODUPDATA));
MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec,
MDBX_NOOVERWRITE | MDBX_NODUPDATA));
MDBX_CHECK(mdbx_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0));
MDBX_CHECK(mdbx_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0));
MDBX_CHECK(mdbx_txn_commit(txn));
mdbx_data_size += (_data_rec.iov_len + _obj_id_rec.iov_len * 4);
mdbx_key_size +=
(_obj_id_rec.iov_len + _session_id1_rec.iov_len +
_session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len);
// transaction commit
mdbx_add_count++;
mdbx_add_time += (getClockUs() - start);
}
static void delete_record(int64_t record_id) {
MDBX_dbi dbi_session;
MDBX_dbi dbi_session_id;
MDBX_dbi dbi_event;
MDBX_dbi dbi_ip;
event_data_t event;
MDBX_txn *txn;
// transaction init
uint64_t start = getClockUs();
MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn));
// open database in read-write mode
MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session));
MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id));
MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event));
MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip));
// put data
MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)};
MDBX_val _data_rec;
// get data
MDBX_CHECK(mdbx_get(txn, dbi_session, &_obj_id_rec, &_data_rec));
session_data_t *data = (session_data_t *)_data_rec.iov_base;
MDBX_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)};
MDBX_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)};
MDBX_val _ip_rec = {data->ip, strlen(data->ip)};
MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id1_rec, NULL));
MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id2_rec, NULL));
MDBX_CHECK(mdbx_del(txn, dbi_ip, &_ip_rec, NULL));
event.obj_id = record_id;
event.event_type = 1;
MDBX_val _event_rec = {&event, sizeof(event)};
MDBX_CHECK(mdbx_del(txn, dbi_event, &_event_rec, NULL));
MDBX_CHECK(mdbx_del(txn, dbi_session, &_obj_id_rec, NULL));
mdbx_data_size -= (_data_rec.iov_len + _obj_id_rec.iov_len * 4);
mdbx_key_size -=
(_obj_id_rec.iov_len + _session_id1_rec.iov_len +
_session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len);
// transaction commit
MDBX_CHECK(mdbx_txn_commit(txn));
mdbx_del_count++;
mdbx_del_time += (getClockUs() - start);
}
static void db_disconnect() {
mdbx_env_close(env);
printf("Connection closed\n");
}
static void get_db_stat(const char *db, int64_t *ms_branch_pages,
int64_t *ms_leaf_pages) {
MDBX_txn *txn;
MDBX_stat stat;
MDBX_dbi dbi;
MDBX_CHECK(mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn));
MDBX_CHECK(mdbx_dbi_open(txn, db, MDBX_CREATE, &dbi));
MDBX_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat)));
mdbx_txn_abort(txn);
printf("%15s | %15ld | %5u | %10ld | %10ld | %11ld |\n", db,
stat.ms_branch_pages, stat.ms_depth, stat.ms_entries,
stat.ms_leaf_pages, stat.ms_overflow_pages);
(*ms_branch_pages) += stat.ms_branch_pages;
(*ms_leaf_pages) += stat.ms_leaf_pages;
}
static void periodic_stat(void) {
int64_t ms_branch_pages = 0;
int64_t ms_leaf_pages = 0;
MDBX_stat mst;
MDBX_envinfo mei;
MDBX_CHECK(mdbx_env_stat(env, &mst, sizeof(mst)));
MDBX_CHECK(mdbx_env_info(env, &mei, sizeof(mei)));
printf("Environment Info\n");
printf(" Pagesize: %u\n", mst.ms_psize);
if (mei.mi_geo.lower != mei.mi_geo.upper) {
printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64
"/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64
"/-%" PRIu64 ")\n",
mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow,
mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize,
mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize,
mei.mi_geo.shrink / mst.ms_psize);
printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
} else {
printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n",
mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize);
}
printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n",
mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize);
printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1);
printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid);
printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n",
mei.mi_latter_reader_txnid,
mei.mi_latter_reader_txnid - mei.mi_recent_txnid);
printf(" Max readers: %u\n", mei.mi_maxreaders);
printf(" Number of readers used: %u\n", mei.mi_numreaders);
printf(" Name | ms_branch_pages | depth | entries | leaf_pages "
"| overf_pages |\n");
get_db_stat("session", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("event", &ms_branch_pages, &ms_leaf_pages);
get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages);
printf("%15s | %15ld | %5s | %10s | %10ld | %11s |\n", "", ms_branch_pages,
"", "", ms_leaf_pages, "");
static int64_t prev_add_count;
static int64_t prev_del_count;
static uint64_t prev_add_time;
static uint64_t prev_del_time;
static int64_t t = -1;
if (t > 0) {
int64_t delta = (getClockUs() - t);
printf(
"CPS: add %ld, delete %ld, items processed - %ldK data=%ldK key=%ldK\n",
(mdbx_add_count - prev_add_count) * 1000000 / delta,
(mdbx_del_count - prev_del_count) * 1000000 / delta, obj_id / 1024,
mdbx_data_size / 1024, mdbx_key_size / 1024);
printf("usage data=%ld%%", ((mdbx_data_size + mdbx_key_size) * 100) /
((ms_leaf_pages + ms_branch_pages) * 4096));
if (prev_add_time != mdbx_add_time) {
printf(" Add : %ld c/s", (mdbx_add_count - prev_add_count) * 1000000 /
(mdbx_add_time - prev_add_time));
}
if (prev_del_time != mdbx_del_time) {
printf(" Del : %ld c/s", (mdbx_del_count - prev_del_count) * 1000000 /
(mdbx_del_time - prev_del_time));
}
if (mdbx_add_time) {
printf(" tAdd : %ld c/s", mdbx_add_count * 1000000 / mdbx_add_time);
}
if (mdbx_del_time) {
printf(" tDel : %ld c/s", mdbx_del_count * 1000000 / mdbx_del_time);
}
puts("");
}
t = getClockUs();
prev_add_count = mdbx_add_count;
prev_del_count = mdbx_del_count;
prev_add_time = mdbx_add_time;
prev_del_time = mdbx_del_time;
}
// static void periodic_add_rec() {
// for (int i = 0; i < 10240; i++) {
// if (ids_count <= REC_COUNT) {
// int64_t id = obj_id++;
// create_record(id);
// add_id_to_pool(id);
// }
// if (ids_count > REC_COUNT) {
// int64_t id = get_id_from_pool();
// delete_record(id);
// }
// }
// periodic_stat();
//}
int main(int argc, char **argv) {
(void)argc;
(void)argv;
char filename[PATH_MAX];
int i;
mkdir(opt_db_path, 0775);
strcpy(filename, opt_db_path);
strcat(filename, "/mdbx.dat");
remove(filename);
strcpy(filename, opt_db_path);
strcat(filename, "/mdbx.lck");
remove(filename);
puts("Open DB...");
db_connect();
puts("Create data...");
int64_t t = getClockUs();
for (i = 0; i < REC_COUNT; i++) {
int64_t id = obj_id++;
create_record(id);
add_id_to_pool(id);
if (i % 1000 == 0) {
int64_t now = getClockUs();
if ((now - t) > 1000000L) {
periodic_stat();
t = now;
}
}
}
periodic_stat();
while (1) {
int i;
for (i = 0; i < 1000; i++) {
int64_t id = obj_id++;
create_record(id);
add_id_to_pool(id);
id = get_id_from_pool();
delete_record(id);
}
// for (i = 0; i < 50; i++) {
// int64_t id = obj_id++;
// create_record(id);
// add_id_to_pool(id);
// }
// int64_t id = obj_id++;
// create_record(id);
// add_id_to_pool(id);
int64_t now = getClockUs();
if ((now - t) > 10000000L) {
periodic_stat();
t = now;
}
}
db_disconnect();
return 0;
}

601
contrib/db/libmdbx/test/test.cc Normal file
View file

@ -0,0 +1,601 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
const char *testcase2str(const actor_testcase testcase) {
switch (testcase) {
default:
assert(false);
return "?!";
case ac_none:
return "none";
case ac_hill:
return "hill";
case ac_deadread:
return "deadread";
case ac_deadwrite:
return "deadwrite";
case ac_jitter:
return "jitter";
case ac_try:
return "try";
case ac_copy:
return "copy";
case ac_append:
return "append";
case ac_ttl:
return "ttl";
}
}
const char *status2str(actor_status status) {
switch (status) {
default:
assert(false);
return "?!";
case as_debuging:
return "debuging";
case as_running:
return "running";
case as_successful:
return "successful";
case as_killed:
return "killed";
case as_failed:
return "failed";
case as_coredump:
return "coredump";
}
}
const char *keygencase2str(const keygen_case keycase) {
switch (keycase) {
default:
assert(false);
return "?!";
case kc_random:
return "random";
case kc_dashes:
return "dashes";
case kc_custom:
return "custom";
}
}
//-----------------------------------------------------------------------------
int testcase::oom_callback(MDBX_env *env, int pid, mdbx_tid_t tid, uint64_t txn,
unsigned gap, int retry) {
testcase *self = (testcase *)mdbx_env_get_userctx(env);
if (retry == 0)
log_notice("oom_callback: waitfor pid %u, thread %" PRIuPTR
", txn #%" PRIu64 ", gap %d",
pid, (size_t)tid, txn, gap);
if (self->should_continue(true)) {
osal_yield();
if (retry > 0)
osal_udelay(retry * 100);
return 0 /* always retry */;
}
return -1;
}
void testcase::db_prepare() {
log_trace(">> db_prepare");
assert(!db_guard);
MDBX_env *env = nullptr;
int rc = mdbx_env_create(&env);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_create()", rc);
assert(env != nullptr);
db_guard.reset(env);
rc = mdbx_env_set_userctx(env, this);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_userctx()", rc);
rc = mdbx_env_set_maxreaders(env, config.params.max_readers);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_maxreaders()", rc);
rc = mdbx_env_set_maxdbs(env, config.params.max_tables);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_maxdbs()", rc);
rc = mdbx_env_set_oomfunc(env, testcase::oom_callback);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_oomfunc()", rc);
rc = mdbx_env_set_geometry(
env, config.params.size_lower, config.params.size_now,
config.params.size_upper, config.params.growth_step,
config.params.shrink_threshold, config.params.pagesize);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_set_mapsize()", rc);
log_trace("<< db_prepare");
}
void testcase::db_open() {
log_trace(">> db_open");
if (!db_guard)
db_prepare();
jitter_delay(true);
int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(),
(unsigned)config.params.mode_flags, 0640);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_env_open()", rc);
log_trace("<< db_open");
}
void testcase::db_close() {
log_trace(">> db_close");
cursor_guard.reset();
txn_guard.reset();
db_guard.reset();
log_trace("<< db_close");
}
void testcase::txn_begin(bool readonly, unsigned flags) {
assert((flags & MDBX_RDONLY) == 0);
log_trace(">> txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write",
flags);
assert(!txn_guard);
MDBX_txn *txn = nullptr;
int rc = mdbx_txn_begin(db_guard.get(), nullptr,
readonly ? flags | MDBX_RDONLY : flags, &txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_begin()", rc);
txn_guard.reset(txn);
log_trace("<< txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write",
flags);
}
int testcase::breakable_commit() {
int rc = MDBX_SUCCESS;
log_trace(">> txn_commit");
assert(txn_guard);
MDBX_txn *txn = txn_guard.release();
txn_inject_writefault(txn);
int err = mdbx_txn_commit(txn);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
rc = err;
err = mdbx_txn_abort(txn);
if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH))
failure_perror("mdbx_txn_abort()", err);
} else
failure_perror("mdbx_txn_commit()", err);
}
log_trace("<< txn_commit: %s", rc ? "failed" : "Ok");
return rc;
}
void testcase::txn_end(bool abort) {
log_trace(">> txn_end(%s)", abort ? "abort" : "commit");
assert(txn_guard);
MDBX_txn *txn = txn_guard.release();
if (abort) {
int err = mdbx_txn_abort(txn);
if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH))
failure_perror("mdbx_txn_abort()", err);
} else {
txn_inject_writefault(txn);
int err = mdbx_txn_commit(txn);
if (unlikely(err != MDBX_SUCCESS))
failure_perror("mdbx_txn_commit()", err);
}
log_trace("<< txn_end(%s)", abort ? "abort" : "commit");
}
void testcase::cursor_open(unsigned dbi) {
log_trace(">> cursor_open(%u)", dbi);
assert(!cursor_guard);
assert(txn_guard);
MDBX_cursor *cursor = nullptr;
int rc = mdbx_cursor_open(txn_guard.get(), dbi, &cursor);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_cursor_open()", rc);
cursor_guard.reset(cursor);
log_trace("<< cursor_open(%u)", dbi);
}
void testcase::cursor_close() {
log_trace(">> cursor_close()");
assert(cursor_guard);
MDBX_cursor *cursor = cursor_guard.release();
mdbx_cursor_close(cursor);
log_trace("<< cursor_close()");
}
int testcase::breakable_restart() {
int rc = MDBX_SUCCESS;
if (txn_guard)
rc = breakable_commit();
if (cursor_guard)
cursor_close();
txn_begin(false, 0);
return rc;
}
void testcase::txn_restart(bool abort, bool readonly, unsigned flags) {
if (txn_guard)
txn_end(abort);
if (cursor_guard)
cursor_close();
txn_begin(readonly, flags);
}
void testcase::txn_inject_writefault(void) {
if (txn_guard)
txn_inject_writefault(txn_guard.get());
}
void testcase::txn_inject_writefault(MDBX_txn *txn) {
if (config.params.inject_writefaultn && txn) {
if (config.params.inject_writefaultn <= nops_completed &&
(mdbx_txn_flags(txn) & MDBX_RDONLY) == 0) {
log_info("== txn_inject_writefault(): got %u nops or more, inject FAULT",
config.params.inject_writefaultn);
log_flush();
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
TerminateProcess(GetCurrentProcess(), 42);
#else
raise(SIGKILL);
#endif
}
}
}
bool testcase::wait4start() {
if (config.wait4id) {
log_trace(">> wait4start(%u)", config.wait4id);
assert(!global::singlemode);
int rc = osal_waitfor(config.wait4id);
if (rc) {
log_trace("<< wait4start(%u), failed %s", config.wait4id,
test_strerror(rc));
return false;
}
} else {
log_trace("== skip wait4start: not needed");
}
if (config.params.delaystart) {
int rc = osal_delay(config.params.delaystart);
if (rc) {
log_trace("<< delay(%u), failed %s", config.params.delaystart,
test_strerror(rc));
return false;
}
} else {
log_trace("== skip delay: not needed");
}
return true;
}
void testcase::kick_progress(bool active) const {
chrono::time now = chrono::now_motonic();
if (active) {
static int last_point = -1;
int point = (now.fixedpoint >> 29) & 3;
if (point != last_point) {
last.progress_timestamp = now;
fprintf(stderr, "%c\b", "-\\|/"[last_point = point]);
fflush(stderr);
}
} else if (now.fixedpoint - last.progress_timestamp.fixedpoint >
chrono::from_seconds(2).fixedpoint) {
last.progress_timestamp = now;
fprintf(stderr, "%c\b", "@*"[now.utc & 1]);
fflush(stderr);
}
}
void testcase::report(size_t nops_done) {
assert(nops_done > 0);
if (!nops_done)
return;
nops_completed += nops_done;
log_verbose("== complete +%" PRIuPTR " iteration, total %" PRIuPTR " done",
nops_done, nops_completed);
if (global::config::progress_indicator)
kick_progress(true);
if (config.signal_nops && !signalled &&
config.signal_nops <= nops_completed) {
log_trace(">> signal(n-ops %" PRIuPTR ")", nops_completed);
if (!global::singlemode)
osal_broadcast(config.actor_id);
signalled = true;
log_trace("<< signal(n-ops %" PRIuPTR ")", nops_completed);
}
}
void testcase::signal() {
if (!signalled) {
log_trace(">> signal(forced)");
if (!global::singlemode)
osal_broadcast(config.actor_id);
signalled = true;
log_trace("<< signal(forced)");
}
}
bool testcase::setup() {
db_prepare();
if (!wait4start())
return false;
start_timestamp = chrono::now_motonic();
nops_completed = 0;
return true;
}
bool testcase::teardown() {
log_trace(">> testcase::teardown");
signal();
db_close();
log_trace("<< testcase::teardown");
return true;
}
bool testcase::should_continue(bool check_timeout_only) const {
bool result = true;
if (config.params.test_duration) {
chrono::time since;
since.fixedpoint =
chrono::now_motonic().fixedpoint - start_timestamp.fixedpoint;
if (since.seconds() >= config.params.test_duration)
result = false;
}
if (!check_timeout_only && config.params.test_nops &&
nops_completed >= config.params.test_nops)
result = false;
if (result && global::config::progress_indicator)
kick_progress(false);
return result;
}
void testcase::fetch_canary() {
mdbx_canary canary_now;
log_trace(">> fetch_canary");
int rc = mdbx_canary_get(txn_guard.get(), &canary_now);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_canary_get()", rc);
if (canary_now.v < last.canary.v)
failure("fetch_canary: %" PRIu64 "(canary-now.v) < %" PRIu64
"(canary-last.v)",
canary_now.v, last.canary.v);
if (canary_now.y < last.canary.y)
failure("fetch_canary: %" PRIu64 "(canary-now.y) < %" PRIu64
"(canary-last.y)",
canary_now.y, last.canary.y);
last.canary = canary_now;
log_trace("<< fetch_canary: db-sequence %" PRIu64
", db-sequence.txnid %" PRIu64,
last.canary.y, last.canary.v);
}
void testcase::update_canary(uint64_t increment) {
mdbx_canary canary_now = last.canary;
log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y,
increment);
canary_now.y += increment;
int rc = mdbx_canary_put(txn_guard.get(), &canary_now);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_canary_put()", rc);
log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y);
}
int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &dbi) {
db_open();
int err, retry_left = 42;
for (;;) {
txn_begin(false);
dbi = db_table_open(true);
db_table_clear(dbi);
err = breakable_commit();
if (likely(err == MDBX_SUCCESS)) {
txn_begin(false);
return MDBX_SUCCESS;
}
if (--retry_left == 0)
break;
jitter_delay(true);
}
log_notice("db_begin_table_create_open_clean: bailout due '%s'",
mdbx_strerror(err));
return err;
}
MDBX_dbi testcase::db_table_open(bool create) {
log_trace(">> testcase::db_table_create");
char tablename_buf[16];
const char *tablename = nullptr;
if (config.space_id) {
int rc = snprintf(tablename_buf, sizeof(tablename_buf), "TBL%04u",
config.space_id);
if (rc < 4 || rc >= (int)sizeof(tablename_buf) - 1)
failure("snprintf(tablename): %d", rc);
tablename = tablename_buf;
}
log_verbose("use %s table", tablename ? tablename : "MAINDB");
MDBX_dbi handle = 0;
int rc = mdbx_dbi_open(txn_guard.get(), tablename,
(create ? MDBX_CREATE : 0) | config.params.table_flags,
&handle);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_dbi_open()", rc);
log_trace("<< testcase::db_table_create, handle %u", handle);
return handle;
}
void testcase::db_table_drop(MDBX_dbi handle) {
log_trace(">> testcase::db_table_drop, handle %u", handle);
if (config.params.drop_table) {
int rc = mdbx_drop(txn_guard.get(), handle, true);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_drop(delete=true)", rc);
log_trace("<< testcase::db_table_drop");
} else {
log_trace("<< testcase::db_table_drop: not needed");
}
}
void testcase::db_table_clear(MDBX_dbi handle) {
log_trace(">> testcase::db_table_clear, handle %u", handle);
int rc = mdbx_drop(txn_guard.get(), handle, false);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_drop(delete=false)", rc);
log_trace("<< testcase::db_table_clear");
}
void testcase::db_table_close(MDBX_dbi handle) {
log_trace(">> testcase::db_table_close, handle %u", handle);
assert(!txn_guard);
int rc = mdbx_dbi_close(db_guard.get(), handle);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_dbi_close()", rc);
log_trace("<< testcase::db_table_close");
}
void testcase::checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check,
MDBX_val expected_valued) {
MDBX_val actual_value = expected_valued;
int rc = mdbx_get2(txn_guard.get(), handle, &key2check, &actual_value);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror(step, rc);
if (!is_samedata(&actual_value, &expected_valued))
failure("%s data mismatch", step);
}
//-----------------------------------------------------------------------------
bool test_execute(const actor_config &config_const) {
const mdbx_pid_t pid = osal_getpid();
actor_config config = config_const;
if (global::singlemode) {
logging::setup(format("single_%s", testcase2str(config.testcase)));
} else {
logging::setup((logging::loglevel)config.params.loglevel,
format("child_%u.%u", config.actor_id, config.space_id));
log_trace(">> wait4barrier");
osal_wait4barrier();
log_trace("<< wait4barrier");
}
try {
std::unique_ptr<testcase> test;
switch (config.testcase) {
case ac_hill:
test.reset(new testcase_hill(config, pid));
break;
case ac_deadread:
test.reset(new testcase_deadread(config, pid));
break;
case ac_deadwrite:
test.reset(new testcase_deadwrite(config, pid));
break;
case ac_jitter:
test.reset(new testcase_jitter(config, pid));
break;
case ac_try:
test.reset(new testcase_try(config, pid));
break;
case ac_copy:
test.reset(new testcase_copy(config, pid));
break;
case ac_append:
test.reset(new testcase_append(config, pid));
break;
case ac_ttl:
test.reset(new testcase_ttl(config, pid));
break;
default:
test.reset(new testcase(config, pid));
break;
}
size_t iter = 0;
do {
iter++;
if (!test->setup()) {
log_notice("test setup failed");
return false;
}
if (!test->run()) {
log_notice("test failed");
return false;
}
if (!test->teardown()) {
log_notice("test teardown failed");
return false;
}
if (config.params.nrepeat == 1)
log_info("test successed");
else {
if (config.params.nrepeat)
log_info("test successed (iteration %zi of %zi)", iter,
size_t(config.params.nrepeat));
else
log_info("test successed (iteration %zi)", iter);
config.params.keygen.seed += INT32_C(0xA4F4D37B);
}
} while (config.params.nrepeat == 0 || iter < config.params.nrepeat);
return true;
} catch (const std::exception &pipets) {
failure("***** Exception: %s *****", pipets.what());
return false;
}
}

218
contrib/db/libmdbx/test/test.h Normal file
View file

@ -0,0 +1,218 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#include "chrono.h"
#include "config.h"
#include "keygen.h"
#include "log.h"
#include "osal.h"
#include "utils.h"
bool test_execute(const actor_config &config);
std::string thunk_param(const actor_config &config);
void testcase_setup(const char *casename, actor_params &params,
unsigned &last_space_id);
void configure_actor(unsigned &last_space_id, const actor_testcase testcase,
const char *space_id_cstr, const actor_params &params);
void keycase_setup(const char *casename, actor_params &params);
namespace global {
extern const char thunk_param_prefix[];
extern std::vector<actor_config> actors;
extern std::unordered_map<unsigned, actor_config *> events;
extern std::unordered_map<mdbx_pid_t, actor_config *> pid2actor;
extern std::set<std::string> databases;
extern unsigned nactors;
extern chrono::time start_motonic;
extern chrono::time deadline_motonic;
extern bool singlemode;
namespace config {
extern unsigned timeout_duration_seconds;
extern bool dump_config;
extern bool cleanup_before;
extern bool cleanup_after;
extern bool failfast;
extern bool progress_indicator;
} /* namespace config */
} /* namespace global */
//-----------------------------------------------------------------------------
struct db_deleter : public std::unary_function<void, MDBX_env *> {
void operator()(MDBX_env *env) const { mdbx_env_close(env); }
};
struct txn_deleter : public std::unary_function<void, MDBX_txn *> {
void operator()(MDBX_txn *txn) const {
int rc = mdbx_txn_abort(txn);
if (rc)
log_trouble(mdbx_func_, "mdbx_txn_abort()", rc);
}
};
struct cursor_deleter : public std::unary_function<void, MDBX_cursor *> {
void operator()(MDBX_cursor *cursor) const { mdbx_cursor_close(cursor); }
};
typedef std::unique_ptr<MDBX_env, db_deleter> scoped_db_guard;
typedef std::unique_ptr<MDBX_txn, txn_deleter> scoped_txn_guard;
typedef std::unique_ptr<MDBX_cursor, cursor_deleter> scoped_cursor_guard;
//-----------------------------------------------------------------------------
class testcase {
protected:
const actor_config &config;
const mdbx_pid_t pid;
scoped_db_guard db_guard;
scoped_txn_guard txn_guard;
scoped_cursor_guard cursor_guard;
bool signalled;
size_t nops_completed;
chrono::time start_timestamp;
keygen::buffer key;
keygen::buffer data;
keygen::maker keyvalue_maker;
struct {
mdbx_canary canary;
mutable chrono::time progress_timestamp;
} last;
static int oom_callback(MDBX_env *env, int pid, mdbx_tid_t tid, uint64_t txn,
unsigned gap, int retry);
void db_prepare();
void db_open();
void db_close();
void txn_begin(bool readonly, unsigned flags = 0);
int breakable_commit();
void txn_end(bool abort);
int breakable_restart();
void txn_restart(bool abort, bool readonly, unsigned flags = 0);
void cursor_open(unsigned dbi);
void cursor_close();
void txn_inject_writefault(void);
void txn_inject_writefault(MDBX_txn *txn);
void fetch_canary();
void update_canary(uint64_t increment);
void kick_progress(bool active) const;
void checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check,
MDBX_val expected_valued);
MDBX_dbi db_table_open(bool create);
void db_table_drop(MDBX_dbi handle);
void db_table_clear(MDBX_dbi handle);
void db_table_close(MDBX_dbi handle);
int db_open__begin__table_create_open_clean(MDBX_dbi &dbi);
bool wait4start();
void report(size_t nops_done);
void signal();
bool should_continue(bool check_timeout_only = false) const;
void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key,
keygen::buffer &out_value, keygen::serial_t data_age = 0) {
keyvalue_maker.pair(serial, out_key, out_value, data_age);
}
void generate_pair(const keygen::serial_t serial,
keygen::serial_t data_age = 0) {
generate_pair(serial, key, data, data_age);
}
bool mode_readonly() const {
return (config.params.mode_flags & MDBX_RDONLY) ? true : false;
}
public:
testcase(const actor_config &config, const mdbx_pid_t pid)
: config(config), pid(pid), signalled(false), nops_completed(0) {
start_timestamp.reset();
memset(&last, 0, sizeof(last));
}
virtual bool setup();
virtual bool run() { return true; }
virtual bool teardown();
virtual ~testcase() {}
};
class testcase_ttl : public testcase {
public:
testcase_ttl(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run();
};
class testcase_hill : public testcase {
public:
testcase_hill(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run();
};
class testcase_append : public testcase {
public:
testcase_append(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run();
};
class testcase_deadread : public testcase {
public:
testcase_deadread(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run();
};
class testcase_deadwrite : public testcase {
public:
testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run();
};
class testcase_jitter : public testcase {
public:
testcase_jitter(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run();
};
class testcase_try : public testcase {
public:
testcase_try(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid) {}
bool run();
};
class testcase_copy : public testcase {
const std::string copy_pathname;
void copy_db(const bool with_compaction);
public:
testcase_copy(const actor_config &config, const mdbx_pid_t pid)
: testcase(config, pid),
copy_pathname(config.params.pathname_db + "-copy") {}
bool run();
};

209
contrib/db/libmdbx/test/test.vcxproj Normal file
View file

@ -0,0 +1,209 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\dll.vcxproj">
<Project>{6d19209b-ece7-4b9c-941c-0aa2b484f199}</Project>
</ProjectReference>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{30E29CE6-E6FC-4D32-AA07-46A55FAF3A31}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>mdbxtest</RootNamespace>
<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v140</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
<TargetName>mdbx_test</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
<TargetName>mdbx_test</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
<TargetName>mdbx_test</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
<TargetName>mdbx_test</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>Use</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;MDBX_DEBUG=1;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<PrecompiledHeaderFile>test.h</PrecompiledHeaderFile>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>Use</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;MDBX_DEBUG=1;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<PrecompiledHeaderFile>test.h</PrecompiledHeaderFile>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>Use</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<PrecompiledHeaderFile>test.h</PrecompiledHeaderFile>
<OmitFramePointers>true</OmitFramePointers>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>Use</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions);LIBMDBX_IMPORTS=1</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<PrecompiledHeaderFile>test.h</PrecompiledHeaderFile>
<OmitFramePointers>true</OmitFramePointers>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>winmm.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="base.h" />
<ClInclude Include="chrono.h" />
<ClInclude Include="config.h" />
<ClInclude Include="keygen.h" />
<ClInclude Include="log.h" />
<ClInclude Include="osal.h" />
<ClInclude Include="test.h" />
<ClInclude Include="utils.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="ttl.cc" />
<ClCompile Include="append.cc" />
<ClCompile Include="cases.cc" />
<ClCompile Include="chrono.cc" />
<ClCompile Include="config.cc" />
<ClCompile Include="copy.cc" />
<ClCompile Include="dead.cc" />
<ClCompile Include="hill.cc" />
<ClCompile Include="try.cc" />
<ClCompile Include="jitter.cc" />
<ClCompile Include="keygen.cc" />
<ClCompile Include="log.cc" />
<ClCompile Include="main.cc" />
<ClCompile Include="osal-windows.cc">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader>
</ClCompile>
<ClCompile Include="test.cc" />
<ClCompile Include="utils.cc" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

20
contrib/db/libmdbx/test/try.cc Normal file
View file

@ -0,0 +1,20 @@
#include "test.h"
bool testcase_try::run() {
db_open();
assert(!txn_guard);
MDBX_txn *txn = nullptr;
MDBX_txn *txn2 = nullptr;
int rc = mdbx_txn_begin(db_guard.get(), nullptr, 0, &txn);
if (unlikely(rc != MDBX_SUCCESS))
failure_perror("mdbx_txn_begin(MDBX_TRYTXN)", rc);
else {
rc = mdbx_txn_begin(db_guard.get(), nullptr, MDBX_TRYTXN, &txn2);
if (unlikely(rc != MDBX_BUSY))
failure_perror("mdbx_txn_begin(MDBX_TRYTXN)", rc);
}
txn_guard.reset(txn);
return true;
}

179
contrib/db/libmdbx/test/ttl.cc Normal file
View file

@ -0,0 +1,179 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <cmath>
#include <deque>
static unsigned edge2window(uint64_t edge, unsigned window_max) {
const double rnd = u64_to_double1(bleach64(edge));
const unsigned window = window_max - std::lrint(std::pow(window_max, rnd));
return window;
}
static unsigned edge2count(uint64_t edge, unsigned count_max) {
const double rnd = u64_to_double1(prng64_map1_white(edge));
const unsigned count = std::lrint(std::pow(count_max, rnd));
return count;
}
bool testcase_ttl::run() {
MDBX_dbi dbi;
int err = db_open__begin__table_create_open_clean(dbi);
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err));
return true;
}
/* LY: тест "эмуляцией time-to-live":
* - организуется "скользящее окно", которое двигается вперед вдоль
* числовой оси каждую транзакцию.
* - по переднему краю "скользящего окна" записи добавляются в таблицу,
* а по заднему удаляются.
* - количество добавляемых/удаляемых записей псевдослучайно зависит
* от номера транзакции, но с экспоненциальным распределением.
* - размер "скользящего окна" также псевдослучайно зависит от номера
* транзакции с "отрицательным" экспоненциальным распределением
* MAX_WIDTH - exp(rnd(N)), при уменьшении окна сдвигается задний
* край и удаляются записи позади него.
*
* Таким образом имитируется поведение таблицы с TTL: записи стохастически
* добавляются и удаляются, но изредка происходят массивные удаления.
*/
/* LY: для параметризации используем подходящие параметры, которые не имеют
* здесь смысла в первоначальном значении. */
const unsigned window_max_lower =
#ifdef __APPLE__
333;
#else
999;
#endif
const unsigned count_max_lower =
#ifdef __APPLE__
333;
#else
999;
#endif
const unsigned window_max = (config.params.batch_read > window_max_lower)
? config.params.batch_read
: window_max_lower;
const unsigned count_max = (config.params.batch_write > count_max_lower)
? config.params.batch_write
: count_max_lower;
log_info("ttl: using `batch_read` value %u for window_max", window_max);
log_info("ttl: using `batch_write` value %u for count_max", count_max);
uint64_t seed =
prng64_map2_white(config.params.keygen.seed) + config.actor_id;
keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */);
key = keygen::alloc(config.params.keylen_max);
data = keygen::alloc(config.params.datalen_max);
const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT)
? MDBX_NODUPDATA
: MDBX_NODUPDATA | MDBX_NOOVERWRITE;
std::deque<std::pair<uint64_t, unsigned>> fifo;
uint64_t serial = 0;
while (should_continue()) {
const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */;
const unsigned window_width = edge2window(salt, window_max);
unsigned head_count = edge2count(salt, count_max);
log_verbose("ttl: step #%zu (serial %" PRIu64
", window %u, count %u) salt %" PRIu64,
nops_completed, serial, window_width, head_count, salt);
if (window_width) {
while (fifo.size() > window_width) {
uint64_t tail_serial = fifo.back().first;
const unsigned tail_count = fifo.back().second;
log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial,
tail_count);
fifo.pop_back();
for (unsigned n = 0; n < tail_count; ++n) {
log_trace("ttl: remove-tail %" PRIu64, serial);
generate_pair(tail_serial);
err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("ttl: tail-bailout due '%s'", mdbx_strerror(err));
goto bailout;
}
failure_perror("mdbx_del(tail)", err);
}
if (unlikely(!keyvalue_maker.increment(tail_serial, 1)))
failure("ttl: unexpected key-space overflow on the tail");
}
}
} else {
log_trace("ttl: purge state");
db_table_clear(dbi);
fifo.clear();
}
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout at commit due '%s'", mdbx_strerror(err));
break;
}
fifo.push_front(std::make_pair(serial, head_count));
retry:
for (unsigned n = 0; n < head_count; ++n) {
log_trace("ttl: insert-head %" PRIu64, serial);
generate_pair(serial);
err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value,
insert_flags);
if (unlikely(err != MDBX_SUCCESS)) {
if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) {
log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err));
txn_restart(true, false);
serial = fifo.front().first;
fifo.front().second = head_count = n;
goto retry;
}
failure_perror("mdbx_put(head)", err);
}
if (unlikely(!keyvalue_maker.increment(serial, 1))) {
log_notice("ttl: unexpected key-space overflow");
goto bailout;
}
}
err = breakable_restart();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err));
serial = fifo.front().first;
fifo.pop_front();
}
report(1);
}
bailout:
txn_end(true);
if (dbi) {
if (config.params.drop_table && !mode_readonly()) {
txn_begin(false);
db_table_drop(dbi);
err = breakable_commit();
if (unlikely(err != MDBX_SUCCESS)) {
log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err));
return true;
}
} else
db_table_close(dbi);
}
return true;
}

367
contrib/db/libmdbx/test/utils.cc Normal file
View file

@ -0,0 +1,367 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "test.h"
#include <float.h>
#if defined(HAVE_IEEE754_H) || __has_include(<ieee754.h>)
#include <ieee754.h>
#endif
#if defined(__APPLE__) || defined(__MACH__)
#include <mach/mach_time.h>
#endif /* defined(__APPLE__) || defined(__MACH__) */
std::string format(const char *fmt, ...) {
va_list ap, ones;
va_start(ap, fmt);
va_copy(ones, ap);
#ifdef _MSC_VER
int needed = _vscprintf(fmt, ap);
#else
int needed = vsnprintf(nullptr, 0, fmt, ap);
#endif
assert(needed >= 0);
va_end(ap);
std::string result;
result.reserve((size_t)needed + 1);
result.resize((size_t)needed, '\0');
int actual = vsnprintf((char *)result.data(), result.capacity(), fmt, ones);
assert(actual == needed);
(void)actual;
va_end(ones);
return result;
}
std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum) {
std::string result;
if (bytes > 0) {
const uint8_t *data = (const uint8_t *)ptr;
checksum.push(data, bytes);
result.reserve(bytes * 2);
const uint8_t *const end = data + bytes;
do {
char h = *data >> 4;
char l = *data & 15;
result.push_back((l < 10) ? l + '0' : l - 10 + 'a');
result.push_back((h < 10) ? h + '0' : h - 10 + 'a');
} while (++data < end);
}
assert(result.size() == bytes * 2);
return result;
}
bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
size_t bytes, simple_checksum &checksum) {
if (bytes * 2 != (size_t)(hex_end - hex_begin))
return false;
uint8_t *data = (uint8_t *)ptr;
for (const char *hex = hex_begin; hex != hex_end; hex += 2, ++data) {
unsigned l = hex[0], h = hex[1];
if (l >= '0' && l <= '9')
l = l - '0';
else if (l >= 'A' && l <= 'F')
l = l - 'A' + 10;
else if (l >= 'a' && l <= 'f')
l = l - 'a' + 10;
else
return false;
if (h >= '0' && h <= '9')
h = h - '0';
else if (h >= 'A' && h <= 'F')
h = h - 'A' + 10;
else if (h >= 'a' && h <= 'f')
h = h - 'a' + 10;
else
return false;
uint32_t c = l + (h << 4);
checksum.push(c);
*data = (uint8_t)c;
}
return true;
}
bool is_samedata(const MDBX_val *a, const MDBX_val *b) {
return a->iov_len == b->iov_len &&
memcmp(a->iov_base, b->iov_base, a->iov_len) == 0;
}
//-----------------------------------------------------------------------------
/* TODO: replace my 'libmera' from t1ha. */
uint64_t entropy_ticks(void) {
#if defined(EMSCRIPTEN)
return (uint64_t)emscripten_get_now();
#endif /* EMSCRIPTEN */
#if defined(__APPLE__) || defined(__MACH__)
return mach_absolute_time();
#endif /* defined(__APPLE__) || defined(__MACH__) */
#if defined(__sun__) || defined(__sun)
return gethrtime();
#endif /* __sun__ */
#if defined(__GNUC__) || defined(__clang__)
#if defined(__ia64__)
uint64_t ticks;
__asm __volatile("mov %0=ar.itc" : "=r"(ticks));
return ticks;
#elif defined(__hppa__)
uint64_t ticks;
__asm __volatile("mfctl 16, %0" : "=r"(ticks));
return ticks;
#elif defined(__s390__)
uint64_t ticks;
__asm __volatile("stck 0(%0)" : : "a"(&(ticks)) : "memory", "cc");
return ticks;
#elif defined(__alpha__)
uint64_t ticks;
__asm __volatile("rpcc %0" : "=r"(ticks));
return ticks;
#elif defined(__sparc__) || defined(__sparc) || defined(__sparc64__) || \
defined(__sparc64) || defined(__sparc_v8plus__) || \
defined(__sparc_v8plus) || defined(__sparc_v8plusa__) || \
defined(__sparc_v8plusa) || defined(__sparc_v9__) || defined(__sparc_v9)
union {
uint64_t u64;
struct {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
uint32_t h, l;
#else
uint32_t l, h;
#endif
} u32;
} cycles;
#if defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || \
defined(__sparc_v9__) || defined(__sparc_v8plus) || \
defined(__sparc_v8plusa) || defined(__sparc_v9)
#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || \
defined(__sparc64__) || defined(__sparc64)
__asm __volatile("rd %%tick, %0" : "=r"(cycles.u64));
#else
__asm __volatile("rd %%tick, %1; srlx %1, 32, %0"
: "=r"(cycles.u32.h), "=r"(cycles.u32.l));
#endif /* __sparc64__ */
#else
__asm __volatile(".byte 0x83, 0x41, 0x00, 0x00; mov %%g1, %0"
: "=r"(cycles.u64)
:
: "%g1");
#endif /* __sparc8plus__ || __sparc_v9__ */
return cycles.u64;
#elif (defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) || \
defined(__powerpc64))
uint64_t ticks;
__asm __volatile("mfspr %0, 268" : "=r"(ticks));
return ticks;
#elif (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) || \
defined(__ppc))
#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul
uint64_t ticks;
__asm __volatile("mftb %0" : "=r"(ticks));
*now = ticks;
#else
uint64_t ticks;
uint32_t low, high_before, high_after;
__asm __volatile("mftbu %0; mftb %1; mftbu %2"
: "=r"(high_before), "=r"(low), "=r"(high_after));
ticks = (uint64_t)high_after << 32;
ticks |= low & /* zeroes if high part has changed */
~(high_before - high_after);
#endif
#elif (defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 7)) && \
!defined(MDBX_SAFE4QEMU)
uint64_t virtual_timer;
__asm __volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer));
return virtual_timer;
#elif (defined(__ARM_ARCH) && __ARM_ARCH > 5 && __ARM_ARCH < 8) || \
defined(_M_ARM)
static uint32_t pmcntenset = 0x00425B00;
if (unlikely(pmcntenset == 0x00425B00)) {
uint32_t pmuseren;
#ifdef _M_ARM
pmuseren = _MoveFromCoprocessor(15, 0, 9, 14, 0);
#else
__asm("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren));
#endif
if (1 & pmuseren /* Is it allowed for user mode code? */) {
#ifdef _M_ARM
pmcntenset = _MoveFromCoprocessor(15, 0, 9, 12, 1);
#else
__asm("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset));
#endif
} else
pmcntenset = 0;
}
if (pmcntenset & 0x80000000ul /* Is it counting? */) {
#ifdef _M_ARM
return __rdpmccntr64();
#else
uint32_t pmccntr;
__asm __volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr));
return pmccntr;
#endif
}
#elif defined(__mips__) || defined(__mips) || defined(_R4000)
unsigned count;
__asm __volatile("rdhwr %0, $2" : "=r"(count));
return count;
#endif /* arch selector */
#endif /* __GNUC__ || __clang__ */
#if defined(__e2k__) || defined(__ia32__)
return __rdtsc();
#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
LARGE_INTEGER PerformanceCount;
if (QueryPerformanceCounter(&PerformanceCount))
return PerformanceCount.QuadPart;
return GetTickCount64();
#else
struct timespec ts;
#if defined(CLOCK_MONOTONIC_COARSE)
clockid_t clk_id = CLOCK_MONOTONIC_COARSE;
#elif defined(CLOCK_MONOTONIC_RAW)
clockid_t clk_id = CLOCK_MONOTONIC_RAW;
#else
clockid_t clk_id = CLOCK_MONOTONIC;
#endif
int rc = clock_gettime(clk_id, &ts);
if (unlikely(rc))
failure_perror("clock_gettime()", rc);
return (((uint64_t)ts.tv_sec) << 32) + ts.tv_nsec;
#endif
}
//-----------------------------------------------------------------------------
uint64_t prng64_white(uint64_t &state) {
state = prng64_map2_careless(state);
return bleach64(state);
}
uint32_t prng32(uint64_t &state) {
return (uint32_t)(prng64_careless(state) >> 32);
}
void prng_fill(uint64_t &state, void *ptr, size_t bytes) {
while (bytes >= 4) {
*((uint32_t *)ptr) = prng32(state);
ptr = (uint32_t *)ptr + 1;
bytes -= 4;
}
switch (bytes & 3) {
case 3: {
uint32_t u32 = prng32(state);
memcpy(ptr, &u32, 3);
} break;
case 2:
*((uint16_t *)ptr) = (uint16_t)prng32(state);
break;
case 1:
*((uint8_t *)ptr) = (uint8_t)prng32(state);
break;
case 0:
break;
}
}
static __thread uint64_t prng_state;
void prng_seed(uint64_t seed) { prng_state = bleach64(seed); }
uint32_t prng32(void) { return prng32(prng_state); }
uint64_t prng64(void) { return prng64_white(prng_state); }
void prng_fill(void *ptr, size_t bytes) { prng_fill(prng_state, ptr, bytes); }
uint64_t entropy_white() { return bleach64(entropy_ticks()); }
double double_from_lower(uint64_t salt) {
#ifdef IEEE754_DOUBLE_BIAS
ieee754_double r;
r.ieee.negative = 0;
r.ieee.exponent = IEEE754_DOUBLE_BIAS;
r.ieee.mantissa0 = (unsigned)(salt >> 32);
r.ieee.mantissa1 = (unsigned)salt;
return r.d;
#else
const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1;
const double scale = 1.0 / (double)top;
return (salt & top) * scale;
#endif
}
double double_from_upper(uint64_t salt) {
#ifdef IEEE754_DOUBLE_BIAS
ieee754_double r;
r.ieee.negative = 0;
r.ieee.exponent = IEEE754_DOUBLE_BIAS;
salt >>= 64 - DBL_MANT_DIG;
r.ieee.mantissa0 = (unsigned)(salt >> 32);
r.ieee.mantissa1 = (unsigned)salt;
return r.d;
#else
const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1;
const double scale = 1.0 / (double)top;
return (salt >> (64 - DBL_MANT_DIG)) * scale;
#endif
}
bool flipcoin() { return bleach32((uint32_t)entropy_ticks()) & 1; }
bool jitter(unsigned probability_percent) {
const uint32_t top = UINT32_MAX - UINT32_MAX % 100;
uint32_t dice, edge = (top) / 100 * probability_percent;
do
dice = bleach32((uint32_t)entropy_ticks());
while (dice >= top);
return dice < edge;
}
void jitter_delay(bool extra) {
unsigned dice = entropy_white() & 3;
if (dice == 0) {
log_trace("== jitter.no-delay");
} else {
log_trace(">> jitter.delay: dice %u", dice);
do {
cpu_relax();
memory_barrier();
cpu_relax();
if (dice > 1) {
osal_yield();
cpu_relax();
if (dice > 2) {
unsigned us = entropy_white() &
(extra ? 0xffff /* 656 ms */ : 0x3ff /* 1 ms */);
log_trace("== jitter.delay: %0.6f", us / 1000000.0);
osal_udelay(us);
}
}
} while (flipcoin());
log_trace("<< jitter.delay: dice %u", dice);
}
}

359
contrib/db/libmdbx/test/utils.h Normal file
View file

@ -0,0 +1,359 @@
/*
* Copyright 2017-2019 Leonid Yuriev <leo@yuriev.ru>
* and other libmdbx authors: please see AUTHORS file.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#pragma once
#include "base.h"
#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \
!defined(__ORDER_BIG_ENDIAN__)
#error __BYTE_ORDER__ should be defined.
#endif
#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \
__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__
#error Unsupported byte order.
#endif
#if __GNUC_PREREQ(4, 4) || defined(__clang__)
#ifndef bswap64
#define bswap64(v) __builtin_bswap64(v)
#endif
#ifndef bswap32
#define bswap32(v) __builtin_bswap32(v)
#endif
#if (__GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)) && \
!defined(bswap16)
#define bswap16(v) __builtin_bswap16(v)
#endif
#elif defined(_MSC_VER)
#if _MSC_FULL_VER < 190024215
#pragma message( \
"It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.")
#endif
#define bswap64(v) _byteswap_uint64(v)
#define bswap32(v) _byteswap_ulong(v)
#define bswap16(v) _byteswap_ushort(v)
#define rot64(v, s) _rotr64(v, s)
#define rot32(v, s) _rotr(v, s)
#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)
#pragma intrinsic(_umul128)
#define mul_64x64_128(a, b, ph) _umul128(a, b, ph)
#pragma intrinsic(__umulh)
#define mul_64x64_high(a, b) __umulh(a, b)
#endif
#if defined(_M_IX86)
#pragma intrinsic(__emulu)
#define mul_32x32_64(a, b) __emulu(a, b)
#elif defined(_M_ARM)
#define mul_32x32_64(a, b) _arm_umull(a, b)
#endif
#endif /* compiler */
#ifndef bswap64
#ifdef __bswap_64
#define bswap64(v) __bswap_64(v)
#else
static __inline uint64_t bswap64(uint64_t v) {
return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) |
((v << 24) & UINT64_C(0x0000ff0000000000)) |
((v << 8) & UINT64_C(0x000000ff00000000)) |
((v >> 8) & UINT64_C(0x00000000ff0000000)) |
((v >> 24) & UINT64_C(0x0000000000ff0000)) |
((v >> 40) & UINT64_C(0x000000000000ff00));
}
#endif
#endif /* bswap64 */
#ifndef bswap32
#ifdef __bswap_32
#define bswap32(v) __bswap_32(v)
#else
static __inline uint32_t bswap32(uint32_t v) {
return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) |
((v >> 8) & UINT32_C(0x0000ff00));
}
#endif
#endif /* bswap32 */
#ifndef bswap16
#ifdef __bswap_16
#define bswap16(v) __bswap_16(v)
#else
static __inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; }
#endif
#endif /* bswap16 */
#define is_byteorder_le() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define is_byteorder_be() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#ifndef htole16
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe16(v) bswap16(v)
#define htole16(v) (v)
#define be16toh(v) bswap16(v)
#define le16toh(v) (v)
#else
#define htobe16(v) (v)
#define htole16(v) bswap16(v)
#define be16toh(v) (v)
#define le16toh(v) bswap16(v)
#endif
#endif /* htole16 */
#ifndef htole32
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe32(v) bswap32(v)
#define htole32(v) (v)
#define be32toh(v) bswap32(v)
#define le32toh(v) (v)
#else
#define htobe32(v) (v)
#define htole32(v) bswap32(v)
#define be32toh(v) (v)
#define le32toh(v) bswap32(v)
#endif
#endif /* htole32 */
#ifndef htole64
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define htobe64(v) bswap64(v)
#define htole64(v) (v)
#define be64toh(v) bswap64(v)
#define le64toh(v) (v)
#else
#define htobe64(v) (v)
#define htole64(v) bswap_64(v)
#define be64toh(v) (v)
#define le64toh(v) bswap_64(v)
#endif
#endif /* htole64 */
namespace unaligned {
template <typename T> static __inline T load(const void *ptr) {
#if defined(_MSC_VER) && \
(defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64))
return *(const T __unaligned *)ptr;
#elif UNALIGNED_OK
return *(const T *)ptr;
#else
T local;
#if defined(__GNUC__) || defined(__clang__)
__builtin_memcpy(&local, (const T *)ptr, sizeof(T));
#else
memcpy(&local, (const T *)ptr, sizeof(T));
#endif /* __GNUC__ || __clang__ */
return local;
#endif /* UNALIGNED_OK */
}
template <typename T> static __inline void store(void *ptr, const T &value) {
#if defined(_MSC_VER) && \
(defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64))
*((T __unaligned *)ptr) = value;
#elif UNALIGNED_OK
*(volatile T *)ptr = value;
#else
#if defined(__GNUC__) || defined(__clang__)
__builtin_memcpy(ptr, &value, sizeof(T));
#else
memcpy(ptr, &value, sizeof(T));
#endif /* __GNUC__ || __clang__ */
#endif /* UNALIGNED_OK */
}
} /* namespace unaligned */
//-----------------------------------------------------------------------------
#ifndef rot64
static __inline uint64_t rot64(uint64_t v, unsigned s) {
return (v >> s) | (v << (64 - s));
}
#endif /* rot64 */
static __inline bool is_power2(size_t x) { return (x & (x - 1)) == 0; }
#undef roundup2
static __inline size_t roundup2(size_t value, size_t granularity) {
assert(is_power2(granularity));
return (value + granularity - 1) & ~(granularity - 1);
}
//-----------------------------------------------------------------------------
static __inline void memory_barrier(void) {
#if __has_extension(c_atomic) || __has_extension(cxx_atomic)
__c11_atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__ATOMIC_SEQ_CST)
__atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif defined(__clang__) || defined(__GNUC__)
__sync_synchronize();
#elif defined(_MSC_VER)
MemoryBarrier();
#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */
#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64)
__mf();
#elif defined(__ia32__)
_mm_mfence();
#else
#error "Unknown target for Intel Compiler, please report to us."
#endif
#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun)
__machine_rw_barrier();
#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \
(defined(HP_IA64) || defined(__ia64))
_Asm_mf();
#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \
defined(__ppc64__) || defined(__powerpc64__)
__lwsync();
#else
#error "Could not guess the kind of compiler, please report to us."
#endif
}
static __inline void cpu_relax() {
#if defined(__ia32__)
_mm_pause();
#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || \
defined(YieldProcessor)
YieldProcessor();
#else
/* nope */
#endif
}
//-----------------------------------------------------------------------------
struct simple_checksum {
uint64_t value;
simple_checksum() : value(0) {}
void push(const uint32_t &data) {
value += data * UINT64_C(9386433910765580089) + 1;
value ^= value >> 41;
value *= UINT64_C(0xBD9CACC22C6E9571);
}
void push(const uint64_t &data) {
push((uint32_t)data);
push((uint32_t)(data >> 32));
}
void push(const bool data) {
push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E));
}
void push(const void *ptr, size_t bytes) {
const uint8_t *data = (const uint8_t *)ptr;
for (size_t i = 0; i < bytes; ++i)
push((uint32_t)data[i]);
}
void push(const double &data) { push(&data, sizeof(double)); }
void push(const char *cstr) { push(cstr, strlen(cstr)); }
void push(const std::string &str) { push(str.data(), str.size()); }
void push(unsigned salt, const MDBX_val &val) {
push(unsigned(val.iov_len));
push(salt);
push(val.iov_base, val.iov_len);
}
#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS)
void push(const HANDLE &handle) { push(&handle, sizeof(handle)); }
#endif /* _WINDOWS */
};
std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum);
bool hex2data(const char *hex_begin, const char *hex_end, void *ptr,
size_t bytes, simple_checksum &checksum);
bool is_samedata(const MDBX_val *a, const MDBX_val *b);
std::string format(const char *fmt, ...);
uint64_t entropy_ticks(void);
uint64_t entropy_white(void);
static inline uint64_t bleach64(uint64_t v) {
// Tommy Ettinger, https://www.blogger.com/profile/04953541827437796598
// http://mostlymangling.blogspot.com/2019/01/better-stronger-mixer-and-test-procedure.html
v ^= rot64(v, 25) ^ rot64(v, 50);
v *= UINT64_C(0xA24BAED4963EE407);
v ^= rot64(v, 24) ^ rot64(v, 49);
v *= UINT64_C(0x9FB21C651E98DF25);
return v ^ v >> 28;
}
static inline uint32_t bleach32(uint32_t x) {
// https://github.com/skeeto/hash-prospector
// exact bias: 0.17353355999581582
x ^= x >> 16;
x *= UINT32_C(0x7feb352d);
x ^= 0x3027C563 ^ (x >> 15);
x *= UINT32_C(0x846ca68b);
x ^= x >> 16;
return x;
}
static inline uint64_t prng64_map1_careless(uint64_t state) {
return state * UINT64_C(6364136223846793005) + 1;
}
static inline uint64_t prng64_map2_careless(uint64_t state) {
return (state + UINT64_C(1442695040888963407)) *
UINT64_C(6364136223846793005);
}
static inline uint64_t prng64_map1_white(uint64_t state) {
return bleach64(prng64_map1_careless(state));
}
static inline uint64_t prng64_map2_white(uint64_t state) {
return bleach64(prng64_map2_careless(state));
}
static inline uint64_t prng64_careless(uint64_t &state) {
state = prng64_map1_careless(state);
return state;
}
static inline double u64_to_double1(uint64_t v) {
union {
uint64_t u64;
double d;
} casting;
casting.u64 = UINT64_C(0x3ff) << 52 | (v >> 12);
assert(casting.d >= 1.0 && casting.d < 2.0);
return casting.d - 1.0;
}
uint64_t prng64_white(uint64_t &state);
uint32_t prng32(uint64_t &state);
void prng_fill(uint64_t &state, void *ptr, size_t bytes);
void prng_seed(uint64_t seed);
uint32_t prng32(void);
uint64_t prng64(void);
void prng_fill(void *ptr, size_t bytes);
bool flipcoin();
bool jitter(unsigned probability_percent);
void jitter_delay(bool extra = false);

7
contrib/db/libmdbx/tutorial/CMakeLists.txt Normal file
View file

@ -0,0 +1,7 @@
set(TARGET mdbx_tutorial)
project(${TARGET})
add_executable(${TARGET} sample-mdbx.c)
target_link_libraries(${TARGET} mdbx)

1
contrib/db/libmdbx/tutorial/README.md Normal file
View file

@ -0,0 +1 @@
This directory is just a placeholder for now. Tutorial and examples will be added later.

77
contrib/db/libmdbx/tutorial/sample-bdb.txt Normal file
View file

@ -0,0 +1,77 @@
/* sample-bdb.txt - BerkeleyDB toy/sample
*
* Do a line-by-line comparison of this and sample-mdb.txt
*/
/*
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* Copyright 2015,2016 Peter-Service R&D LLC.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include <stdio.h>
#include <string.h>
#include <db.h>
int main(int argc,char * argv[])
{
int rc;
DB_ENV *env;
DB *dbi;
DBT key, data;
DB_TXN *txn;
DBC *cursor;
char sval[32], kval[32];
/* Note: Most error checking omitted for simplicity */
#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD)
rc = db_env_create(&env, 0);
rc = env->open(env, "./testdb", FLAGS, 0664);
rc = db_create(&dbi, env, 0);
rc = env->txn_begin(env, NULL, &txn, 0);
rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664);
memset(&key, 0, sizeof(DBT));
memset(&data, 0, sizeof(DBT));
key.size = sizeof(int);
key.data = sval;
data.size = sizeof(sval);
data.data = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = dbi->put(dbi, txn, &key, &data, 0);
rc = txn->commit(txn, 0);
if (rc) {
fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc));
goto leave;
}
rc = env->txn_begin(env, NULL, &txn, 0);
rc = dbi->cursor(dbi, txn, &cursor, 0);
key.flags = DB_DBT_USERMEM;
key.data = kval;
key.ulen = sizeof(kval);
data.flags = DB_DBT_USERMEM;
data.data = sval;
data.ulen = sizeof(sval);
while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
key.data, (int) key.size, (char *) key.data,
data.data, (int) data.size, (char *) data.data);
}
rc = cursor->c_close(cursor);
rc = txn->abort(txn);
leave:
rc = dbi->close(dbi, 0);
rc = env->close(env, 0);
return rc;
}

112
contrib/db/libmdbx/tutorial/sample-mdbx.c Normal file
View file

@ -0,0 +1,112 @@
/* sample-mdb.txt - MDB toy/sample
*
* Do a line-by-line comparison of this and sample-bdb.txt
*/
/*
* Copyright 2017 Ilya Shipitsin <chipitsine@gmail.com>.
* Copyright 2015-2019 Leonid Yuriev <leo@yuriev.ru>.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
* Public License.
*
* A copy of this license is available in the file LICENSE in the
* top-level directory of the distribution or, alternatively, at
* <http://www.OpenLDAP.org/license.html>.
*/
#include "mdbx.h"
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char *argv[]) {
(void)argc;
(void)argv;
int rc;
MDBX_env *env = NULL;
MDBX_dbi dbi = 0;
MDBX_val key, data;
MDBX_txn *txn = NULL;
MDBX_cursor *cursor = NULL;
char sval[32];
rc = mdbx_env_create(&env);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_env_open(env, "./example-db",
MDBX_NOSUBDIR | MDBX_COALESCE | MDBX_LIFORECLAIM, 0664);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_txn_begin(env, NULL, 0, &txn);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_dbi_open(txn, NULL, 0, &dbi);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
key.iov_len = sizeof(int);
key.iov_base = sval;
data.iov_len = sizeof(sval);
data.iov_base = sval;
sprintf(sval, "%03x %d foo bar", 32, 3141592);
rc = mdbx_put(txn, dbi, &key, &data, 0);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_put: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_txn_commit(txn);
if (rc) {
fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
txn = NULL;
rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
rc = mdbx_cursor_open(txn, dbi, &cursor);
if (rc != MDBX_SUCCESS) {
fprintf(stderr, "mdbx_cursor_open: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
}
int found = 0;
while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len,
(char *)key.iov_base, data.iov_base, (int)data.iov_len,
(char *)data.iov_base);
found += 1;
}
if (rc != MDBX_NOTFOUND || found == 0) {
fprintf(stderr, "mdbx_cursor_get: (%d) %s\n", rc, mdbx_strerror(rc));
goto bailout;
} else {
rc = MDBX_SUCCESS;
}
bailout:
if (cursor)
mdbx_cursor_close(cursor);
if (txn)
mdbx_txn_abort(txn);
if (dbi)
mdbx_dbi_close(env, dbi);
if (env)
mdbx_env_close(env);
return (rc != MDBX_SUCCESS) ? EXIT_FAILURE : EXIT_SUCCESS;
}