forked from lthn/blockchain
lmdb: reverting back to v 18 due to complex issues with resizing/synchronization
lmdb v 24 requires manual resizing/growing during it's normal function Zano core is purely async so it's not easy to prevent all DB txs from starting on lmdb adapter level, because it will lead to random deadlocks in the core due to many high-level cross-thread dependencies. We will rethink this later. Many thanks to @leo-yuriev who helped us to discover these issues!
This commit is contained in:
sowle
parent
4936c6bb20
commit
28659827c3
33 changed files with 1806 additions and 993 deletions
|
|
@ -1,70 +1,14 @@
|
|||
LMDB 0.9 Change Log
|
||||
|
||||
LMDB 0.9.24 Release (2019/07/24)
|
||||
ITS#8969 Tweak mdb_page_split
|
||||
ITS#8975 WIN32 fix writemap set_mapsize crash
|
||||
ITS#9007 Fix loose pages in WRITEMAP
|
||||
|
||||
LMDB 0.9.23 Release (2018/12/19)
|
||||
ITS#8756 Fix loose pages in dirty list
|
||||
ITS#8831 Fix mdb_load flag init
|
||||
ITS#8844 Fix mdb_env_close in forked process
|
||||
Documentation
|
||||
ITS#8857 mdb_cursor_del doesn't invalidate cursor
|
||||
ITS#8908 GET_MULTIPLE etc don't change passed in key
|
||||
|
||||
LMDB 0.9.22 Release (2018/03/22)
|
||||
Fix MDB_DUPSORT alignment bug (ITS#8819)
|
||||
Fix regression with new db from 0.9.19 (ITS#8760)
|
||||
Fix liblmdb to build on Solaris (ITS#8612)
|
||||
Fix delete behavior with DUPSORT DB (ITS#8622)
|
||||
Fix mdb_cursor_get/mdb_cursor_del behavior (ITS#8722)
|
||||
|
||||
LMDB 0.9.21 Release (2017/06/01)
|
||||
Fix xcursor after cursor_del (ITS#8622)
|
||||
|
||||
LMDB 0.9.20 (Withdrawn)
|
||||
Fix mdb_load with escaped plaintext (ITS#8558)
|
||||
Fix mdb_cursor_last / mdb_put interaction (ITS#8557)
|
||||
|
||||
LMDB 0.9.19 Release (2016/12/28)
|
||||
Fix mdb_env_cwalk cursor init (ITS#8424)
|
||||
Fix robust mutexes on Solaris 10/11 (ITS#8339)
|
||||
Tweak Win32 error message buffer
|
||||
Fix MDB_GET_BOTH on non-dup record (ITS#8393)
|
||||
Optimize mdb_drop
|
||||
Fix xcursors after mdb_cursor_del (ITS#8406)
|
||||
Fix MDB_NEXT_DUP after mdb_cursor_del (ITS#8412)
|
||||
Fix mdb_cursor_put resetting C_EOF (ITS#8489)
|
||||
Fix mdb_env_copyfd2 to return EPIPE on SIGPIPE (ITS#8504)
|
||||
Fix mdb_env_copy with empty DB (ITS#8209)
|
||||
Fix behaviors with fork (ITS#8505)
|
||||
Fix mdb_dbi_open with mainDB cursors (ITS#8542)
|
||||
Fix robust mutexes on kFreeBSD (ITS#8554)
|
||||
Fix utf8_to_utf16 error checks (ITS#7992)
|
||||
Fix F_NOCACHE on MacOS, error is non-fatal (ITS#7682)
|
||||
Build
|
||||
Make shared lib suffix overridable (ITS#8481)
|
||||
Documentation
|
||||
Cleanup doxygen nits
|
||||
Note reserved vs actual mem/disk usage
|
||||
|
||||
|
||||
LMDB 0.9.18 Release (2016/02/05)
|
||||
LMDB 0.9.18 Release Engineering
|
||||
Fix robust mutex detection on glibc 2.10-11 (ITS#8330)
|
||||
Fix page_search_root assert on FreeDB (ITS#8336)
|
||||
Fix MDB_APPENDDUP vs. rewrite(single item) (ITS#8334)
|
||||
Fix mdb_copy of large files on Windows
|
||||
Fix subcursor move after delete (ITS#8355)
|
||||
Fix mdb_midl_shirnk off-by-one (ITS#8363)
|
||||
Check for utf8_to_utf16 failures (ITS#7992)
|
||||
Catch strdup failure in mdb_dbi_open
|
||||
Build
|
||||
Additional makefile var tweaks (ITS#8169)
|
||||
Documentation
|
||||
Add Getting Started page
|
||||
Update WRITEMAP description
|
||||
|
||||
|
||||
|
||||
LMDB 0.9.17 Release (2015/11/30)
|
||||
Fix ITS#7377 catch calloc failure
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
Copyright 2011-2019 Howard Chu, Symas Corp.
|
||||
Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -253,7 +253,7 @@ IDL_PROPERTY_SUPPORT = YES
|
|||
# member in the group (if any) for the other members of the group. By default
|
||||
# all members of a group must be documented explicitly.
|
||||
|
||||
DISTRIBUTE_GROUP_DOC = YES
|
||||
DISTRIBUTE_GROUP_DOC = NO
|
||||
|
||||
# Set the SUBGROUPING tag to YES (the default) to allow class member groups of
|
||||
# the same type (for instance a group of public functions) to be put as a
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
# platforms; you should not need to change any of these.
|
||||
# Read their descriptions in mdb.c if you do:
|
||||
#
|
||||
# - MDB_USE_POSIX_SEM
|
||||
# - MDB_USE_POSIX_MUTEX, MDB_USE_POSIX_SEM, MDB_USE_SYSV_SEM
|
||||
# - MDB_DSYNC
|
||||
# - MDB_FDATASYNC
|
||||
# - MDB_FDATASYNC_WORKS
|
||||
|
|
@ -24,9 +24,8 @@ W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
|
|||
THREADS = -pthread
|
||||
OPT = -O2 -g
|
||||
CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS)
|
||||
LDLIBS =
|
||||
SOLIBS =
|
||||
SOEXT = .so
|
||||
LDLIBS = # -lntdll # Windows needs ntdll
|
||||
SOLIBS = # -lntdll
|
||||
prefix = /usr/local
|
||||
exec_prefix = $(prefix)
|
||||
bindir = $(exec_prefix)/bin
|
||||
|
|
@ -38,7 +37,7 @@ mandir = $(datarootdir)/man
|
|||
########################################################################
|
||||
|
||||
IHDRS = lmdb.h
|
||||
ILIBS = liblmdb.a liblmdb$(SOEXT)
|
||||
ILIBS = liblmdb.a liblmdb.so
|
||||
IPROGS = mdb_stat mdb_copy mdb_dump mdb_load
|
||||
IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1
|
||||
PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
|
||||
|
|
@ -64,7 +63,7 @@ test: all
|
|||
liblmdb.a: mdb.o midl.o
|
||||
$(AR) rs $@ mdb.o midl.o
|
||||
|
||||
liblmdb$(SOEXT): mdb.lo midl.lo
|
||||
liblmdb.so: mdb.lo midl.lo
|
||||
# $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
|
||||
$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright 2015-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -53,14 +53,15 @@
|
|||
*
|
||||
* Fix: Check for stale readers periodically, using the
|
||||
* #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool.
|
||||
* Stale writers will be cleared automatically on some systems:
|
||||
* Stale writers will be cleared automatically on most systems:
|
||||
* - Windows - automatic
|
||||
* - BSD, systems using SysV semaphores - automatic
|
||||
* - Linux, systems using POSIX mutexes with Robust option - automatic
|
||||
* - not on BSD, systems using POSIX semaphores.
|
||||
* Otherwise just make all programs using the database close it;
|
||||
* the lockfile is always reset on first open of the environment.
|
||||
*
|
||||
* - On BSD systems or others configured with MDB_USE_POSIX_SEM,
|
||||
* - On BSD systems or others configured with MDB_USE_SYSV_SEM or
|
||||
* MDB_USE_POSIX_SEM,
|
||||
* startup can fail due to semaphores owned by another userid.
|
||||
*
|
||||
* Fix: Open and close the database as the user which owns the
|
||||
|
|
@ -96,12 +97,11 @@
|
|||
* transactions. Each transaction belongs to one thread. See below.
|
||||
* The #MDB_NOTLS flag changes this for read-only transactions.
|
||||
*
|
||||
* - Use an MDB_env* in the process which opened it, not after fork().
|
||||
* - Use an MDB_env* in the process which opened it, without fork()ing.
|
||||
*
|
||||
* - Do not have open an LMDB database twice in the same process at
|
||||
* the same time. Not even from a plain open() call - close()ing it
|
||||
* breaks fcntl() advisory locking. (It is OK to reopen it after
|
||||
* fork() - exec*(), since the lockfile has FD_CLOEXEC set.)
|
||||
* breaks flock() advisory locking.
|
||||
*
|
||||
* - Avoid long-lived transactions. Read transactions prevent
|
||||
* reuse of pages freed by newer write transactions, thus the
|
||||
|
|
@ -135,7 +135,7 @@
|
|||
*
|
||||
* @author Howard Chu, Symas Corporation.
|
||||
*
|
||||
* @copyright Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved.
|
||||
* @copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted only as authorized by the OpenLDAP
|
||||
|
|
@ -166,6 +166,7 @@
|
|||
#define _LMDB_H_
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -178,6 +179,13 @@ typedef int mdb_mode_t;
|
|||
typedef mode_t mdb_mode_t;
|
||||
#endif
|
||||
|
||||
#ifdef MDB_VL32
|
||||
typedef uint64_t mdb_size_t;
|
||||
#define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */
|
||||
#else
|
||||
typedef size_t mdb_size_t;
|
||||
#endif
|
||||
|
||||
/** An abstraction for a file handle.
|
||||
* On POSIX systems file handles are small integers. On Windows
|
||||
* they're opaque pointers.
|
||||
|
|
@ -200,7 +208,7 @@ typedef int mdb_filehandle_t;
|
|||
/** Library minor version */
|
||||
#define MDB_VERSION_MINOR 9
|
||||
/** Library patch version */
|
||||
#define MDB_VERSION_PATCH 24
|
||||
#define MDB_VERSION_PATCH 70
|
||||
|
||||
/** Combine args a,b,c into a single integer for easy version comparisons */
|
||||
#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c))
|
||||
|
|
@ -210,7 +218,7 @@ typedef int mdb_filehandle_t;
|
|||
MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
|
||||
|
||||
/** The release date of this library version */
|
||||
#define MDB_VERSION_DATE "July 24, 2019"
|
||||
#define MDB_VERSION_DATE "December 19, 2015"
|
||||
|
||||
/** A stringifier for the version info */
|
||||
#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")"
|
||||
|
|
@ -303,6 +311,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
|
|||
#define MDB_NORDAHEAD 0x800000
|
||||
/** don't initialize malloc'd memory before writing to datafile */
|
||||
#define MDB_NOMEMINIT 0x1000000
|
||||
/** use the previous snapshot rather than the latest one */
|
||||
#define MDB_PREVSNAPSHOT 0x2000000
|
||||
/** @} */
|
||||
|
||||
/** @defgroup mdb_dbi_open Database Flags
|
||||
|
|
@ -370,7 +380,7 @@ typedef enum MDB_cursor_op {
|
|||
MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */
|
||||
MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */
|
||||
MDB_GET_CURRENT, /**< Return key/data at current cursor position */
|
||||
MDB_GET_MULTIPLE, /**< Return up to a page of duplicate data items
|
||||
MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items
|
||||
from current cursor position. Move cursor to prepare
|
||||
for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */
|
||||
MDB_LAST, /**< Position at last key/data item */
|
||||
|
|
@ -379,7 +389,7 @@ typedef enum MDB_cursor_op {
|
|||
MDB_NEXT, /**< Position at next data item */
|
||||
MDB_NEXT_DUP, /**< Position at next data item of current key.
|
||||
Only for #MDB_DUPSORT */
|
||||
MDB_NEXT_MULTIPLE, /**< Return up to a page of duplicate data items
|
||||
MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items
|
||||
from next cursor position. Move cursor to prepare
|
||||
for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */
|
||||
MDB_NEXT_NODUP, /**< Position at first data item of next key */
|
||||
|
|
@ -390,7 +400,7 @@ typedef enum MDB_cursor_op {
|
|||
MDB_SET, /**< Position at specified key */
|
||||
MDB_SET_KEY, /**< Position at specified key, return key + data */
|
||||
MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */
|
||||
MDB_PREV_MULTIPLE /**< Position at previous page and return up to
|
||||
MDB_PREV_MULTIPLE /**< Position at previous page and return key and up to
|
||||
a page of duplicate data items. Only for #MDB_DUPFIXED */
|
||||
} MDB_cursor_op;
|
||||
|
||||
|
|
@ -457,18 +467,18 @@ typedef struct MDB_stat {
|
|||
unsigned int ms_psize; /**< Size of a database page.
|
||||
This is currently the same for all databases. */
|
||||
unsigned int ms_depth; /**< Depth (height) of the B-tree */
|
||||
size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */
|
||||
size_t ms_leaf_pages; /**< Number of leaf pages */
|
||||
size_t ms_overflow_pages; /**< Number of overflow pages */
|
||||
size_t ms_entries; /**< Number of data items */
|
||||
mdb_size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */
|
||||
mdb_size_t ms_leaf_pages; /**< Number of leaf pages */
|
||||
mdb_size_t ms_overflow_pages; /**< Number of overflow pages */
|
||||
mdb_size_t ms_entries; /**< Number of data items */
|
||||
} MDB_stat;
|
||||
|
||||
/** @brief Information about the environment */
|
||||
typedef struct MDB_envinfo {
|
||||
void *me_mapaddr; /**< Address of map, if fixed */
|
||||
size_t me_mapsize; /**< Size of the data memory map */
|
||||
size_t me_last_pgno; /**< ID of the last used page */
|
||||
size_t me_last_txnid; /**< ID of the last committed transaction */
|
||||
mdb_size_t me_mapsize; /**< Size of the data memory map */
|
||||
mdb_size_t me_last_pgno; /**< ID of the last used page */
|
||||
mdb_size_t me_last_txnid; /**< ID of the last committed transaction */
|
||||
unsigned int me_maxreaders; /**< max reader slots in the environment */
|
||||
unsigned int me_numreaders; /**< max reader slots used in the environment */
|
||||
} MDB_envinfo;
|
||||
|
|
@ -614,6 +624,12 @@ int mdb_env_create(MDB_env **env);
|
|||
* caller is expected to overwrite all of the memory that was
|
||||
* reserved in that case.
|
||||
* This flag may be changed at any time using #mdb_env_set_flags().
|
||||
* <li>#MDB_PREVSNAPSHOT
|
||||
* Open the environment with the previous snapshot rather than the latest
|
||||
* one. This loses the latest transaction, but may help work around some
|
||||
* types of corruption. If opened with write access, this must be the
|
||||
* only process using the environment. This flag is automatically reset
|
||||
* after a write transaction is successfully committed.
|
||||
* </ul>
|
||||
* @param[in] mode The UNIX permissions to set on created files and semaphores.
|
||||
* This parameter is ignored on Windows.
|
||||
|
|
@ -680,7 +696,6 @@ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
|
|||
* <li>#MDB_CP_COMPACT - Perform compaction while copying: omit free
|
||||
* pages and sequentially renumber all pages in output. This option
|
||||
* consumes more CPU and runs more slowly than the default.
|
||||
* Currently it fails if the environment has suffered a page leak.
|
||||
* </ul>
|
||||
* @return A non-zero error value on failure and 0 on success.
|
||||
*/
|
||||
|
|
@ -795,10 +810,6 @@ int mdb_env_get_flags(MDB_env *env, unsigned int *flags);
|
|||
int mdb_env_get_path(MDB_env *env, const char **path);
|
||||
|
||||
/** @brief Return the filedescriptor for the given environment.
|
||||
*
|
||||
* This function may be called after fork(), so the descriptor can be
|
||||
* closed before exec*(). Other LMDB file descriptors have FD_CLOEXEC.
|
||||
* (Until LMDB 0.9.18, only the lockfile had that.)
|
||||
*
|
||||
* @param[in] env An environment handle returned by #mdb_env_create()
|
||||
* @param[out] fd Address of a mdb_filehandle_t to contain the descriptor.
|
||||
|
|
@ -842,7 +853,7 @@ int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd);
|
|||
* an active write transaction.
|
||||
* </ul>
|
||||
*/
|
||||
int mdb_env_set_mapsize(MDB_env *env, size_t size);
|
||||
int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size);
|
||||
|
||||
/** @brief Set the maximum number of threads/reader slots for the environment.
|
||||
*
|
||||
|
|
@ -955,6 +966,10 @@ int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
|
|||
* <ul>
|
||||
* <li>#MDB_RDONLY
|
||||
* This transaction will not perform any write operations.
|
||||
* <li>#MDB_NOSYNC
|
||||
* Don't flush system buffers to disk when committing this transaction.
|
||||
* <li>#MDB_NOMETASYNC
|
||||
* Flush system buffers but omit metadata flush when committing this transaction.
|
||||
* </ul>
|
||||
* @param[out] txn Address where the new #MDB_txn handle will be stored
|
||||
* @return A non-zero error value on failure and 0 on success. Some possible
|
||||
|
|
@ -987,7 +1002,7 @@ MDB_env *mdb_txn_env(MDB_txn *txn);
|
|||
* @param[in] txn A transaction handle returned by #mdb_txn_begin()
|
||||
* @return A transaction ID, valid if input is an active transaction.
|
||||
*/
|
||||
size_t mdb_txn_id(MDB_txn *txn);
|
||||
mdb_size_t mdb_txn_id(MDB_txn *txn);
|
||||
|
||||
/** @brief Commit all the operations of a transaction into the database.
|
||||
*
|
||||
|
|
@ -1103,9 +1118,8 @@ int mdb_txn_renew(MDB_txn *txn);
|
|||
* This flag may only be used in combination with #MDB_DUPSORT. This option
|
||||
* tells the library that the data items for this database are all the same
|
||||
* size, which allows further optimizations in storage and retrieval. When
|
||||
* all data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE
|
||||
* and #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple
|
||||
* items at once.
|
||||
* all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
|
||||
* cursor operations may be used to retrieve multiple items at once.
|
||||
* <li>#MDB_INTEGERDUP
|
||||
* This option specifies that duplicate data items are binary integers,
|
||||
* similar to #MDB_INTEGERKEY keys.
|
||||
|
|
@ -1510,10 +1524,6 @@ int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
|
|||
/** @brief Delete current key/data pair
|
||||
*
|
||||
* This function deletes the key/data pair to which the cursor refers.
|
||||
* This does not invalidate the cursor, so operations such as MDB_NEXT
|
||||
* can still be used on it.
|
||||
* Both MDB_NEXT and MDB_GET_CURRENT will return the same record after
|
||||
* this operation.
|
||||
* @param[in] cursor A cursor handle returned by #mdb_cursor_open()
|
||||
* @param[in] flags Options for this operation. This parameter
|
||||
* must be set to 0 or one of the values described here.
|
||||
|
|
@ -1542,7 +1552,7 @@ int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags);
|
|||
* <li>EINVAL - cursor is not initialized, or an invalid parameter was specified.
|
||||
* </ul>
|
||||
*/
|
||||
int mdb_cursor_count(MDB_cursor *cursor, size_t *countp);
|
||||
int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp);
|
||||
|
||||
/** @brief Compare two data items according to a particular database.
|
||||
*
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,5 +1,5 @@
|
|||
.TH MDB_COPY 1 "2014/07/01" "LMDB 0.9.14"
|
||||
.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved.
|
||||
.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
|
||||
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
|
||||
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
|
||||
.SH NAME
|
||||
mdb_copy \- LMDB environment copy tool
|
||||
|
|
@ -11,6 +11,8 @@ mdb_copy \- LMDB environment copy tool
|
|||
.BR \-c ]
|
||||
[\c
|
||||
.BR \-n ]
|
||||
[\c
|
||||
.BR \-v ]
|
||||
.B srcpath
|
||||
[\c
|
||||
.BR dstpath ]
|
||||
|
|
@ -36,10 +38,13 @@ Write the library version number to the standard output, and exit.
|
|||
Compact while copying. Only current data pages will be copied; freed
|
||||
or unused pages will be omitted from the copy. This option will
|
||||
slow down the backup process as it is more CPU-intensive.
|
||||
Currently it fails if the environment has suffered a page leak.
|
||||
.TP
|
||||
.BR \-n
|
||||
Open LDMB environment(s) which do not use subdirectories.
|
||||
.TP
|
||||
.BR \-v
|
||||
Use the previous environment state instead of the latest state.
|
||||
This may be useful if the latest state has been corrupted.
|
||||
|
||||
.SH DIAGNOSTICS
|
||||
Exit status is zero if no errors occur.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mdb_copy.c - memory-mapped database backup tool */
|
||||
/*
|
||||
* Copyright 2012-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2012-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -38,6 +38,8 @@ int main(int argc,char * argv[])
|
|||
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
|
||||
if (argv[1][1] == 'n' && argv[1][2] == '\0')
|
||||
flags |= MDB_NOSUBDIR;
|
||||
else if (argv[1][1] == 'v' && argv[1][2] == '\0')
|
||||
flags |= MDB_PREVSNAPSHOT;
|
||||
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
|
||||
cpflags |= MDB_CP_COMPACT;
|
||||
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
|
||||
|
|
@ -48,7 +50,7 @@ int main(int argc,char * argv[])
|
|||
}
|
||||
|
||||
if (argc<2 || argc>3) {
|
||||
fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname);
|
||||
fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] srcpath [dstpath]\n", progname);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
.TH MDB_DUMP 1 "2015/09/30" "LMDB 0.9.17"
|
||||
.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
|
||||
.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
|
||||
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
|
||||
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
|
||||
.SH NAME
|
||||
mdb_dump \- LMDB environment export tool
|
||||
|
|
@ -14,6 +14,8 @@ mdb_dump \- LMDB environment export tool
|
|||
[\c
|
||||
.BR \-n ]
|
||||
[\c
|
||||
.BR \-v ]
|
||||
[\c
|
||||
.BR \-p ]
|
||||
[\c
|
||||
.BR \-a \ |
|
||||
|
|
@ -42,6 +44,10 @@ names will be listed, no data will be output.
|
|||
.BR \-n
|
||||
Dump an LMDB database which does not use subdirectories.
|
||||
.TP
|
||||
.BR \-v
|
||||
Use the previous environment state instead of the latest state.
|
||||
This may be useful if the latest state has been corrupted.
|
||||
.TP
|
||||
.BR \-p
|
||||
If characters in either the key or data items are printing characters (as
|
||||
defined by isprint(3)), output them directly. This option permits users to
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mdb_dump.c - memory-mapped database dump tool */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -25,6 +25,15 @@
|
|||
#else
|
||||
#define Z "z"
|
||||
#endif
|
||||
#ifdef MDB_VL32
|
||||
#ifdef _WIN32
|
||||
#define Y "I64"
|
||||
#else
|
||||
#define Y "ll"
|
||||
#endif
|
||||
#else
|
||||
#define Y Z
|
||||
#endif
|
||||
|
||||
#define PRINT 1
|
||||
static int mode;
|
||||
|
|
@ -115,7 +124,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
|
|||
if (name)
|
||||
printf("database=%s\n", name);
|
||||
printf("type=btree\n");
|
||||
printf("mapsize=%" Z "u\n", info.me_mapsize);
|
||||
printf("mapsize=%" Y "u\n", info.me_mapsize);
|
||||
if (info.me_mapaddr)
|
||||
printf("mapaddr=%p\n", info.me_mapaddr);
|
||||
printf("maxreaders=%u\n", info.me_maxreaders);
|
||||
|
|
@ -155,7 +164,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
|
|||
|
||||
static void usage(char *prog)
|
||||
{
|
||||
fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n", prog);
|
||||
fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-a|-s subdb] dbpath\n", prog);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
|
|
@ -179,6 +188,7 @@ int main(int argc, char *argv[])
|
|||
* -n: use NOSUBDIR flag on env_open
|
||||
* -p: use printable characters
|
||||
* -f: write to file instead of stdout
|
||||
* -v: use previous snapshot
|
||||
* -V: print version and exit
|
||||
* (default) dump only the main DB
|
||||
*/
|
||||
|
|
@ -206,6 +216,9 @@ int main(int argc, char *argv[])
|
|||
case 'n':
|
||||
envflags |= MDB_NOSUBDIR;
|
||||
break;
|
||||
case 'v':
|
||||
envflags |= MDB_PREVSNAPSHOT;
|
||||
break;
|
||||
case 'p':
|
||||
mode |= PRINT;
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
.TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.17"
|
||||
.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
|
||||
.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
|
||||
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
|
||||
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
|
||||
.SH NAME
|
||||
mdb_load \- LMDB environment import tool
|
||||
|
|
@ -37,6 +37,13 @@ option below.
|
|||
.BR \-V
|
||||
Write the library version number to the standard output, and exit.
|
||||
.TP
|
||||
.BR \-a
|
||||
Append all records in the order they appear in the input. The input is assumed to already be
|
||||
in correctly sorted order and no sorting or checking for redundant values will be performed.
|
||||
This option must be used to reload data that was produced by running
|
||||
.B mdb_dump
|
||||
on a database that uses custom compare functions.
|
||||
.TP
|
||||
.BR \-f \ file
|
||||
Read from the specified file instead of from the standard input.
|
||||
.TP
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mdb_load.c - memory-mapped database load tool */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -37,12 +37,22 @@ static int Eof;
|
|||
static MDB_envinfo info;
|
||||
|
||||
static MDB_val kbuf, dbuf;
|
||||
static MDB_val k0buf;
|
||||
|
||||
#ifdef _WIN32
|
||||
#define Z "I"
|
||||
#else
|
||||
#define Z "z"
|
||||
#endif
|
||||
#ifdef MDB_VL32
|
||||
#ifdef _WIN32
|
||||
#define Y "I64"
|
||||
#else
|
||||
#define Y "ll"
|
||||
#endif
|
||||
#else
|
||||
#define Y Z
|
||||
#endif
|
||||
|
||||
#define STRLENOF(s) (sizeof(s)-1)
|
||||
|
||||
|
|
@ -68,7 +78,6 @@ static void readhdr(void)
|
|||
{
|
||||
char *ptr;
|
||||
|
||||
flags = 0;
|
||||
while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
|
||||
lineno++;
|
||||
if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
|
||||
|
|
@ -113,7 +122,7 @@ static void readhdr(void)
|
|||
int i;
|
||||
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
|
||||
if (ptr) *ptr = '\0';
|
||||
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize);
|
||||
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize);
|
||||
if (i != 1) {
|
||||
fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
|
||||
prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
|
||||
|
|
@ -249,8 +258,7 @@ badend:
|
|||
c2 += 2;
|
||||
}
|
||||
} else {
|
||||
/* copies are redundant when no escapes were used */
|
||||
*c1++ = *c2++;
|
||||
c1++; c2++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
@ -278,10 +286,15 @@ badend:
|
|||
|
||||
static void usage(void)
|
||||
{
|
||||
fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
|
||||
fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
static int greater(const MDB_val *a, const MDB_val *b)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
int i, rc;
|
||||
|
|
@ -291,7 +304,8 @@ int main(int argc, char *argv[])
|
|||
MDB_dbi dbi;
|
||||
char *envname;
|
||||
int envflags = 0, putflags = 0;
|
||||
int dohdr = 0;
|
||||
int dohdr = 0, append = 0;
|
||||
MDB_val prevk;
|
||||
|
||||
prog = argv[0];
|
||||
|
||||
|
|
@ -299,19 +313,23 @@ int main(int argc, char *argv[])
|
|||
usage();
|
||||
}
|
||||
|
||||
/* -f: load file instead of stdin
|
||||
/* -a: append records in input order
|
||||
* -f: load file instead of stdin
|
||||
* -n: use NOSUBDIR flag on env_open
|
||||
* -s: load into named subDB
|
||||
* -N: use NOOVERWRITE on puts
|
||||
* -T: read plaintext
|
||||
* -V: print version and exit
|
||||
*/
|
||||
while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
|
||||
while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
|
||||
switch(i) {
|
||||
case 'V':
|
||||
printf("%s\n", MDB_VERSION_STRING);
|
||||
exit(0);
|
||||
break;
|
||||
case 'a':
|
||||
append = 1;
|
||||
break;
|
||||
case 'f':
|
||||
if (freopen(optarg, "r", stdin) == NULL) {
|
||||
fprintf(stderr, "%s: %s: reopen: %s\n",
|
||||
|
|
@ -370,11 +388,17 @@ int main(int argc, char *argv[])
|
|||
}
|
||||
|
||||
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
|
||||
kbuf.mv_data = malloc(kbuf.mv_size);
|
||||
kbuf.mv_data = malloc(kbuf.mv_size * 2);
|
||||
k0buf.mv_size = kbuf.mv_size;
|
||||
k0buf.mv_data = (char *)kbuf.mv_data + kbuf.mv_size;
|
||||
prevk.mv_size = 0;
|
||||
prevk.mv_data = k0buf.mv_data;
|
||||
|
||||
while(!Eof) {
|
||||
MDB_val key, data;
|
||||
int batch = 0;
|
||||
flags = 0;
|
||||
int appflag;
|
||||
|
||||
if (!dohdr) {
|
||||
dohdr = 1;
|
||||
|
|
@ -392,6 +416,11 @@ int main(int argc, char *argv[])
|
|||
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
|
||||
goto txn_abort;
|
||||
}
|
||||
if (append) {
|
||||
mdb_set_compare(txn, dbi, greater);
|
||||
if (flags & MDB_DUPSORT)
|
||||
mdb_set_dupsort(txn, dbi, greater);
|
||||
}
|
||||
|
||||
rc = mdb_cursor_open(txn, dbi, &mc);
|
||||
if (rc) {
|
||||
|
|
@ -410,7 +439,20 @@ int main(int argc, char *argv[])
|
|||
goto txn_abort;
|
||||
}
|
||||
|
||||
rc = mdb_cursor_put(mc, &key, &data, putflags);
|
||||
if (append) {
|
||||
appflag = MDB_APPEND;
|
||||
if (flags & MDB_DUPSORT) {
|
||||
if (prevk.mv_size == key.mv_size && !memcmp(prevk.mv_data, key.mv_data, key.mv_size))
|
||||
appflag = MDB_APPENDDUP;
|
||||
else {
|
||||
memcpy(prevk.mv_data, key.mv_data, key.mv_size);
|
||||
prevk.mv_size = key.mv_size;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
appflag = 0;
|
||||
}
|
||||
rc = mdb_cursor_put(mc, &key, &data, putflags|appflag);
|
||||
if (rc == MDB_KEYEXIST && putflags)
|
||||
continue;
|
||||
if (rc) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
.TH MDB_STAT 1 "2015/09/30" "LMDB 0.9.17"
|
||||
.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved.
|
||||
.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
|
||||
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
|
||||
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
|
||||
.SH NAME
|
||||
mdb_stat \- LMDB environment status tool
|
||||
|
|
@ -14,6 +14,8 @@ mdb_stat \- LMDB environment status tool
|
|||
[\c
|
||||
.BR \-n ]
|
||||
[\c
|
||||
.BR \-v ]
|
||||
[\c
|
||||
.BR \-r [ r ]]
|
||||
[\c
|
||||
.BR \-a \ |
|
||||
|
|
@ -39,6 +41,10 @@ If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
|
|||
.BR \-n
|
||||
Display the status of an LMDB database which does not use subdirectories.
|
||||
.TP
|
||||
.BR \-v
|
||||
Use the previous environment state instead of the latest state.
|
||||
This may be useful if the latest state has been corrupted.
|
||||
.TP
|
||||
.BR \-r
|
||||
Display information about the environment reader table.
|
||||
Shows the process ID, thread ID, and transaction ID for each active
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mdb_stat.c - memory-mapped database status tool */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -22,6 +22,15 @@
|
|||
#else
|
||||
#define Z "z"
|
||||
#endif
|
||||
#ifdef MDB_VL32
|
||||
#ifdef _WIN32
|
||||
#define Y "I64"
|
||||
#else
|
||||
#define Y "ll"
|
||||
#endif
|
||||
#else
|
||||
#define Y Z
|
||||
#endif
|
||||
|
||||
static void prstat(MDB_stat *ms)
|
||||
{
|
||||
|
|
@ -29,15 +38,15 @@ static void prstat(MDB_stat *ms)
|
|||
printf(" Page size: %u\n", ms->ms_psize);
|
||||
#endif
|
||||
printf(" Tree depth: %u\n", ms->ms_depth);
|
||||
printf(" Branch pages: %"Z"u\n", ms->ms_branch_pages);
|
||||
printf(" Leaf pages: %"Z"u\n", ms->ms_leaf_pages);
|
||||
printf(" Overflow pages: %"Z"u\n", ms->ms_overflow_pages);
|
||||
printf(" Entries: %"Z"u\n", ms->ms_entries);
|
||||
printf(" Branch pages: %"Y"u\n", ms->ms_branch_pages);
|
||||
printf(" Leaf pages: %"Y"u\n", ms->ms_leaf_pages);
|
||||
printf(" Overflow pages: %"Y"u\n", ms->ms_overflow_pages);
|
||||
printf(" Entries: %"Y"u\n", ms->ms_entries);
|
||||
}
|
||||
|
||||
static void usage(char *prog)
|
||||
{
|
||||
fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n", prog);
|
||||
fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-a|-s subdb] dbpath\n", prog);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
|
|
@ -64,6 +73,7 @@ int main(int argc, char *argv[])
|
|||
* -f: print freelist info
|
||||
* -r: print reader info
|
||||
* -n: use NOSUBDIR flag on env_open
|
||||
* -v: use previous snapshot
|
||||
* -V: print version and exit
|
||||
* (default) print stat of only the main DB
|
||||
*/
|
||||
|
|
@ -87,6 +97,9 @@ int main(int argc, char *argv[])
|
|||
case 'n':
|
||||
envflags |= MDB_NOSUBDIR;
|
||||
break;
|
||||
case 'v':
|
||||
envflags |= MDB_PREVSNAPSHOT;
|
||||
break;
|
||||
case 'r':
|
||||
rdrinfo++;
|
||||
break;
|
||||
|
|
@ -125,11 +138,11 @@ int main(int argc, char *argv[])
|
|||
(void)mdb_env_info(env, &mei);
|
||||
printf("Environment Info\n");
|
||||
printf(" Map address: %p\n", mei.me_mapaddr);
|
||||
printf(" Map size: %"Z"u\n", mei.me_mapsize);
|
||||
printf(" Map size: %"Y"u\n", mei.me_mapsize);
|
||||
printf(" Page size: %u\n", mst.ms_psize);
|
||||
printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize);
|
||||
printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1);
|
||||
printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid);
|
||||
printf(" Max pages: %"Y"u\n", mei.me_mapsize / mst.ms_psize);
|
||||
printf(" Number of pages used: %"Y"u\n", mei.me_last_pgno+1);
|
||||
printf(" Last transaction ID: %"Y"u\n", mei.me_last_txnid);
|
||||
printf(" Max readers: %u\n", mei.me_maxreaders);
|
||||
printf(" Number of readers used: %u\n", mei.me_numreaders);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,7 @@
|
|||
/* $OpenLDAP$ */
|
||||
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
|
||||
*
|
||||
* Copyright 2000-2019 The OpenLDAP Foundation.
|
||||
* Portions Copyright 2001-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2000-2015 The OpenLDAP Foundation.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -355,5 +354,67 @@ int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
|
|||
return 0;
|
||||
}
|
||||
|
||||
#ifdef MDB_VL32
|
||||
unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id )
|
||||
{
|
||||
/*
|
||||
* binary search of id in ids
|
||||
* if found, returns position of id
|
||||
* if not found, returns first position greater than id
|
||||
*/
|
||||
unsigned base = 0;
|
||||
unsigned cursor = 1;
|
||||
int val = 0;
|
||||
unsigned n = (unsigned)ids[0].mid;
|
||||
|
||||
while( 0 < n ) {
|
||||
unsigned pivot = n >> 1;
|
||||
cursor = base + pivot + 1;
|
||||
val = CMP( id, ids[cursor].mid );
|
||||
|
||||
if( val < 0 ) {
|
||||
n = pivot;
|
||||
|
||||
} else if ( val > 0 ) {
|
||||
base = cursor;
|
||||
n -= pivot + 1;
|
||||
|
||||
} else {
|
||||
return cursor;
|
||||
}
|
||||
}
|
||||
|
||||
if( val > 0 ) {
|
||||
++cursor;
|
||||
}
|
||||
return cursor;
|
||||
}
|
||||
|
||||
int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id )
|
||||
{
|
||||
unsigned x, i;
|
||||
|
||||
x = mdb_mid3l_search( ids, id->mid );
|
||||
|
||||
if( x < 1 ) {
|
||||
/* internal error */
|
||||
return -2;
|
||||
}
|
||||
|
||||
if ( x <= ids[0].mid && ids[x].mid == id->mid ) {
|
||||
/* duplicate */
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* insert id */
|
||||
ids[0].mid++;
|
||||
for (i=(unsigned)ids[0].mid; i>x; i--)
|
||||
ids[i] = ids[i-1];
|
||||
ids[x] = *id;
|
||||
|
||||
return 0;
|
||||
}
|
||||
#endif /* MDB_VL32 */
|
||||
|
||||
/** @} */
|
||||
/** @} */
|
||||
|
|
|
|||
|
|
@ -11,8 +11,7 @@
|
|||
/* $OpenLDAP$ */
|
||||
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
|
||||
*
|
||||
* Copyright 2000-2019 The OpenLDAP Foundation.
|
||||
* Portions Copyright 2001-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2000-2015 The OpenLDAP Foundation.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -28,6 +27,7 @@
|
|||
#define _MDB_MIDL_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -43,7 +43,11 @@ extern "C" {
|
|||
/** A generic unsigned ID number. These were entryIDs in back-bdb.
|
||||
* Preferably it should have the same size as a pointer.
|
||||
*/
|
||||
#ifdef MDB_VL32
|
||||
typedef uint64_t MDB_ID;
|
||||
#else
|
||||
typedef size_t MDB_ID;
|
||||
#endif
|
||||
|
||||
/** An IDL is an ID List, a sorted array of IDs. The first
|
||||
* element of the array is a counter for how many actual
|
||||
|
|
@ -178,6 +182,20 @@ int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id );
|
|||
*/
|
||||
int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
|
||||
|
||||
#ifdef MDB_VL32
|
||||
typedef struct MDB_ID3 {
|
||||
MDB_ID mid; /**< The ID */
|
||||
void *mptr; /**< The pointer */
|
||||
unsigned int mcnt; /**< Number of pages */
|
||||
unsigned int mref; /**< Refcounter */
|
||||
} MDB_ID3;
|
||||
|
||||
typedef MDB_ID3 *MDB_ID3L;
|
||||
|
||||
unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id );
|
||||
int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id );
|
||||
|
||||
#endif /* MDB_VL32 */
|
||||
/** @} */
|
||||
/** @} */
|
||||
#ifdef __cplusplus
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mtest.c - memory-mapped database tester/toy */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mtest2.c - memory-mapped database tester/toy */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mtest3.c - memory-mapped database tester/toy */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mtest4.c - memory-mapped database tester/toy */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mtest5.c - memory-mapped database tester/toy */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* mtest6.c - memory-mapped database tester/toy */
|
||||
/*
|
||||
* Copyright 2011-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2011-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
* Do a line-by-line comparison of this and sample-mdb.txt
|
||||
*/
|
||||
/*
|
||||
* Copyright 2012-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2012-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
* Do a line-by-line comparison of this and sample-bdb.txt
|
||||
*/
|
||||
/*
|
||||
* Copyright 2012-2018 Howard Chu, Symas Corp.
|
||||
* Copyright 2012-2015 Howard Chu, Symas Corp.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
|
|||
|
|
@ -241,9 +241,9 @@ namespace tools
|
|||
m_is_open = false;
|
||||
return m_backend->close();
|
||||
}
|
||||
bool open(const std::string& path, uint64_t flags = 0)
|
||||
bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE)
|
||||
{
|
||||
bool r = m_backend->open(path, flags);
|
||||
bool r = m_backend->open(path, cache_sz);
|
||||
if(r)
|
||||
m_is_open = true;
|
||||
|
||||
|
|
@ -558,10 +558,10 @@ namespace tools
|
|||
bool init(const std::string& container_name)
|
||||
{
|
||||
#ifdef ENABLE_PROFILING
|
||||
m_get_profiler.m_name = container_name +":get";
|
||||
m_set_profiler.m_name = container_name + ":set";
|
||||
m_explicit_get_profiler.m_name = container_name + ":explicit_get";
|
||||
m_explicit_set_profiler.m_name = container_name + ":explicit_set";
|
||||
m_get_profiler.m_name = container_name +":get";
|
||||
m_set_profiler.m_name = container_name + ":set";
|
||||
m_explicit_get_profiler.m_name = container_name + ":explicit_get";
|
||||
m_explicit_set_profiler.m_name = container_name + ":explicit_set";
|
||||
m_commit_profiler.m_name = container_name + ":commit";
|
||||
#endif
|
||||
return bdb.get_backend()->open_container(container_name, m_h);
|
||||
|
|
|
|||
|
|
@ -5,6 +5,13 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#ifndef ENV32BIT
|
||||
#define CACHE_SIZE uint64_t(uint64_t(1UL * 128UL) * 1024UL * 1024UL * 1024UL)
|
||||
#else
|
||||
#define CACHE_SIZE (1 * 1024UL * 1024UL * 1024UL)
|
||||
#endif
|
||||
|
||||
|
||||
namespace tools
|
||||
{
|
||||
namespace db
|
||||
|
|
@ -25,20 +32,20 @@ namespace tools
|
|||
|
||||
struct i_db_backend
|
||||
{
|
||||
virtual bool close() = 0;
|
||||
virtual bool close()=0;
|
||||
virtual bool begin_transaction(bool read_only = false) = 0;
|
||||
virtual bool commit_transaction() = 0;
|
||||
virtual void abort_transaction() = 0;
|
||||
virtual bool open(const std::string& path, uint64_t flags = 0) = 0;
|
||||
virtual bool open_container(const std::string& name, container_handle& h) = 0;
|
||||
virtual bool commit_transaction()=0;
|
||||
virtual void abort_transaction()=0;
|
||||
virtual bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) = 0;
|
||||
virtual bool open_container(const std::string& name, container_handle& h)=0;
|
||||
virtual bool erase(container_handle h, const char* k, size_t s) = 0;
|
||||
virtual uint64_t size(container_handle h) = 0;
|
||||
virtual bool get(container_handle h, const char* k, size_t s, std::string& res_buff) = 0;
|
||||
virtual bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs) = 0;
|
||||
virtual bool clear(container_handle h) = 0;
|
||||
virtual bool enumerate(container_handle h, i_db_callback* pcb) = 0;
|
||||
virtual bool enumerate(container_handle h, i_db_callback* pcb)=0;
|
||||
virtual bool get_stat_info(stat_info& si) = 0;
|
||||
virtual ~i_db_backend() {};
|
||||
virtual ~i_db_backend(){};
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -76,6 +76,7 @@ DISABLE_VS_WARNINGS(4267)
|
|||
|
||||
namespace
|
||||
{
|
||||
const command_line::arg_descriptor<uint32_t> arg_db_cache_l1 = { "db-cache-l1", "Specify size of memory mapped db cache file", 0, true };
|
||||
const command_line::arg_descriptor<uint32_t> arg_db_cache_l2 = { "db-cache-l2", "Specify cached elements in db helpers", 0, true };
|
||||
}
|
||||
|
||||
|
|
@ -151,6 +152,7 @@ std::shared_ptr<transaction> blockchain_storage::get_tx(const crypto::hash &id)
|
|||
//------------------------------------------------------------------
|
||||
void blockchain_storage::init_options(boost::program_options::options_description& desc)
|
||||
{
|
||||
command_line::add_arg(desc, arg_db_cache_l1);
|
||||
command_line::add_arg(desc, arg_db_cache_l2);
|
||||
}
|
||||
//------------------------------------------------------------------
|
||||
|
|
@ -213,20 +215,36 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro
|
|||
return false;
|
||||
}
|
||||
|
||||
uint64_t cache_size_l1 = CACHE_SIZE;
|
||||
if (command_line::has_arg(vm, arg_db_cache_l1))
|
||||
{
|
||||
cache_size_l1 = command_line::get_arg(vm, arg_db_cache_l1);
|
||||
}
|
||||
LOG_PRINT_GREEN("Using db file cache size(L1): " << cache_size_l1, LOG_LEVEL_0);
|
||||
|
||||
m_config_folder = config_folder;
|
||||
|
||||
// remove old incompartible DB
|
||||
const std::string old_db_folder_path = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD;
|
||||
if (boost::filesystem::exists(old_db_folder_path))
|
||||
{
|
||||
LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0);
|
||||
boost::filesystem::remove_all(old_db_folder_path);
|
||||
}
|
||||
|
||||
const std::string db_folder_path = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME;
|
||||
LOG_PRINT_L0("Loading blockchain from " << db_folder_path);
|
||||
|
||||
bool db_opened_okay = false;
|
||||
for(size_t loading_attempt_no = 0; loading_attempt_no < 2; ++loading_attempt_no)
|
||||
{
|
||||
bool res = m_db.open(db_folder_path);
|
||||
bool res = m_db.open(db_folder_path, cache_size_l1);
|
||||
if (!res)
|
||||
{
|
||||
// if DB could not be opened -- try to remove the whole folder and re-open DB
|
||||
LOG_PRINT_YELLOW("Failed to initialize database in folder: " << db_folder_path << ", first attempt", LOG_LEVEL_0);
|
||||
boost::filesystem::remove_all(db_folder_path);
|
||||
res = m_db.open(db_folder_path);
|
||||
res = m_db.open(db_folder_path, cache_size_l1);
|
||||
CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << db_folder_path << ", second attempt");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -187,8 +187,10 @@
|
|||
#define CURRENCY_CORE_INSTANCE_LOCK_FILE "lock.lck"
|
||||
|
||||
|
||||
#define CURRENCY_POOLDATA_FOLDERNAME "poolstate"
|
||||
#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain"
|
||||
#define CURRENCY_POOLDATA_FOLDERNAME_OLD "poolstate"
|
||||
#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD "blockchain"
|
||||
#define CURRENCY_POOLDATA_FOLDERNAME "poolstate_lmdb_v1"
|
||||
#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain_lmdb_v1"
|
||||
#define P2P_NET_DATA_FILENAME "p2pstate.bin"
|
||||
#define MINER_CONFIG_FILENAME "miner_conf.json"
|
||||
#define GUI_SECURE_CONFIG_FILENAME "gui_secure_conf.bin"
|
||||
|
|
|
|||
|
|
@ -1120,19 +1120,30 @@ namespace currency
|
|||
{
|
||||
m_config_folder = config_folder;
|
||||
|
||||
LOG_PRINT_L0("Loading blockchain...");
|
||||
uint64_t cache_size_l1 = CACHE_SIZE;
|
||||
LOG_PRINT_GREEN("Using pool db file cache size(L1): " << cache_size_l1, LOG_LEVEL_0);
|
||||
|
||||
// remove old incompartible DB
|
||||
const std::string old_db_folder_path = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME_OLD;
|
||||
if (boost::filesystem::exists(old_db_folder_path))
|
||||
{
|
||||
LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0);
|
||||
boost::filesystem::remove_all(old_db_folder_path);
|
||||
}
|
||||
|
||||
const std::string db_folder_path = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME;
|
||||
|
||||
LOG_PRINT_L0("Loading blockchain from " << db_folder_path << "...");
|
||||
|
||||
bool db_opened_okay = false;
|
||||
for(size_t loading_attempt_no = 0; loading_attempt_no < 2; ++loading_attempt_no)
|
||||
{
|
||||
bool res = m_db.open(db_folder_path);
|
||||
bool res = m_db.open(db_folder_path, cache_size_l1);
|
||||
if (!res)
|
||||
{
|
||||
// if DB could not be opened -- try to remove the whole folder and re-open DB
|
||||
LOG_PRINT_YELLOW("Failed to initialize database in folder: " << db_folder_path << ", first attempt", LOG_LEVEL_0);
|
||||
boost::filesystem::remove_all(db_folder_path);
|
||||
res = m_db.open(db_folder_path);
|
||||
res = m_db.open(db_folder_path, cache_size_l1);
|
||||
CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << db_folder_path << ", second attempt");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ int main(int argc, char* argv[])
|
|||
#endif
|
||||
log_space::get_set_log_detalisation_level(true, LOG_LEVEL_2);
|
||||
log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
|
||||
log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,wallet,lmdb");
|
||||
log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,wallet");
|
||||
LOG_PRINT_L0("Starting...");
|
||||
|
||||
tools::signal_handler::install_fatal([](int sig_number, void* address) {
|
||||
|
|
|
|||
|
|
@ -29,8 +29,9 @@ TEST(db_accessor_tests, cached_key_value_accessor_test)
|
|||
tools::db::cached_key_value_accessor<uint64_t, uint64_t, false, true> m_container(m_db);
|
||||
const std::string folder_name = "./TEST_cached_key_value_accessor_test";
|
||||
tools::create_directories_if_necessary(folder_name);
|
||||
uint64_t cache_size = CACHE_SIZE;
|
||||
|
||||
ASSERT_TRUE(m_db.open(folder_name));
|
||||
ASSERT_TRUE(m_db.open(folder_name, cache_size));
|
||||
ASSERT_TRUE(m_container.init("container"));
|
||||
|
||||
... TODO ...
|
||||
|
|
@ -46,7 +47,8 @@ TEST(db_accessor_tests_2, recoursive_tx_test)
|
|||
|
||||
const std::string folder_name = "./TEST_db_recursive_tx";
|
||||
tools::create_directories_if_necessary(folder_name);
|
||||
ASSERT_TRUE(m_db.open(folder_name));
|
||||
uint64_t cache_size = CACHE_SIZE;
|
||||
ASSERT_TRUE(m_db.open(folder_name, cache_size));
|
||||
ASSERT_TRUE(m_container.init("zzzz") );
|
||||
|
||||
bool tx_result = m_container.begin_transaction();
|
||||
|
|
@ -317,7 +319,8 @@ TEST(db_accessor_tests, median_db_cache_test)
|
|||
const std::string folder_name = "./TEST_median_db_cache";
|
||||
const std::string naive_median_serialization_filename = folder_name + "/naive_median";
|
||||
tools::create_directories_if_necessary(folder_name);
|
||||
ASSERT_TRUE(m_db.open(folder_name));
|
||||
uint64_t cache_size = CACHE_SIZE;
|
||||
ASSERT_TRUE(m_db.open(folder_name, cache_size));
|
||||
ASSERT_TRUE(m_tx_fee_median.init("median_fee"));
|
||||
m_db.begin_transaction();
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ namespace lmdb_test
|
|||
// write data
|
||||
//
|
||||
|
||||
r = bdba.open(db_file_path);
|
||||
r = bdba.open(db_file_path, CACHE_SIZE);
|
||||
ASSERT_TRUE(r);
|
||||
|
||||
db::container_handle h;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue