Charon/blockchain
1
0
Fork 0
forked from lthn/blockchain
Code Pull requests Activity

Merge branch 'lmdb_18_revert' into develop

Browse source
This commit is contained in:
cryptozoidberg 2019-08-31 16:05:15 +02:00
commit c5820ce55e
No known key found for this signature in database
GPG key ID: 22DEB97A54C6FDEC
49 changed files with 2241 additions and 1168 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
contrib/db/liblmdb/CHANGES
View file

@ -1,70 +1,14 @@
LMDB 0.9 Change Log
LMDB 0.9.24 Release (2019/07/24)
ITS#8969 Tweak mdb_page_split
ITS#8975 WIN32 fix writemap set_mapsize crash
ITS#9007 Fix loose pages in WRITEMAP
LMDB 0.9.23 Release (2018/12/19)
ITS#8756 Fix loose pages in dirty list
ITS#8831 Fix mdb_load flag init
ITS#8844 Fix mdb_env_close in forked process
Documentation
ITS#8857 mdb_cursor_del doesn't invalidate cursor
ITS#8908 GET_MULTIPLE etc don't change passed in key
LMDB 0.9.22 Release (2018/03/22)
Fix MDB_DUPSORT alignment bug (ITS#8819)
Fix regression with new db from 0.9.19 (ITS#8760)
Fix liblmdb to build on Solaris (ITS#8612)
Fix delete behavior with DUPSORT DB (ITS#8622)
Fix mdb_cursor_get/mdb_cursor_del behavior (ITS#8722)
LMDB 0.9.21 Release (2017/06/01)
Fix xcursor after cursor_del (ITS#8622)
LMDB 0.9.20 (Withdrawn)
Fix mdb_load with escaped plaintext (ITS#8558)
Fix mdb_cursor_last / mdb_put interaction (ITS#8557)
LMDB 0.9.19 Release (2016/12/28)
Fix mdb_env_cwalk cursor init (ITS#8424)
Fix robust mutexes on Solaris 10/11 (ITS#8339)
Tweak Win32 error message buffer
Fix MDB_GET_BOTH on non-dup record (ITS#8393)
Optimize mdb_drop
Fix xcursors after mdb_cursor_del (ITS#8406)
Fix MDB_NEXT_DUP after mdb_cursor_del (ITS#8412)
Fix mdb_cursor_put resetting C_EOF (ITS#8489)
Fix mdb_env_copyfd2 to return EPIPE on SIGPIPE (ITS#8504)
Fix mdb_env_copy with empty DB (ITS#8209)
Fix behaviors with fork (ITS#8505)
Fix mdb_dbi_open with mainDB cursors (ITS#8542)
Fix robust mutexes on kFreeBSD (ITS#8554)
Fix utf8_to_utf16 error checks (ITS#7992)
Fix F_NOCACHE on MacOS, error is non-fatal (ITS#7682)
Build
Make shared lib suffix overridable (ITS#8481)
Documentation
Cleanup doxygen nits
Note reserved vs actual mem/disk usage
LMDB 0.9.18 Release (2016/02/05)
LMDB 0.9.18 Release Engineering
Fix robust mutex detection on glibc 2.10-11 (ITS#8330)
Fix page_search_root assert on FreeDB (ITS#8336)
Fix MDB_APPENDDUP vs. rewrite(single item) (ITS#8334)
Fix mdb_copy of large files on Windows
Fix subcursor move after delete (ITS#8355)
Fix mdb_midl_shirnk off-by-one (ITS#8363)
Check for utf8_to_utf16 failures (ITS#7992)
Catch strdup failure in mdb_dbi_open
Build
Additional makefile var tweaks (ITS#8169)
Documentation
Add Getting Started page
Update WRITEMAP description
LMDB 0.9.17 Release (2015/11/30)
Fix ITS#7377 catch calloc failure

2
contrib/db/liblmdb/COPYRIGHT
View file

@ -1,4 +1,4 @@
Copyright 2011-2019 Howard Chu, Symas Corp.
Copyright 2011-2015 Howard Chu, Symas Corp.
All rights reserved.
Redistribution and use in source and binary forms, with or without

2
contrib/db/liblmdb/Doxyfile
View file

@ -253,7 +253,7 @@ IDL_PROPERTY_SUPPORT = YES
# member in the group (if any) for the other members of the group. By default
# all members of a group must be documented explicitly.
DISTRIBUTE_GROUP_DOC = YES
DISTRIBUTE_GROUP_DOC = NO
# Set the SUBGROUPING tag to YES (the default) to allow class member groups of
# the same type (for instance a group of public functions) to be put as a

11
contrib/db/liblmdb/Makefile
View file

@ -8,7 +8,7 @@
# platforms; you should not need to change any of these.
# Read their descriptions in mdb.c if you do:
#
# - MDB_USE_POSIX_SEM
# - MDB_USE_POSIX_MUTEX, MDB_USE_POSIX_SEM, MDB_USE_SYSV_SEM
# - MDB_DSYNC
# - MDB_FDATASYNC
# - MDB_FDATASYNC_WORKS
@ -24,9 +24,8 @@ W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
THREADS = -pthread
OPT = -O2 -g
CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS)
LDLIBS =
SOLIBS =
SOEXT = .so
LDLIBS = # -lntdll # Windows needs ntdll
SOLIBS = # -lntdll
prefix = /usr/local
exec_prefix = $(prefix)
bindir = $(exec_prefix)/bin
@ -38,7 +37,7 @@ mandir = $(datarootdir)/man
########################################################################
IHDRS = lmdb.h
ILIBS = liblmdb.a liblmdb$(SOEXT)
ILIBS = liblmdb.a liblmdb.so
IPROGS = mdb_stat mdb_copy mdb_dump mdb_load
IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1
PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
@ -64,7 +63,7 @@ test: all
liblmdb.a: mdb.o midl.o
$(AR) rs $@ mdb.o midl.o
liblmdb$(SOEXT): mdb.lo midl.lo
liblmdb.so: mdb.lo midl.lo
# $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)

2
contrib/db/liblmdb/intro.doc
View file

@ -1,5 +1,5 @@
/*
* Copyright 2015-2018 Howard Chu, Symas Corp.
* Copyright 2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

78
contrib/db/liblmdb/lmdb.h
View file

@ -53,14 +53,15 @@
*
* Fix: Check for stale readers periodically, using the
* #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool.
* Stale writers will be cleared automatically on some systems:
* Stale writers will be cleared automatically on most systems:
* - Windows - automatic
* - BSD, systems using SysV semaphores - automatic
* - Linux, systems using POSIX mutexes with Robust option - automatic
* - not on BSD, systems using POSIX semaphores.
* Otherwise just make all programs using the database close it;
* the lockfile is always reset on first open of the environment.
*
* - On BSD systems or others configured with MDB_USE_POSIX_SEM,
* - On BSD systems or others configured with MDB_USE_SYSV_SEM or
* MDB_USE_POSIX_SEM,
* startup can fail due to semaphores owned by another userid.
*
* Fix: Open and close the database as the user which owns the
@ -96,12 +97,11 @@
* transactions. Each transaction belongs to one thread. See below.
* The #MDB_NOTLS flag changes this for read-only transactions.
*
* - Use an MDB_env* in the process which opened it, not after fork().
* - Use an MDB_env* in the process which opened it, without fork()ing.
*
* - Do not have open an LMDB database twice in the same process at
* the same time. Not even from a plain open() call - close()ing it
* breaks fcntl() advisory locking. (It is OK to reopen it after
* fork() - exec*(), since the lockfile has FD_CLOEXEC set.)
* breaks flock() advisory locking.
*
* - Avoid long-lived transactions. Read transactions prevent
* reuse of pages freed by newer write transactions, thus the
@ -135,7 +135,7 @@
*
* @author Howard Chu, Symas Corporation.
*
* @copyright Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved.
* @copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
@ -166,6 +166,7 @@
#define _LMDB_H_
#include <sys/types.h>
#include <inttypes.h>
#ifdef __cplusplus
extern "C" {
@ -178,6 +179,13 @@ typedef int mdb_mode_t;
typedef mode_t mdb_mode_t;
#endif
#ifdef MDB_VL32
typedef uint64_t mdb_size_t;
#define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */
#else
typedef size_t mdb_size_t;
#endif
/** An abstraction for a file handle.
* On POSIX systems file handles are small integers. On Windows
* they're opaque pointers.
@ -200,7 +208,7 @@ typedef int mdb_filehandle_t;
/** Library minor version */
#define MDB_VERSION_MINOR 9
/** Library patch version */
#define MDB_VERSION_PATCH 24
#define MDB_VERSION_PATCH 70
/** Combine args a,b,c into a single integer for easy version comparisons */
#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c))
@ -210,7 +218,7 @@ typedef int mdb_filehandle_t;
MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
/** The release date of this library version */
#define MDB_VERSION_DATE "July 24, 2019"
#define MDB_VERSION_DATE "December 19, 2015"
/** A stringifier for the version info */
#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")"
@ -303,6 +311,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
#define MDB_NORDAHEAD 0x800000
/** don't initialize malloc'd memory before writing to datafile */
#define MDB_NOMEMINIT 0x1000000
/** use the previous snapshot rather than the latest one */
#define MDB_PREVSNAPSHOT 0x2000000
/** @} */
/** @defgroup mdb_dbi_open Database Flags
@ -370,7 +380,7 @@ typedef enum MDB_cursor_op {
MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */
MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */
MDB_GET_CURRENT, /**< Return key/data at current cursor position */
MDB_GET_MULTIPLE, /**< Return up to a page of duplicate data items
MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items
from current cursor position. Move cursor to prepare
for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */
MDB_LAST, /**< Position at last key/data item */
@ -379,7 +389,7 @@ typedef enum MDB_cursor_op {
MDB_NEXT, /**< Position at next data item */
MDB_NEXT_DUP, /**< Position at next data item of current key.
Only for #MDB_DUPSORT */
MDB_NEXT_MULTIPLE, /**< Return up to a page of duplicate data items
MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items
from next cursor position. Move cursor to prepare
for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */
MDB_NEXT_NODUP, /**< Position at first data item of next key */
@ -390,7 +400,7 @@ typedef enum MDB_cursor_op {
MDB_SET, /**< Position at specified key */
MDB_SET_KEY, /**< Position at specified key, return key + data */
MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */
MDB_PREV_MULTIPLE /**< Position at previous page and return up to
MDB_PREV_MULTIPLE /**< Position at previous page and return key and up to
a page of duplicate data items. Only for #MDB_DUPFIXED */
} MDB_cursor_op;
@ -457,18 +467,18 @@ typedef struct MDB_stat {
unsigned int ms_psize; /**< Size of a database page.
This is currently the same for all databases. */
unsigned int ms_depth; /**< Depth (height) of the B-tree */
size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */
size_t ms_leaf_pages; /**< Number of leaf pages */
size_t ms_overflow_pages; /**< Number of overflow pages */
size_t ms_entries; /**< Number of data items */
mdb_size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */
mdb_size_t ms_leaf_pages; /**< Number of leaf pages */
mdb_size_t ms_overflow_pages; /**< Number of overflow pages */
mdb_size_t ms_entries; /**< Number of data items */
} MDB_stat;
/** @brief Information about the environment */
typedef struct MDB_envinfo {
void *me_mapaddr; /**< Address of map, if fixed */
size_t me_mapsize; /**< Size of the data memory map */
size_t me_last_pgno; /**< ID of the last used page */
size_t me_last_txnid; /**< ID of the last committed transaction */
mdb_size_t me_mapsize; /**< Size of the data memory map */
mdb_size_t me_last_pgno; /**< ID of the last used page */
mdb_size_t me_last_txnid; /**< ID of the last committed transaction */
unsigned int me_maxreaders; /**< max reader slots in the environment */
unsigned int me_numreaders; /**< max reader slots used in the environment */
} MDB_envinfo;
@ -614,6 +624,12 @@ int mdb_env_create(MDB_env **env);
* caller is expected to overwrite all of the memory that was
* reserved in that case.
* This flag may be changed at any time using #mdb_env_set_flags().
* <li>#MDB_PREVSNAPSHOT
* Open the environment with the previous snapshot rather than the latest
* one. This loses the latest transaction, but may help work around some
* types of corruption. If opened with write access, this must be the
* only process using the environment. This flag is automatically reset
* after a write transaction is successfully committed.
* </ul>
* @param[in] mode The UNIX permissions to set on created files and semaphores.
* This parameter is ignored on Windows.
@ -680,7 +696,6 @@ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
* <li>#MDB_CP_COMPACT - Perform compaction while copying: omit free
* pages and sequentially renumber all pages in output. This option
* consumes more CPU and runs more slowly than the default.
* Currently it fails if the environment has suffered a page leak.
* </ul>
* @return A non-zero error value on failure and 0 on success.
*/
@ -795,10 +810,6 @@ int mdb_env_get_flags(MDB_env *env, unsigned int *flags);
int mdb_env_get_path(MDB_env *env, const char **path);
/** @brief Return the filedescriptor for the given environment.
*
* This function may be called after fork(), so the descriptor can be
* closed before exec*(). Other LMDB file descriptors have FD_CLOEXEC.
* (Until LMDB 0.9.18, only the lockfile had that.)
*
* @param[in] env An environment handle returned by #mdb_env_create()
* @param[out] fd Address of a mdb_filehandle_t to contain the descriptor.
@ -842,7 +853,7 @@ int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd);
* an active write transaction.
* </ul>
*/
int mdb_env_set_mapsize(MDB_env *env, size_t size);
int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size);
/** @brief Set the maximum number of threads/reader slots for the environment.
*
@ -955,6 +966,10 @@ int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
* <ul>
* <li>#MDB_RDONLY
* This transaction will not perform any write operations.
* <li>#MDB_NOSYNC
* Don't flush system buffers to disk when committing this transaction.
* <li>#MDB_NOMETASYNC
* Flush system buffers but omit metadata flush when committing this transaction.
* </ul>
* @param[out] txn Address where the new #MDB_txn handle will be stored
* @return A non-zero error value on failure and 0 on success. Some possible
@ -987,7 +1002,7 @@ MDB_env *mdb_txn_env(MDB_txn *txn);
* @param[in] txn A transaction handle returned by #mdb_txn_begin()
* @return A transaction ID, valid if input is an active transaction.
*/
size_t mdb_txn_id(MDB_txn *txn);
mdb_size_t mdb_txn_id(MDB_txn *txn);
/** @brief Commit all the operations of a transaction into the database.
*
@ -1103,9 +1118,8 @@ int mdb_txn_renew(MDB_txn *txn);
* This flag may only be used in combination with #MDB_DUPSORT. This option
* tells the library that the data items for this database are all the same
* size, which allows further optimizations in storage and retrieval. When
* all data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE
* and #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple
* items at once.
* all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
* cursor operations may be used to retrieve multiple items at once.
* <li>#MDB_INTEGERDUP
* This option specifies that duplicate data items are binary integers,
* similar to #MDB_INTEGERKEY keys.
@ -1510,10 +1524,6 @@ int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
/** @brief Delete current key/data pair
*
* This function deletes the key/data pair to which the cursor refers.
* This does not invalidate the cursor, so operations such as MDB_NEXT
* can still be used on it.
* Both MDB_NEXT and MDB_GET_CURRENT will return the same record after
* this operation.
* @param[in] cursor A cursor handle returned by #mdb_cursor_open()
* @param[in] flags Options for this operation. This parameter
* must be set to 0 or one of the values described here.
@ -1542,7 +1552,7 @@ int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags);
* <li>EINVAL - cursor is not initialized, or an invalid parameter was specified.
* </ul>
*/
int mdb_cursor_count(MDB_cursor *cursor, size_t *countp);
int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp);
/** @brief Compare two data items according to a particular database.
*

2282
contrib/db/liblmdb/mdb.c
View file

File diff suppressed because it is too large Load diff

11
contrib/db/liblmdb/mdb_copy.1
View file

@ -1,5 +1,5 @@
.TH MDB_COPY 1 "2014/07/01" "LMDB 0.9.14"
.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved.
.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_copy \- LMDB environment copy tool
@ -11,6 +11,8 @@ mdb_copy \- LMDB environment copy tool
.BR \-c ]
[\c
.BR \-n ]
[\c
.BR \-v ]
.B srcpath
[\c
.BR dstpath ]
@ -36,10 +38,13 @@ Write the library version number to the standard output, and exit.
Compact while copying. Only current data pages will be copied; freed
or unused pages will be omitted from the copy. This option will
slow down the backup process as it is more CPU-intensive.
Currently it fails if the environment has suffered a page leak.
.TP
.BR \-n
Open LDMB environment(s) which do not use subdirectories.
.TP
.BR \-v
Use the previous environment state instead of the latest state.
This may be useful if the latest state has been corrupted.
.SH DIAGNOSTICS
Exit status is zero if no errors occur.

6
contrib/db/liblmdb/mdb_copy.c
View file

@ -1,6 +1,6 @@
/* mdb_copy.c - memory-mapped database backup tool */
/*
* Copyright 2012-2018 Howard Chu, Symas Corp.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -38,6 +38,8 @@ int main(int argc,char * argv[])
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
if (argv[1][1] == 'n' && argv[1][2] == '\0')
flags |= MDB_NOSUBDIR;
else if (argv[1][1] == 'v' && argv[1][2] == '\0')
flags |= MDB_PREVSNAPSHOT;
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
cpflags |= MDB_CP_COMPACT;
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
@ -48,7 +50,7 @@ int main(int argc,char * argv[])
}
if (argc<2 || argc>3) {
fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname);
fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] srcpath [dstpath]\n", progname);
exit(EXIT_FAILURE);
}

10
contrib/db/liblmdb/mdb_dump.1
View file

@ -1,5 +1,5 @@
.TH MDB_DUMP 1 "2015/09/30" "LMDB 0.9.17"
.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_dump \- LMDB environment export tool
@ -14,6 +14,8 @@ mdb_dump \- LMDB environment export tool
[\c
.BR \-n ]
[\c
.BR \-v ]
[\c
.BR \-p ]
[\c
.BR \-a \ |
@ -42,6 +44,10 @@ names will be listed, no data will be output.
.BR \-n
Dump an LMDB database which does not use subdirectories.
.TP
.BR \-v
Use the previous environment state instead of the latest state.
This may be useful if the latest state has been corrupted.
.TP
.BR \-p
If characters in either the key or data items are printing characters (as
defined by isprint(3)), output them directly. This option permits users to

19
contrib/db/liblmdb/mdb_dump.c
View file

@ -1,6 +1,6 @@
/* mdb_dump.c - memory-mapped database dump tool */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -25,6 +25,15 @@
#else
#define Z "z"
#endif
#ifdef MDB_VL32
#ifdef _WIN32
#define Y "I64"
#else
#define Y "ll"
#endif
#else
#define Y Z
#endif
#define PRINT 1
static int mode;
@ -115,7 +124,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
if (name)
printf("database=%s\n", name);
printf("type=btree\n");
printf("mapsize=%" Z "u\n", info.me_mapsize);
printf("mapsize=%" Y "u\n", info.me_mapsize);
if (info.me_mapaddr)
printf("mapaddr=%p\n", info.me_mapaddr);
printf("maxreaders=%u\n", info.me_maxreaders);
@ -155,7 +164,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
static void usage(char *prog)
{
fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n", prog);
fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-a|-s subdb] dbpath\n", prog);
exit(EXIT_FAILURE);
}
@ -179,6 +188,7 @@ int main(int argc, char *argv[])
* -n: use NOSUBDIR flag on env_open
* -p: use printable characters
* -f: write to file instead of stdout
* -v: use previous snapshot
* -V: print version and exit
* (default) dump only the main DB
*/
@ -206,6 +216,9 @@ int main(int argc, char *argv[])
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'v':
envflags |= MDB_PREVSNAPSHOT;
break;
case 'p':
mode |= PRINT;
break;

11
contrib/db/liblmdb/mdb_load.1
View file

@ -1,5 +1,5 @@
.TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.17"
.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_load \- LMDB environment import tool
@ -37,6 +37,13 @@ option below.
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-a
Append all records in the order they appear in the input. The input is assumed to already be
in correctly sorted order and no sorting or checking for redundant values will be performed.
This option must be used to reload data that was produced by running
.B mdb_dump
on a database that uses custom compare functions.
.TP
.BR \-f \ file
Read from the specified file instead of from the standard input.
.TP

64
contrib/db/liblmdb/mdb_load.c
View file

@ -1,6 +1,6 @@
/* mdb_load.c - memory-mapped database load tool */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -37,12 +37,22 @@ static int Eof;
static MDB_envinfo info;
static MDB_val kbuf, dbuf;
static MDB_val k0buf;
#ifdef _WIN32
#define Z "I"
#else
#define Z "z"
#endif
#ifdef MDB_VL32
#ifdef _WIN32
#define Y "I64"
#else
#define Y "ll"
#endif
#else
#define Y Z
#endif
#define STRLENOF(s) (sizeof(s)-1)
@ -68,7 +78,6 @@ static void readhdr(void)
{
char *ptr;
flags = 0;
while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
lineno++;
if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
@ -113,7 +122,7 @@ static void readhdr(void)
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize);
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize);
if (i != 1) {
fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
@ -249,8 +258,7 @@ badend:
c2 += 2;
}
} else {
/* copies are redundant when no escapes were used */
*c1++ = *c2++;
c1++; c2++;
}
}
} else {
@ -278,10 +286,15 @@ badend:
static void usage(void)
{
fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
exit(EXIT_FAILURE);
}
static int greater(const MDB_val *a, const MDB_val *b)
{
return 1;
}
int main(int argc, char *argv[])
{
int i, rc;
@ -291,7 +304,8 @@ int main(int argc, char *argv[])
MDB_dbi dbi;
char *envname;
int envflags = 0, putflags = 0;
int dohdr = 0;
int dohdr = 0, append = 0;
MDB_val prevk;
prog = argv[0];
@ -299,19 +313,23 @@ int main(int argc, char *argv[])
usage();
}
/* -f: load file instead of stdin
/* -a: append records in input order
* -f: load file instead of stdin
* -n: use NOSUBDIR flag on env_open
* -s: load into named subDB
* -N: use NOOVERWRITE on puts
* -T: read plaintext
* -V: print version and exit
*/
while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
switch(i) {
case 'V':
printf("%s\n", MDB_VERSION_STRING);
exit(0);
break;
case 'a':
append = 1;
break;
case 'f':
if (freopen(optarg, "r", stdin) == NULL) {
fprintf(stderr, "%s: %s: reopen: %s\n",
@ -370,11 +388,17 @@ int main(int argc, char *argv[])
}
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
kbuf.mv_data = malloc(kbuf.mv_size);
kbuf.mv_data = malloc(kbuf.mv_size * 2);
k0buf.mv_size = kbuf.mv_size;
k0buf.mv_data = (char *)kbuf.mv_data + kbuf.mv_size;
prevk.mv_size = 0;
prevk.mv_data = k0buf.mv_data;
while(!Eof) {
MDB_val key, data;
int batch = 0;
flags = 0;
int appflag;
if (!dohdr) {
dohdr = 1;
@ -392,6 +416,11 @@ int main(int argc, char *argv[])
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
if (append) {
mdb_set_compare(txn, dbi, greater);
if (flags & MDB_DUPSORT)
mdb_set_dupsort(txn, dbi, greater);
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
@ -410,7 +439,20 @@ int main(int argc, char *argv[])
goto txn_abort;
}
rc = mdb_cursor_put(mc, &key, &data, putflags);
if (append) {
appflag = MDB_APPEND;
if (flags & MDB_DUPSORT) {
if (prevk.mv_size == key.mv_size && !memcmp(prevk.mv_data, key.mv_data, key.mv_size))
appflag = MDB_APPENDDUP;
else {
memcpy(prevk.mv_data, key.mv_data, key.mv_size);
prevk.mv_size = key.mv_size;
}
}
} else {
appflag = 0;
}
rc = mdb_cursor_put(mc, &key, &data, putflags|appflag);
if (rc == MDB_KEYEXIST && putflags)
continue;
if (rc) {

10
contrib/db/liblmdb/mdb_stat.1
View file

@ -1,5 +1,5 @@
.TH MDB_STAT 1 "2015/09/30" "LMDB 0.9.17"
.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved.
.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_stat \- LMDB environment status tool
@ -14,6 +14,8 @@ mdb_stat \- LMDB environment status tool
[\c
.BR \-n ]
[\c
.BR \-v ]
[\c
.BR \-r [ r ]]
[\c
.BR \-a \ |
@ -39,6 +41,10 @@ If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
.BR \-n
Display the status of an LMDB database which does not use subdirectories.
.TP
.BR \-v
Use the previous environment state instead of the latest state.
This may be useful if the latest state has been corrupted.
.TP
.BR \-r
Display information about the environment reader table.
Shows the process ID, thread ID, and transaction ID for each active

33
contrib/db/liblmdb/mdb_stat.c
View file

@ -1,6 +1,6 @@
/* mdb_stat.c - memory-mapped database status tool */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -22,6 +22,15 @@
#else
#define Z "z"
#endif
#ifdef MDB_VL32
#ifdef _WIN32
#define Y "I64"
#else
#define Y "ll"
#endif
#else
#define Y Z
#endif
static void prstat(MDB_stat *ms)
{
@ -29,15 +38,15 @@ static void prstat(MDB_stat *ms)
printf(" Page size: %u\n", ms->ms_psize);
#endif
printf(" Tree depth: %u\n", ms->ms_depth);
printf(" Branch pages: %"Z"u\n", ms->ms_branch_pages);
printf(" Leaf pages: %"Z"u\n", ms->ms_leaf_pages);
printf(" Overflow pages: %"Z"u\n", ms->ms_overflow_pages);
printf(" Entries: %"Z"u\n", ms->ms_entries);
printf(" Branch pages: %"Y"u\n", ms->ms_branch_pages);
printf(" Leaf pages: %"Y"u\n", ms->ms_leaf_pages);
printf(" Overflow pages: %"Y"u\n", ms->ms_overflow_pages);
printf(" Entries: %"Y"u\n", ms->ms_entries);
}
static void usage(char *prog)
{
fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n", prog);
fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-a|-s subdb] dbpath\n", prog);
exit(EXIT_FAILURE);
}
@ -64,6 +73,7 @@ int main(int argc, char *argv[])
* -f: print freelist info
* -r: print reader info
* -n: use NOSUBDIR flag on env_open
* -v: use previous snapshot
* -V: print version and exit
* (default) print stat of only the main DB
*/
@ -87,6 +97,9 @@ int main(int argc, char *argv[])
case 'n':
envflags |= MDB_NOSUBDIR;
break;
case 'v':
envflags |= MDB_PREVSNAPSHOT;
break;
case 'r':
rdrinfo++;
break;
@ -125,11 +138,11 @@ int main(int argc, char *argv[])
(void)mdb_env_info(env, &mei);
printf("Environment Info\n");
printf(" Map address: %p\n", mei.me_mapaddr);
printf(" Map size: %"Z"u\n", mei.me_mapsize);
printf(" Map size: %"Y"u\n", mei.me_mapsize);
printf(" Page size: %u\n", mst.ms_psize);
printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize);
printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1);
printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid);
printf(" Max pages: %"Y"u\n", mei.me_mapsize / mst.ms_psize);
printf(" Number of pages used: %"Y"u\n", mei.me_last_pgno+1);
printf(" Last transaction ID: %"Y"u\n", mei.me_last_txnid);
printf(" Max readers: %u\n", mei.me_maxreaders);
printf(" Number of readers used: %u\n", mei.me_numreaders);
}

65
contrib/db/liblmdb/midl.c
View file

@ -3,8 +3,7 @@
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
*
* Copyright 2000-2019 The OpenLDAP Foundation.
* Portions Copyright 2001-2018 Howard Chu, Symas Corp.
* Copyright 2000-2015 The OpenLDAP Foundation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -355,5 +354,67 @@ int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
return 0;
}
#ifdef MDB_VL32
unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id )
{
/*
* binary search of id in ids
* if found, returns position of id
* if not found, returns first position greater than id
*/
unsigned base = 0;
unsigned cursor = 1;
int val = 0;
unsigned n = (unsigned)ids[0].mid;
while( 0 < n ) {
unsigned pivot = n >> 1;
cursor = base + pivot + 1;
val = CMP( id, ids[cursor].mid );
if( val < 0 ) {
n = pivot;
} else if ( val > 0 ) {
base = cursor;
n -= pivot + 1;
} else {
return cursor;
}
}
if( val > 0 ) {
++cursor;
}
return cursor;
}
int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id )
{
unsigned x, i;
x = mdb_mid3l_search( ids, id->mid );
if( x < 1 ) {
/* internal error */
return -2;
}
if ( x <= ids[0].mid && ids[x].mid == id->mid ) {
/* duplicate */
return -1;
}
/* insert id */
ids[0].mid++;
for (i=(unsigned)ids[0].mid; i>x; i--)
ids[i] = ids[i-1];
ids[x] = *id;
return 0;
}
#endif /* MDB_VL32 */
/** @} */
/** @} */

22
contrib/db/liblmdb/midl.h
View file

@ -11,8 +11,7 @@
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
*
* Copyright 2000-2019 The OpenLDAP Foundation.
* Portions Copyright 2001-2018 Howard Chu, Symas Corp.
* Copyright 2000-2015 The OpenLDAP Foundation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -28,6 +27,7 @@
#define _MDB_MIDL_H_
#include <stddef.h>
#include <inttypes.h>
#ifdef __cplusplus
extern "C" {
@ -43,7 +43,11 @@ extern "C" {
/** A generic unsigned ID number. These were entryIDs in back-bdb.
* Preferably it should have the same size as a pointer.
*/
#ifdef MDB_VL32
typedef uint64_t MDB_ID;
#else
typedef size_t MDB_ID;
#endif
/** An IDL is an ID List, a sorted array of IDs. The first
* element of the array is a counter for how many actual
@ -178,6 +182,20 @@ int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id );
*/
int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
#ifdef MDB_VL32
typedef struct MDB_ID3 {
MDB_ID mid; /**< The ID */
void *mptr; /**< The pointer */
unsigned int mcnt; /**< Number of pages */
unsigned int mref; /**< Refcounter */
} MDB_ID3;
typedef MDB_ID3 *MDB_ID3L;
unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id );
int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id );
#endif /* MDB_VL32 */
/** @} */
/** @} */
#ifdef __cplusplus

2
contrib/db/liblmdb/mtest.c
View file

@ -1,6 +1,6 @@
/* mtest.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
contrib/db/liblmdb/mtest2.c
View file

@ -1,6 +1,6 @@
/* mtest2.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
contrib/db/liblmdb/mtest3.c
View file

@ -1,6 +1,6 @@
/* mtest3.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
contrib/db/liblmdb/mtest4.c
View file

@ -1,6 +1,6 @@
/* mtest4.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
contrib/db/liblmdb/mtest5.c
View file

@ -1,6 +1,6 @@
/* mtest5.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
contrib/db/liblmdb/mtest6.c
View file

@ -1,6 +1,6 @@
/* mtest6.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2018 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
contrib/db/liblmdb/sample-bdb.txt
View file

@ -3,7 +3,7 @@
* Do a line-by-line comparison of this and sample-mdb.txt
*/
/*
* Copyright 2012-2018 Howard Chu, Symas Corp.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
contrib/db/liblmdb/sample-mdb.txt
View file

@ -3,7 +3,7 @@
* Do a line-by-line comparison of this and sample-bdb.txt
*/
/*
* Copyright 2012-2018 Howard Chu, Symas Corp.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
contrib/epee/include/misc_log_ex.h
View file

@ -150,7 +150,7 @@ DISABLE_VS_WARNINGS(4100)
{std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, true, log_name);}}
#define LOG_ERROR2(log_name, x) { \
std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "[ERROR] Location: " << std::endl << LOCATION_SS << epee::misc_utils::print_trace() << " Message:" << std::endl << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), LOG_LEVEL_0, epee::log_space::console_color_red, true, log_name); LOCAL_ASSERT(0); epee::log_space::increase_error_count(LOG_DEFAULT_CHANNEL); }
std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "[ERROR] Location: " << std::endl << LOCATION_SS << epee::misc_utils::get_callstack() << " Message:" << std::endl << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), LOG_LEVEL_0, epee::log_space::console_color_red, true, log_name); LOCAL_ASSERT(0); epee::log_space::increase_error_count(LOG_DEFAULT_CHANNEL); }
#define LOG_FRAME2(log_name, x, y) epee::log_space::log_frame frame(x, y, log_name)

30
contrib/epee/include/misc_os_dependent.h
View file

@ -113,7 +113,7 @@ namespace misc_utils
#include <execinfo.h>
#include <boost/core/demangle.hpp>
#endif
inline std::string print_trace()
inline std::string print_trace_default()
{
std::stringstream ss;
#if defined(__GNUC__)
@ -134,5 +134,33 @@ namespace misc_utils
#endif
return ss.str();
}
typedef std::string (stack_retrieving_function_t)();
//
// To get stack trace call it with the defaults.
//
inline std::string get_callstack(stack_retrieving_function_t* p_stack_retrieving_function_to_be_added = nullptr, bool remove_func = false)
{
static stack_retrieving_function_t* p_srf = nullptr;
if (remove_func)
{
p_srf = nullptr;
return "";
}
if (p_stack_retrieving_function_to_be_added != nullptr)
{
p_srf = p_stack_retrieving_function_to_be_added;
return "";
}
if (p_srf != nullptr)
return p_srf();
return print_trace_default();
}
}
}

4
contrib/epee/include/storages/levin_abstract_invoke2.h
View file

@ -126,7 +126,7 @@ namespace epee
t_result result_struct = AUTO_VAL_INIT(result_struct);
if( code <=0 )
{
LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::print_trace());
LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::get_callstack());
LOG_PRINT_L1("Failed to invoke command " << command << " return code " << code << "(" << epee::levin::get_err_descr(code) << ")context:" << print_connection_context(context));
TRY_ENTRY()
cb(code, result_struct, context);
@ -150,7 +150,7 @@ namespace epee
}, inv_timeout);
if( res <=0 )
{
LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::print_trace());
LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::get_callstack());
LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res << "(" << epee::levin::get_err_descr(res) << ") conn_id=" << conn_id);
return false;
}

35
contrib/ethereum/libethash/ethash.cpp
View file

@ -145,7 +145,11 @@ epoch_context_full* create_epoch_context(
char* const alloc_data = static_cast<char*>(std::calloc(1, alloc_size));
if (!alloc_data)
return nullptr; // Signal out-of-memory by returning null pointer.
{
LOG_CUSTOM_WITH_CALLSTACK("CRITICAL: std::calloc(" << alloc_size << ") failed in create_epoch_context()", 0);
return nullptr; // Signal out-of-memory by returning null pointer.
}
LOG_CUSTOM("context for epoch " << epoch_number << " allocated, size: " << alloc_size << " bytes", 0);
hash512* const light_cache = reinterpret_cast<hash512*>(alloc_data + context_alloc_size);
const hash256 epoch_seed = calculate_epoch_seed(epoch_number);
@ -373,6 +377,33 @@ search_result search(const epoch_context_full& context, const hash256& header_ha
}
return {};
}
custom_log_level_function*& access_custom_log_level_function()
{
static custom_log_level_function* p_custom_log_level_function = nullptr;
return p_custom_log_level_function;
}
custom_log_function*& access_custom_log_function()
{
static custom_log_function* p_custom_log_function = nullptr;
return p_custom_log_function;
}
int get_custom_log_level()
{
if (access_custom_log_level_function() != nullptr)
return access_custom_log_level_function()();
return -1;
}
void custom_log(const std::string& m, bool add_callstack)
{
if (access_custom_log_function() != nullptr)
access_custom_log_function()(m, add_callstack);
}
} // namespace ethash
using namespace ethash;
@ -434,6 +465,8 @@ void ethash_destroy_epoch_context_full(epoch_context_full* context) noexcept
void ethash_destroy_epoch_context(epoch_context* context) noexcept
{
LOG_CUSTOM("context for epoch " << context->epoch_number << " is about to be freed", 0);
context->~epoch_context();
std::free(context);
}

33
contrib/ethereum/libethash/ethash/ethash.hpp
View file

@ -20,6 +20,8 @@
#include <cstdint>
#include <cstring>
#include <memory>
#include <string>
#include <sstream>
namespace ethash
{
@ -156,5 +158,34 @@ int find_epoch_number(const hash256& seed) noexcept;
const epoch_context& get_global_epoch_context(int epoch_number);
/// Get global shared epoch context with full dataset initialized.
const epoch_context_full& get_global_epoch_context_full(int epoch_number);
std::shared_ptr<epoch_context_full> get_global_epoch_context_full(int epoch_number);
typedef int (custom_log_level_function)();
typedef void (custom_log_function)(const std::string& m, bool add_callstack);
custom_log_level_function*& access_custom_log_level_function();
custom_log_function*& access_custom_log_function();
int get_custom_log_level();
void custom_log(const std::string& m, bool add_callstack);
#define LOG_CUSTOM(msg, level) \
{ \
if (level <= ethash::get_custom_log_level()) \
{ \
std::stringstream ss; \
ss << msg << std::endl; \
ethash::custom_log(ss.str(), false); \
} \
}
#define LOG_CUSTOM_WITH_CALLSTACK(msg, level) \
{ \
if (level <= ethash::get_custom_log_level()) \
{ \
std::stringstream ss; \
ss << msg << std::endl; \
ethash::custom_log(ss.str(), true); \
} \
}
} // namespace ethash

4
contrib/ethereum/libethash/managed.cpp
View file

@ -89,12 +89,12 @@ const epoch_context& get_global_epoch_context(int epoch_number)
return *thread_local_context;
}
const epoch_context_full& get_global_epoch_context_full(int epoch_number)
std::shared_ptr<epoch_context_full> get_global_epoch_context_full(int epoch_number)
{
// Check if local context matches epoch number.
if (!thread_local_context_full || thread_local_context_full->epoch_number != epoch_number)
update_local_context_full(epoch_number);
return *thread_local_context_full;
return thread_local_context_full;
}
} // namespace ethash

179
src/common/callstack_helper.cpp Normal file
View file

@ -0,0 +1,179 @@
// Copyright (c) 2019 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#if defined(WIN32)
#define WIN32_LEAN_AND_MEAN 1
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <Psapi.h>
#pragma comment(lib, "psapi.lib")
#pragma comment(lib, "dbghelp.lib")
#pragma pack( push, before_imagehlp, 8 )
#include <imagehlp.h>
#pragma pack( pop, before_imagehlp )
#include "include_base_utils.h"
#include "callstack_helper.h"
namespace
{
struct module_data
{
std::string image_name;
std::string module_name;
void *base_address;
DWORD load_size;
};
class get_mod_info
{
HANDLE process;
static const int buffer_length = 4096;
public:
get_mod_info(HANDLE h) : process(h) {}
module_data operator()(HMODULE module)
{
module_data ret;
char temp[buffer_length];
MODULEINFO mi;
GetModuleInformation(process, module, &mi, sizeof(mi));
ret.base_address = mi.lpBaseOfDll;
ret.load_size = mi.SizeOfImage;
GetModuleFileNameEx(process, module, temp, sizeof(temp));
ret.image_name = temp;
GetModuleBaseName(process, module, temp, sizeof(temp));
ret.module_name = temp;
std::vector<char> img(ret.image_name.begin(), ret.image_name.end());
std::vector<char> mod(ret.module_name.begin(), ret.module_name.end());
SymLoadModule64(process, 0, &img[0], &mod[0], (DWORD64)ret.base_address, ret.load_size);
return ret;
}
};
std::string get_symbol_undecorated_name(HANDLE process, DWORD64 address, std::stringstream& ss)
{
SYMBOL_INFO* sym;
static const int max_name_len = 1024;
std::vector<char> sym_buffer(sizeof(SYMBOL_INFO) + max_name_len, '\0');
sym = (SYMBOL_INFO *)sym_buffer.data();
sym->SizeOfStruct = sizeof(SYMBOL_INFO);
sym->MaxNameLen = max_name_len;
DWORD64 displacement;
if (!SymFromAddr(process, address, &displacement, sym))
return std::string("SymFromAddr failed1: ") + epee::string_tools::num_to_string_fast(GetLastError());
if (*sym->Name == '\0')
return std::string("SymFromAddr failed2: ") + epee::string_tools::num_to_string_fast(GetLastError());
/*
ss << " SizeOfStruct : " << sym->SizeOfStruct << ENDL;
ss << " TypeIndex : " << sym->TypeIndex << ENDL; // Type Index of symbol
ss << " Index : " << sym->Index << ENDL;
ss << " Size : " << sym->Size << ENDL;
ss << " ModBase : " << sym->ModBase << ENDL; // Base Address of module comtaining this symbol
ss << " Flags : " << sym->Flags << ENDL;
ss << " Value : " << sym->Value << ENDL; // Value of symbol, ValuePresent should be 1
ss << " Address : " << sym->Address << ENDL; // Address of symbol including base address of module
ss << " Register : " << sym->Register << ENDL; // register holding value or pointer to value
ss << " Scope : " << sym->Scope << ENDL; // scope of the symbol
ss << " Tag : " << sym->Tag << ENDL; // pdb classification
ss << " NameLen : " << sym->NameLen << ENDL; // Actual length of name
ss << " MaxNameLen : " << sym->MaxNameLen << ENDL;
ss << " Name[1] : " << &sym->Name << ENDL; // Name of symbol
*/
std::string und_name(max_name_len, '\0');
DWORD chars_written = UnDecorateSymbolName(sym->Name, &und_name.front(), max_name_len, UNDNAME_COMPLETE);
und_name.resize(chars_written);
return und_name;
}
} // namespace
namespace tools
{
std::string get_callstack_win_x64()
{
HANDLE h_process = GetCurrentProcess();
HANDLE h_thread = GetCurrentThread();
PCSTR user_search_path = NULL; // may be path to a pdb?
if (!SymInitialize(h_process, user_search_path, false))
return "<win callstack error 1>";
DWORD sym_options = SymGetOptions();
sym_options |= SYMOPT_LOAD_LINES | SYMOPT_UNDNAME;
SymSetOptions(sym_options);
DWORD cb_needed;
std::vector<HMODULE> module_handles(1);
EnumProcessModules(h_process, &module_handles[0], module_handles.size() * sizeof(HMODULE), &cb_needed);
module_handles.resize(cb_needed / sizeof(HMODULE));
EnumProcessModules(h_process, &module_handles[0], module_handles.size() * sizeof(HMODULE), &cb_needed);
std::vector<module_data> modules;
std::transform(module_handles.begin(), module_handles.end(), std::back_inserter(modules), get_mod_info(h_process));
void *base = modules[0].base_address;
CONTEXT context;
memset(&context, 0, sizeof context);
RtlCaptureContext( &context );
STACKFRAME64 frame;
memset(&frame, 0, sizeof frame);
frame.AddrPC.Offset = context.Rip;
frame.AddrPC.Mode = AddrModeFlat;
frame.AddrStack.Offset = context.Rsp;
frame.AddrStack.Mode = AddrModeFlat;
frame.AddrFrame.Offset = context.Rbp;
frame.AddrFrame.Mode = AddrModeFlat;
IMAGEHLP_LINE64 line = { 0 };
line.SizeOfStruct = sizeof line;
IMAGE_NT_HEADERS *image_nt_header = ImageNtHeader(base);
std::stringstream ss;
ss << ENDL;
// ss << "main module loaded at 0x" << std::hex << std::setw(16) << std::setfill('0') << base << std::dec << " from " << modules[0].image_name << ENDL;
for (size_t n = 0; n < 250; ++n)
{
if (!StackWalk64(image_nt_header->FileHeader.Machine, h_process, h_thread, &frame, &context, NULL, SymFunctionTableAccess64, SymGetModuleBase64, NULL))
break;
if (frame.AddrReturn.Offset == 0)
break;
std::string fnName = get_symbol_undecorated_name(h_process, frame.AddrPC.Offset, ss);
ss << "0x" << std::setw(16) << std::setfill('0') << std::hex << frame.AddrPC.Offset << " " << std::dec << fnName;
DWORD offset_from_line = 0;
if (SymGetLineFromAddr64(h_process, frame.AddrPC.Offset, &offset_from_line, &line))
ss << "+" << offset_from_line << " " << line.FileName << "(" << line.LineNumber << ")";
for (auto el : modules)
{
if ((DWORD64)el.base_address <= frame.AddrPC.Offset && frame.AddrPC.Offset < (DWORD64)el.base_address + (DWORD64)el.load_size)
{
ss << " : " << el.module_name << " @ 0x" << std::setw(0) << std::hex << (DWORD64)el.base_address << ENDL;
break;
}
}
}
SymCleanup(h_process);
return ss.str();
}
} // namespace tools
#endif // #if defined(WIN32)

24
src/common/callstack_helper.h Normal file
View file

@ -0,0 +1,24 @@
// Copyright (c) 2019 Zano Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#pragma once
#include <epee/include/misc_os_dependent.h>
namespace tools
{
#if defined(WIN32)
extern std::string get_callstack_win_x64();
#endif
inline std::string get_callstack()
{
#if defined(__GNUC__)
return epee::misc_utils::print_trace_default();
#elif defined(WIN32)
return get_callstack_win_x64();
#else
return "";
#endif
}
} // namespace tools

12
src/common/db_abstract_accessor.h
View file

@ -241,9 +241,9 @@ namespace tools
m_is_open = false;
return m_backend->close();
}
bool open(const std::string& path, uint64_t flags = 0)
bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE)
{
bool r = m_backend->open(path, flags);
bool r = m_backend->open(path, cache_sz);
if(r)
m_is_open = true;
@ -558,10 +558,10 @@ namespace tools
bool init(const std::string& container_name)
{
#ifdef ENABLE_PROFILING
m_get_profiler.m_name = container_name +":get";
m_set_profiler.m_name = container_name + ":set";
m_explicit_get_profiler.m_name = container_name + ":explicit_get";
m_explicit_set_profiler.m_name = container_name + ":explicit_set";
m_get_profiler.m_name = container_name +":get";
m_set_profiler.m_name = container_name + ":set";
m_explicit_get_profiler.m_name = container_name + ":explicit_get";
m_explicit_set_profiler.m_name = container_name + ":explicit_set";
m_commit_profiler.m_name = container_name + ":commit";
#endif
return bdb.get_backend()->open_container(container_name, m_h);

23
src/common/db_backend_base.h
View file

@ -5,6 +5,13 @@
#pragma once
#ifndef ENV32BIT
#define CACHE_SIZE uint64_t(uint64_t(1UL * 128UL) * 1024UL * 1024UL * 1024UL)
#else
#define CACHE_SIZE (1 * 1024UL * 1024UL * 1024UL)
#endif
namespace tools
{
namespace db
@ -25,20 +32,20 @@ namespace tools
struct i_db_backend
{
virtual bool close() = 0;
virtual bool close()=0;
virtual bool begin_transaction(bool read_only = false) = 0;
virtual bool commit_transaction() = 0;
virtual void abort_transaction() = 0;
virtual bool open(const std::string& path, uint64_t flags = 0) = 0;
virtual bool open_container(const std::string& name, container_handle& h) = 0;
virtual bool commit_transaction()=0;
virtual void abort_transaction()=0;
virtual bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) = 0;
virtual bool open_container(const std::string& name, container_handle& h)=0;
virtual bool erase(container_handle h, const char* k, size_t s) = 0;
virtual uint64_t size(container_handle h) = 0;
virtual bool get(container_handle h, const char* k, size_t s, std::string& res_buff) = 0;
virtual bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs) = 0;
virtual bool clear(container_handle h) = 0;
virtual bool enumerate(container_handle h, i_db_callback* pcb) = 0;
virtual bool enumerate(container_handle h, i_db_callback* pcb)=0;
virtual bool get_stat_info(stat_info& si) = 0;
virtual ~i_db_backend() {};
virtual ~i_db_backend(){};
};
}
}
}

69
src/common/db_backend_lmdb.cpp
View file

@ -10,10 +10,6 @@
#include "util.h"
#define BUF_SIZE 1024
#define DB_RESIZE_MIN_FREE_SIZE (100 * 1024 * 1024) // DB map size will grow if that much space left on DB
#define DB_RESIZE_MIN_MAX_SIZE (50 * 1024 * 1024) // Minimum DB map size (starting size)
#define DB_RESIZE_INCREMENT_SIZE (100 * 1024 * 1024) // Grow step size
#define DB_RESIZE_COMMITS_TO_CHECK 50
#define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
#define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess);
@ -27,13 +23,10 @@ namespace tools
{
namespace db
{
lmdb_db_backend::lmdb_db_backend()
: m_penv(AUTO_VAL_INIT(m_penv))
, m_commits_count(0)
lmdb_db_backend::lmdb_db_backend() : m_penv(AUTO_VAL_INIT(m_penv))
{
}
lmdb_db_backend::~lmdb_db_backend()
{
NESTED_TRY_ENTRY();
@ -43,16 +36,18 @@ namespace tools
NESTED_CATCH_ENTRY(__func__);
}
bool lmdb_db_backend::open(const std::string& path_, uint64_t /* flags -- unused atm */)
bool lmdb_db_backend::open(const std::string& path_, uint64_t cache_sz)
{
int res = 0;
res = mdb_env_create(&m_penv);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_create");
MDB_dbi max_dbs = 15;
res = mdb_env_set_maxdbs(m_penv, max_dbs);
res = mdb_env_set_maxdbs(m_penv, 15);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_maxdbs");
res = mdb_env_set_mapsize(m_penv, cache_sz);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize");
m_path = path_;
#ifdef WIN32
m_path = epee::string_encoding::convert_ansii_to_utf8(m_path);
@ -60,14 +55,8 @@ namespace tools
CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path);
// TODO: convert flags to lmdb_flags (implement fast lmdb modes)
unsigned int lmdb_flags = MDB_NORDAHEAD /*| MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC*/;
mdb_mode_t lmdb_mode = 0644;
res = mdb_env_open(m_penv, m_path.c_str(), lmdb_flags, lmdb_mode);
res = mdb_env_open(m_penv, m_path.c_str(), MDB_NORDAHEAD , 0644);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_open, m_path=" << m_path);
resize_if_needed();
return true;
}
@ -116,13 +105,6 @@ namespace tools
{
LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3);
CRITICAL_SECTION_LOCK(m_write_exclusive_lock);
if (m_commits_count.fetch_add(1, std::memory_order_relaxed) % DB_RESIZE_COMMITS_TO_CHECK == DB_RESIZE_COMMITS_TO_CHECK - 1)
{
if (!resize_if_needed())
m_commits_count.store(DB_RESIZE_COMMITS_TO_CHECK - 1, std::memory_order_relaxed); // if failed, try again on next commit
}
}
PROFILE_FUNC("lmdb_db_backend::begin_transaction");
{
@ -347,7 +329,6 @@ namespace tools
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put");
return true;
}
bool lmdb_db_backend::enumerate(container_handle h, i_db_callback* pcb)
{
CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate");
@ -402,42 +383,6 @@ namespace tools
}
return true;
}
bool lmdb_db_backend::resize_if_needed()
{
LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED in resize_if_needed()", LOG_LEVEL_3);
CRITICAL_REGION_LOCAL(m_write_exclusive_lock);
if (have_tx())
{
LOG_PRINT_RED("[DB " << m_path << "] : resize_if_needed(): Have txs on stack, unable to resize!", LOG_LEVEL_0);
return false;
}
MDB_stat st = AUTO_VAL_INIT(st);
mdb_env_stat(m_penv, &st);
MDB_envinfo ei = AUTO_VAL_INIT(ei);
mdb_env_info(m_penv, &ei);
uint64_t dirty_size = ei.me_last_pgno * st.ms_psize;
int64_t size_diff = ei.me_mapsize - dirty_size;
if (size_diff >= DB_RESIZE_MIN_FREE_SIZE && ei.me_mapsize >= DB_RESIZE_MIN_MAX_SIZE)
return true; // resize is not needed
double gigabyte = 1024 * 1024 * 1024;
const uint64_t increment_size_pg_aligned = DB_RESIZE_INCREMENT_SIZE - (DB_RESIZE_INCREMENT_SIZE % st.ms_psize);
// need to resize DB
uint64_t new_size = ei.me_mapsize - (ei.me_mapsize % increment_size_pg_aligned) + increment_size_pg_aligned;
int res = mdb_env_set_mapsize(m_penv, new_size);
CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize");
LOG_PRINT_CYAN("[DB " << m_path << "] has grown: " << std::fixed << std::setprecision(2) << ei.me_mapsize / gigabyte << " GiB -> " << std::fixed << std::setprecision(2) << new_size / gigabyte << " GiB", LOG_LEVEL_0);
return true;
}
}
}

15
src/common/db_backend_lmdb.h
View file

@ -1,4 +1,4 @@
// Copyright (c) 2014-2018 Zano Project
// Copyright (c) 2014-2019 Zano Project
// Copyright (c) 2014-2018 The Louisdor Project
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
@ -6,7 +6,6 @@
#pragma once
#include <thread>
#include "include_base_utils.h"
#include "db_backend_base.h"
@ -17,9 +16,6 @@ namespace tools
{
namespace db
{
class lmdb_db_backend : public i_db_backend
{
@ -38,11 +34,7 @@ namespace tools
boost::recursive_mutex m_cs;
boost::recursive_mutex m_write_exclusive_lock;
std::map<std::thread::id, transactions_list> m_txs; // size_t -> count of nested read_only transactions
std::atomic<uint64_t> m_commits_count;
bool pop_tx_entry(tx_entry& txe);
public:
lmdb_db_backend();
~lmdb_db_backend();
@ -52,7 +44,7 @@ namespace tools
bool begin_transaction(bool read_only = false);
bool commit_transaction();
void abort_transaction();
bool open(const std::string& path, uint64_t flags = 0);
bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE);
bool open_container(const std::string& name, container_handle& h);
bool erase(container_handle h, const char* k, size_t s);
bool get(container_handle h, const char* k, size_t s, std::string& res_buff);
@ -64,8 +56,7 @@ namespace tools
//-------------------------------------------------------------------------------------
bool have_tx();
MDB_txn* get_current_tx();
bool resize_if_needed();
};
}
}
}

65
src/common/util.cpp
View file

@ -10,6 +10,7 @@ using namespace epee;
#include "util.h"
#include "currency_core/currency_config.h"
#include "version.h"
#ifdef WIN32
#include <windows.h>
@ -362,7 +363,69 @@ namespace tools
return pszOS;
}
}
#else
void signal_handler::GenerateCrashDump(EXCEPTION_POINTERS *pep /* = NULL*/)
{
SYSTEMTIME sysTime = { 0 };
GetSystemTime(&sysTime);
// get the computer name
char compName[MAX_COMPUTERNAME_LENGTH + 1] = { 0 };
DWORD compNameLen = ARRAYSIZE(compName);
GetComputerNameA(compName, &compNameLen);
// build the filename: APPNAME_COMPUTERNAME_DATE_TIME.DMP
char path[MAX_PATH*10] = { 0 };
std::string folder = epee::log_space::log_singletone::get_default_log_folder();
sprintf_s(path, ARRAYSIZE(path),"%s\\crashdump_" PROJECT_VERSION_LONG "_%s_%04u-%02u-%02u_%02u-%02u-%02u.dmp",
folder.c_str(), compName, sysTime.wYear, sysTime.wMonth, sysTime.wDay,
sysTime.wHour, sysTime.wMinute, sysTime.wSecond);
HANDLE hFile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE,
0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE))
{
// Create the minidump
MINIDUMP_EXCEPTION_INFORMATION mdei;
mdei.ThreadId = GetCurrentThreadId();
mdei.ExceptionPointers = pep;
mdei.ClientPointers = FALSE;
MINIDUMP_CALLBACK_INFORMATION mci;
mci.CallbackRoutine = (MINIDUMP_CALLBACK_ROUTINE)MyMiniDumpCallback;
mci.CallbackParam = 0;
MINIDUMP_TYPE mdt = (MINIDUMP_TYPE)(MiniDumpWithPrivateReadWriteMemory |
MiniDumpWithDataSegs |
MiniDumpWithHandleData |
MiniDumpWithFullMemoryInfo |
MiniDumpWithThreadInfo |
MiniDumpWithUnloadedModules);
BOOL rv = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(),
hFile, mdt, (pep != 0) ? &mdei : 0, 0, &mci);
if (!rv)
{
LOG_ERROR("Minidump file create FAILED(error " << GetLastError() << ") on path: " << path);
}
else
{
LOG_PRINT_L0("Minidump file created on path: " << path);
}
// Close the file
CloseHandle(hFile);
}
else
{
LOG_ERROR("Minidump FAILED to create file (error " << GetLastError() << ") on path: " << path);
}
}
#else // ifdef WIN32
std::string get_nix_version_display_string()
{
utsname un;

62
src/common/util.h
View file

@ -231,67 +231,7 @@ namespace tools
}
static void GenerateCrashDump(EXCEPTION_POINTERS *pep = NULL)
{
SYSTEMTIME sysTime = { 0 };
GetSystemTime(&sysTime);
// get the computer name
char compName[MAX_COMPUTERNAME_LENGTH + 1] = { 0 };
DWORD compNameLen = ARRAYSIZE(compName);
GetComputerNameA(compName, &compNameLen);
// build the filename: APPNAME_COMPUTERNAME_DATE_TIME.DMP
char path[MAX_PATH*10] = { 0 };
std::string folder = epee::log_space::log_singletone::get_default_log_folder();
sprintf_s(path, ARRAYSIZE(path),"%s\\crashdump_%s_%04u-%02u-%02u_%02u-%02u-%02u.dmp",
folder.c_str(), compName, sysTime.wYear, sysTime.wMonth, sysTime.wDay,
sysTime.wHour, sysTime.wMinute, sysTime.wSecond);
HANDLE hFile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE,
0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE))
{
// Create the minidump
MINIDUMP_EXCEPTION_INFORMATION mdei;
mdei.ThreadId = GetCurrentThreadId();
mdei.ExceptionPointers = pep;
mdei.ClientPointers = FALSE;
MINIDUMP_CALLBACK_INFORMATION mci;
mci.CallbackRoutine = (MINIDUMP_CALLBACK_ROUTINE)MyMiniDumpCallback;
mci.CallbackParam = 0;
MINIDUMP_TYPE mdt = (MINIDUMP_TYPE)(MiniDumpWithPrivateReadWriteMemory |
MiniDumpWithDataSegs |
MiniDumpWithHandleData |
MiniDumpWithFullMemoryInfo |
MiniDumpWithThreadInfo |
MiniDumpWithUnloadedModules);
BOOL rv = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(),
hFile, mdt, (pep != 0) ? &mdei : 0, 0, &mci);
if (!rv)
{
LOG_ERROR("Minidump file create FAILED(error " << GetLastError() << ") on path: " << path);
}
else
{
LOG_PRINT_L0("Minidump file created on path: " << path);
}
// Close the file
CloseHandle(hFile);
}
else
{
LOG_ERROR("Minidump FAILED to create file (error " << GetLastError() << ") on path: " << path);
}
}
static void GenerateCrashDump(EXCEPTION_POINTERS *pep = NULL);
static LONG WINAPI win_unhandled_exception_handler(_In_ struct _EXCEPTION_POINTERS *ep)
{

42
src/currency_core/basic_pow_helpers.cpp
View file

@ -22,17 +22,31 @@ namespace currency
{
//--------------------------------------------------------------
//global object
// crypto::ethash::cache_manager cache;
// void ethash_set_use_dag(bool use_dag)
// {
// cache.set_use_dag(use_dag);
// }
// //------------------------------------------------------------------
// const uint8_t* ethash_get_dag(uint64_t epoch, uint64_t& dag_size)
// {
// return cache.get_dag(epoch, dag_size);
// }
int ethash_custom_log_get_level()
{
return epee::log_space::get_set_log_detalisation_level();
}
//--------------------------------------------------------------
void ethash_custom_log(const std::string& m, bool add_callstack)
{
std::string msg = epee::log_space::log_singletone::get_prefix_entry() + "[ETHASH]" + m;
if (add_callstack)
msg = msg + "callstask: " + epee::misc_utils::get_callstack();
epee::log_space::log_singletone::do_log_message(msg, LOG_LEVEL_0, epee::log_space::console_color_default, true, LOG_DEFAULT_TARGET);
}
//--------------------------------------------------------------
void init_ethash_log_if_necessary()
{
static bool inited = false;
if (inited)
return;
ethash::access_custom_log_level_function() = &ethash_custom_log_get_level;
ethash::access_custom_log_function() = &ethash_custom_log;
inited = true;
}
//------------------------------------------------------------------
int ethash_height_to_epoch(uint64_t height)
{
@ -49,9 +63,11 @@ namespace currency
//--------------------------------------------------------------
crypto::hash get_block_longhash(uint64_t height, const crypto::hash& block_header_hash, uint64_t nonce)
{
init_ethash_log_if_necessary();
int epoch = ethash_height_to_epoch(height);
const auto& context = progpow::get_global_epoch_context_full(static_cast<int>(epoch));
auto res_eth = progpow::hash(context, static_cast<int>(height), *(ethash::hash256*)&block_header_hash, nonce);
std::shared_ptr<ethash::epoch_context_full> p_context = progpow::get_global_epoch_context_full(static_cast<int>(epoch));
CHECK_AND_ASSERT_THROW_MES(p_context, "progpow::get_global_epoch_context_full returned null");
auto res_eth = progpow::hash(*p_context, static_cast<int>(height), *(ethash::hash256*)&block_header_hash, nonce);
crypto::hash result = currency::null_hash;
memcpy(&result.data, &res_eth.final_hash, sizeof(res_eth.final_hash));
return result;

22
src/currency_core/blockchain_storage.cpp
View file

@ -76,6 +76,7 @@ DISABLE_VS_WARNINGS(4267)
namespace
{
const command_line::arg_descriptor<uint32_t> arg_db_cache_l1 = { "db-cache-l1", "Specify size of memory mapped db cache file", 0, true };
const command_line::arg_descriptor<uint32_t> arg_db_cache_l2 = { "db-cache-l2", "Specify cached elements in db helpers", 0, true };
}
@ -151,6 +152,7 @@ std::shared_ptr<transaction> blockchain_storage::get_tx(const crypto::hash &id)
//------------------------------------------------------------------
void blockchain_storage::init_options(boost::program_options::options_description& desc)
{
command_line::add_arg(desc, arg_db_cache_l1);
command_line::add_arg(desc, arg_db_cache_l2);
}
//------------------------------------------------------------------
@ -213,20 +215,36 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro
return false;
}
uint64_t cache_size_l1 = CACHE_SIZE;
if (command_line::has_arg(vm, arg_db_cache_l1))
{
cache_size_l1 = command_line::get_arg(vm, arg_db_cache_l1);
}
LOG_PRINT_GREEN("Using db file cache size(L1): " << cache_size_l1, LOG_LEVEL_0);
m_config_folder = config_folder;
// remove old incompartible DB
const std::string old_db_folder_path = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD;
if (boost::filesystem::exists(old_db_folder_path))
{
LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0);
boost::filesystem::remove_all(old_db_folder_path);
}
const std::string db_folder_path = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME;
LOG_PRINT_L0("Loading blockchain from " << db_folder_path);
bool db_opened_okay = false;
for(size_t loading_attempt_no = 0; loading_attempt_no < 2; ++loading_attempt_no)
{
bool res = m_db.open(db_folder_path);
bool res = m_db.open(db_folder_path, cache_size_l1);
if (!res)
{
// if DB could not be opened -- try to remove the whole folder and re-open DB
LOG_PRINT_YELLOW("Failed to initialize database in folder: " << db_folder_path << ", first attempt", LOG_LEVEL_0);
boost::filesystem::remove_all(db_folder_path);
res = m_db.open(db_folder_path);
res = m_db.open(db_folder_path, cache_size_l1);
CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << db_folder_path << ", second attempt");
}

6
src/currency_core/currency_config.h
View file

@ -187,8 +187,10 @@
#define CURRENCY_CORE_INSTANCE_LOCK_FILE "lock.lck"
#define CURRENCY_POOLDATA_FOLDERNAME "poolstate"
#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain"
#define CURRENCY_POOLDATA_FOLDERNAME_OLD "poolstate"
#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD "blockchain"
#define CURRENCY_POOLDATA_FOLDERNAME "poolstate_lmdb_v1"
#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain_lmdb_v1"
#define P2P_NET_DATA_FILENAME "p2pstate.bin"
#define MINER_CONFIG_FILENAME "miner_conf.json"
#define GUI_SECURE_CONFIG_FILENAME "gui_secure_conf.bin"

19
src/currency_core/tx_pool.cpp
View file

@ -1120,19 +1120,30 @@ namespace currency
{
m_config_folder = config_folder;
LOG_PRINT_L0("Loading blockchain...");
uint64_t cache_size_l1 = CACHE_SIZE;
LOG_PRINT_GREEN("Using pool db file cache size(L1): " << cache_size_l1, LOG_LEVEL_0);
// remove old incompartible DB
const std::string old_db_folder_path = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME_OLD;
if (boost::filesystem::exists(old_db_folder_path))
{
LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0);
boost::filesystem::remove_all(old_db_folder_path);
}
const std::string db_folder_path = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME;
LOG_PRINT_L0("Loading blockchain from " << db_folder_path << "...");
bool db_opened_okay = false;
for(size_t loading_attempt_no = 0; loading_attempt_no < 2; ++loading_attempt_no)
{
bool res = m_db.open(db_folder_path);
bool res = m_db.open(db_folder_path, cache_size_l1);
if (!res)
{
// if DB could not be opened -- try to remove the whole folder and re-open DB
LOG_PRINT_YELLOW("Failed to initialize database in folder: " << db_folder_path << ", first attempt", LOG_LEVEL_0);
boost::filesystem::remove_all(db_folder_path);
res = m_db.open(db_folder_path);
res = m_db.open(db_folder_path, cache_size_l1);
CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << db_folder_path << ", second attempt");
}

6
src/daemon/daemon.cpp
View file

@ -26,6 +26,7 @@ using namespace epee;
#include "common/miniupnp_helper.h"
#include "version.h"
#include "currency_core/core_tools.h"
#include "common/callstack_helper.h"
#include <cstdlib>
@ -100,7 +101,7 @@ int main(int argc, char* argv[])
#endif
log_space::get_set_log_detalisation_level(true, LOG_LEVEL_2);
log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL);
log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,wallet,lmdb");
log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,wallet");
LOG_PRINT_L0("Starting...");
tools::signal_handler::install_fatal([](int sig_number, void* address) {
@ -108,6 +109,9 @@ int main(int argc, char* argv[])
std::fflush(nullptr); // all open output streams are flushed
});
// setup custom callstack retrieving function
epee::misc_utils::get_callstack(tools::get_callstack);
po::options_description desc_cmd_only("Command line options");
po::options_description desc_cmd_sett("Command line options and settings options", 130, 83);

4
src/gui/qt-daemon/application/daemon_backend.cpp
View file

@ -9,7 +9,7 @@
#include "core_fast_rpc_proxy.h"
#include "string_coding.h"
#include "currency_core/core_tools.h"
//#include <codecvt>
#include "common/callstack_helper.h"
#define GET_WALLET_OPT_BY_ID(wallet_id, name) \
CRITICAL_REGION_LOCAL(m_wallets_lock); \
@ -85,6 +85,8 @@ bool daemon_backend::init(int argc, char* argv[], view::i_view* pview_handler)
std::fflush(nullptr); // all open output streams are flushed
});
// setup custom callstack retrieving function
epee::misc_utils::get_callstack(tools::get_callstack);
//#if !defined(NDEBUG)
// log_space::log_singletone::add_logger(LOGGER_DEBUGGER, nullptr, nullptr);

36
src/gui/qt-daemon/main.cpp
View file

@ -42,25 +42,29 @@ int main(int argc, char *argv[])
#endif
epee::string_tools::set_module_name_and_folder(argv[0]);
QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling);
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling);
#ifdef _MSC_VER
#if _MSC_VER >= 1910
QCoreApplication::setAttribute(Qt::AA_UseHighDpiPixmaps); //HiDPI pixmaps
qputenv("QT_SCALE_FACTOR", "0.75");
QCoreApplication::setAttribute(Qt::AA_UseHighDpiPixmaps); //HiDPI pixmaps
qputenv("QT_SCALE_FACTOR", "0.75");
#endif
#endif
QApplication app(argc, argv);
MainWindow viewer;
if (!viewer.init_backend(argc, argv))
{
static_cast<view::i_view*>(&viewer)->show_msg_box("Failed to initialize backend, check debug logs for more details.");
return 1;
}
app.installNativeEventFilter(&viewer);
viewer.setWindowTitle(CURRENCY_NAME_BASE);
viewer.show_inital();
return app.exec();
QApplication app(argc, argv);
MainWindow viewer;
if (!viewer.init_backend(argc, argv))
{
static_cast<view::i_view*>(&viewer)->show_msg_box("Failed to initialize backend, check debug logs for more details.");
return 1;
}
app.installNativeEventFilter(&viewer);
viewer.setWindowTitle(CURRENCY_NAME_BASE);
viewer.show_inital();
int res = app.exec();
LOG_PRINT_L0("Process exit with code: " << res);
return res;
CATCH_ENTRY2(0);
}

2
src/version.h.in
View file

@ -2,6 +2,6 @@
#define BUILD_COMMIT_ID "@VERSION@"
#define PROJECT_VERSION "1.0"
#define PROJECT_VERSION_BUILD_NO 45
#define PROJECT_VERSION_BUILD_NO 50
#define PROJECT_VERSION_BUILD_NO_STR STRINGIFY_EXPAND(PROJECT_VERSION_BUILD_NO)
#define PROJECT_VERSION_LONG PROJECT_VERSION "." PROJECT_VERSION_BUILD_NO_STR "[" BUILD_COMMIT_ID "]"

9
tests/unit_tests/db_accessors.cpp
View file

@ -29,8 +29,9 @@ TEST(db_accessor_tests, cached_key_value_accessor_test)
tools::db::cached_key_value_accessor<uint64_t, uint64_t, false, true> m_container(m_db);
const std::string folder_name = "./TEST_cached_key_value_accessor_test";
tools::create_directories_if_necessary(folder_name);
uint64_t cache_size = CACHE_SIZE;
ASSERT_TRUE(m_db.open(folder_name));
ASSERT_TRUE(m_db.open(folder_name, cache_size));
ASSERT_TRUE(m_container.init("container"));
... TODO ...
@ -46,7 +47,8 @@ TEST(db_accessor_tests_2, recoursive_tx_test)
const std::string folder_name = "./TEST_db_recursive_tx";
tools::create_directories_if_necessary(folder_name);
ASSERT_TRUE(m_db.open(folder_name));
uint64_t cache_size = CACHE_SIZE;
ASSERT_TRUE(m_db.open(folder_name, cache_size));
ASSERT_TRUE(m_container.init("zzzz") );
bool tx_result = m_container.begin_transaction();
@ -317,7 +319,8 @@ TEST(db_accessor_tests, median_db_cache_test)
const std::string folder_name = "./TEST_median_db_cache";
const std::string naive_median_serialization_filename = folder_name + "/naive_median";
tools::create_directories_if_necessary(folder_name);
ASSERT_TRUE(m_db.open(folder_name));
uint64_t cache_size = CACHE_SIZE;
ASSERT_TRUE(m_db.open(folder_name, cache_size));
ASSERT_TRUE(m_tx_fee_median.init("median_fee"));
m_db.begin_transaction();

2
tests/unit_tests/lmdb_tests.cpp
View file

@ -30,7 +30,7 @@ namespace lmdb_test
// write data
//
r = bdba.open(db_file_path);
r = bdba.open(db_file_path, CACHE_SIZE);
ASSERT_TRUE(r);
db::container_handle h;