diff --git a/contrib/db/liblmdb/CHANGES b/contrib/db/liblmdb/CHANGES index aabb1adf..f00efbb9 100644 --- a/contrib/db/liblmdb/CHANGES +++ b/contrib/db/liblmdb/CHANGES @@ -1,14 +1,70 @@ LMDB 0.9 Change Log -LMDB 0.9.18 Release Engineering +LMDB 0.9.24 Release (2019/07/24) + ITS#8969 Tweak mdb_page_split + ITS#8975 WIN32 fix writemap set_mapsize crash + ITS#9007 Fix loose pages in WRITEMAP + +LMDB 0.9.23 Release (2018/12/19) + ITS#8756 Fix loose pages in dirty list + ITS#8831 Fix mdb_load flag init + ITS#8844 Fix mdb_env_close in forked process + Documentation + ITS#8857 mdb_cursor_del doesn't invalidate cursor + ITS#8908 GET_MULTIPLE etc don't change passed in key + +LMDB 0.9.22 Release (2018/03/22) + Fix MDB_DUPSORT alignment bug (ITS#8819) + Fix regression with new db from 0.9.19 (ITS#8760) + Fix liblmdb to build on Solaris (ITS#8612) + Fix delete behavior with DUPSORT DB (ITS#8622) + Fix mdb_cursor_get/mdb_cursor_del behavior (ITS#8722) + +LMDB 0.9.21 Release (2017/06/01) + Fix xcursor after cursor_del (ITS#8622) + +LMDB 0.9.20 (Withdrawn) + Fix mdb_load with escaped plaintext (ITS#8558) + Fix mdb_cursor_last / mdb_put interaction (ITS#8557) + +LMDB 0.9.19 Release (2016/12/28) + Fix mdb_env_cwalk cursor init (ITS#8424) + Fix robust mutexes on Solaris 10/11 (ITS#8339) + Tweak Win32 error message buffer + Fix MDB_GET_BOTH on non-dup record (ITS#8393) + Optimize mdb_drop + Fix xcursors after mdb_cursor_del (ITS#8406) + Fix MDB_NEXT_DUP after mdb_cursor_del (ITS#8412) + Fix mdb_cursor_put resetting C_EOF (ITS#8489) + Fix mdb_env_copyfd2 to return EPIPE on SIGPIPE (ITS#8504) + Fix mdb_env_copy with empty DB (ITS#8209) + Fix behaviors with fork (ITS#8505) + Fix mdb_dbi_open with mainDB cursors (ITS#8542) + Fix robust mutexes on kFreeBSD (ITS#8554) + Fix utf8_to_utf16 error checks (ITS#7992) + Fix F_NOCACHE on MacOS, error is non-fatal (ITS#7682) + Build + Make shared lib suffix overridable (ITS#8481) + Documentation + Cleanup doxygen nits + Note reserved vs actual mem/disk usage + + +LMDB 0.9.18 Release (2016/02/05) Fix robust mutex detection on glibc 2.10-11 (ITS#8330) + Fix page_search_root assert on FreeDB (ITS#8336) + Fix MDB_APPENDDUP vs. rewrite(single item) (ITS#8334) + Fix mdb_copy of large files on Windows + Fix subcursor move after delete (ITS#8355) + Fix mdb_midl_shirnk off-by-one (ITS#8363) Check for utf8_to_utf16 failures (ITS#7992) Catch strdup failure in mdb_dbi_open Build Additional makefile var tweaks (ITS#8169) Documentation Add Getting Started page - + Update WRITEMAP description + LMDB 0.9.17 Release (2015/11/30) Fix ITS#7377 catch calloc failure diff --git a/contrib/db/liblmdb/COPYRIGHT b/contrib/db/liblmdb/COPYRIGHT index 722d1a51..f076556e 100644 --- a/contrib/db/liblmdb/COPYRIGHT +++ b/contrib/db/liblmdb/COPYRIGHT @@ -1,4 +1,4 @@ -Copyright 2011-2015 Howard Chu, Symas Corp. +Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/Doxyfile b/contrib/db/liblmdb/Doxyfile index 5047c0bb..5ca2cfe8 100644 --- a/contrib/db/liblmdb/Doxyfile +++ b/contrib/db/liblmdb/Doxyfile @@ -253,7 +253,7 @@ IDL_PROPERTY_SUPPORT = YES # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. -DISTRIBUTE_GROUP_DOC = NO +DISTRIBUTE_GROUP_DOC = YES # Set the SUBGROUPING tag to YES (the default) to allow class member groups of # the same type (for instance a group of public functions) to be put as a diff --git a/contrib/db/liblmdb/Makefile b/contrib/db/liblmdb/Makefile index f3c93a2f..f254511f 100644 --- a/contrib/db/liblmdb/Makefile +++ b/contrib/db/liblmdb/Makefile @@ -8,7 +8,7 @@ # platforms; you should not need to change any of these. # Read their descriptions in mdb.c if you do: # -# - MDB_USE_POSIX_MUTEX, MDB_USE_POSIX_SEM, MDB_USE_SYSV_SEM +# - MDB_USE_POSIX_SEM # - MDB_DSYNC # - MDB_FDATASYNC # - MDB_FDATASYNC_WORKS @@ -24,8 +24,9 @@ W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized THREADS = -pthread OPT = -O2 -g CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) -LDLIBS = # -lntdll # Windows needs ntdll -SOLIBS = # -lntdll +LDLIBS = +SOLIBS = +SOEXT = .so prefix = /usr/local exec_prefix = $(prefix) bindir = $(exec_prefix)/bin @@ -37,7 +38,7 @@ mandir = $(datarootdir)/man ######################################################################## IHDRS = lmdb.h -ILIBS = liblmdb.a liblmdb.so +ILIBS = liblmdb.a liblmdb$(SOEXT) IPROGS = mdb_stat mdb_copy mdb_dump mdb_load IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 @@ -63,7 +64,7 @@ test: all liblmdb.a: mdb.o midl.o $(AR) rs $@ mdb.o midl.o -liblmdb.so: mdb.lo midl.lo +liblmdb$(SOEXT): mdb.lo midl.lo # $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS) $(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS) diff --git a/contrib/db/liblmdb/intro.doc b/contrib/db/liblmdb/intro.doc index 870c7bb8..64dfcaad 100644 --- a/contrib/db/liblmdb/intro.doc +++ b/contrib/db/liblmdb/intro.doc @@ -1,5 +1,5 @@ /* - * Copyright 2015 Howard Chu, Symas Corp. + * Copyright 2015-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/lmdb.h b/contrib/db/liblmdb/lmdb.h index 0bca3eb7..2f55290d 100644 --- a/contrib/db/liblmdb/lmdb.h +++ b/contrib/db/liblmdb/lmdb.h @@ -53,15 +53,14 @@ * * Fix: Check for stale readers periodically, using the * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. - * Stale writers will be cleared automatically on most systems: + * Stale writers will be cleared automatically on some systems: * - Windows - automatic - * - BSD, systems using SysV semaphores - automatic * - Linux, systems using POSIX mutexes with Robust option - automatic + * - not on BSD, systems using POSIX semaphores. * Otherwise just make all programs using the database close it; * the lockfile is always reset on first open of the environment. * - * - On BSD systems or others configured with MDB_USE_SYSV_SEM or - * MDB_USE_POSIX_SEM, + * - On BSD systems or others configured with MDB_USE_POSIX_SEM, * startup can fail due to semaphores owned by another userid. * * Fix: Open and close the database as the user which owns the @@ -97,11 +96,12 @@ * transactions. Each transaction belongs to one thread. See below. * The #MDB_NOTLS flag changes this for read-only transactions. * - * - Use an MDB_env* in the process which opened it, without fork()ing. + * - Use an MDB_env* in the process which opened it, not after fork(). * * - Do not have open an LMDB database twice in the same process at * the same time. Not even from a plain open() call - close()ing it - * breaks flock() advisory locking. + * breaks fcntl() advisory locking. (It is OK to reopen it after + * fork() - exec*(), since the lockfile has FD_CLOEXEC set.) * * - Avoid long-lived transactions. Read transactions prevent * reuse of pages freed by newer write transactions, thus the @@ -135,7 +135,7 @@ * * @author Howard Chu, Symas Corporation. * - * @copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved. + * @copyright Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted only as authorized by the OpenLDAP @@ -166,7 +166,6 @@ #define _LMDB_H_ #include -#include #ifdef __cplusplus extern "C" { @@ -179,13 +178,6 @@ typedef int mdb_mode_t; typedef mode_t mdb_mode_t; #endif -#ifdef MDB_VL32 -typedef uint64_t mdb_size_t; -#define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */ -#else -typedef size_t mdb_size_t; -#endif - /** An abstraction for a file handle. * On POSIX systems file handles are small integers. On Windows * they're opaque pointers. @@ -208,7 +200,7 @@ typedef int mdb_filehandle_t; /** Library minor version */ #define MDB_VERSION_MINOR 9 /** Library patch version */ -#define MDB_VERSION_PATCH 70 +#define MDB_VERSION_PATCH 24 /** Combine args a,b,c into a single integer for easy version comparisons */ #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) @@ -218,7 +210,7 @@ typedef int mdb_filehandle_t; MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) /** The release date of this library version */ -#define MDB_VERSION_DATE "December 19, 2015" +#define MDB_VERSION_DATE "July 24, 2019" /** A stringifier for the version info */ #define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" @@ -311,8 +303,6 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel #define MDB_NORDAHEAD 0x800000 /** don't initialize malloc'd memory before writing to datafile */ #define MDB_NOMEMINIT 0x1000000 - /** use the previous snapshot rather than the latest one */ -#define MDB_PREVSNAPSHOT 0x2000000 /** @} */ /** @defgroup mdb_dbi_open Database Flags @@ -380,7 +370,7 @@ typedef enum MDB_cursor_op { MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ MDB_GET_CURRENT, /**< Return key/data at current cursor position */ - MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items + MDB_GET_MULTIPLE, /**< Return up to a page of duplicate data items from current cursor position. Move cursor to prepare for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ MDB_LAST, /**< Position at last key/data item */ @@ -389,7 +379,7 @@ typedef enum MDB_cursor_op { MDB_NEXT, /**< Position at next data item */ MDB_NEXT_DUP, /**< Position at next data item of current key. Only for #MDB_DUPSORT */ - MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items + MDB_NEXT_MULTIPLE, /**< Return up to a page of duplicate data items from next cursor position. Move cursor to prepare for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ MDB_NEXT_NODUP, /**< Position at first data item of next key */ @@ -400,7 +390,7 @@ typedef enum MDB_cursor_op { MDB_SET, /**< Position at specified key */ MDB_SET_KEY, /**< Position at specified key, return key + data */ MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */ - MDB_PREV_MULTIPLE /**< Position at previous page and return key and up to + MDB_PREV_MULTIPLE /**< Position at previous page and return up to a page of duplicate data items. Only for #MDB_DUPFIXED */ } MDB_cursor_op; @@ -467,18 +457,18 @@ typedef struct MDB_stat { unsigned int ms_psize; /**< Size of a database page. This is currently the same for all databases. */ unsigned int ms_depth; /**< Depth (height) of the B-tree */ - mdb_size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ - mdb_size_t ms_leaf_pages; /**< Number of leaf pages */ - mdb_size_t ms_overflow_pages; /**< Number of overflow pages */ - mdb_size_t ms_entries; /**< Number of data items */ + size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ + size_t ms_leaf_pages; /**< Number of leaf pages */ + size_t ms_overflow_pages; /**< Number of overflow pages */ + size_t ms_entries; /**< Number of data items */ } MDB_stat; /** @brief Information about the environment */ typedef struct MDB_envinfo { void *me_mapaddr; /**< Address of map, if fixed */ - mdb_size_t me_mapsize; /**< Size of the data memory map */ - mdb_size_t me_last_pgno; /**< ID of the last used page */ - mdb_size_t me_last_txnid; /**< ID of the last committed transaction */ + size_t me_mapsize; /**< Size of the data memory map */ + size_t me_last_pgno; /**< ID of the last used page */ + size_t me_last_txnid; /**< ID of the last committed transaction */ unsigned int me_maxreaders; /**< max reader slots in the environment */ unsigned int me_numreaders; /**< max reader slots used in the environment */ } MDB_envinfo; @@ -624,12 +614,6 @@ int mdb_env_create(MDB_env **env); * caller is expected to overwrite all of the memory that was * reserved in that case. * This flag may be changed at any time using #mdb_env_set_flags(). - *
  • #MDB_PREVSNAPSHOT - * Open the environment with the previous snapshot rather than the latest - * one. This loses the latest transaction, but may help work around some - * types of corruption. If opened with write access, this must be the - * only process using the environment. This flag is automatically reset - * after a write transaction is successfully committed. * * @param[in] mode The UNIX permissions to set on created files and semaphores. * This parameter is ignored on Windows. @@ -696,6 +680,7 @@ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); *
  • #MDB_CP_COMPACT - Perform compaction while copying: omit free * pages and sequentially renumber all pages in output. This option * consumes more CPU and runs more slowly than the default. + * Currently it fails if the environment has suffered a page leak. * * @return A non-zero error value on failure and 0 on success. */ @@ -810,6 +795,10 @@ int mdb_env_get_flags(MDB_env *env, unsigned int *flags); int mdb_env_get_path(MDB_env *env, const char **path); /** @brief Return the filedescriptor for the given environment. + * + * This function may be called after fork(), so the descriptor can be + * closed before exec*(). Other LMDB file descriptors have FD_CLOEXEC. + * (Until LMDB 0.9.18, only the lockfile had that.) * * @param[in] env An environment handle returned by #mdb_env_create() * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. @@ -853,7 +842,7 @@ int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); * an active write transaction. * */ -int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size); +int mdb_env_set_mapsize(MDB_env *env, size_t size); /** @brief Set the maximum number of threads/reader slots for the environment. * @@ -966,10 +955,6 @@ int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); *
      *
    • #MDB_RDONLY * This transaction will not perform any write operations. - *
    • #MDB_NOSYNC - * Don't flush system buffers to disk when committing this transaction. - *
    • #MDB_NOMETASYNC - * Flush system buffers but omit metadata flush when committing this transaction. *
    * @param[out] txn Address where the new #MDB_txn handle will be stored * @return A non-zero error value on failure and 0 on success. Some possible @@ -1002,7 +987,7 @@ MDB_env *mdb_txn_env(MDB_txn *txn); * @param[in] txn A transaction handle returned by #mdb_txn_begin() * @return A transaction ID, valid if input is an active transaction. */ -mdb_size_t mdb_txn_id(MDB_txn *txn); +size_t mdb_txn_id(MDB_txn *txn); /** @brief Commit all the operations of a transaction into the database. * @@ -1118,8 +1103,9 @@ int mdb_txn_renew(MDB_txn *txn); * This flag may only be used in combination with #MDB_DUPSORT. This option * tells the library that the data items for this database are all the same * size, which allows further optimizations in storage and retrieval. When - * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE - * cursor operations may be used to retrieve multiple items at once. + * all data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE + * and #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple + * items at once. *
  • #MDB_INTEGERDUP * This option specifies that duplicate data items are binary integers, * similar to #MDB_INTEGERKEY keys. @@ -1524,6 +1510,10 @@ int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, /** @brief Delete current key/data pair * * This function deletes the key/data pair to which the cursor refers. + * This does not invalidate the cursor, so operations such as MDB_NEXT + * can still be used on it. + * Both MDB_NEXT and MDB_GET_CURRENT will return the same record after + * this operation. * @param[in] cursor A cursor handle returned by #mdb_cursor_open() * @param[in] flags Options for this operation. This parameter * must be set to 0 or one of the values described here. @@ -1552,7 +1542,7 @@ int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); *
  • EINVAL - cursor is not initialized, or an invalid parameter was specified. * */ -int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp); +int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); /** @brief Compare two data items according to a particular database. * diff --git a/contrib/db/liblmdb/mdb.c b/contrib/db/liblmdb/mdb.c index ca9f3b12..692feaa3 100644 --- a/contrib/db/liblmdb/mdb.c +++ b/contrib/db/liblmdb/mdb.c @@ -5,7 +5,7 @@ * BerkeleyDB API, but much simplified. */ /* - * Copyright 2011-2016 Howard Chu, Symas Corp. + * Copyright 2011-2019 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,50 +35,13 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif -#if defined(MDB_VL32) || defined(__WIN64__) +#if defined(__WIN64__) #define _FILE_OFFSET_BITS 64 #endif #ifdef _WIN32 #include #include -#include - -#ifndef _NTDEF_ -typedef _Return_type_success_(return >= 0) LONG NTSTATUS; -typedef NTSTATUS *PNTSTATUS; -#endif - -/** Visual studio big files support -*/ - -/* We use native NT APIs to setup the memory map, so that we can - * let the DB file grow incrementally instead of always preallocating - * the full size. These APIs are defined in and - * but those headers are meant for driver-level development and - * conflict with the regular user-level headers, so we explicitly - * declare them here. Using these APIs also means we must link to - * ntdll.dll, which is not linked by default in user code. - */ -NTSTATUS WINAPI -NtCreateSection(OUT PHANDLE sh, IN ACCESS_MASK acc, - IN void * oa OPTIONAL, - IN PLARGE_INTEGER ms OPTIONAL, - IN ULONG pp, IN ULONG aa, IN HANDLE fh OPTIONAL); - -typedef enum _SECTION_INHERIT { - ViewShare = 1, - ViewUnmap = 2 -} SECTION_INHERIT; - -NTSTATUS WINAPI -NtMapViewOfSection(IN PHANDLE sh, IN HANDLE ph, - IN OUT PVOID *addr, IN ULONG_PTR zbits, - IN SIZE_T cs, IN OUT PLARGE_INTEGER off OPTIONAL, - IN OUT PSIZE_T vs, IN SECTION_INHERIT ih, - IN ULONG at, IN ULONG pp); - -NTSTATUS WINAPI -NtClose(HANDLE h); +#include /* get wcscpy() */ /** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it * as int64 which is wrong. MSVC doesn't define it at all, so just @@ -130,13 +93,6 @@ extern int cacheflush(char *addr, int nbytes, int cache); #define BROKEN_FDATASYNC #endif -#ifdef _WIN32 -typedef int64_t off64_t; -#else -typedef off_t off64_t; -#endif - - #include #include #include @@ -153,10 +109,14 @@ typedef SSIZE_T ssize_t; #include #endif -#if defined(__sun) || defined(__ANDROID__) +#if defined(__sun) || defined(ANDROID) /* Most platforms have posix_memalign, older may only have memalign */ #define HAVE_MEMALIGN 1 #include +/* On Solaris, we need the POSIX sigwait function */ +#if defined (__sun) +# define _POSIX_PTHREAD_SEMANTICS 1 +#endif #endif #if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) @@ -164,36 +124,25 @@ typedef SSIZE_T ssize_t; #include /* defines BYTE_ORDER on HPUX and Solaris */ #endif -#if defined(__APPLE__) || defined (BSD) -# if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM)) -# define MDB_USE_SYSV_SEM 1 -# endif +#if defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__) +# define MDB_USE_POSIX_SEM 1 # define MDB_FDATASYNC fsync -#elif defined(__ANDROID__) +#elif defined(ANDROID) # define MDB_FDATASYNC fsync #endif #ifndef _WIN32 #include +#include #ifdef MDB_USE_POSIX_SEM # define MDB_USE_HASH 1 #include -#elif defined(MDB_USE_SYSV_SEM) -#include -#include -#ifdef _SEM_SEMUN_UNDEFINED -union semun { - int val; - struct semid_ds *buf; - unsigned short *array; -}; -#endif /* _SEM_SEMUN_UNDEFINED */ #else #define MDB_USE_POSIX_MUTEX 1 -#endif /* MDB_USE_POSIX_SEM */ -#endif /* !_WIN32 */ +#endif +#endif -#if defined(_WIN32) + defined(MDB_USE_POSIX_SEM) + defined(MDB_USE_SYSV_SEM) \ +#if defined(_WIN32) + defined(MDB_USE_POSIX_SEM) \ + defined(MDB_USE_POSIX_MUTEX) != 1 # error "Ambiguous shared-lock implementation" #endif @@ -295,8 +244,6 @@ union semun { #define MDB_NO_ROOT (MDB_LAST_ERRCODE + 10) #ifdef _WIN32 #define MDB_OWNERDEAD ((int) WAIT_ABANDONED) -#elif defined MDB_USE_SYSV_SEM -#define MDB_OWNERDEAD (MDB_LAST_ERRCODE + 11) #elif defined(MDB_USE_POSIX_MUTEX) && defined(EOWNERDEAD) #define MDB_OWNERDEAD EOWNERDEAD /**< #LOCK_MUTEX0() result if dead owner */ #endif @@ -307,28 +254,31 @@ union semun { /** Some platforms define the EOWNERDEAD error code * even though they don't support Robust Mutexes. * Compile with -DMDB_USE_ROBUST=0, or use some other - * mechanism like -DMDB_USE_SYSV_SEM instead of - * -DMDB_USE_POSIX_MUTEX. (SysV semaphores are - * also Robust, but some systems don't support them - * either.) + * mechanism like -DMDB_USE_POSIX_SEM instead of + * -DMDB_USE_POSIX_MUTEX. + * (Posix semaphores are not robust.) */ #ifndef MDB_USE_ROBUST /* Android currently lacks Robust Mutex support. So does glibc < 2.4. */ -# if defined(MDB_USE_POSIX_MUTEX) && (defined(__ANDROID__) || \ +# if defined(MDB_USE_POSIX_MUTEX) && (defined(ANDROID) || \ (defined(__GLIBC__) && GLIBC_VER < 0x020004)) # define MDB_USE_ROBUST 0 # else # define MDB_USE_ROBUST 1 +# endif +#endif /* !MDB_USE_ROBUST */ + +#if defined(MDB_USE_POSIX_MUTEX) && (MDB_USE_ROBUST) /* glibc < 2.12 only provided _np API */ -# if defined(__GLIBC__) && GLIBC_VER < 0x02000c +# if (defined(__GLIBC__) && GLIBC_VER < 0x02000c) || \ + (defined(PTHREAD_MUTEX_ROBUST_NP) && !defined(PTHREAD_MUTEX_ROBUST)) # define PTHREAD_MUTEX_ROBUST PTHREAD_MUTEX_ROBUST_NP # define pthread_mutexattr_setrobust(attr, flag) pthread_mutexattr_setrobust_np(attr, flag) # define pthread_mutex_consistent(mutex) pthread_mutex_consistent_np(mutex) # endif -# endif -#endif /* MDB_USE_ROBUST */ +#endif /* MDB_USE_POSIX_MUTEX && MDB_USE_ROBUST */ -#if defined(MDB_OWNERDEAD) && MDB_USE_ROBUST +#if defined(MDB_OWNERDEAD) && (MDB_USE_ROBUST) #define MDB_ROBUST_SUPPORTED 1 #endif @@ -351,8 +301,10 @@ typedef HANDLE mdb_mutex_t, mdb_mutexref_t; #define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) #define pthread_cond_signal(x) SetEvent(*x) #define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) -#define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,start,arg,0,NULL) -#define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) +#define THREAD_CREATE(thr,start,arg) \ + (((thr) = CreateThread(NULL, 0, start, arg, 0, NULL)) ? 0 : ErrCode()) +#define THREAD_FINISH(thr) \ + (WaitForSingleObject(thr, INFINITE) ? ErrCode() : 0) #define LOCK_MUTEX0(mutex) WaitForSingleObject(mutex, INFINITE) #define UNLOCK_MUTEX(mutex) ReleaseMutex(mutex) #define mdb_mutex_consistent(mutex) 0 @@ -392,50 +344,16 @@ mdb_sem_wait(sem_t *sem) return rc; } -#elif defined MDB_USE_SYSV_SEM - -typedef struct mdb_mutex { - int semid; - int semnum; - int *locked; -} mdb_mutex_t[1], *mdb_mutexref_t; - -#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) -#define UNLOCK_MUTEX(mutex) do { \ - struct sembuf sb = { 0, 1, SEM_UNDO }; \ - sb.sem_num = (mutex)->semnum; \ - *(mutex)->locked = 0; \ - semop((mutex)->semid, &sb, 1); \ -} while(0) - -static int -mdb_sem_wait(mdb_mutexref_t sem) -{ - int rc, *locked = sem->locked; - struct sembuf sb = { 0, -1, SEM_UNDO }; - sb.sem_num = sem->semnum; - do { - if (!semop(sem->semid, &sb, 1)) { - rc = *locked ? MDB_OWNERDEAD : MDB_SUCCESS; - *locked = 1; - break; - } - } while ((rc = errno) == EINTR); - return rc; -} - -#define mdb_mutex_consistent(mutex) 0 - #else /* MDB_USE_POSIX_MUTEX: */ - /** Shared mutex/semaphore as it is stored (mdb_mutex_t), and as - * local variables keep it (mdb_mutexref_t). + /** Shared mutex/semaphore as the original is stored. * - * An mdb_mutex_t can be assigned to an mdb_mutexref_t. They can - * be the same, or an array[size 1] and a pointer. - * @{ + * Not for copies. Instead it can be assigned to an #mdb_mutexref_t. + * When mdb_mutexref_t is a pointer and mdb_mutex_t is not, then it + * is array[size 1] so it can be assigned to the pointer. */ -typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t; - /* @} */ +typedef pthread_mutex_t mdb_mutex_t[1]; + /** Reference to an #mdb_mutex_t */ +typedef pthread_mutex_t *mdb_mutexref_t; /** Lock the reader or writer mutex. * Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX(). */ @@ -446,7 +364,7 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t; /** Mark mutex-protected data as repaired, after death of previous owner. */ #define mdb_mutex_consistent(mutex) pthread_mutex_consistent(mutex) -#endif /* MDB_USE_POSIX_SEM || MDB_USE_SYSV_SEM */ +#endif /* MDB_USE_POSIX_SEM */ /** Get the error code for the last failed system function. */ @@ -471,30 +389,12 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t; #define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) #endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif - #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) #define MNAME_LEN 32 -#elif defined(MDB_USE_SYSV_SEM) -#define MNAME_LEN (sizeof(int)) #else #define MNAME_LEN (sizeof(pthread_mutex_t)) #endif -#ifdef MDB_USE_SYSV_SEM -#define SYSV_SEM_FLAG 1 /**< SysV sems in lockfile format */ -#else -#define SYSV_SEM_FLAG 0 -#endif - /** @} */ #ifdef MDB_ROBUST_SUPPORTED @@ -636,7 +536,7 @@ static txnid_t mdb_debug_start; /** The version number for a database's datafile format. */ #define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) /** The version number for a database's lockfile format. */ -#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 1) +#define MDB_LOCK_VERSION 1 /** @brief The max size of a key we can write, or 0 for computed max. * @@ -825,6 +725,14 @@ typedef struct MDB_txbody { uint32_t mtb_magic; /** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */ uint32_t mtb_format; +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mtb_rmname[MNAME_LEN]; +#else + /** Mutex protecting access to this table. + * This is the reader table lock used with LOCK_MUTEX(). + */ + mdb_mutex_t mtb_rmutex; +#endif /** The ID of the last transaction committed to the database. * This is recorded here only for convenience; the value can always * be determined by reading the main database meta pages. @@ -835,17 +743,6 @@ typedef struct MDB_txbody { * when readers release their slots. */ volatile unsigned mtb_numreaders; -#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) - char mtb_rmname[MNAME_LEN]; -#elif defined(MDB_USE_SYSV_SEM) - int mtb_semid; - int mtb_rlocked; -#else - /** Mutex protecting access to this table. - * This is the reader table lock used with LOCK_MUTEX(). - */ - mdb_mutex_t mtb_rmutex; -#endif } MDB_txbody; /** The actual reader table definition. */ @@ -858,19 +755,12 @@ typedef struct MDB_txninfo { #define mti_rmname mt1.mtb.mtb_rmname #define mti_txnid mt1.mtb.mtb_txnid #define mti_numreaders mt1.mtb.mtb_numreaders -#ifdef MDB_USE_SYSV_SEM -#define mti_semid mt1.mtb.mtb_semid -#define mti_rlocked mt1.mtb.mtb_rlocked -#endif char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)]; } mt1; union { #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) char mt2_wmname[MNAME_LEN]; #define mti_wmname mt2.mt2_wmname -#elif defined MDB_USE_SYSV_SEM - int mt2_wlocked; -#define mti_wlocked mt2.mt2_wlocked #else mdb_mutex_t mt2_wmutex; #define mti_wmutex mt2.mt2_wmutex @@ -885,13 +775,26 @@ typedef struct MDB_txninfo { ((uint32_t) \ ((MDB_LOCK_VERSION) \ /* Flags which describe functionality */ \ - + (SYSV_SEM_FLAG << 18) \ + (((MDB_PIDLOCK) != 0) << 16))) /** @} */ -/** Common header for all page types. - * Overflow records occupy a number of contiguous pages with no - * headers on any page after the first. +/** Common header for all page types. The page type depends on #mp_flags. + * + * #P_BRANCH and #P_LEAF pages have unsorted '#MDB_node's at the end, with + * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages + * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header. + * + * #P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of #F_BIGDATA nodes. + * + * #P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * #P_META pages contain #MDB_meta, the start point of an LMDB snapshot. + * + * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a freeDB record. */ typedef struct MDB_page { #define mp_pgno mp_p.p_pgno @@ -900,7 +803,7 @@ typedef struct MDB_page { pgno_t p_pgno; /**< page number */ struct MDB_page *p_next; /**< for in-memory list of freed pages */ } mp_p; - uint16_t mp_pad; + uint16_t mp_pad; /**< key size if this is a LEAF2 page */ /** @defgroup mdb_page Page Flags * @ingroup internal * Flags for the page headers. @@ -967,25 +870,34 @@ typedef struct MDB_page { /** The number of overflow pages needed to store the given size. */ #define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) - /** Link in #MDB_txn.%mt_loose_pgs list */ + /** Link in #MDB_txn.%mt_loose_pgs list. + * Kept outside the page header, which is needed when reusing the page. + */ #define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) /** Header for a single key/data pair within a page. * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. * We guarantee 2-byte alignment for 'MDB_node's. + * + * #mn_lo and #mn_hi are used for data size on leaf nodes, and for child + * pgno on branch nodes. On 64 bit platforms, #mn_flags is also used + * for pgno. (Branch nodes have no flags). Lo and hi are in host byte + * order in case some accesses can be optimized to 32-bit word access. + * + * Leaf node flags describe node contents. #F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * #F_DUPDATA and #F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just #F_SUBDATA). */ typedef struct MDB_node { - /** lo and hi are used for data size on leaf nodes and for - * child pgno on branch nodes. On 64 bit platforms, flags - * is also used for pgno. (Branch nodes have no flags). - * They are in host byte order in case that lets some - * accesses be optimized into a 32-bit word access. - */ + /** part of data size or pgno + * @{ */ #if BYTE_ORDER == LITTLE_ENDIAN - unsigned short mn_lo, mn_hi; /**< part of data size or pgno */ + unsigned short mn_lo, mn_hi; #else unsigned short mn_hi, mn_lo; #endif + /** @} */ /** @defgroup mdb_node Node Flags * @ingroup internal * Flags for node headers. @@ -1091,13 +1003,13 @@ typedef struct MDB_db { pgno_t md_branch_pages; /**< number of internal pages */ pgno_t md_leaf_pages; /**< number of leaf pages */ pgno_t md_overflow_pages; /**< number of overflow pages */ - mdb_size_t md_entries; /**< number of data items */ + size_t md_entries; /**< number of data items */ pgno_t md_root; /**< the root page of this tree */ } MDB_db; - /** mdb_dbi_open flags */ #define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ #define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) + /** #mdb_dbi_open() flags */ #define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) @@ -1121,22 +1033,17 @@ typedef struct MDB_meta { uint32_t mm_magic; /** Version number of this file. Must be set to #MDB_DATA_VERSION. */ uint32_t mm_version; -#ifdef MDB_VL32 - union { /* always zero since we don't support fixed mapping in MDB_VL32 */ - MDB_ID mmun_ull; - void *mmun_address; - } mm_un; -#define mm_address mm_un.mmun_address -#else void *mm_address; /**< address for fixed mapping */ -#endif - pgno_t mm_mapsize; /**< size of mmap region */ + size_t mm_mapsize; /**< size of mmap region */ MDB_db mm_dbs[CORE_DBS]; /**< first is free space, 2nd is main db */ /** The size of pages used in this DB */ #define mm_psize mm_dbs[FREE_DBI].md_pad /** Any persistent environment flags. @ref mdb_env */ #define mm_flags mm_dbs[FREE_DBI].md_flags - pgno_t mm_last_pg; /**< last used page in file */ + /** Last used page in the datafile. + * Actually the file may be shorter if the freeDB lists the final pages. + */ + pgno_t mm_last_pg; volatile txnid_t mm_txnid; /**< txnid that committed this page */ } MDB_meta; @@ -1173,9 +1080,6 @@ struct MDB_txn { /** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */ MDB_txn *mt_child; pgno_t mt_next_pgno; /**< next unallocated page */ -#ifdef MDB_VL32 - pgno_t mt_last_pgno; /**< last written page */ -#endif /** The ID of this transaction. IDs are integers incrementing from 1. * Only committed write transactions increment the ID. If a transaction * aborts, the ID may be re-used by the next writer. @@ -1189,7 +1093,7 @@ struct MDB_txn { * in this transaction, linked through #NEXT_LOOSE_PAGE(page). */ MDB_page *mt_loose_pgs; - /* #Number of loose pages (#mt_loose_pgs) */ + /** Number of loose pages (#mt_loose_pgs) */ int mt_loose_count; /** The sorted list of dirty pages we temporarily wrote to disk * because the dirty list was full. page numbers in here are @@ -1212,29 +1116,17 @@ struct MDB_txn { * @ingroup internal * @{ */ -#define DB_DIRTY 0x01 /**< DB was modified or is DUPSORT data */ +#define DB_DIRTY 0x01 /**< DB was written in this txn */ #define DB_STALE 0x02 /**< Named-DB record is older than txnID */ #define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ #define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ #define DB_USRVALID 0x10 /**< As #DB_VALID, but not set for #FREE_DBI */ +#define DB_DUPDATA 0x20 /**< DB is #MDB_DUPSORT data */ /** @} */ /** In write txns, array of cursors for each DB */ MDB_cursor **mt_cursors; /** Array of flags for each DB */ unsigned char *mt_dbflags; -#ifdef MDB_VL32 - /** List of read-only pages (actually chunks) */ - MDB_ID3L mt_rpages; - /** We map chunks of 16 pages. Even though Windows uses 4KB pages, all - * mappings must begin on 64KB boundaries. So we round off all pgnos to - * a chunk boundary. We do the same on Linux for symmetry, and also to - * reduce the frequency of mmap/munmap calls. - */ -#define MDB_RPAGE_CHUNK 16 -#define MDB_TRPAGE_SIZE 4096 /**< size of #mt_rpages array of chunks */ -#define MDB_TRPAGE_MAX (MDB_TRPAGE_SIZE-1) /**< maximum chunk index */ - unsigned int mt_rpcheck; /**< threshold for reclaiming unref'd chunks */ -#endif /** Number of DB records in use, or 0 when the txn is finished. * This number only ever increments until the txn finishes; we * don't decrement it when individual DB handles are closed. @@ -1246,9 +1138,7 @@ struct MDB_txn { * @{ */ /** #mdb_txn_begin() flags */ -#define MDB_TXN_BEGIN_FLAGS (MDB_NOMETASYNC|MDB_NOSYNC|MDB_RDONLY) -#define MDB_TXN_NOMETASYNC MDB_NOMETASYNC /**< don't sync meta for this txn on commit */ -#define MDB_TXN_NOSYNC MDB_NOSYNC /**< don't sync this txn on commit */ +#define MDB_TXN_BEGIN_FLAGS MDB_RDONLY #define MDB_TXN_RDONLY MDB_RDONLY /**< read-only transaction */ /* internal txn flags */ #define MDB_TXN_WRITEMAP MDB_WRITEMAP /**< copy of #MDB_env flag in writers */ @@ -1314,19 +1204,10 @@ struct MDB_cursor { #define C_SUB 0x04 /**< Cursor is a sub-cursor */ #define C_DEL 0x08 /**< last op was a cursor_del */ #define C_UNTRACK 0x40 /**< Un-track cursor when closing */ -#define C_WRITEMAP MDB_TXN_WRITEMAP /**< Copy of txn flag */ -/** Read-only cursor into the txn's original snapshot in the map. - * Set for read-only txns, and in #mdb_page_alloc() for #FREE_DBI when - * #MDB_DEVEL & 2. Only implements code which is necessary for this. - */ -#define C_ORIG_RDONLY MDB_TXN_RDONLY /** @} */ unsigned int mc_flags; /**< @ref mdb_cursor */ MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ -#ifdef MDB_VL32 - MDB_page *mc_ovpg; /**< a referenced overflow page */ -#endif }; /** Context for sorted-dup records. @@ -1345,6 +1226,23 @@ typedef struct MDB_xcursor { unsigned char mx_dbflag; } MDB_xcursor; + /** Check if there is an inited xcursor */ +#define XCURSOR_INITED(mc) \ + ((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + + /** Update the xcursor's sub-page pointer, if any, in \b mc. Needed + * when the node which contains the sub-page may have moved. Called + * with leaf page \b mp = mc->mc_pg[\b top]. + */ +#define XCURSOR_REFRESH(mc, top, mp) do { \ + MDB_page *xr_pg = (mp); \ + MDB_node *xr_node; \ + if (!XCURSOR_INITED(mc) || (mc)->mc_ki[top] >= NUMKEYS(xr_pg)) break; \ + xr_node = NODEPTR(xr_pg, (mc)->mc_ki[top]); \ + if ((xr_node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) \ + (mc)->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(xr_node); \ +} while (0) + /** State of FreeDB old pages, stored in the MDB_env */ typedef struct MDB_pgstate { pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ @@ -1355,10 +1253,7 @@ typedef struct MDB_pgstate { struct MDB_env { HANDLE me_fd; /**< The main data file */ HANDLE me_lfd; /**< The lock file */ - HANDLE me_mfd; /**< just for writing the meta pages */ -#if defined(MDB_VL32) && defined(_WIN32) - HANDLE me_fmh; /**< File Mapping handle */ -#endif + HANDLE me_mfd; /**< For writing and syncing the meta pages */ /** Failed to update the meta page. Probably an I/O error. */ #define MDB_FATAL_ERROR 0x80000000U /** Some fields are initialized. */ @@ -1383,8 +1278,8 @@ struct MDB_env { void *me_pbuf; /**< scratch area for DUPSORT put() */ MDB_txn *me_txn; /**< current write transaction */ MDB_txn *me_txn0; /**< prealloc'd write transaction */ - mdb_size_t me_mapsize; /**< size of the data memory map */ - off64_t me_size; /**< current file size */ + size_t me_mapsize; /**< size of the data memory map */ + off_t me_size; /**< current file size */ pgno_t me_maxpg; /**< me_mapsize / me_psize */ MDB_dbx *me_dbxs; /**< array of static DB info */ uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ @@ -1416,13 +1311,6 @@ struct MDB_env { #else mdb_mutex_t me_rmutex; mdb_mutex_t me_wmutex; -#endif -#ifdef MDB_VL32 - MDB_ID3L me_rpages; /**< like #mt_rpages, but global to env */ - pthread_mutex_t me_rpmutex; /**< control access to #me_rpages */ -#define MDB_ERPAGE_SIZE 16384 -#define MDB_ERPAGE_MAX (MDB_ERPAGE_SIZE-1) - unsigned int me_rpcheck; #endif void *me_userctx; /**< User-settable context */ MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ @@ -1484,7 +1372,7 @@ static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, unsigned int nflags); -static int mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta); +static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); static MDB_meta *mdb_env_pick_meta(const MDB_env *env); static int mdb_env_write_meta(MDB_txn *txn); #ifdef MDB_USE_POSIX_MUTEX /* Drop unused excl arg */ @@ -1543,7 +1431,8 @@ static SECURITY_DESCRIPTOR mdb_null_sd; static SECURITY_ATTRIBUTES mdb_all_sa; static int mdb_sec_inited; -static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsize); +struct MDB_name; +static int utf8_to_utf16(const char *src, struct MDB_name *dst, int xtra); #endif /** Return the library version info. */ @@ -1722,20 +1611,20 @@ mdb_page_list(MDB_page *mp) case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; case P_OVERFLOW: - fprintf(stderr, "Overflow page %"Y"u pages %u%s\n", + fprintf(stderr, "Overflow page %"Z"u pages %u%s\n", pgno, mp->mp_pages, state); return; case P_META: - fprintf(stderr, "Meta-page %"Y"u txnid %"Y"u\n", + fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n", pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); return; default: - fprintf(stderr, "Bad page %"Y"u flags 0x%u\n", pgno, mp->mp_flags); + fprintf(stderr, "Bad page %"Z"u flags 0x%X\n", pgno, mp->mp_flags); return; } nkeys = NUMKEYS(mp); - fprintf(stderr, "%s %"Y"u numkeys %d%s\n", type, pgno, nkeys, state); + fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state); for (i=0; imn_data; nsize = NODESIZE + key.mv_size; if (IS_BRANCH(mp)) { - fprintf(stderr, "key %d: page %"Y"u, %s\n", i, NODEPGNO(node), + fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node), DKEY(&key)); total += nsize; } else { @@ -1785,7 +1674,7 @@ mdb_cursor_chk(MDB_cursor *mc) } if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) printf("ack!\n"); - if (mc->mc_xcursor && (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + if (XCURSOR_INITED(mc)) { node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) && mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) { @@ -1846,7 +1735,7 @@ static void mdb_audit(MDB_txn *txn) } } if (freecount + count + NUM_METAS != txn->mt_next_pgno) { - fprintf(stderr, "audit: %"Y"u freecount: %"Y"u count: %"Y"u total: %"Y"u next_pgno: %"Y"u\n", + fprintf(stderr, "audit: %"Z"u freecount: %"Z"u count: %"Z"u total: %"Z"u next_pgno: %"Z"u\n", txn->mt_txnid, freecount, count+NUM_METAS, freecount+count+NUM_METAS, txn->mt_next_pgno); } @@ -1863,8 +1752,8 @@ int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) { MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && a->mv_size == sizeof(mdb_size_t)) +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t)) dcmp = mdb_cmp_clong; #endif return dcmp(a, b); @@ -1872,6 +1761,7 @@ mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) /** Allocate memory for a page. * Re-use old malloc'd pages first for singletons, otherwise just malloc. + * Set #MDB_TXN_ERROR on failure. */ static MDB_page * mdb_page_malloc(MDB_txn *txn, unsigned num) @@ -1946,47 +1836,6 @@ mdb_dlist_free(MDB_txn *txn) dl[0].mid = 0; } -#ifdef MDB_VL32 -static void -mdb_page_unref(MDB_txn *txn, MDB_page *mp) -{ - pgno_t pgno; - MDB_ID3L tl = txn->mt_rpages; - unsigned x, rem; - if (mp->mp_flags & (P_SUBP|P_DIRTY)) - return; - rem = mp->mp_pgno & (MDB_RPAGE_CHUNK-1); - pgno = mp->mp_pgno ^ rem; - x = mdb_mid3l_search(tl, pgno); - if (x != tl[0].mid && tl[x+1].mid == mp->mp_pgno) - x++; - if (tl[x].mref) - tl[x].mref--; -} -#define MDB_PAGE_UNREF(txn, mp) mdb_page_unref(txn, mp) - -static void -mdb_cursor_unref(MDB_cursor *mc) -{ - int i; - if (mc->mc_txn->mt_rpages[0].mid) { - if (!mc->mc_snum || !mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0])) - return; - for (i=0; imc_snum; i++) - mdb_page_unref(mc->mc_txn, mc->mc_pg[i]); - if (mc->mc_ovpg) { - mdb_page_unref(mc->mc_txn, mc->mc_ovpg); - mc->mc_ovpg = 0; - } - } - mc->mc_snum = mc->mc_top = 0; - mc->mc_pg[0] = NULL; - mc->mc_flags &= ~C_INITIALIZED; -} -#else -#define MDB_PAGE_UNREF(txn, mp) -#endif /* MDB_VL32 */ - /** Loosen or free a single page. * Saves single pages to a list for future reuse * in this same txn. It has been pulled from the freeDB @@ -2028,7 +1877,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp) } } if (loose) { - DPRINTF(("loosen db %d page %"Y"u", DDBI(mc), + DPRINTF(("loosen db %d page %"Z"u", DDBI(mc), mp->mp_pgno)); NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; txn->mt_loose_pgs = mp; @@ -2280,15 +2129,13 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp) } /** Allocate page numbers and memory for writing. Maintain me_pglast, - * me_pghead and mt_next_pgno. + * me_pghead and mt_next_pgno. Set #MDB_TXN_ERROR on failure. * * If there are free pages available from older transactions, they * are re-used first. Otherwise allocate a new page at mt_next_pgno. * Do not modify the freedB, just merge freeDB records into me_pghead[] * and move me_pglast to say which records were consumed. Only this * function can create me_pghead and move me_pglast/mt_next_pgno. - * When #MDB_DEVEL & 2, it is not affected by #mdb_freelist_save(): it - * then uses the transaction's original snapshot of the freeDB. * @param[in] mc cursor A cursor handle identifying the transaction and * database for which we are allocating. * @param[in] num the number of pages to allocate. @@ -2326,7 +2173,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) np = txn->mt_loose_pgs; txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); txn->mt_loose_count--; - DPRINTF(("db %d use loose page %"Y"u", DDBI(mc), + DPRINTF(("db %d use loose page %"Z"u", DDBI(mc), np->mp_pgno)); *mp = np; return MDB_SUCCESS; @@ -2364,14 +2211,6 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) last = env->me_pglast; oldest = env->me_pgoldest; mdb_cursor_init(&m2, txn, FREE_DBI, NULL); -#if (MDB_DEVEL) & 2 /* "& 2" so MDB_DEVEL=1 won't hide bugs breaking freeDB */ - /* Use original snapshot. TODO: Should need less care in code - * which modifies the database. Maybe we can delete some code? - */ - m2.mc_flags |= C_ORIG_RDONLY; - m2.mc_db = &env->me_metas[(txn->mt_txnid-1) & 1]->mm_dbs[FREE_DBI]; - m2.mc_dbflag = (unsigned char *)""; /* probably unnecessary */ -#endif if (last) { op = MDB_SET_RANGE; key.mv_data = &last; /* will look up last+1 */ @@ -2413,7 +2252,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) np = m2.mc_pg[m2.mc_top]; leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); if ((rc = mdb_node_read(&m2, leaf, &data)) != MDB_SUCCESS) - return rc; + goto fail; idl = (MDB_ID *) data.mv_data; i = idl[0]; @@ -2429,10 +2268,10 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } env->me_pglast = last; #if (MDB_DEBUG) > 1 - DPRINTF(("IDL read txn %"Y"u root %"Y"u num %u", + DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u", last, txn->mt_dbs[FREE_DBI].md_root, i)); for (j = i; j; j--) - DPRINTF(("IDL %"Y"u", idl[j])); + DPRINTF(("IDL %"Z"u", idl[j])); #endif /* Merge in descending sorted order */ mdb_midl_xmerge(mop, idl); @@ -2447,20 +2286,6 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) rc = MDB_MAP_FULL; goto fail; } -#if defined(_WIN32) && !defined(MDB_VL32) - if (!(env->me_flags & MDB_RDONLY)) { - void *p; - p = (MDB_page *)(env->me_map + env->me_psize * pgno); - p = VirtualAlloc(p, env->me_psize * num, MEM_COMMIT, - (env->me_flags & MDB_WRITEMAP) ? PAGE_READWRITE: - PAGE_READONLY); - if (!p) { - DPUTS("VirtualAlloc failed"); - rc = ErrCode(); - goto fail; - } - } -#endif search_done: if (env->me_flags & MDB_WRITEMAP) { @@ -2577,6 +2402,7 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) } /** Touch a page: make it dirty and re-insert into tree with updated pgno. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc cursor pointing to the page to be touched * @return 0 on success, non-zero on failure. */ @@ -2602,7 +2428,7 @@ mdb_page_touch(MDB_cursor *mc) (rc = mdb_page_alloc(mc, 1, &np))) goto fail; pgno = np->mp_pgno; - DPRINTF(("touched db %d page %"Y"u -> %"Y"u", DDBI(mc), + DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc), mp->mp_pgno, pgno)); mdb_cassert(mc, mp->mp_pgno != pgno); mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); @@ -2665,18 +2491,11 @@ done: if (m2 == mc) continue; if (m2->mc_pg[mc->mc_top] == mp) { m2->mc_pg[mc->mc_top] = np; - if ((mc->mc_db->md_flags & MDB_DUPSORT) && - IS_LEAF(np) && - (m2->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) - { - MDB_node *leaf = NODEPTR(np, m2->mc_ki[mc->mc_top]); - if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } + if (IS_LEAF(np)) + XCURSOR_REFRESH(m2, mc->mc_top, np); } } } - MDB_PAGE_UNREF(mc->mc_txn, mp); return 0; fail: @@ -2685,7 +2504,7 @@ fail: } int -mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs) +mdb_env_sync(MDB_env *env, int force) { int rc = 0; if (env->me_flags & MDB_RDONLY) @@ -2694,7 +2513,7 @@ mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs) if (env->me_flags & MDB_WRITEMAP) { int flags = ((env->me_flags & MDB_MAPASYNC) && !force) ? MS_ASYNC : MS_SYNC; - if (MDB_MSYNC(env->me_map, env->me_psize * numpgs, flags)) + if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) rc = ErrCode(); #ifdef _WIN32 else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) @@ -2714,13 +2533,6 @@ mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs) return rc; } -int -mdb_env_sync(MDB_env *env, int force) -{ - MDB_meta *m = mdb_env_pick_meta(env); - return mdb_env_sync0(env, force, m->mm_last_pg+1); -} - /** Back up parent txn's cursors, then grab the originals for tracking */ static int mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) @@ -2965,9 +2777,6 @@ mdb_txn_renew0(MDB_txn *txn) /* Moved to here to avoid a data race in read TXNs */ txn->mt_next_pgno = meta->mm_last_pg+1; -#ifdef MDB_VL32 - txn->mt_last_pgno = txn->mt_next_pgno - 1; -#endif txn->mt_flags = flags; @@ -3003,7 +2812,7 @@ mdb_txn_renew(MDB_txn *txn) rc = mdb_txn_renew0(txn); if (rc == MDB_SUCCESS) { - DPRINTF(("renew txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); } @@ -3046,17 +2855,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) DPRINTF(("calloc: %s", strerror(errno))); return ENOMEM; } -#ifdef MDB_VL32 - if (!parent) { - txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3)); - if (!txn->mt_rpages) { - free(txn); - return ENOMEM; - } - txn->mt_rpages[0].mid = 0; - txn->mt_rpcheck = MDB_TRPAGE_SIZE/2; - } -#endif txn->mt_dbxs = env->me_dbxs; /* static */ txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs; @@ -3084,9 +2882,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) parent->mt_child = txn; txn->mt_parent = parent; txn->mt_numdbs = parent->mt_numdbs; -#ifdef MDB_VL32 - txn->mt_rpages = parent->mt_rpages; -#endif memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); /* Copy parent's mt_dbflags, but clear DB_NEW */ for (i=0; imt_numdbs; i++) @@ -3112,16 +2907,12 @@ renew: rc = mdb_txn_renew0(txn); } if (rc) { - if (txn != env->me_txn0) { -#ifdef MDB_VL32 - free(txn->mt_rpages); -#endif + if (txn != env->me_txn0) free(txn); - } } else { txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ *ret = txn; - DPRINTF(("begin txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w', (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); } @@ -3136,7 +2927,7 @@ mdb_txn_env(MDB_txn *txn) return txn->mt_env; } -mdb_size_t +size_t mdb_txn_id(MDB_txn *txn) { if(!txn) return 0; @@ -3188,7 +2979,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode) /* Export or close DBI handles opened in this txn */ mdb_dbis_update(txn, mode & MDB_END_UPDATE); - DPRINTF(("%s txn %"Y"u%c %p on mdbenv %p, root page %"Y"u", + DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", names[mode & MDB_END_OPMASK], txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); @@ -3242,31 +3033,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode) mdb_midl_free(pghead); } -#ifdef MDB_VL32 - if (!txn->mt_parent) { - MDB_ID3L el = env->me_rpages, tl = txn->mt_rpages; - unsigned i, x, n = tl[0].mid; - pthread_mutex_lock(&env->me_rpmutex); - for (i = 1; i <= n; i++) { - if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) { - /* tmp overflow pages that we didn't share in env */ - munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); - } else { - x = mdb_mid3l_search(el, tl[i].mid); - if (tl[i].mptr == el[x].mptr) { - el[x].mref--; - } else { - /* another tmp overflow page */ - munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); - } - } - } - pthread_mutex_unlock(&env->me_rpmutex); - tl[0].mid = 0; - if (mode & MDB_END_FREE) - free(tl); - } -#endif + if (mode & MDB_END_FREE) free(txn); } @@ -3298,9 +3065,6 @@ mdb_txn_abort(MDB_txn *txn) /** Save the freelist as of this transaction to the freeDB. * This changes the freelist. Keep trying until it stabilizes. - * - * When (MDB_DEVEL) & 2, the changes do not affect #mdb_page_alloc(), - * it then uses the transaction's original snapshot of the freeDB. */ static int mdb_freelist_save(MDB_txn *txn) @@ -3330,10 +3094,41 @@ mdb_freelist_save(MDB_txn *txn) * we may be unable to return them to me_pghead. */ MDB_page *mp = txn->mt_loose_pgs; + MDB_ID2 *dl = txn->mt_u.dirty_list; + unsigned x; if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) return rc; - for (; mp; mp = NEXT_LOOSE_PAGE(mp)) + for (; mp; mp = NEXT_LOOSE_PAGE(mp)) { mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* must also remove from dirty list */ + if (txn->mt_flags & MDB_TXN_WRITEMAP) { + for (x=1; x<=dl[0].mid; x++) + if (dl[x].mid == mp->mp_pgno) + break; + mdb_tassert(txn, x <= dl[0].mid); + } else { + x = mdb_mid2l_search(dl, mp->mp_pgno); + mdb_tassert(txn, dl[x].mid == mp->mp_pgno); + mdb_dpage_free(env, mp); + } + dl[x].mptr = NULL; + } + { + /* squash freed slots out of the dirty list */ + unsigned y; + for (y=1; dl[y].mptr && y <= dl[0].mid; y++); + if (y <= dl[0].mid) { + for(x=y, y++;;) { + while (!dl[y].mptr && y <= dl[0].mid) y++; + if (y > dl[0].mid) break; + dl[x++] = dl[y++]; + } + dl[0].mid = x-1; + } else { + /* all slots freed */ + dl[0].mid = 0; + } + } txn->mt_loose_pgs = NULL; txn->mt_loose_count = 0; } @@ -3389,10 +3184,10 @@ mdb_freelist_save(MDB_txn *txn) #if (MDB_DEBUG) > 1 { unsigned int i = free_pgs[0]; - DPRINTF(("IDL write txn %"Y"u root %"Y"u num %u", + DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u", txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); for (; i; i--) - DPRINTF(("IDL %"Y"u", free_pgs[i])); + DPRINTF(("IDL %"Z"u", free_pgs[i])); } #endif continue; @@ -3503,16 +3298,15 @@ mdb_page_flush(MDB_txn *txn, int keep) MDB_ID2L dl = txn->mt_u.dirty_list; unsigned psize = env->me_psize, j; int i, pagecount = dl[0].mid, rc; - size_t size = 0; - off64_t pos = 0; + size_t size = 0, pos = 0; pgno_t pgno = 0; MDB_page *dp = NULL; #ifdef _WIN32 OVERLAPPED ov; #else struct iovec iov[MDB_COMMIT_PAGES]; - ssize_t wsize = 0, wres; - off64_t wpos = 0, next_pos = 1; /* impossible pos, so pos != next_pos */ + ssize_t wpos = 0, wsize = 0, wres; + size_t next_pos = 1; /* impossible pos, so pos != next_pos */ int n = 0; #endif @@ -3611,7 +3405,7 @@ retry_seek: wpos = pos; wsize = 0; } - DPRINTF(("committing page %"Y"u", pgno)); + DPRINTF(("committing page %"Z"u", pgno)); next_pos = pos + size; iov[n].iov_len = size; iov[n].iov_base = (char *)dp; @@ -3619,10 +3413,6 @@ retry_seek: n++; #endif /* _WIN32 */ } -#ifdef MDB_VL32 - if (pgno > txn->mt_last_pgno) - txn->mt_last_pgno = pgno; -#endif /* MIPS has cache coherency issues, this is a no-op everywhere else * Note: for any size >= on-chip cache size, entire on-chip cache is @@ -3648,8 +3438,6 @@ done: return MDB_SUCCESS; } -static int ESECT mdb_env_share_locks(MDB_env *env, int *excl); - int mdb_txn_commit(MDB_txn *txn) { @@ -3826,7 +3614,7 @@ mdb_txn_commit(MDB_txn *txn) !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) goto done; - DPRINTF(("committing txn %"Y"u %p on mdbenv %p, root page %"Y"u", + DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); /* Update DB root pointers */ @@ -3864,23 +3652,11 @@ mdb_txn_commit(MDB_txn *txn) mdb_audit(txn); #endif - if ((rc = mdb_page_flush(txn, 0))) - goto fail; - if (!F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC) && - (rc = mdb_env_sync0(env, 0, txn->mt_next_pgno))) - goto fail; - if ((rc = mdb_env_write_meta(txn))) + if ((rc = mdb_page_flush(txn, 0)) || + (rc = mdb_env_sync(env, 0)) || + (rc = mdb_env_write_meta(txn))) goto fail; end_mode = MDB_END_COMMITTED|MDB_END_UPDATE; - if (env->me_flags & MDB_PREVSNAPSHOT) { - if (!(env->me_flags & MDB_NOLOCK)) { - int excl; - rc = mdb_env_share_locks(env, &excl); - if (rc) - goto fail; - } - env->me_flags ^= MDB_PREVSNAPSHOT; - } done: mdb_txn_end(txn, end_mode); @@ -3894,12 +3670,11 @@ fail: /** Read the environment parameters of a DB environment before * mapping it into memory. * @param[in] env the environment handle - * @param[in] prev whether to read the backup meta page * @param[out] meta address of where to store the meta information * @return 0 on success, non-zero on failure. */ static int ESECT -mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) +mdb_env_read_header(MDB_env *env, MDB_meta *meta) { MDB_metabuf pbuf; MDB_page *p; @@ -3934,7 +3709,7 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) p = (MDB_page *)&pbuf; if (!F_ISSET(p->mp_flags, P_META)) { - DPRINTF(("page %"Y"u not a meta page", p->mp_pgno)); + DPRINTF(("page %"Z"u not a meta page", p->mp_pgno)); return MDB_INVALID; } @@ -3950,7 +3725,7 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) return MDB_VERSION_MISMATCH; } - if (off == 0 || (prev ? m->mm_txnid < meta->mm_txnid : m->mm_txnid > meta->mm_txnid)) + if (off == 0 || m->mm_txnid > meta->mm_txnid) *meta = *m; } return 0; @@ -4004,6 +3779,7 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta) p = calloc(NUM_METAS, psize); if (!p) return ENOMEM; + p->mp_pgno = 0; p->mp_flags = P_META; *(MDB_meta *)METADATA(p) = *meta; @@ -4034,8 +3810,8 @@ mdb_env_write_meta(MDB_txn *txn) MDB_env *env; MDB_meta meta, metab, *mp; unsigned flags; - mdb_size_t mapsize; - off64_t off; + size_t mapsize; + off_t off; int rc, len, toggle; char *ptr; HANDLE mfd; @@ -4046,11 +3822,11 @@ mdb_env_write_meta(MDB_txn *txn) #endif toggle = txn->mt_txnid & 1; - DPRINTF(("writing meta page %d for root page %"Y"u", + DPRINTF(("writing meta page %d for root page %"Z"u", toggle, txn->mt_dbs[MAIN_DBI].md_root)); env = txn->mt_env; - flags = txn->mt_flags | env->me_flags; + flags = env->me_flags; mp = env->me_metas[toggle]; mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; /* Persist any increases of mapsize config */ @@ -4098,7 +3874,10 @@ mdb_env_write_meta(MDB_txn *txn) len = sizeof(MDB_meta) - off; off += (char *)mp - env->me_map; - /* Write to the SYNC fd */ + /* Write to the SYNC fd unless MDB_NOSYNC/MDB_NOMETASYNC. + * (me_mfd goes to the same file as me_fd, but writing to it + * also syncs to disk. Avoids a separate fdatasync() call.) + */ mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd; #ifdef _WIN32 { @@ -4159,8 +3938,7 @@ static MDB_meta * mdb_env_pick_meta(const MDB_env *env) { MDB_meta *const *metas = env->me_metas; - return metas[ (metas[0]->mm_txnid < metas[1]->mm_txnid) ^ - ((env->me_flags & MDB_PREVSNAPSHOT) != 0) ]; + return metas[ metas[0]->mm_txnid < metas[1]->mm_txnid ]; } int ESECT @@ -4180,9 +3958,6 @@ mdb_env_create(MDB_env **env) #ifdef MDB_USE_POSIX_SEM e->me_rmutex = SEM_FAILED; e->me_wmutex = SEM_FAILED; -#elif defined MDB_USE_SYSV_SEM - e->me_rmutex->semid = -1; - e->me_wmutex->semid = -1; #endif e->me_pid = getpid(); GET_PAGESIZE(e->me_os_psize); @@ -4191,19 +3966,6 @@ mdb_env_create(MDB_env **env) return MDB_SUCCESS; } -#ifdef _WIN32 -/** @brief Map a result from an NTAPI call to WIN32. */ -static DWORD -mdb_nt2win32(NTSTATUS st) -{ - OVERLAPPED o = {0}; - DWORD br; - o.Internal = st; - GetOverlappedResult(NULL, &o, &br, FALSE); - return GetLastError(); -} -#endif - static int ESECT mdb_env_map(MDB_env *env, void *addr) { @@ -4211,51 +3973,42 @@ mdb_env_map(MDB_env *env, void *addr) unsigned int flags = env->me_flags; #ifdef _WIN32 int rc; - int access = SECTION_MAP_READ; HANDLE mh; - void *map; - SIZE_T msize; - ULONG pageprot = PAGE_READONLY, secprot, alloctype; + LONG sizelo, sizehi; + size_t msize; - if (flags & MDB_WRITEMAP) { - access |= SECTION_MAP_WRITE; - pageprot = PAGE_READWRITE; - } if (flags & MDB_RDONLY) { - secprot = PAGE_READONLY; + /* Don't set explicit map size, use whatever exists */ msize = 0; - alloctype = 0; + sizelo = 0; + sizehi = 0; } else { - secprot = PAGE_READWRITE; msize = env->me_mapsize; - alloctype = MEM_RESERVE; + sizelo = msize & 0xffffffff; + sizehi = msize >> 16 >> 16; /* only needed on Win64 */ + + /* Windows won't create mappings for zero length files. + * and won't map more than the file size. + * Just set the maxsize right now. + */ + if (!(flags & MDB_WRITEMAP) && (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo + || !SetEndOfFile(env->me_fd) + || SetFilePointer(env->me_fd, 0, NULL, 0) != 0)) + return ErrCode(); } - rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd); - if (rc) - return mdb_nt2win32(rc); - map = addr; -#ifdef MDB_VL32 - msize = NUM_METAS * env->me_psize; -#endif - rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, alloctype, pageprot); -#ifdef MDB_VL32 - env->me_fmh = mh; -#else - NtClose(mh); -#endif - if (rc) - return mdb_nt2win32(rc); - env->me_map = map; -#else -#ifdef MDB_VL32 - (void) flags; - env->me_map = mmap(addr, NUM_METAS * env->me_psize, PROT_READ, MAP_SHARED, - env->me_fd, 0); - if (env->me_map == MAP_FAILED) { - env->me_map = NULL; + mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? + PAGE_READWRITE : PAGE_READONLY, + sizehi, sizelo, NULL); + if (!mh) return ErrCode(); - } + env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? + FILE_MAP_WRITE : FILE_MAP_READ, + 0, 0, msize, addr); + rc = env->me_map ? 0 : ErrCode(); + CloseHandle(mh); + if (rc) + return rc; #else int prot = PROT_READ; if (flags & MDB_WRITEMAP) { @@ -4289,7 +4042,6 @@ mdb_env_map(MDB_env *env, void *addr) */ if (addr && env->me_map != addr) return EBUSY; /* TODO: Make a new MDB_* error code? */ -#endif p = (MDB_page *)env->me_map; env->me_metas[0] = METADATA(p); @@ -4299,17 +4051,15 @@ mdb_env_map(MDB_env *env, void *addr) } int ESECT -mdb_env_set_mapsize(MDB_env *env, mdb_size_t size) +mdb_env_set_mapsize(MDB_env *env, size_t size) { /* If env is already open, caller is responsible for making * sure there are no active txns. */ if (env->me_map) { - MDB_meta *meta; -#ifndef MDB_VL32 - void *old; int rc; -#endif + MDB_meta *meta; + void *old; if (env->me_txn) return EINVAL; meta = mdb_env_pick_meta(env); @@ -4317,21 +4067,16 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size) size = meta->mm_mapsize; { /* Silently round up to minimum if the size is too small */ - mdb_size_t minsize = (meta->mm_last_pg + 1) * env->me_psize; + size_t minsize = (meta->mm_last_pg + 1) * env->me_psize; if (size < minsize) size = minsize; } -#ifndef MDB_VL32 - /* For MDB_VL32 this bit is a noop since we dynamically remap - * chunks of the DB anyway. - */ munmap(env->me_map, env->me_mapsize); env->me_mapsize = size; old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; rc = mdb_env_map(env, old); if (rc) return rc; -#endif /* !MDB_VL32 */ } env->me_mapsize = size; if (env->me_psize) @@ -4367,7 +4112,7 @@ mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) } static int ESECT -mdb_fsize(HANDLE fd, mdb_size_t *size) +mdb_fsize(HANDLE fd, size_t *size) { #ifdef _WIN32 LARGE_INTEGER fsize; @@ -4387,6 +4132,189 @@ mdb_fsize(HANDLE fd, mdb_size_t *size) return MDB_SUCCESS; } + +#ifdef _WIN32 +typedef wchar_t mdb_nchar_t; +# define MDB_NAME(str) L##str +# define mdb_name_cpy wcscpy +#else +/** Character type for file names: char on Unix, wchar_t on Windows */ +typedef char mdb_nchar_t; +# define MDB_NAME(str) str /**< #mdb_nchar_t[] string literal */ +# define mdb_name_cpy strcpy /**< Copy name (#mdb_nchar_t string) */ +#endif + +/** Filename - string of #mdb_nchar_t[] */ +typedef struct MDB_name { + int mn_len; /**< Length */ + int mn_alloced; /**< True if #mn_val was malloced */ + mdb_nchar_t *mn_val; /**< Contents */ +} MDB_name; + +/** Filename suffixes [datafile,lockfile][without,with MDB_NOSUBDIR] */ +static const mdb_nchar_t *const mdb_suffixes[2][2] = { + { MDB_NAME("/data.mdb"), MDB_NAME("") }, + { MDB_NAME("/lock.mdb"), MDB_NAME("-lock") } +}; + +#define MDB_SUFFLEN 9 /**< Max string length in #mdb_suffixes[] */ + +/** Set up filename + scratch area for filename suffix, for opening files. + * It should be freed with #mdb_fname_destroy(). + * On Windows, paths are converted from char *UTF-8 to wchar_t *UTF-16. + * + * @param[in] path Pathname for #mdb_env_open(). + * @param[in] envflags Whether a subdir and/or lockfile will be used. + * @param[out] fname Resulting filename, with room for a suffix if necessary. + */ +static int ESECT +mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname) +{ + int no_suffix = F_ISSET(envflags, MDB_NOSUBDIR|MDB_NOLOCK); + fname->mn_alloced = 0; +#ifdef _WIN32 + return utf8_to_utf16(path, fname, no_suffix ? 0 : MDB_SUFFLEN); +#else + fname->mn_len = strlen(path); + if (no_suffix) + fname->mn_val = (char *) path; + else if ((fname->mn_val = malloc(fname->mn_len + MDB_SUFFLEN+1)) != NULL) { + fname->mn_alloced = 1; + strcpy(fname->mn_val, path); + } + else + return ENOMEM; + return MDB_SUCCESS; +#endif +} + +/** Destroy \b fname from #mdb_fname_init() */ +#define mdb_fname_destroy(fname) \ + do { if ((fname).mn_alloced) free((fname).mn_val); } while (0) + +#ifdef O_CLOEXEC /* POSIX.1-2008: Set FD_CLOEXEC atomically at open() */ +# define MDB_CLOEXEC O_CLOEXEC +#else +# define MDB_CLOEXEC 0 +#endif + +/** File type, access mode etc. for #mdb_fopen() */ +enum mdb_fopen_type { +#ifdef _WIN32 + MDB_O_RDONLY, MDB_O_RDWR, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS +#else + /* A comment in mdb_fopen() explains some O_* flag choices. */ + MDB_O_RDONLY= O_RDONLY, /**< for RDONLY me_fd */ + MDB_O_RDWR = O_RDWR |O_CREAT, /**< for me_fd */ + MDB_O_META = O_WRONLY|MDB_DSYNC |MDB_CLOEXEC, /**< for me_mfd */ + MDB_O_COPY = O_WRONLY|O_CREAT|O_EXCL|MDB_CLOEXEC, /**< for #mdb_env_copy() */ + /** Bitmask for open() flags in enum #mdb_fopen_type. The other bits + * distinguish otherwise-equal MDB_O_* constants from each other. + */ + MDB_O_MASK = MDB_O_RDWR|MDB_CLOEXEC | MDB_O_RDONLY|MDB_O_META|MDB_O_COPY, + MDB_O_LOCKS = MDB_O_RDWR|MDB_CLOEXEC | ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */ +#endif +}; + +/** Open an LMDB file. + * @param[in] env The LMDB environment. + * @param[in,out] fname Path from from #mdb_fname_init(). A suffix is + * appended if necessary to create the filename, without changing mn_len. + * @param[in] which Determines file type, access mode, etc. + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[out] res Resulting file handle. + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_fopen(const MDB_env *env, MDB_name *fname, + enum mdb_fopen_type which, mdb_mode_t mode, + HANDLE *res) +{ + int rc = MDB_SUCCESS; + HANDLE fd; +#ifdef _WIN32 + DWORD acc, share, disp, attrs; +#else + int flags; +#endif + + if (fname->mn_alloced) /* modifiable copy */ + mdb_name_cpy(fname->mn_val + fname->mn_len, + mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]); + + /* The directory must already exist. Usually the file need not. + * MDB_O_META requires the file because we already created it using + * MDB_O_RDWR. MDB_O_COPY must not overwrite an existing file. + * + * With MDB_O_COPY we do not want the OS to cache the writes, since + * the source data is already in the OS cache. + * + * The lockfile needs FD_CLOEXEC (close file descriptor on exec*()) + * to avoid the flock() issues noted under Caveats in lmdb.h. + * Also set it for other filehandles which the user cannot get at + * and close himself, which he may need after fork(). I.e. all but + * me_fd, which programs do use via mdb_env_get_fd(). + */ + +#ifdef _WIN32 + acc = GENERIC_READ|GENERIC_WRITE; + share = FILE_SHARE_READ|FILE_SHARE_WRITE; + disp = OPEN_ALWAYS; + attrs = FILE_ATTRIBUTE_NORMAL; + switch (which) { + case MDB_O_RDONLY: /* read-only datafile */ + acc = GENERIC_READ; + disp = OPEN_EXISTING; + break; + case MDB_O_META: /* for writing metapages */ + acc = GENERIC_WRITE; + disp = OPEN_EXISTING; + attrs = FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH; + break; + case MDB_O_COPY: /* mdb_env_copy() & co */ + acc = GENERIC_WRITE; + share = 0; + disp = CREATE_NEW; + attrs = FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH; + break; + default: break; /* silence gcc -Wswitch (not all enum values handled) */ + } + fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL); +#else + fd = open(fname->mn_val, which & MDB_O_MASK, mode); +#endif + + if (fd == INVALID_HANDLE_VALUE) + rc = ErrCode(); +#ifndef _WIN32 + else { + if (which != MDB_O_RDONLY && which != MDB_O_RDWR) { + /* Set CLOEXEC if we could not pass it to open() */ + if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1) + (void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC); + } + if (which == MDB_O_COPY && env->me_psize >= env->me_os_psize) { + /* This may require buffer alignment. There is no portable + * way to ask how much, so we require OS pagesize alignment. + */ +# ifdef F_NOCACHE /* __APPLE__ */ + (void) fcntl(fd, F_NOCACHE, 1); +# elif defined O_DIRECT + /* open(...O_DIRECT...) would break on filesystems without + * O_DIRECT support (ITS#7682). Try to set it here instead. + */ + if ((flags = fcntl(fd, F_GETFL)) != -1) + (void) fcntl(fd, F_SETFL, flags | O_DIRECT); +# endif + } + } +#endif /* !_WIN32 */ + + *res = fd; + return rc; +} + + #ifdef BROKEN_FDATASYNC #include #include @@ -4395,7 +4323,7 @@ mdb_fsize(HANDLE fd, mdb_size_t *size) /** Further setup required for opening an LMDB environment */ static int ESECT -mdb_env_open2(MDB_env *env, int prev) +mdb_env_open2(MDB_env *env) { unsigned int flags = env->me_flags; int i, newenv = 0, rc; @@ -4458,7 +4386,7 @@ mdb_env_open2(MDB_env *env, int prev) } #endif - if ((i = mdb_env_read_header(env, prev, &meta)) != 0) { + if ((i = mdb_env_read_header(env, &meta)) != 0) { if (i != ENOENT) return i; DPUTS("new mdbenv"); @@ -4481,7 +4409,7 @@ mdb_env_open2(MDB_env *env, int prev) /* Make sure mapsize >= committed data size. Even when using * mm_mapsize, which could be broken in old files (ITS#7789). */ - mdb_size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; if (env->me_mapsize < minsize) env->me_mapsize = minsize; } @@ -4500,18 +4428,6 @@ mdb_env_open2(MDB_env *env, int prev) return rc; newenv = 0; } -#ifdef _WIN32 - /* For FIXEDMAP, make sure the file is non-empty before we attempt to map it */ - if (newenv) { - char dummy = 0; - DWORD len; - rc = WriteFile(env->me_fd, &dummy, 1, &len, NULL); - if (!rc) { - rc = ErrCode(); - return rc; - } - } -#endif rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); if (rc) @@ -4534,9 +4450,6 @@ mdb_env_open2(MDB_env *env, int prev) #endif env->me_maxpg = env->me_mapsize / env->me_psize; - if (env->me_txns) - env->me_txns->mti_txnid = meta.mm_txnid; - #if MDB_DEBUG { MDB_meta *meta = mdb_env_pick_meta(env); @@ -4546,11 +4459,11 @@ mdb_env_open2(MDB_env *env, int prev) meta->mm_version, env->me_psize)); DPRINTF(("using meta page %d", (int) (meta->mm_txnid & 1))); DPRINTF(("depth: %u", db->md_depth)); - DPRINTF(("entries: %"Y"u", db->md_entries)); - DPRINTF(("branch pages: %"Y"u", db->md_branch_pages)); - DPRINTF(("leaf pages: %"Y"u", db->md_leaf_pages)); - DPRINTF(("overflow pages: %"Y"u", db->md_overflow_pages)); - DPRINTF(("root: %"Y"u", db->md_root)); + DPRINTF(("entries: %"Z"u", db->md_entries)); + DPRINTF(("branch pages: %"Z"u", db->md_branch_pages)); + DPRINTF(("leaf pages: %"Z"u", db->md_leaf_pages)); + DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages)); + DPRINTF(("root: %"Z"u", db->md_root)); } #endif @@ -4567,7 +4480,11 @@ mdb_env_reader_dest(void *ptr) { MDB_reader *reader = ptr; - reader->mr_pid = 0; +#ifndef _WIN32 + if (reader->mr_pid == getpid()) /* catch pthread_exit() in child process */ +#endif + /* We omit the mutex, so do this atomically (i.e. skip mr_txnid) */ + reader->mr_pid = 0; } #ifdef _WIN32 @@ -4632,6 +4549,9 @@ static int ESECT mdb_env_share_locks(MDB_env *env, int *excl) { int rc = 0; + MDB_meta *meta = mdb_env_pick_meta(env); + + env->me_txns->mti_txnid = meta->mm_txnid; #ifdef _WIN32 { @@ -4804,56 +4724,30 @@ mdb_hash_enc(MDB_val *val, char *encbuf) /** Open and/or initialize the lock region for the environment. * @param[in] env The LMDB environment. - * @param[in] lpath The pathname of the file used for the lock region. + * @param[in] fname Filename + scratch area, from #mdb_fname_init(). * @param[in] mode The Unix permissions for the file, if we create it. * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive * @return 0 on success, non-zero on failure. */ static int ESECT -mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) +mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl) { #ifdef _WIN32 # define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT #else # define MDB_ERRCODE_ROFS EROFS -#ifdef O_CLOEXEC /* Linux: Open file and set FD_CLOEXEC atomically */ -# define MDB_CLOEXEC O_CLOEXEC -#else - int fdflags; -# define MDB_CLOEXEC 0 -#endif -#endif -#ifdef MDB_USE_SYSV_SEM - int semid; - union semun semu; #endif int rc; - off64_t size, rsize; + off_t size, rsize; -#ifdef _WIN32 - wchar_t *wlpath; - rc = utf8_to_utf16(lpath, -1, &wlpath, NULL); - if (rc) - return rc; - env->me_lfd = CreateFileW(wlpath, GENERIC_READ|GENERIC_WRITE, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL, NULL); - free(wlpath); -#else - env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode); -#endif - if (env->me_lfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); + rc = mdb_fopen(env, fname, MDB_O_LOCKS, mode, &env->me_lfd); + if (rc) { + /* Omit lockfile if read-only env on read-only filesystem */ if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { return MDB_SUCCESS; } - goto fail_errno; + goto fail; } -#if ! ((MDB_CLOEXEC) || defined(_WIN32)) - /* Lose record locks when exec*() */ - if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0) - fcntl(env->me_lfd, F_SETFD, fdflags); -#endif if (!(env->me_flags & MDB_NOTLS)) { rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); @@ -4978,31 +4872,29 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) env->me_wmutex = sem_open(env->me_txns->mti_wmname, O_CREAT|O_EXCL, mode, 1); if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#elif defined(MDB_USE_SYSV_SEM) - unsigned short vals[2] = {1, 1}; - key_t key = ftok(lpath, 'M'); - if (key == -1) - goto fail_errno; - semid = semget(key, 2, (mode & 0777) | IPC_CREAT); - if (semid < 0) - goto fail_errno; - semu.array = vals; - if (semctl(semid, 0, SETALL, semu) < 0) - goto fail_errno; - env->me_txns->mti_semid = semid; #else /* MDB_USE_POSIX_MUTEX: */ pthread_mutexattr_t mattr; - if ((rc = pthread_mutexattr_init(&mattr)) - || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) -#ifdef MDB_ROBUST_SUPPORTED - || (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST)) -#endif - || (rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr)) - || (rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr))) + /* Solaris needs this before initing a robust mutex. Otherwise + * it may skip the init and return EBUSY "seems someone already + * inited" or EINVAL "it was inited differently". + */ + memset(env->me_txns->mti_rmutex, 0, sizeof(*env->me_txns->mti_rmutex)); + memset(env->me_txns->mti_wmutex, 0, sizeof(*env->me_txns->mti_wmutex)); + + if ((rc = pthread_mutexattr_init(&mattr))) goto fail; + + rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); +#ifdef MDB_ROBUST_SUPPORTED + if (!rc) rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); +#endif + if (!rc) rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr); + if (!rc) rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr); pthread_mutexattr_destroy(&mattr); -#endif /* _WIN32 || ... */ + if (rc) + goto fail; +#endif /* _WIN32 || MDB_USE_POSIX_SEM */ env->me_txns->mti_magic = MDB_MAGIC; env->me_txns->mti_format = MDB_LOCK_FORMAT; @@ -5010,9 +4902,6 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) env->me_txns->mti_numreaders = 0; } else { -#ifdef MDB_USE_SYSV_SEM - struct semid_ds buf; -#endif if (env->me_txns->mti_magic != MDB_MAGIC) { DPUTS("lock region has invalid magic"); rc = MDB_INVALID; @@ -5038,33 +4927,8 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) if (env->me_rmutex == SEM_FAILED) goto fail_errno; env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#elif defined(MDB_USE_SYSV_SEM) - semid = env->me_txns->mti_semid; - semu.buf = &buf; - /* check for read access */ - if (semctl(semid, 0, IPC_STAT, semu) < 0) - goto fail_errno; - /* check for write access */ - if (semctl(semid, 0, IPC_SET, semu) < 0) - goto fail_errno; #endif } -#ifdef MDB_USE_SYSV_SEM - env->me_rmutex->semid = semid; - env->me_wmutex->semid = semid; - env->me_rmutex->semnum = 0; - env->me_wmutex->semnum = 1; - env->me_rmutex->locked = &env->me_txns->mti_rlocked; - env->me_wmutex->locked = &env->me_txns->mti_wlocked; -#endif -#ifdef MDB_VL32 -#ifdef _WIN32 - env->me_rpmutex = CreateMutex(NULL, FALSE, NULL); -#else - pthread_mutex_init(&env->me_rpmutex, NULL); -#endif -#endif - return MDB_SUCCESS; fail_errno: @@ -5073,19 +4937,13 @@ fail: return rc; } - /** The name of the lock file in the DB environment */ -#define LOCKNAME "/lock.mdb" - /** The name of the data file in the DB environment */ -#define DATANAME "/data.mdb" - /** The suffix of the lock file when no subdir is used */ -#define LOCKSUFF "-lock" /** Only a subset of the @ref mdb_env flags can be changed * at runtime. Changing other flags requires closing the * environment and re-opening it with the new flags. */ #define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) #define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \ - MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVSNAPSHOT) + MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) #if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) # error "Persistent DB flags & env flags overlap, but both go in mm_flags" @@ -5094,47 +4952,18 @@ fail: int ESECT mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) { - int oflags, rc, len, excl = -1; - char *lpath, *dpath; -#ifdef _WIN32 - wchar_t *wpath; -#endif + int rc, excl = -1; + MDB_name fname; if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) return EINVAL; -#ifdef MDB_VL32 - if (flags & MDB_WRITEMAP) { - /* silently ignore WRITEMAP in 32 bit mode */ - flags ^= MDB_WRITEMAP; - } - if (flags & MDB_FIXEDMAP) { - /* cannot support FIXEDMAP */ - return EINVAL; - } -#endif - - len = strlen(path); - if (flags & MDB_NOSUBDIR) { - rc = len + sizeof(LOCKSUFF) + len + 1; - } else { - rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME); - } - lpath = malloc(rc); - if (!lpath) - return ENOMEM; - if (flags & MDB_NOSUBDIR) { - dpath = lpath + len + sizeof(LOCKSUFF); - sprintf(lpath, "%s" LOCKSUFF, path); - strcpy(dpath, path); - } else { - dpath = lpath + len + sizeof(LOCKNAME); - sprintf(lpath, "%s" LOCKNAME, path); - sprintf(dpath, "%s" DATANAME, path); - } - - rc = MDB_SUCCESS; flags |= env->me_flags; + + rc = mdb_fname_init(path, flags, &fname); + if (rc) + return rc; + if (flags & MDB_RDONLY) { /* silently ignore WRITEMAP when we're only getting read access */ flags &= ~MDB_WRITEMAP; @@ -5143,17 +4972,6 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) rc = ENOMEM; } -#ifdef MDB_VL32 - if (!rc) { - env->me_rpages = malloc(MDB_ERPAGE_SIZE * sizeof(MDB_ID3)); - if (!env->me_rpages) { - rc = ENOMEM; - goto leave; - } - env->me_rpages[0].mid = 0; - env->me_rpcheck = MDB_ERPAGE_SIZE/2; - } -#endif env->me_flags = flags |= MDB_ENV_ACTIVE; if (rc) goto leave; @@ -5170,76 +4988,34 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode /* For RDONLY, get lockfile after we know datafile exists */ if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { - rc = mdb_env_setup_locks(env, lpath, mode, &excl); + rc = mdb_env_setup_locks(env, &fname, mode, &excl); if (rc) goto leave; } -#ifdef _WIN32 - if (F_ISSET(flags, MDB_RDONLY)) { - oflags = GENERIC_READ; - len = OPEN_EXISTING; - } else { - oflags = GENERIC_READ|GENERIC_WRITE; - len = OPEN_ALWAYS; - } - mode = FILE_ATTRIBUTE_NORMAL; - rc = utf8_to_utf16(dpath, -1, &wpath, NULL); + rc = mdb_fopen(env, &fname, + (flags & MDB_RDONLY) ? MDB_O_RDONLY : MDB_O_RDWR, + mode, &env->me_fd); if (rc) goto leave; - env->me_fd = CreateFileW(wpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, - NULL, len, mode, NULL); - free(wpath); -#else - if (F_ISSET(flags, MDB_RDONLY)) - oflags = O_RDONLY; - else - oflags = O_RDWR | O_CREAT; - - env->me_fd = open(dpath, oflags, mode); -#endif - if (env->me_fd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { - rc = mdb_env_setup_locks(env, lpath, mode, &excl); + rc = mdb_env_setup_locks(env, &fname, mode, &excl); if (rc) goto leave; - if ((flags & MDB_PREVSNAPSHOT) && !excl) { - rc = EAGAIN; - goto leave; - } } - if ((rc = mdb_env_open2(env, flags & MDB_PREVSNAPSHOT)) == MDB_SUCCESS) { - if (flags & (MDB_RDONLY|MDB_WRITEMAP)) { - env->me_mfd = env->me_fd; - } else { + if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) { + if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) { /* Synchronous fd for meta writes. Needed even with * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. */ -#ifdef _WIN32 - len = OPEN_EXISTING; - rc = utf8_to_utf16(dpath, -1, &wpath, NULL); + rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd); if (rc) goto leave; - env->me_mfd = CreateFileW(wpath, oflags, - FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, - mode | FILE_FLAG_WRITE_THROUGH, NULL); - free(wpath); -#else - oflags &= ~O_CREAT; - env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode); -#endif - if (env->me_mfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } } DPRINTF(("opened dbenv %p", (void *) env)); - if (excl > 0 && !(flags & MDB_PREVSNAPSHOT)) { + if (excl > 0) { rc = mdb_env_share_locks(env, &excl); if (rc) goto leave; @@ -5256,16 +5032,6 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); txn->mt_env = env; -#ifdef MDB_VL32 - txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3)); - if (!txn->mt_rpages) { - free(txn); - rc = ENOMEM; - goto leave; - } - txn->mt_rpages[0].mid = 0; - txn->mt_rpcheck = MDB_TRPAGE_SIZE/2; -#endif txn->mt_dbxs = env->me_dbxs; txn->mt_flags = MDB_TXN_FINISHED; env->me_txn0 = txn; @@ -5279,7 +5045,7 @@ leave: if (rc) { mdb_env_close0(env, excl); } - free(lpath); + mdb_fname_destroy(fname); return rc; } @@ -5304,15 +5070,6 @@ mdb_env_close0(MDB_env *env, int excl) free(env->me_dbflags); free(env->me_path); free(env->me_dirty_list); -#ifdef MDB_VL32 - if (env->me_txn0 && env->me_txn0->mt_rpages) - free(env->me_txn0->mt_rpages); - { unsigned int x; - for (x=1; x<=env->me_rpages[0].mid; x++) - munmap(env->me_rpages[x].mptr, env->me_rpages[x].mcnt * env->me_psize); - } - free(env->me_rpages); -#endif free(env->me_txn0); mdb_midl_free(env->me_free_pgs); @@ -5330,18 +5087,14 @@ mdb_env_close0(MDB_env *env, int excl) } if (env->me_map) { -#ifdef MDB_VL32 - munmap(env->me_map, NUM_METAS*env->me_psize); -#else munmap(env->me_map, env->me_mapsize); -#endif } - if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE) + if (env->me_mfd != INVALID_HANDLE_VALUE) (void) close(env->me_mfd); if (env->me_fd != INVALID_HANDLE_VALUE) (void) close(env->me_fd); if (env->me_txns) { - MDB_PID_T pid = env->me_pid; + MDB_PID_T pid = getpid(); /* Clearing readers is done in this function because * me_txkey with its destructor must be disabled first. * @@ -5375,16 +5128,6 @@ mdb_env_close0(MDB_env *env, int excl) sem_unlink(env->me_txns->mti_wmname); } } -#elif defined(MDB_USE_SYSV_SEM) - if (env->me_rmutex->semid != -1) { - /* If we have the filelock: If we are the - * only remaining user, clean up semaphores. - */ - if (excl == 0) - mdb_env_excl_lock(env, &excl); - if (excl > 0) - semctl(env->me_rmutex->semid, 0, IPC_RMID); - } #endif munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); } @@ -5399,14 +5142,6 @@ mdb_env_close0(MDB_env *env, int excl) #endif (void) close(env->me_lfd); } -#ifdef MDB_VL32 -#ifdef _WIN32 - if (env->me_fmh) CloseHandle(env->me_fmh); - if (env->me_rpmutex) CloseHandle(env->me_rpmutex); -#else - pthread_mutex_destroy(&env->me_rpmutex); -#endif -#endif env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); } @@ -5430,18 +5165,18 @@ mdb_env_close(MDB_env *env) free(env); } -/** Compare two items pointing at aligned mdb_size_t's */ +/** Compare two items pointing at aligned size_t's */ static int mdb_cmp_long(const MDB_val *a, const MDB_val *b) { - return (*(mdb_size_t *)a->mv_data < *(mdb_size_t *)b->mv_data) ? -1 : - *(mdb_size_t *)a->mv_data > *(mdb_size_t *)b->mv_data; + return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 : + *(size_t *)a->mv_data > *(size_t *)b->mv_data; } /** Compare two items pointing at aligned unsigned int's. * * This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp, - * but #mdb_cmp_clong() is called instead if the data type is mdb_size_t. + * but #mdb_cmp_clong() is called instead if the data type is size_t. */ static int mdb_cmp_int(const MDB_val *a, const MDB_val *b) @@ -5546,7 +5281,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) nkeys = NUMKEYS(mp); - DPRINTF(("searching %u keys in %s %spage %"Y"u", + DPRINTF(("searching %u keys in %s %spage %"Z"u", nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", mdb_dbg_pgno(mp))); @@ -5558,7 +5293,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) * alignment is guaranteed. Use faster mdb_cmp_int. */ if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { - if (NODEPTR(mp, 1)->mn_ksize == sizeof(mdb_size_t)) + if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t)) cmp = mdb_cmp_long; else cmp = mdb_cmp_int; @@ -5594,7 +5329,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) DPRINTF(("found leaf index %u [%s], rc = %i", i, DKEY(&nodekey), rc)); else - DPRINTF(("found branch index %u [%s -> %"Y"u], rc = %i", + DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i", i, DKEY(&nodekey), NODEPGNO(node), rc)); #endif if (rc == 0) @@ -5642,7 +5377,7 @@ static void mdb_cursor_pop(MDB_cursor *mc) { if (mc->mc_snum) { - DPRINTF(("popping page %"Y"u off db %d cursor %p", + DPRINTF(("popping page %"Z"u off db %d cursor %p", mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc)); mc->mc_snum--; @@ -5654,11 +5389,13 @@ mdb_cursor_pop(MDB_cursor *mc) } } -/** Push a page onto the top of the cursor's stack. */ +/** Push a page onto the top of the cursor's stack. + * Set #MDB_TXN_ERROR on failure. + */ static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) { - DPRINTF(("pushing page %"Y"u on db %d cursor %p", mp->mp_pgno, + DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno, DDBI(mc), (void *) mc)); if (mc->mc_snum >= CURSOR_STACK) { @@ -5673,295 +5410,8 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) return MDB_SUCCESS; } -#ifdef MDB_VL32 -/** Map a read-only page. - * There are two levels of tracking in use, a per-txn list and a per-env list. - * ref'ing and unref'ing the per-txn list is faster since it requires no - * locking. Pages are cached in the per-env list for global reuse, and a lock - * is required. Pages are not immediately unmapped when their refcnt goes to - * zero; they hang around in case they will be reused again soon. - * - * When the per-txn list gets full, all pages with refcnt=0 are purged from the - * list and their refcnts in the per-env list are decremented. - * - * When the per-env list gets full, all pages with refcnt=0 are purged from the - * list and their pages are unmapped. - * - * @note "full" means the list has reached its respective rpcheck threshold. - * This threshold slowly raises if no pages could be purged on a given check, - * and returns to its original value when enough pages were purged. - * - * If purging doesn't free any slots, filling the per-txn list will return - * MDB_TXN_FULL, and filling the per-env list returns MDB_MAP_FULL. - * - * Reference tracking in a txn is imperfect, pages can linger with non-zero - * refcnt even without active references. It was deemed to be too invasive - * to add unrefs in every required location. However, all pages are unref'd - * at the end of the transaction. This guarantees that no stale references - * linger in the per-env list. - * - * Usually we map chunks of 16 pages at a time, but if an overflow page begins - * at the tail of the chunk we extend the chunk to include the entire overflow - * page. Unfortunately, pages can be turned into overflow pages after their - * chunk was already mapped. In that case we must remap the chunk if the - * overflow page is referenced. If the chunk's refcnt is 0 we can just remap - * it, otherwise we temporarily map a new chunk just for the overflow page. - * - * @note this chunk handling means we cannot guarantee that a data item - * returned from the DB will stay alive for the duration of the transaction: - * We unref pages as soon as a cursor moves away from the page - * A subsequent op may cause a purge, which may unmap any unref'd chunks - * The caller must copy the data if it must be used later in the same txn. - * - * Also - our reference counting revolves around cursors, but overflow pages - * aren't pointed to by a cursor's page stack. We have to remember them - * explicitly, in the added mc_ovpg field. A single cursor can only hold a - * reference to one overflow page at a time. - * - * @param[in] txn the transaction for this access. - * @param[in] pgno the page number for the page to retrieve. - * @param[out] ret address of a pointer where the page's address will be stored. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret) -{ - MDB_env *env = txn->mt_env; - MDB_page *p; - MDB_ID3L tl = txn->mt_rpages; - MDB_ID3L el = env->me_rpages; - MDB_ID3 id3; - unsigned x, rem; - pgno_t pgno; - int rc, retries = 1; -#ifdef _WIN32 - LARGE_INTEGER off; - SIZE_T len; -#define SET_OFF(off,val) off.QuadPart = val -#define MAP(rc,env,addr,len,off) \ - addr = NULL; \ - rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \ - len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \ - if (rc) rc = mdb_nt2win32(rc) -#else - off64_t off; - size_t len; -#define SET_OFF(off,val) off = val -#define MAP(rc,env,addr,len,off) \ - addr = mmap(NULL, len, PROT_READ, MAP_SHARED, env->me_fd, off); \ - rc = (addr == MAP_FAILED) ? errno : 0 -#endif - - /* remember the offset of the actual page number, so we can - * return the correct pointer at the end. - */ - rem = pg0 & (MDB_RPAGE_CHUNK-1); - pgno = pg0 ^ rem; - - id3.mid = 0; - x = mdb_mid3l_search(tl, pgno); - if (x <= tl[0].mid && tl[x].mid == pgno) { - if (x != tl[0].mid && tl[x+1].mid == pg0) - x++; - /* check for overflow size */ - p = (MDB_page *)((char *)tl[x].mptr + rem * env->me_psize); - if (IS_OVERFLOW(p) && p->mp_pages + rem > tl[x].mcnt) { - id3.mcnt = p->mp_pages + rem; - len = id3.mcnt * env->me_psize; - SET_OFF(off, pgno * env->me_psize); - MAP(rc, env, id3.mptr, len, off); - if (rc) - return rc; - /* check for local-only page */ - if (rem) { - mdb_tassert(txn, tl[x].mid != pg0); - /* hope there's room to insert this locally. - * setting mid here tells later code to just insert - * this id3 instead of searching for a match. - */ - id3.mid = pg0; - goto notlocal; - } else { - /* ignore the mapping we got from env, use new one */ - tl[x].mptr = id3.mptr; - tl[x].mcnt = id3.mcnt; - /* if no active ref, see if we can replace in env */ - if (!tl[x].mref) { - unsigned i; - pthread_mutex_lock(&env->me_rpmutex); - i = mdb_mid3l_search(el, tl[x].mid); - if (el[i].mref == 1) { - /* just us, replace it */ - munmap(el[i].mptr, el[i].mcnt * env->me_psize); - el[i].mptr = tl[x].mptr; - el[i].mcnt = tl[x].mcnt; - } else { - /* there are others, remove ourself */ - el[i].mref--; - } - pthread_mutex_unlock(&env->me_rpmutex); - } - } - } - id3.mptr = tl[x].mptr; - id3.mcnt = tl[x].mcnt; - tl[x].mref++; - goto ok; - } - -notlocal: - if (tl[0].mid >= MDB_TRPAGE_MAX - txn->mt_rpcheck) { - unsigned i, y; - /* purge unref'd pages from our list and unref in env */ - pthread_mutex_lock(&env->me_rpmutex); -retry: - y = 0; - for (i=1; i<=tl[0].mid; i++) { - if (!tl[i].mref) { - if (!y) y = i; - /* tmp overflow pages don't go to env */ - if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) { - munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); - continue; - } - x = mdb_mid3l_search(el, tl[i].mid); - el[x].mref--; - } - } - pthread_mutex_unlock(&env->me_rpmutex); - if (!y) { - /* we didn't find any unref'd chunks. - * if we're out of room, fail. - */ - if (tl[0].mid >= MDB_TRPAGE_MAX) - return MDB_TXN_FULL; - /* otherwise, raise threshold for next time around - * and let this go. - */ - txn->mt_rpcheck /= 2; - } else { - /* we found some unused; consolidate the list */ - for (i=y+1; i<= tl[0].mid; i++) - if (tl[i].mref) - tl[y++] = tl[i]; - tl[0].mid = y-1; - /* decrease the check threshold toward its original value */ - if (!txn->mt_rpcheck) - txn->mt_rpcheck = 1; - while (txn->mt_rpcheck < tl[0].mid && txn->mt_rpcheck < MDB_TRPAGE_SIZE/2) - txn->mt_rpcheck *= 2; - } - } - if (tl[0].mid < MDB_TRPAGE_SIZE) { - id3.mref = 1; - if (id3.mid) - goto found; - /* don't map past last written page in read-only envs */ - if ((env->me_flags & MDB_RDONLY) && pgno + MDB_RPAGE_CHUNK-1 > txn->mt_last_pgno) - id3.mcnt = txn->mt_last_pgno + 1 - pgno; - else - id3.mcnt = MDB_RPAGE_CHUNK; - len = id3.mcnt * env->me_psize; - id3.mid = pgno; - - /* search for page in env */ - pthread_mutex_lock(&env->me_rpmutex); - x = mdb_mid3l_search(el, pgno); - if (x <= el[0].mid && el[x].mid == pgno) { - id3.mptr = el[x].mptr; - id3.mcnt = el[x].mcnt; - /* check for overflow size */ - p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize); - if (IS_OVERFLOW(p) && p->mp_pages + rem > id3.mcnt) { - id3.mcnt = p->mp_pages + rem; - len = id3.mcnt * env->me_psize; - SET_OFF(off, pgno * env->me_psize); - MAP(rc, env, id3.mptr, len, off); - if (rc) - goto fail; - if (!el[x].mref) { - munmap(el[x].mptr, env->me_psize * el[x].mcnt); - el[x].mptr = id3.mptr; - el[x].mcnt = id3.mcnt; - } else { - id3.mid = pg0; - pthread_mutex_unlock(&env->me_rpmutex); - goto found; - } - } - el[x].mref++; - pthread_mutex_unlock(&env->me_rpmutex); - goto found; - } - if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) { - /* purge unref'd pages */ - unsigned i, y = 0; - for (i=1; i<=el[0].mid; i++) { - if (!el[i].mref) { - if (!y) y = i; - munmap(el[i].mptr, env->me_psize * el[i].mcnt); - } - } - if (!y) { - if (retries) { - /* see if we can unref some local pages */ - retries--; - id3.mid = 0; - goto retry; - } - if (el[0].mid >= MDB_ERPAGE_MAX) { - pthread_mutex_unlock(&env->me_rpmutex); - return MDB_MAP_FULL; - } - env->me_rpcheck /= 2; - } else { - for (i=y+1; i<= el[0].mid; i++) - if (el[i].mref) - el[y++] = el[i]; - el[0].mid = y-1; - if (!env->me_rpcheck) - env->me_rpcheck = 1; - while (env->me_rpcheck < el[0].mid && env->me_rpcheck < MDB_ERPAGE_SIZE/2) - env->me_rpcheck *= 2; - } - } - SET_OFF(off, pgno * env->me_psize); - MAP(rc, env, id3.mptr, len, off); - if (rc) { -fail: - pthread_mutex_unlock(&env->me_rpmutex); - return rc; - } - /* check for overflow size */ - p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize); - if (IS_OVERFLOW(p) && p->mp_pages + rem > id3.mcnt) { - id3.mcnt = p->mp_pages + rem; - munmap(id3.mptr, len); - len = id3.mcnt * env->me_psize; - MAP(rc, env, id3.mptr, len, off); - if (rc) - goto fail; - } - mdb_mid3l_insert(el, &id3); - pthread_mutex_unlock(&env->me_rpmutex); -found: - mdb_mid3l_insert(tl, &id3); - } else { - return MDB_TXN_FULL; - } -ok: - p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize); -#if MDB_DEBUG /* we don't need this check any more */ - if (IS_OVERFLOW(p)) { - mdb_tassert(txn, p->mp_pages + rem <= id3.mcnt); - } -#endif - *ret = p; - return MDB_SUCCESS; -} -#endif - /** Find the address of the page corresponding to a given page number. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc the cursor accessing the page. * @param[in] pgno the page number for the page to retrieve. * @param[out] ret address of a pointer where the page's address will be stored. @@ -5972,13 +5422,11 @@ static int mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) { MDB_txn *txn = mc->mc_txn; -#ifndef MDB_VL32 MDB_env *env = txn->mt_env; -#endif MDB_page *p = NULL; int level; - if (! (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP))) { + if (! (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_WRITEMAP))) { MDB_txn *tx2 = txn; level = 1; do { @@ -5993,13 +5441,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) MDB_ID pn = pgno << 1; x = mdb_midl_search(tx2->mt_spill_pgs, pn); if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { -#ifdef MDB_VL32 - int rc = mdb_rpage_get(txn, pgno, &p); - if (rc) - return rc; -#else p = (MDB_page *)(env->me_map + env->me_psize * pgno); -#endif goto done; } } @@ -6016,17 +5458,9 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) if (pgno < txn->mt_next_pgno) { level = 0; -#ifdef MDB_VL32 - { - int rc = mdb_rpage_get(txn, pgno, &p); - if (rc) - return rc; - } -#else p = (MDB_page *)(env->me_map + env->me_psize * pgno); -#endif } else { - DPRINTF(("page %"Y"u not found", pgno)); + DPRINTF(("page %"Z"u not found", pgno)); txn->mt_flags |= MDB_TXN_ERROR; return MDB_PAGE_NOTFOUND; } @@ -6052,18 +5486,27 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) MDB_node *node; indx_t i; - DPRINTF(("branch page %"Y"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); + DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); /* Don't assert on branch pages in the FreeDB. We can get here * while in the process of rebalancing a FreeDB branch page; we must * let that proceed. ITS#8336 */ mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1); - DPRINTF(("found index 0 to page %"Y"u", NODEPGNO(NODEPTR(mp, 0)))); + DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0)))); if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { i = 0; - if (flags & MDB_PS_LAST) + if (flags & MDB_PS_LAST) { i = NUMKEYS(mp) - 1; + /* if already init'd, see if we're already in right place */ + if (mc->mc_flags & C_INITIALIZED) { + if (mc->mc_ki[mc->mc_top] == i) { + mc->mc_top = mc->mc_snum++; + mp = mc->mc_pg[mc->mc_top]; + goto ready; + } + } + } } else { int exact; node = mdb_node_search(mc, key, &exact); @@ -6089,6 +5532,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) if ((rc = mdb_cursor_push(mc, mp))) return rc; +ready: if (flags & MDB_PS_MODIFY) { if ((rc = mdb_page_touch(mc)) != 0) return rc; @@ -6103,7 +5547,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) return MDB_CORRUPTED; } - DPRINTF(("found leaf page %"Y"u for key [%s]", mp->mp_pgno, + DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno, key ? DKEY(key) : "null")); mc->mc_flags |= C_INITIALIZED; mc->mc_flags &= ~C_EOF; @@ -6199,26 +5643,14 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) } mdb_cassert(mc, root > 1); - if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) { -#ifdef MDB_VL32 - if (mc->mc_pg[0]) - MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[0]); -#endif + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) if ((rc = mdb_page_get(mc, root, &mc->mc_pg[0], NULL)) != 0) return rc; - } -#ifdef MDB_VL32 - { - int i; - for (i=1; imc_snum; i++) - MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[i]); - } -#endif mc->mc_snum = 1; mc->mc_top = 0; - DPRINTF(("db %d root page %"Y"u has flags 0x%X", + DPRINTF(("db %d root page %"Z"u has flags 0x%X", DDBI(mc), root, mc->mc_pg[0]->mp_flags)); if (flags & MDB_PS_MODIFY) { @@ -6243,7 +5675,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) MDB_ID pn = pg << 1; int rc; - DPRINTF(("free ov page %"Y"u (%d)", pg, ovpages)); + DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages)); /* If the page is dirty or on the spill list we just acquired it, * so we should give it back to our current free list, if any. * Otherwise put it onto the list of pages we freed in this txn. @@ -6304,10 +5736,6 @@ release: if (rc) return rc; } -#ifdef MDB_VL32 - if (mc->mc_ovpg == mp) - mc->mc_ovpg = NULL; -#endif mc->mc_db->md_overflow_pages -= ovpages; return 0; } @@ -6325,12 +5753,6 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) pgno_t pgno; int rc; -#ifdef MDB_VL32 - if (mc->mc_ovpg) { - MDB_PAGE_UNREF(mc->mc_txn, mc->mc_ovpg); - mc->mc_ovpg = 0; - } -#endif if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { data->mv_size = NODEDSZ(leaf); data->mv_data = NODEDATA(leaf); @@ -6342,13 +5764,10 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) data->mv_size = NODEDSZ(leaf); memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) { - DPRINTF(("read overflow page %"Y"u failed", pgno)); + DPRINTF(("read overflow page %"Z"u failed", pgno)); return rc; } data->mv_data = METADATA(omp); -#ifdef MDB_VL32 - mc->mc_ovpg = omp; -#endif return MDB_SUCCESS; } @@ -6359,7 +5778,7 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi, { MDB_cursor mc; MDB_xcursor mx; - int exact = 0, rc; + int exact = 0; DKBUF; DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); @@ -6371,16 +5790,7 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi, return MDB_BAD_TXN; mdb_cursor_init(&mc, txn, dbi, &mx); - rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact); -#ifdef MDB_VL32 - { - /* unref all the pages - caller must copy the data - * before doing anything else - */ - mdb_cursor_unref(&mc); - } -#endif - return rc; + return mdb_cursor_set(&mc, key, data, MDB_SET, &exact); } /** Find a sibling for a page. @@ -6397,19 +5807,13 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right) int rc; MDB_node *indx; MDB_page *mp; -#ifdef MDB_VL32 - MDB_page *op; -#endif if (mc->mc_snum < 2) { return MDB_NOTFOUND; /* root has no siblings */ } -#ifdef MDB_VL32 - op = mc->mc_pg[mc->mc_top]; -#endif mdb_cursor_pop(mc); - DPRINTF(("parent page is page %"Y"u, index %u", + DPRINTF(("parent page is page %"Z"u, index %u", mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) @@ -6432,8 +5836,6 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right) } mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); - MDB_PAGE_UNREF(mc->mc_txn, op); - indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if ((rc = mdb_page_get(mc, NODEPGNO(indx), &mp, NULL)) != 0) { /* mc will be inconsistent if caller does mc_snum++ as above */ @@ -6456,14 +5858,20 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) MDB_node *leaf; int rc; - if (mc->mc_flags & C_EOF) { + if ((mc->mc_flags & C_DEL && op == MDB_NEXT_DUP)) return MDB_NOTFOUND; - } - mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + if (!(mc->mc_flags & C_INITIALIZED)) + return mdb_cursor_first(mc, key, data); mp = mc->mc_pg[mc->mc_top]; + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1) + return MDB_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + if (mc->mc_db->md_flags & MDB_DUPSORT) { leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { @@ -6475,13 +5883,6 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } } -#ifdef MDB_VL32 - else { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } - } -#endif } else { mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); if (op == MDB_NEXT_DUP) @@ -6489,7 +5890,7 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } } - DPRINTF(("cursor_next: top page is %"Y"u in cursor %p", + DPRINTF(("cursor_next: top page is %"Z"u in cursor %p", mdb_dbg_pgno(mp), (void *) mc)); if (mc->mc_flags & C_DEL) { mc->mc_flags ^= C_DEL; @@ -6503,12 +5904,12 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } mp = mc->mc_pg[mc->mc_top]; - DPRINTF(("next page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); } else mc->mc_ki[mc->mc_top]++; skip: - DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u", + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); if (IS_LEAF2(mp)) { @@ -6546,7 +5947,12 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) MDB_node *leaf; int rc; - mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdb_cursor_last(mc, key, data); + if (rc) + return rc; + mc->mc_ki[mc->mc_top]++; + } mp = mc->mc_pg[mc->mc_top]; @@ -6563,13 +5969,6 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) return rc; } } -#ifdef MDB_VL32 - else { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } - } -#endif } else { mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); if (op == MDB_PREV_DUP) @@ -6577,7 +5976,7 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } } - DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p", + DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p", mdb_dbg_pgno(mp), (void *) mc)); mc->mc_flags &= ~(C_EOF|C_DEL); @@ -6589,13 +5988,11 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) } mp = mc->mc_pg[mc->mc_top]; mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; - DPRINTF(("prev page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); } else mc->mc_ki[mc->mc_top]--; - mc->mc_flags &= ~C_EOF; - - DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u", + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); if (IS_LEAF2(mp)) { @@ -6638,14 +6035,8 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, if (key->mv_size == 0) return MDB_BAD_VALSIZE; - if (mc->mc_xcursor) { -#ifdef MDB_VL32 - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif + if (mc->mc_xcursor) mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - } /* See if we're already on the right page */ if (mc->mc_flags & C_INITIALIZED) { @@ -6711,6 +6102,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, } } rc = 0; + mc->mc_flags &= ~C_EOF; goto set2; } } @@ -6802,8 +6194,8 @@ set1: if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS) return rc; dcmp = mc->mc_dbx->md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t)) +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) dcmp = mdb_cmp_clong; #endif rc = dcmp(data, &olddata); @@ -6837,14 +6229,8 @@ mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) int rc; MDB_node *leaf; - if (mc->mc_xcursor) { -#ifdef MDB_VL32 - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif + if (mc->mc_xcursor) mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - } if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); @@ -6887,25 +6273,16 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) int rc; MDB_node *leaf; - if (mc->mc_xcursor) { -#ifdef MDB_VL32 - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - mdb_cursor_unref(&mc->mc_xcursor->mx_cursor); - } -#endif + if (mc->mc_xcursor) mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_LAST); + if (rc != MDB_SUCCESS) + return rc; } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - if (!(mc->mc_flags & C_EOF)) { - - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = mdb_page_search(mc, NULL, MDB_PS_LAST); - if (rc != MDB_SUCCESS) - return rc; - } - mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - - } mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; mc->mc_flags |= C_INITIALIZED|C_EOF; leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); @@ -7019,10 +6396,7 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, rc = MDB_INCOMPATIBLE; break; } - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_first(mc, key, data); - else - rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); + rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); if (rc == MDB_SUCCESS) { if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { MDB_cursor *mx; @@ -7064,21 +6438,11 @@ fetchm: case MDB_NEXT: case MDB_NEXT_DUP: case MDB_NEXT_NODUP: - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_first(mc, key, data); - else - rc = mdb_cursor_next(mc, key, data, op); + rc = mdb_cursor_next(mc, key, data, op); break; case MDB_PREV: case MDB_PREV_DUP: case MDB_PREV_NODUP: - if (!(mc->mc_flags & C_INITIALIZED)) { - rc = mdb_cursor_last(mc, key, data); - if (rc) - break; - mc->mc_flags |= C_INITIALIZED; - mc->mc_ki[mc->mc_top]++; - } rc = mdb_cursor_prev(mc, key, data, op); break; case MDB_FIRST: @@ -7095,6 +6459,11 @@ fetchm: rc = MDB_INCOMPATIBLE; break; } + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) { + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + rc = MDB_NOTFOUND; + break; + } { MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { @@ -7136,7 +6505,8 @@ mdb_cursor_touch(MDB_cursor *mc) { int rc = MDB_SUCCESS; - if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & DB_DIRTY)) { + if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & (DB_DIRTY|DB_DUPDATA))) { + /* Touch DB record of named DB */ MDB_cursor mc2; MDB_xcursor mcx; if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) @@ -7361,8 +6731,8 @@ more: if (flags == MDB_CURRENT) goto current; dcmp = mc->mc_dbx->md_dcmp; -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t)) +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) dcmp = mdb_cmp_clong; #endif /* does data match? */ @@ -7460,8 +6830,9 @@ prep_subDB: } else { memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, olddata.mv_size - fp->mp_upper - PAGEBASE); + memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0])); for (i=0; imp_ptrs[i] = fp->mp_ptrs[i] + offset; + mp->mp_ptrs[i] += offset; } } @@ -7515,8 +6886,13 @@ current: /* Note - this page is already counted in parent's dirty_room */ rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); mdb_cassert(mc, rc2 == 0); + /* Currently we make the page look as with put() in the + * parent txn, in case the user peeks at MDB_RESERVEd + * or unused parts. Some users treat ovpages specially. + */ if (!(flags & MDB_RESERVE)) { - /* Copy end of page, adjusting alignment so + /* Skip the part where LMDB will put *data. + * Copy end of page, adjusting alignment so * compiler may copy words instead of bytes. */ off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); @@ -7585,11 +6961,7 @@ new_sub: if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) { m3->mc_ki[i]++; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { - MDB_node *n2 = NODEPTR(mp, m3->mc_ki[i]); - if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); - } + XCURSOR_REFRESH(m3, i, mp); } } } @@ -7602,7 +6974,7 @@ new_sub: */ if (do_sub) { int xflags, new_dupdata; - mdb_size_t ecount; + size_t ecount; put_sub: xdata.mv_size = 0; xdata.mv_data = ""; @@ -7631,7 +7003,6 @@ put_sub: MDB_xcursor *mx = mc->mc_xcursor; unsigned i = mc->mc_top; MDB_page *mp = mc->mc_pg[i]; - int nkeys = NUMKEYS(mp); for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; @@ -7639,10 +7010,8 @@ put_sub: if (m2->mc_pg[i] == mp) { if (m2->mc_ki[i] == mc->mc_ki[i]) { mdb_xcursor_init2(m2, mx, new_dupdata); - } else if (!insert_key && m2->mc_ki[i] < nkeys) { - MDB_node *n2 = NODEPTR(mp, m2->mc_ki[i]); - if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); + } else if (!insert_key) { + XCURSOR_REFRESH(m2, i, mp); } } } @@ -7747,13 +7116,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags) if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; if (!(m2->mc_flags & C_INITIALIZED)) continue; if (m2->mc_pg[mc->mc_top] == mp) { - if (m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) { - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } else { - MDB_node *n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]); - if (!(n2->mn_flags & F_SUBDATA)) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); - } + XCURSOR_REFRESH(m2, mc->mc_top, mp); } } } @@ -7798,6 +7161,7 @@ fail: } /** Allocate and initialize new pages for a database. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc a cursor on the database being added to. * @param[in] flags flags defining what type of page is being allocated. * @param[in] num the number of pages to allocate. This is usually 1, @@ -7813,7 +7177,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) if ((rc = mdb_page_alloc(mc, num, &np))) return rc; - DPRINTF(("allocated new mpage %"Y"u, page size %u", + DPRINTF(("allocated new mpage %"Z"u, page size %u", np->mp_pgno, mc->mc_txn->mt_env->me_psize)); np->mp_flags = flags | P_DIRTY; np->mp_lower = (PAGEHDRSZ-PAGEBASE); @@ -7883,6 +7247,7 @@ mdb_branch_size(MDB_env *env, MDB_val *key) } /** Add a node to the page pointed to by the cursor. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc The cursor for this operation. * @param[in] indx The index on the page where the new node should be added. * @param[in] key The key for the new node. @@ -7913,7 +7278,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); - DPRINTF(("add to %s %spage %"Y"u index %i, data size %"Z"u key size %"Z"u [%s]", + DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]", IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, @@ -7954,7 +7319,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, goto full; if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) return rc; - DPRINTF(("allocated overflow page %"Y"u", ofp->mp_pgno)); + DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno)); flags |= F_BIGDATA; goto update; } else { @@ -8011,7 +7376,7 @@ update: return MDB_SUCCESS; full: - DPRINTF(("not enough room in page %"Y"u, got %u ptrs", + DPRINTF(("not enough room in page %"Z"u, got %u ptrs", mdb_dbg_pgno(mp), NUMKEYS(mp))); DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); DPRINTF(("node size = %"Z"u", node_size)); @@ -8034,7 +7399,7 @@ mdb_node_del(MDB_cursor *mc, int ksize) MDB_node *node; char *base; - DPRINTF(("delete node %u on %s page %"Y"u", indx, + DPRINTF(("delete node %u on %s page %"Z"u", indx, IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); numkeys = NUMKEYS(mp); mdb_cassert(mc, indx < numkeys); @@ -8143,10 +7508,7 @@ mdb_xcursor_init0(MDB_cursor *mc) mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; mx->mx_cursor.mc_snum = 0; mx->mx_cursor.mc_top = 0; -#ifdef MDB_VL32 - mx->mx_cursor.mc_ovpg = 0; -#endif - mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP)); + mx->mx_cursor.mc_flags = C_SUB; mx->mx_dbx.md_name.mv_size = 0; mx->mx_dbx.md_name.mv_data = NULL; mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; @@ -8165,12 +7527,12 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) { MDB_xcursor *mx = mc->mc_xcursor; - mx->mx_cursor.mc_flags &= C_SUB|C_ORIG_RDONLY|C_WRITEMAP; if (node->mn_flags & F_SUBDATA) { memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); mx->mx_cursor.mc_pg[0] = 0; mx->mx_cursor.mc_snum = 0; mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; } else { MDB_page *fp = NODEDATA(node); mx->mx_db.md_pad = 0; @@ -8183,7 +7545,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); mx->mx_cursor.mc_snum = 1; mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags |= C_INITIALIZED; + mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; mx->mx_cursor.mc_pg[0] = fp; mx->mx_cursor.mc_ki[0] = 0; if (mc->mc_db->md_flags & MDB_DUPFIXED) { @@ -8193,11 +7555,11 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) mx->mx_db.md_flags |= MDB_INTEGERKEY; } } - DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi, + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, mx->mx_db.md_root)); - mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ -#if UINT_MAX < SIZE_MAX || defined(MDB_VL32) - if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(mdb_size_t)) + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; +#if UINT_MAX < SIZE_MAX + if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) mx->mx_dbx.md_cmp = mdb_cmp_clong; #endif } @@ -8221,7 +7583,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) mx->mx_cursor.mc_top = 0; mx->mx_cursor.mc_flags |= C_INITIALIZED; mx->mx_cursor.mc_ki[0] = 0; - mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; #if UINT_MAX < SIZE_MAX mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; #endif @@ -8230,7 +7592,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) } mx->mx_db = src_mx->mx_db; mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; - DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi, + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, mx->mx_db.md_root)); } @@ -8249,10 +7611,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) mc->mc_top = 0; mc->mc_pg[0] = 0; mc->mc_ki[0] = 0; -#ifdef MDB_VL32 - mc->mc_ovpg = 0; -#endif - mc->mc_flags = txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP); + mc->mc_flags = 0; if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { mdb_tassert(txn, mx != NULL); mc->mc_xcursor = mx; @@ -8317,7 +7676,7 @@ mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) /* Return the count of duplicate data items for the current key */ int -mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) +mdb_cursor_count(MDB_cursor *mc, size_t *countp) { MDB_node *leaf; @@ -8333,9 +7692,15 @@ mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) if (!(mc->mc_flags & C_INITIALIZED)) return EINVAL; - if (!mc->mc_snum || (mc->mc_flags & C_EOF)) + if (!mc->mc_snum) return MDB_NOTFOUND; + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { *countp = 1; @@ -8351,11 +7716,6 @@ mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) void mdb_cursor_close(MDB_cursor *mc) { -#ifdef MDB_VL32 - if (mc) { - mdb_cursor_unref(mc); - } -#endif if (mc && !mc->mc_backup) { /* remove from txn, if tracked */ if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { @@ -8382,6 +7742,7 @@ mdb_cursor_dbi(MDB_cursor *mc) } /** Replace the key for a branch node with a new key. + * Set #MDB_TXN_ERROR on failure. * @param[in] mc Cursor pointing to the node to operate on. * @param[in] key The new key to use. * @return 0 on success, non-zero on failure. @@ -8407,7 +7768,7 @@ mdb_update_key(MDB_cursor *mc, MDB_val *key) char kbuf2[DKBUF_MAXKEYSIZE*2+1]; k2.mv_data = NODEKEY(node); k2.mv_size = node->mn_ksize; - DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Y"u", + DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u", indx, ptr, mdb_dkey(&k2, kbuf2), DKEY(key), @@ -8555,7 +7916,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) return rc; } - DPRINTF(("moving %s node %u [%s] on page %"Y"u to node %u on page %"Y"u", + DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u", IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", csrc->mc_ki[csrc->mc_top], DKEY(&key), @@ -8601,12 +7962,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; m3->mc_ki[csrc->mc_top-1]++; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && - IS_LEAF(mps)) { - MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + if (IS_LEAF(mps)) + XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); } } else /* Adding on the right, bump others down */ @@ -8627,12 +7984,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) } else { m3->mc_ki[csrc->mc_top]--; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && - IS_LEAF(mps)) { - MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + if (IS_LEAF(mps)) + XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); } } } @@ -8649,7 +8002,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) key.mv_size = NODEKSZ(srcnode); key.mv_data = NODEKEY(srcnode); } - DPRINTF(("update separator for source page %"Y"u to [%s]", + DPRINTF(("update separator for source page %"Z"u to [%s]", csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); mdb_cursor_copy(csrc, &mn); mn.mc_snum--; @@ -8680,7 +8033,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) key.mv_size = NODEKSZ(srcnode); key.mv_data = NODEKEY(srcnode); } - DPRINTF(("update separator for destination page %"Y"u to [%s]", + DPRINTF(("update separator for destination page %"Z"u to [%s]", cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); mdb_cursor_copy(cdst, &mn); mn.mc_snum--; @@ -8726,7 +8079,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) psrc = csrc->mc_pg[csrc->mc_top]; pdst = cdst->mc_pg[cdst->mc_top]; - DPRINTF(("merging page %"Y"u into %"Y"u", psrc->mp_pgno, pdst->mp_pgno)); + DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno)); mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ mdb_cassert(csrc, cdst->mc_snum > 1); @@ -8783,7 +8136,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) } } - DPRINTF(("dst page %"Y"u now has %u keys (%.1f%% filled)", + DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)", pdst->mp_pgno, NUMKEYS(pdst), (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); @@ -8833,12 +8186,8 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) m3->mc_ki[top-1] > csrc->mc_ki[top-1]) { m3->mc_ki[top-1]--; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && - IS_LEAF(psrc)) { - MDB_node *node = NODEPTR(m3->mc_pg[top], m3->mc_ki[top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + if (IS_LEAF(psrc)) + XCURSOR_REFRESH(m3, top, m3->mc_pg[top]); } } { @@ -8871,9 +8220,6 @@ mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) cdst->mc_snum = csrc->mc_snum; cdst->mc_top = csrc->mc_top; cdst->mc_flags = csrc->mc_flags; -#ifdef MDB_VL32 - cdst->mc_ovpg = csrc->mc_ovpg; -#endif for (i=0; imc_snum; i++) { cdst->mc_pg[i] = csrc->mc_pg[i]; @@ -8902,14 +8248,14 @@ mdb_rebalance(MDB_cursor *mc) minkeys = 1; thresh = FILL_THRESHOLD; } - DPRINTF(("rebalancing %s page %"Y"u (has %u keys, %.1f%% full)", + DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)", IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh && NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { - DPRINTF(("no need to rebalance page %"Y"u, above fill threshold", + DPRINTF(("no need to rebalance page %"Z"u, above fill threshold", mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); return MDB_SUCCESS; } @@ -9038,7 +8384,7 @@ mdb_rebalance(MDB_cursor *mc) fromleft = 1; } - DPRINTF(("found neighbor page %"Y"u (%u keys, %.1f%% full)", + DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)", mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); @@ -9095,16 +8441,15 @@ mdb_cursor_del0(MDB_cursor *mc) if (m3->mc_pg[mc->mc_top] == mp) { if (m3->mc_ki[mc->mc_top] == ki) { m3->mc_flags |= C_DEL; - if (mc->mc_db->md_flags & MDB_DUPSORT) - m3->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + if (mc->mc_db->md_flags & MDB_DUPSORT) { + /* Sub-cursor referred into dataset which is gone */ + m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + } + continue; } else if (m3->mc_ki[mc->mc_top] > ki) { m3->mc_ki[mc->mc_top]--; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { - MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + XCURSOR_REFRESH(m3, mc->mc_top, mp); } } } @@ -9130,11 +8475,32 @@ mdb_cursor_del0(MDB_cursor *mc) continue; if (m3->mc_pg[mc->mc_top] == mp) { /* if m3 points past last node in page, find next sibling */ - if (m3->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(m3, 1); - if (rc == MDB_NOTFOUND) { - m3->mc_flags |= C_EOF; - rc = MDB_SUCCESS; + if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) { + if (m3->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(m3, 1); + if (rc == MDB_NOTFOUND) { + m3->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + continue; + } + } + if (mc->mc_db->md_flags & MDB_DUPSORT) { + MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]); + /* If this node has dupdata, it may need to be reinited + * because its data has moved. + * If the xcursor was not initd it must be reinited. + * Else if node points to a subDB, nothing is needed. + * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset. + */ + if (node->mn_flags & F_DUPDATA) { + if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + if (!(node->mn_flags & F_SUBDATA)) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); + } else { + mdb_xcursor_init1(m3, node); + m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL; + } + } } } } @@ -9209,6 +8575,7 @@ mdb_del0(MDB_txn *txn, MDB_dbi dbi, } /** Split a page and insert a new node. + * Set #MDB_TXN_ERROR on failure. * @param[in,out] mc Cursor pointing to the page and desired insertion index. * The cursor will be updated to point to the actual page and index where * the node got inserted after the split. @@ -9240,7 +8607,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno newindx = mc->mc_ki[mc->mc_top]; nkeys = NUMKEYS(mp); - DPRINTF(("-----> splitting %s page %"Y"u and adding [%s] at index %i/%i", + DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i", IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); @@ -9248,7 +8615,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) return rc; rp->mp_pad = mp->mp_pad; - DPRINTF(("new right sibling: page %"Y"u", rp->mp_pgno)); + DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno)); /* Usually when splitting the root page, the cursor * height is 1. But when called from mdb_update_key, @@ -9266,7 +8633,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno mc->mc_pg[0] = pp; mc->mc_ki[0] = 0; mc->mc_db->md_root = pp->mp_pgno; - DPRINTF(("root split! new root = %"Y"u", pp->mp_pgno)); + DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno)); new_root = mc->mc_db->md_depth++; /* Add left (implicit) pointer. */ @@ -9283,7 +8650,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno ptop = 0; } else { ptop = mc->mc_top-1; - DPRINTF(("parent branch page is %"Y"u", mc->mc_pg[ptop]->mp_pgno)); + DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno)); } mdb_cursor_copy(mc, &mn); @@ -9382,7 +8749,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno * the split so the new page is emptier than the old page. * This yields better packing during sequential inserts. */ - if (nkeys < 20 || nsize > pmax/16 || newindx >= nkeys) { + if (nkeys < 32 || nsize > pmax/16 || newindx >= nkeys) { /* Find split point */ psize = 0; if (newindx <= split_indx || newindx >= nkeys) { @@ -9618,12 +8985,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { m3->mc_ki[ptop]++; } - if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) && - IS_LEAF(mp)) { - MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } + if (IS_LEAF(mp)) + XCURSOR_REFRESH(m3, mc->mc_top, m3->mc_pg[mc->mc_top]); } } DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); @@ -9664,23 +9027,26 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi, #ifndef MDB_WBUF #define MDB_WBUF (1024*1024) #endif +#define MDB_EOF 0x10 /**< #mdb_env_copyfd1() is done reading */ - /** State needed for a compacting copy. */ + /** State needed for a double-buffering compacting copy. */ typedef struct mdb_copy { - pthread_mutex_t mc_mutex; - pthread_cond_t mc_cond; - char *mc_wbuf[2]; - char *mc_over[2]; MDB_env *mc_env; MDB_txn *mc_txn; + pthread_mutex_t mc_mutex; + pthread_cond_t mc_cond; /**< Condition variable for #mc_new */ + char *mc_wbuf[2]; + char *mc_over[2]; int mc_wlen[2]; int mc_olen[2]; pgno_t mc_next_pgno; HANDLE mc_fd; - int mc_status; - volatile int mc_new; - int mc_toggle; - + int mc_toggle; /**< Buffer number in provider */ + int mc_new; /**< (0-2 buffers to write) | (#MDB_EOF at end) */ + /** Error code. Never cleared if set. Both threads can set nonzero + * to fail the copy. Not mutex-protected, LMDB expects atomic int. + */ + volatile int mc_error; } mdb_copy; /** Dedicated writer thread for compacting copy. */ @@ -9696,26 +9062,38 @@ mdb_env_copythr(void *arg) #else int len; #define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#ifdef SIGPIPE + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGPIPE); + if ((rc = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0) + my->mc_error = rc; +#endif #endif pthread_mutex_lock(&my->mc_mutex); - my->mc_new = 0; - pthread_cond_signal(&my->mc_cond); for(;;) { while (!my->mc_new) pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - if (my->mc_new < 0) { - my->mc_new = 0; + if (my->mc_new == 0 + MDB_EOF) /* 0 buffers, just EOF */ break; - } - my->mc_new = 0; wsize = my->mc_wlen[toggle]; ptr = my->mc_wbuf[toggle]; again: - while (wsize > 0) { + rc = MDB_SUCCESS; + while (wsize > 0 && !my->mc_error) { DO_WRITE(rc, my->mc_fd, ptr, wsize, len); if (!rc) { rc = ErrCode(); +#if defined(SIGPIPE) && !defined(_WIN32) + if (rc == EPIPE) { + /* Collect the pending SIGPIPE, otherwise at least OS X + * gives it to the process on thread-exit (ITS#8504). + */ + int tmp; + sigwait(&set, &tmp); + } +#endif break; } else if (len > 0) { rc = MDB_SUCCESS; @@ -9728,8 +9106,7 @@ again: } } if (rc) { - my->mc_status = rc; - break; + my->mc_error = rc; } /* If there's an overflow page tail, write it too */ if (my->mc_olen[toggle]) { @@ -9740,38 +9117,45 @@ again: } my->mc_wlen[toggle] = 0; toggle ^= 1; + /* Return the empty buffer to provider */ + my->mc_new--; pthread_cond_signal(&my->mc_cond); } - pthread_cond_signal(&my->mc_cond); pthread_mutex_unlock(&my->mc_mutex); return (THREAD_RET)0; #undef DO_WRITE } - /** Tell the writer thread there's a buffer ready to write */ + /** Give buffer and/or #MDB_EOF to writer thread, await unused buffer. + * + * @param[in] my control structure. + * @param[in] adjust (1 to hand off 1 buffer) | (MDB_EOF when ending). + */ static int ESECT -mdb_env_cthr_toggle(mdb_copy *my, int st) +mdb_env_cthr_toggle(mdb_copy *my, int adjust) { - int toggle = my->mc_toggle ^ 1; pthread_mutex_lock(&my->mc_mutex); - if (my->mc_status) { - pthread_mutex_unlock(&my->mc_mutex); - return my->mc_status; - } - while (my->mc_new == 1) - pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - my->mc_new = st; - my->mc_toggle = toggle; + my->mc_new += adjust; pthread_cond_signal(&my->mc_cond); + while (my->mc_new & 2) /* both buffers in use */ + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); pthread_mutex_unlock(&my->mc_mutex); - return 0; + + my->mc_toggle ^= (adjust & 1); + /* Both threads reset mc_wlen, to be safe from threading errors */ + my->mc_wlen[my->mc_toggle] = 0; + return my->mc_error; } - /** Depth-first tree traversal for compacting copy. */ + /** Depth-first tree traversal for compacting copy. + * @param[in] my control structure. + * @param[in,out] pg database root. + * @param[in] flags includes #F_DUPDATA if it is a sorted-duplicate sub-DB. + */ static int ESECT mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) { - MDB_cursor mc; + MDB_cursor mc = {0}; MDB_node *ni; MDB_page *mo, *mp, *leaf; char *buf, *ptr; @@ -9783,7 +9167,6 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) return MDB_SUCCESS; mc.mc_snum = 1; - mc.mc_top = 0; mc.mc_txn = my->mc_txn; rc = mdb_page_get(&mc, *pg, &mc.mc_pg[0], NULL); @@ -9830,6 +9213,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) } memcpy(&pg, NODEDATA(ni), sizeof(pg)); + memcpy(NODEDATA(ni), &my->mc_next_pgno, sizeof(pgno_t)); rc = mdb_page_get(&mc, pg, &omp, NULL); if (rc) goto done; @@ -9852,7 +9236,6 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) goto done; toggle = my->mc_toggle; } - memcpy(NODEDATA(ni), &mo->mp_pgno, sizeof(pgno_t)); } else if (ni->mn_flags & F_SUBDATA) { MDB_db db; @@ -9930,47 +9313,56 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) { MDB_meta *mm; MDB_page *mp; - mdb_copy my; + mdb_copy my = {0}; MDB_txn *txn = NULL; pthread_t thr; - int rc; + pgno_t root, new_root; + int rc = MDB_SUCCESS; #ifdef _WIN32 - my.mc_mutex = CreateMutex(NULL, FALSE, NULL); - my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!(my.mc_mutex = CreateMutex(NULL, FALSE, NULL)) || + !(my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL))) { + rc = ErrCode(); + goto done; + } my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); - if (my.mc_wbuf[0] == NULL) - return errno; + if (my.mc_wbuf[0] == NULL) { + /* _aligned_malloc() sets errno, but we use Windows error codes */ + rc = ERROR_NOT_ENOUGH_MEMORY; + goto done; + } #else - pthread_mutex_init(&my.mc_mutex, NULL); - pthread_cond_init(&my.mc_cond, NULL); + if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) != 0) + return rc; + if ((rc = pthread_cond_init(&my.mc_cond, NULL)) != 0) + goto done2; #ifdef HAVE_MEMALIGN my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); - if (my.mc_wbuf[0] == NULL) - return errno; + if (my.mc_wbuf[0] == NULL) { + rc = errno; + goto done; + } #else - rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); - if (rc) - return rc; + { + void *p; + if ((rc = posix_memalign(&p, env->me_os_psize, MDB_WBUF*2)) != 0) + goto done; + my.mc_wbuf[0] = p; + } #endif #endif memset(my.mc_wbuf[0], 0, MDB_WBUF*2); my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; - my.mc_wlen[0] = 0; - my.mc_wlen[1] = 0; - my.mc_olen[0] = 0; - my.mc_olen[1] = 0; my.mc_next_pgno = NUM_METAS; - my.mc_status = 0; - my.mc_new = 1; - my.mc_toggle = 0; my.mc_env = env; my.mc_fd = fd; - THREAD_CREATE(thr, mdb_env_copythr, &my); + rc = THREAD_CREATE(thr, mdb_env_copythr, &my); + if (rc) + goto done; rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (rc) - return rc; + goto finish; mp = (MDB_page *)my.mc_wbuf[0]; memset(mp, 0, NUM_METAS * env->me_psize); @@ -9986,57 +9378,64 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) *(MDB_meta *)METADATA(mp) = *mm; mm = (MDB_meta *)METADATA(mp); - /* Count the number of free pages, subtract from lastpg to find - * number of active pages - */ - { + /* Set metapage 1 with current main DB */ + root = new_root = txn->mt_dbs[MAIN_DBI].md_root; + if (root != P_INVALID) { + /* Count free pages + freeDB pages. Subtract from last_pg + * to find the new last_pg, which also becomes the new root. + */ MDB_ID freecount = 0; MDB_cursor mc; MDB_val key, data; mdb_cursor_init(&mc, txn, FREE_DBI, NULL); while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) freecount += *(MDB_ID *)data.mv_data; + if (rc != MDB_NOTFOUND) + goto finish; freecount += txn->mt_dbs[FREE_DBI].md_branch_pages + txn->mt_dbs[FREE_DBI].md_leaf_pages + txn->mt_dbs[FREE_DBI].md_overflow_pages; - /* Set metapage 1 */ - mm->mm_last_pg = txn->mt_next_pgno - freecount - 1; + new_root = txn->mt_next_pgno - 1 - freecount; + mm->mm_last_pg = new_root; mm->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; - if (mm->mm_last_pg > NUM_METAS-1) { - mm->mm_dbs[MAIN_DBI].md_root = mm->mm_last_pg; - mm->mm_txnid = 1; - } else { - mm->mm_dbs[MAIN_DBI].md_root = P_INVALID; - } + mm->mm_dbs[MAIN_DBI].md_root = new_root; + } else { + /* When the DB is empty, handle it specially to + * fix any breakage like page leaks from ITS#8174. + */ + mm->mm_dbs[MAIN_DBI].md_flags = txn->mt_dbs[MAIN_DBI].md_flags; } + if (root != P_INVALID || mm->mm_dbs[MAIN_DBI].md_flags) { + mm->mm_txnid = 1; /* use metapage 1 */ + } + my.mc_wlen[0] = env->me_psize * NUM_METAS; my.mc_txn = txn; - pthread_mutex_lock(&my.mc_mutex); - while(my.mc_new) - pthread_cond_wait(&my.mc_cond, &my.mc_mutex); - pthread_mutex_unlock(&my.mc_mutex); - rc = mdb_env_cwalk(&my, &txn->mt_dbs[MAIN_DBI].md_root, 0); - if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle]) - rc = mdb_env_cthr_toggle(&my, 1); - mdb_env_cthr_toggle(&my, -1); - pthread_mutex_lock(&my.mc_mutex); - while(my.mc_new) - pthread_cond_wait(&my.mc_cond, &my.mc_mutex); - pthread_mutex_unlock(&my.mc_mutex); - THREAD_FINISH(thr); + rc = mdb_env_cwalk(&my, &root, 0); + if (rc == MDB_SUCCESS && root != new_root) { + rc = MDB_INCOMPATIBLE; /* page leak or corrupt DB */ + } +finish: + if (rc) + my.mc_error = rc; + mdb_env_cthr_toggle(&my, 1 | MDB_EOF); + rc = THREAD_FINISH(thr); mdb_txn_abort(txn); + +done: #ifdef _WIN32 - CloseHandle(my.mc_cond); - CloseHandle(my.mc_mutex); - _aligned_free(my.mc_wbuf[0]); + if (my.mc_wbuf[0]) _aligned_free(my.mc_wbuf[0]); + if (my.mc_cond) CloseHandle(my.mc_cond); + if (my.mc_mutex) CloseHandle(my.mc_mutex); #else - pthread_cond_destroy(&my.mc_cond); - pthread_mutex_destroy(&my.mc_mutex); free(my.mc_wbuf[0]); + pthread_cond_destroy(&my.mc_cond); +done2: + pthread_mutex_destroy(&my.mc_mutex); #endif - return rc; + return rc ? rc : my.mc_error; } /** Copy environment as-is. */ @@ -10046,7 +9445,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) MDB_txn *txn = NULL; mdb_mutexref_t wmutex = NULL; int rc; - mdb_size_t wsize, w3; + size_t wsize, w3; char *ptr; #ifdef _WIN32 DWORD len, w2; @@ -10107,7 +9506,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) w3 = txn->mt_next_pgno * env->me_psize; { - mdb_size_t fsize = 0; + size_t fsize = 0; if ((rc = mdb_fsize(env->me_fd, &fsize))) goto leave; if (w3 > fsize) @@ -10157,67 +9556,20 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd) int ESECT mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) { - int rc, len; - char *lpath; + int rc; + MDB_name fname; HANDLE newfd = INVALID_HANDLE_VALUE; -#ifdef _WIN32 - wchar_t *wpath; -#endif - if (env->me_flags & MDB_NOSUBDIR) { - lpath = (char *)path; - } else { - len = strlen(path); - len += sizeof(DATANAME); - lpath = malloc(len); - if (!lpath) - return ENOMEM; - sprintf(lpath, "%s" DATANAME, path); + rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname); + if (rc == MDB_SUCCESS) { + rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, &newfd); + mdb_fname_destroy(fname); } - - /* The destination path must exist, but the destination file must not. - * We don't want the OS to cache the writes, since the source data is - * already in the OS cache. - */ -#ifdef _WIN32 - rc = utf8_to_utf16(lpath, -1, &wpath, NULL); - if (rc) - goto leave; - newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, - FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); - free(wpath); -#else - newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666); -#endif - if (newfd == INVALID_HANDLE_VALUE) { - rc = ErrCode(); - goto leave; - } - - if (env->me_psize >= env->me_os_psize) { -#ifdef O_DIRECT - /* Set O_DIRECT if the file system supports it */ - if ((rc = fcntl(newfd, F_GETFL)) != -1) - (void) fcntl(newfd, F_SETFL, rc | O_DIRECT); -#endif -#ifdef F_NOCACHE /* __APPLE__ */ - rc = fcntl(newfd, F_NOCACHE, 1); - if (rc) { - rc = ErrCode(); - goto leave; - } -#endif - } - - rc = mdb_env_copyfd2(env, newfd, flags); - -leave: - if (!(env->me_flags & MDB_NOSUBDIR)) - free(lpath); - if (newfd != INVALID_HANDLE_VALUE) + if (rc == MDB_SUCCESS) { + rc = mdb_env_copyfd2(env, newfd, flags); if (close(newfd) < 0 && rc == MDB_SUCCESS) rc = ErrCode(); - + } return rc; } @@ -10439,8 +9791,11 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) return MDB_INCOMPATIBLE; - } else if (! (rc == MDB_NOTFOUND && (flags & MDB_CREATE))) { - return rc; + } else { + if (rc != MDB_NOTFOUND || !(flags & MDB_CREATE)) + return rc; + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; } /* Done here so we cannot fail after creating a new DB */ @@ -10454,7 +9809,8 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db memset(&dummy, 0, sizeof(dummy)); dummy.md_root = P_INVALID; dummy.md_flags = flags & PERSISTENT_FLAGS; - rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA); + WITH_CURSOR_TRACKING(mc, + rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA)); dbflag |= DB_DIRTY; } @@ -10554,11 +9910,6 @@ mdb_drop0(MDB_cursor *mc, int subs) mdb_cursor_pop(mc); mdb_cursor_copy(mc, &mx); -#ifdef MDB_VL32 - /* bump refcount for mx's pages */ - for (i=0; imc_snum; i++) - mdb_page_get(&mx, mc->mc_pg[i]->mp_pgno, &mx.mc_pg[i], NULL); -#endif while (mc->mc_snum > 0) { MDB_page *mp = mc->mc_pg[mc->mc_top]; unsigned n = NUMKEYS(mp); @@ -10624,10 +9975,6 @@ pop: done: if (rc) txn->mt_flags |= MDB_TXN_ERROR; -#ifdef MDB_VL32 - /* drop refcount for mx's pages */ - mdb_cursor_unref(&mx); -#endif } else if (rc == MDB_NOTFOUND) { rc = MDB_SUCCESS; } @@ -10747,7 +10094,7 @@ mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) if (mr[i].mr_pid) { txnid_t txnid = mr[i].mr_txnid; sprintf(buf, txnid == (txnid_t)-1 ? - "%10d %"Z"x -\n" : "%10d %"Z"x %"Y"u\n", + "%10d %"Z"x -\n" : "%10d %"Z"x %"Z"u\n", (int)mr[i].mr_pid, (size_t)mr[i].mr_tid, txnid); if (first) { first = 0; @@ -10816,7 +10163,7 @@ mdb_reader_check(MDB_env *env, int *dead) return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS; } -/** As #mdb_reader_check(). rlocked = . */ +/** As #mdb_reader_check(). \b rlocked is set if caller locked #me_rmutex. */ static int ESECT mdb_reader_check0(MDB_env *env, int rlocked, int *dead) { @@ -10852,7 +10199,7 @@ mdb_reader_check0(MDB_env *env, int rlocked, int *dead) } for (; jmn_alloced = 1; + dst->mn_len = need - 1; + dst->mn_val = result; + return MDB_SUCCESS; + } } #endif /* defined(_WIN32) */ +/** @} */ diff --git a/contrib/db/liblmdb/mdb_copy.1 b/contrib/db/liblmdb/mdb_copy.1 index 401e47ab..594ff124 100644 --- a/contrib/db/liblmdb/mdb_copy.1 +++ b/contrib/db/liblmdb/mdb_copy.1 @@ -1,5 +1,5 @@ -.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_COPY 1 "2014/07/01" "LMDB 0.9.14" +.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_copy \- LMDB environment copy tool @@ -11,8 +11,6 @@ mdb_copy \- LMDB environment copy tool .BR \-c ] [\c .BR \-n ] -[\c -.BR \-v ] .B srcpath [\c .BR dstpath ] @@ -38,13 +36,10 @@ Write the library version number to the standard output, and exit. Compact while copying. Only current data pages will be copied; freed or unused pages will be omitted from the copy. This option will slow down the backup process as it is more CPU-intensive. +Currently it fails if the environment has suffered a page leak. .TP .BR \-n Open LDMB environment(s) which do not use subdirectories. -.TP -.BR \-v -Use the previous environment state instead of the latest state. -This may be useful if the latest state has been corrupted. .SH DIAGNOSTICS Exit status is zero if no errors occur. diff --git a/contrib/db/liblmdb/mdb_copy.c b/contrib/db/liblmdb/mdb_copy.c index 95a6e713..1b89396e 100644 --- a/contrib/db/liblmdb/mdb_copy.c +++ b/contrib/db/liblmdb/mdb_copy.c @@ -1,6 +1,6 @@ /* mdb_copy.c - memory-mapped database backup tool */ /* - * Copyright 2012-2015 Howard Chu, Symas Corp. + * Copyright 2012-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,8 +38,6 @@ int main(int argc,char * argv[]) for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { if (argv[1][1] == 'n' && argv[1][2] == '\0') flags |= MDB_NOSUBDIR; - else if (argv[1][1] == 'v' && argv[1][2] == '\0') - flags |= MDB_PREVSNAPSHOT; else if (argv[1][1] == 'c' && argv[1][2] == '\0') cpflags |= MDB_CP_COMPACT; else if (argv[1][1] == 'V' && argv[1][2] == '\0') { @@ -50,7 +48,7 @@ int main(int argc,char * argv[]) } if (argc<2 || argc>3) { - fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] srcpath [dstpath]\n", progname); + fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname); exit(EXIT_FAILURE); } diff --git a/contrib/db/liblmdb/mdb_dump.1 b/contrib/db/liblmdb/mdb_dump.1 index a25fb92e..72cf6ca8 100644 --- a/contrib/db/liblmdb/mdb_dump.1 +++ b/contrib/db/liblmdb/mdb_dump.1 @@ -1,5 +1,5 @@ -.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_DUMP 1 "2015/09/30" "LMDB 0.9.17" +.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_dump \- LMDB environment export tool @@ -14,8 +14,6 @@ mdb_dump \- LMDB environment export tool [\c .BR \-n ] [\c -.BR \-v ] -[\c .BR \-p ] [\c .BR \-a \ | @@ -44,10 +42,6 @@ names will be listed, no data will be output. .BR \-n Dump an LMDB database which does not use subdirectories. .TP -.BR \-v -Use the previous environment state instead of the latest state. -This may be useful if the latest state has been corrupted. -.TP .BR \-p If characters in either the key or data items are printing characters (as defined by isprint(3)), output them directly. This option permits users to diff --git a/contrib/db/liblmdb/mdb_dump.c b/contrib/db/liblmdb/mdb_dump.c index 7a42bc0b..9df5dc0b 100644 --- a/contrib/db/liblmdb/mdb_dump.c +++ b/contrib/db/liblmdb/mdb_dump.c @@ -1,6 +1,6 @@ /* mdb_dump.c - memory-mapped database dump tool */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -25,15 +25,6 @@ #else #define Z "z" #endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif #define PRINT 1 static int mode; @@ -124,7 +115,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) if (name) printf("database=%s\n", name); printf("type=btree\n"); - printf("mapsize=%" Y "u\n", info.me_mapsize); + printf("mapsize=%" Z "u\n", info.me_mapsize); if (info.me_mapaddr) printf("mapaddr=%p\n", info.me_mapaddr); printf("maxreaders=%u\n", info.me_maxreaders); @@ -164,7 +155,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) static void usage(char *prog) { - fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-a|-s subdb] dbpath\n", prog); + fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n", prog); exit(EXIT_FAILURE); } @@ -188,7 +179,6 @@ int main(int argc, char *argv[]) * -n: use NOSUBDIR flag on env_open * -p: use printable characters * -f: write to file instead of stdout - * -v: use previous snapshot * -V: print version and exit * (default) dump only the main DB */ @@ -216,9 +206,6 @@ int main(int argc, char *argv[]) case 'n': envflags |= MDB_NOSUBDIR; break; - case 'v': - envflags |= MDB_PREVSNAPSHOT; - break; case 'p': mode |= PRINT; break; diff --git a/contrib/db/liblmdb/mdb_load.1 b/contrib/db/liblmdb/mdb_load.1 index ede3702d..998acc12 100644 --- a/contrib/db/liblmdb/mdb_load.1 +++ b/contrib/db/liblmdb/mdb_load.1 @@ -1,5 +1,5 @@ -.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.17" +.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_load \- LMDB environment import tool @@ -37,13 +37,6 @@ option below. .BR \-V Write the library version number to the standard output, and exit. .TP -.BR \-a -Append all records in the order they appear in the input. The input is assumed to already be -in correctly sorted order and no sorting or checking for redundant values will be performed. -This option must be used to reload data that was produced by running -.B mdb_dump -on a database that uses custom compare functions. -.TP .BR \-f \ file Read from the specified file instead of from the standard input. .TP diff --git a/contrib/db/liblmdb/mdb_load.c b/contrib/db/liblmdb/mdb_load.c index 797c2f97..0f177f1e 100644 --- a/contrib/db/liblmdb/mdb_load.c +++ b/contrib/db/liblmdb/mdb_load.c @@ -1,6 +1,6 @@ /* mdb_load.c - memory-mapped database load tool */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,22 +37,12 @@ static int Eof; static MDB_envinfo info; static MDB_val kbuf, dbuf; -static MDB_val k0buf; #ifdef _WIN32 #define Z "I" #else #define Z "z" #endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif #define STRLENOF(s) (sizeof(s)-1) @@ -78,6 +68,7 @@ static void readhdr(void) { char *ptr; + flags = 0; while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) { lineno++; if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { @@ -122,7 +113,7 @@ static void readhdr(void) int i; ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); if (ptr) *ptr = '\0'; - i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize); + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize); if (i != 1) { fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize=")); @@ -258,7 +249,8 @@ badend: c2 += 2; } } else { - c1++; c2++; + /* copies are redundant when no escapes were used */ + *c1++ = *c2++; } } } else { @@ -286,15 +278,10 @@ badend: static void usage(void) { - fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog); + fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog); exit(EXIT_FAILURE); } -static int greater(const MDB_val *a, const MDB_val *b) -{ - return 1; -} - int main(int argc, char *argv[]) { int i, rc; @@ -304,8 +291,7 @@ int main(int argc, char *argv[]) MDB_dbi dbi; char *envname; int envflags = 0, putflags = 0; - int dohdr = 0, append = 0; - MDB_val prevk; + int dohdr = 0; prog = argv[0]; @@ -313,23 +299,19 @@ int main(int argc, char *argv[]) usage(); } - /* -a: append records in input order - * -f: load file instead of stdin + /* -f: load file instead of stdin * -n: use NOSUBDIR flag on env_open * -s: load into named subDB * -N: use NOOVERWRITE on puts * -T: read plaintext * -V: print version and exit */ - while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) { + while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) { switch(i) { case 'V': printf("%s\n", MDB_VERSION_STRING); exit(0); break; - case 'a': - append = 1; - break; case 'f': if (freopen(optarg, "r", stdin) == NULL) { fprintf(stderr, "%s: %s: reopen: %s\n", @@ -388,17 +370,11 @@ int main(int argc, char *argv[]) } kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; - kbuf.mv_data = malloc(kbuf.mv_size * 2); - k0buf.mv_size = kbuf.mv_size; - k0buf.mv_data = (char *)kbuf.mv_data + kbuf.mv_size; - prevk.mv_size = 0; - prevk.mv_data = k0buf.mv_data; + kbuf.mv_data = malloc(kbuf.mv_size); while(!Eof) { MDB_val key, data; int batch = 0; - flags = 0; - int appflag; if (!dohdr) { dohdr = 1; @@ -416,11 +392,6 @@ int main(int argc, char *argv[]) fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } - if (append) { - mdb_set_compare(txn, dbi, greater); - if (flags & MDB_DUPSORT) - mdb_set_dupsort(txn, dbi, greater); - } rc = mdb_cursor_open(txn, dbi, &mc); if (rc) { @@ -439,20 +410,7 @@ int main(int argc, char *argv[]) goto txn_abort; } - if (append) { - appflag = MDB_APPEND; - if (flags & MDB_DUPSORT) { - if (prevk.mv_size == key.mv_size && !memcmp(prevk.mv_data, key.mv_data, key.mv_size)) - appflag = MDB_APPENDDUP; - else { - memcpy(prevk.mv_data, key.mv_data, key.mv_size); - prevk.mv_size = key.mv_size; - } - } - } else { - appflag = 0; - } - rc = mdb_cursor_put(mc, &key, &data, putflags|appflag); + rc = mdb_cursor_put(mc, &key, &data, putflags); if (rc == MDB_KEYEXIST && putflags) continue; if (rc) { diff --git a/contrib/db/liblmdb/mdb_stat.1 b/contrib/db/liblmdb/mdb_stat.1 index bf49bd3b..7c3f2846 100644 --- a/contrib/db/liblmdb/mdb_stat.1 +++ b/contrib/db/liblmdb/mdb_stat.1 @@ -1,5 +1,5 @@ -.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14" -.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_STAT 1 "2015/09/30" "LMDB 0.9.17" +.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_stat \- LMDB environment status tool @@ -14,8 +14,6 @@ mdb_stat \- LMDB environment status tool [\c .BR \-n ] [\c -.BR \-v ] -[\c .BR \-r [ r ]] [\c .BR \-a \ | @@ -41,10 +39,6 @@ If \fB\-fff\fP is given, display the full list of page IDs in the freelist. .BR \-n Display the status of an LMDB database which does not use subdirectories. .TP -.BR \-v -Use the previous environment state instead of the latest state. -This may be useful if the latest state has been corrupted. -.TP .BR \-r Display information about the environment reader table. Shows the process ID, thread ID, and transaction ID for each active diff --git a/contrib/db/liblmdb/mdb_stat.c b/contrib/db/liblmdb/mdb_stat.c index 30ec81fe..f4c0dc1e 100644 --- a/contrib/db/liblmdb/mdb_stat.c +++ b/contrib/db/liblmdb/mdb_stat.c @@ -1,6 +1,6 @@ /* mdb_stat.c - memory-mapped database status tool */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -22,15 +22,6 @@ #else #define Z "z" #endif -#ifdef MDB_VL32 -#ifdef _WIN32 -#define Y "I64" -#else -#define Y "ll" -#endif -#else -#define Y Z -#endif static void prstat(MDB_stat *ms) { @@ -38,15 +29,15 @@ static void prstat(MDB_stat *ms) printf(" Page size: %u\n", ms->ms_psize); #endif printf(" Tree depth: %u\n", ms->ms_depth); - printf(" Branch pages: %"Y"u\n", ms->ms_branch_pages); - printf(" Leaf pages: %"Y"u\n", ms->ms_leaf_pages); - printf(" Overflow pages: %"Y"u\n", ms->ms_overflow_pages); - printf(" Entries: %"Y"u\n", ms->ms_entries); + printf(" Branch pages: %"Z"u\n", ms->ms_branch_pages); + printf(" Leaf pages: %"Z"u\n", ms->ms_leaf_pages); + printf(" Overflow pages: %"Z"u\n", ms->ms_overflow_pages); + printf(" Entries: %"Z"u\n", ms->ms_entries); } static void usage(char *prog) { - fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-a|-s subdb] dbpath\n", prog); + fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n", prog); exit(EXIT_FAILURE); } @@ -73,7 +64,6 @@ int main(int argc, char *argv[]) * -f: print freelist info * -r: print reader info * -n: use NOSUBDIR flag on env_open - * -v: use previous snapshot * -V: print version and exit * (default) print stat of only the main DB */ @@ -97,9 +87,6 @@ int main(int argc, char *argv[]) case 'n': envflags |= MDB_NOSUBDIR; break; - case 'v': - envflags |= MDB_PREVSNAPSHOT; - break; case 'r': rdrinfo++; break; @@ -138,11 +125,11 @@ int main(int argc, char *argv[]) (void)mdb_env_info(env, &mei); printf("Environment Info\n"); printf(" Map address: %p\n", mei.me_mapaddr); - printf(" Map size: %"Y"u\n", mei.me_mapsize); + printf(" Map size: %"Z"u\n", mei.me_mapsize); printf(" Page size: %u\n", mst.ms_psize); - printf(" Max pages: %"Y"u\n", mei.me_mapsize / mst.ms_psize); - printf(" Number of pages used: %"Y"u\n", mei.me_last_pgno+1); - printf(" Last transaction ID: %"Y"u\n", mei.me_last_txnid); + printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize); + printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1); + printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid); printf(" Max readers: %u\n", mei.me_maxreaders); printf(" Number of readers used: %u\n", mei.me_numreaders); } diff --git a/contrib/db/liblmdb/midl.c b/contrib/db/liblmdb/midl.c index 152a1ec0..75420c41 100644 --- a/contrib/db/liblmdb/midl.c +++ b/contrib/db/liblmdb/midl.c @@ -3,7 +3,8 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2015 The OpenLDAP Foundation. + * Copyright 2000-2019 The OpenLDAP Foundation. + * Portions Copyright 2001-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -354,67 +355,5 @@ int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) return 0; } -#ifdef MDB_VL32 -unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id ) -{ - /* - * binary search of id in ids - * if found, returns position of id - * if not found, returns first position greater than id - */ - unsigned base = 0; - unsigned cursor = 1; - int val = 0; - unsigned n = (unsigned)ids[0].mid; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( id, ids[cursor].mid ); - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - return cursor; - } - } - - if( val > 0 ) { - ++cursor; - } - return cursor; -} - -int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id ) -{ - unsigned x, i; - - x = mdb_mid3l_search( ids, id->mid ); - - if( x < 1 ) { - /* internal error */ - return -2; - } - - if ( x <= ids[0].mid && ids[x].mid == id->mid ) { - /* duplicate */ - return -1; - } - - /* insert id */ - ids[0].mid++; - for (i=(unsigned)ids[0].mid; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = *id; - - return 0; -} -#endif /* MDB_VL32 */ - /** @} */ /** @} */ diff --git a/contrib/db/liblmdb/midl.h b/contrib/db/liblmdb/midl.h index 1555ecb1..462c3497 100644 --- a/contrib/db/liblmdb/midl.h +++ b/contrib/db/liblmdb/midl.h @@ -11,7 +11,8 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2015 The OpenLDAP Foundation. + * Copyright 2000-2019 The OpenLDAP Foundation. + * Portions Copyright 2001-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -27,7 +28,6 @@ #define _MDB_MIDL_H_ #include -#include #ifdef __cplusplus extern "C" { @@ -43,11 +43,7 @@ extern "C" { /** A generic unsigned ID number. These were entryIDs in back-bdb. * Preferably it should have the same size as a pointer. */ -#ifdef MDB_VL32 -typedef uint64_t MDB_ID; -#else typedef size_t MDB_ID; -#endif /** An IDL is an ID List, a sorted array of IDs. The first * element of the array is a counter for how many actual @@ -182,20 +178,6 @@ int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ); */ int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ); -#ifdef MDB_VL32 -typedef struct MDB_ID3 { - MDB_ID mid; /**< The ID */ - void *mptr; /**< The pointer */ - unsigned int mcnt; /**< Number of pages */ - unsigned int mref; /**< Refcounter */ -} MDB_ID3; - -typedef MDB_ID3 *MDB_ID3L; - -unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id ); -int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id ); - -#endif /* MDB_VL32 */ /** @} */ /** @} */ #ifdef __cplusplus diff --git a/contrib/db/liblmdb/mtest.c b/contrib/db/liblmdb/mtest.c index 9d15088b..6fc5840c 100644 --- a/contrib/db/liblmdb/mtest.c +++ b/contrib/db/liblmdb/mtest.c @@ -1,6 +1,6 @@ /* mtest.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/mtest2.c b/contrib/db/liblmdb/mtest2.c index eacbe59d..64b742aa 100644 --- a/contrib/db/liblmdb/mtest2.c +++ b/contrib/db/liblmdb/mtest2.c @@ -1,6 +1,6 @@ /* mtest2.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/mtest3.c b/contrib/db/liblmdb/mtest3.c index 9db79e62..81e4bbf9 100644 --- a/contrib/db/liblmdb/mtest3.c +++ b/contrib/db/liblmdb/mtest3.c @@ -1,6 +1,6 @@ /* mtest3.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/mtest4.c b/contrib/db/liblmdb/mtest4.c index 6df890e2..c355cf10 100644 --- a/contrib/db/liblmdb/mtest4.c +++ b/contrib/db/liblmdb/mtest4.c @@ -1,6 +1,6 @@ /* mtest4.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/mtest5.c b/contrib/db/liblmdb/mtest5.c index 14e3c0da..95793ec1 100644 --- a/contrib/db/liblmdb/mtest5.c +++ b/contrib/db/liblmdb/mtest5.c @@ -1,6 +1,6 @@ /* mtest5.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/mtest6.c b/contrib/db/liblmdb/mtest6.c index ae3c7f26..cb0d4d73 100644 --- a/contrib/db/liblmdb/mtest6.c +++ b/contrib/db/liblmdb/mtest6.c @@ -1,6 +1,6 @@ /* mtest6.c - memory-mapped database tester/toy */ /* - * Copyright 2011-2015 Howard Chu, Symas Corp. + * Copyright 2011-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/sample-bdb.txt b/contrib/db/liblmdb/sample-bdb.txt index 563807a2..97220f0e 100644 --- a/contrib/db/liblmdb/sample-bdb.txt +++ b/contrib/db/liblmdb/sample-bdb.txt @@ -3,7 +3,7 @@ * Do a line-by-line comparison of this and sample-mdb.txt */ /* - * Copyright 2012-2015 Howard Chu, Symas Corp. + * Copyright 2012-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/db/liblmdb/sample-mdb.txt b/contrib/db/liblmdb/sample-mdb.txt index 10a25687..1d20ed3d 100644 --- a/contrib/db/liblmdb/sample-mdb.txt +++ b/contrib/db/liblmdb/sample-mdb.txt @@ -3,7 +3,7 @@ * Do a line-by-line comparison of this and sample-bdb.txt */ /* - * Copyright 2012-2015 Howard Chu, Symas Corp. + * Copyright 2012-2018 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index c8bd69e1..c620daec 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -251,6 +251,21 @@ DISABLE_VS_WARNINGS(4100) custom_code; \ } + +#define CATCH_ENTRY_WITH_FORWARDING_EXCEPTION() } \ + catch(const std::exception& ex) \ +{ \ + LOG_ERROR("Exception at [" << LOCATION_SS << "], what=" << ex.what()); \ + throw std::runtime_error(std::string("[EXCEPTION FORWARDED]: ") + ex.what()); \ +} \ + catch(...) \ +{ \ + LOG_ERROR("Exception at [" << LOCATION_SS << "], generic unknown exception \"...\""); \ + throw std::runtime_error("[EXCEPTION FORWARDED]"); \ +} + + + #define NESTED_TRY_ENTRY() try { TRY_ENTRY(); #define NESTED_CATCH_ENTRY(location) \ diff --git a/src/common/db_backend_lmdb.cpp b/src/common/db_backend_lmdb.cpp index 4bac3eb7..f27a696d 100644 --- a/src/common/db_backend_lmdb.cpp +++ b/src/common/db_backend_lmdb.cpp @@ -7,8 +7,13 @@ #include "misc_language.h" #include "string_coding.h" #include "profile_tools.h" +#include "util.h" #define BUF_SIZE 1024 +#define DB_RESIZE_MIN_FREE_SIZE (100 * 1024 * 1024) // DB map size will grow if that much space left on DB +#define DB_RESIZE_MIN_MAX_SIZE (50 * 1024 * 1024) // Minimum DB map size (starting size) +#define DB_RESIZE_INCREMENT_SIZE (100 * 1024 * 1024) // Grow step size +#define DB_RESIZE_COMMITS_TO_CHECK 50 #define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); #define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); @@ -22,10 +27,13 @@ namespace tools { namespace db { - lmdb_db_backend::lmdb_db_backend() : m_penv(AUTO_VAL_INIT(m_penv)) + lmdb_db_backend::lmdb_db_backend() + : m_penv(AUTO_VAL_INIT(m_penv)) + , m_commits_count(0) { } + lmdb_db_backend::~lmdb_db_backend() { NESTED_TRY_ENTRY(); @@ -44,16 +52,17 @@ namespace tools res = mdb_env_set_maxdbs(m_penv, 15); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_maxdbs"); - res = mdb_env_set_mapsize(m_penv, cache_sz); - CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize"); - m_path = path_; #ifdef WIN32 m_path = epee::string_encoding::convert_ansii_to_utf8(m_path); #endif - res = mdb_env_open(m_penv, m_path.c_str(), MDB_NORDAHEAD , 0644); + CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path); + + res = mdb_env_open(m_penv, m_path.c_str(), MDB_NORDAHEAD /*| MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC*/, 0644); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_open, m_path=" << m_path); + + resize_if_needed(); return true; } @@ -102,6 +111,13 @@ namespace tools { LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3); CRITICAL_SECTION_LOCK(m_write_exclusive_lock); + + if (m_commits_count.fetch_add(1, std::memory_order_relaxed) % DB_RESIZE_COMMITS_TO_CHECK == DB_RESIZE_COMMITS_TO_CHECK - 1) + { + if (!resize_if_needed()) + m_commits_count.store(DB_RESIZE_COMMITS_TO_CHECK - 1, std::memory_order_relaxed); // if failed, try again on next commit + } + } PROFILE_FUNC("lmdb_db_backend::begin_transaction"); { @@ -326,6 +342,7 @@ namespace tools CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put"); return true; } + bool lmdb_db_backend::enumerate(container_handle h, i_db_callback* pcb) { CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate"); @@ -380,6 +397,42 @@ namespace tools } return true; } + + bool lmdb_db_backend::resize_if_needed() + { + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED in resize_if_needed()", LOG_LEVEL_3); + CRITICAL_REGION_LOCAL(m_write_exclusive_lock); + + if (have_tx()) + { + LOG_PRINT_RED("[DB " << m_path << "] : resize_if_needed(): Have txs on stack, unable to resize!", LOG_LEVEL_0); + return false; + } + + MDB_stat st = AUTO_VAL_INIT(st); + mdb_env_stat(m_penv, &st); + MDB_envinfo ei = AUTO_VAL_INIT(ei); + mdb_env_info(m_penv, &ei); + + uint64_t dirty_size = ei.me_last_pgno * st.ms_psize; + int64_t size_diff = ei.me_mapsize - dirty_size; + if (size_diff >= DB_RESIZE_MIN_FREE_SIZE && ei.me_mapsize >= DB_RESIZE_MIN_MAX_SIZE) + return true; // resize is not needed + + double gigabyte = 1024 * 1024 * 1024; + const uint64_t increment_size_pg_aligned = DB_RESIZE_INCREMENT_SIZE - (DB_RESIZE_INCREMENT_SIZE % st.ms_psize); + + // need to resize DB + uint64_t new_size = ei.me_mapsize - (ei.me_mapsize % increment_size_pg_aligned) + increment_size_pg_aligned; + + int res = mdb_env_set_mapsize(m_penv, new_size); + CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize"); + + LOG_PRINT_CYAN("[DB " << m_path << "] has grown: " << std::fixed << std::setprecision(2) << ei.me_mapsize / gigabyte << " GiB -> " << std::fixed << std::setprecision(2) << new_size / gigabyte << " GiB", LOG_LEVEL_0); + + return true; + } + } } diff --git a/src/common/db_backend_lmdb.h b/src/common/db_backend_lmdb.h index f41f3e35..987f61e1 100644 --- a/src/common/db_backend_lmdb.h +++ b/src/common/db_backend_lmdb.h @@ -38,7 +38,11 @@ namespace tools boost::recursive_mutex m_cs; boost::recursive_mutex m_write_exclusive_lock; std::map m_txs; // size_t -> count of nested read_only transactions + std::atomic m_commits_count; + bool pop_tx_entry(tx_entry& txe); + + public: lmdb_db_backend(); ~lmdb_db_backend(); @@ -60,6 +64,7 @@ namespace tools //------------------------------------------------------------------------------------- bool have_tx(); MDB_txn* get_current_tx(); + bool resize_if_needed(); }; } diff --git a/src/currency_core/basic_pow_helpers.cpp b/src/currency_core/basic_pow_helpers.cpp index 6b3e53f1..7c2275cc 100644 --- a/src/currency_core/basic_pow_helpers.cpp +++ b/src/currency_core/basic_pow_helpers.cpp @@ -36,7 +36,7 @@ namespace currency //------------------------------------------------------------------ int ethash_height_to_epoch(uint64_t height) { - return height / ETHASH_EPOCH_LENGTH; + return static_cast(height / ETHASH_EPOCH_LENGTH); } //-------------------------------------------------------------- crypto::hash ethash_epoch_to_seed(int epoch) @@ -51,7 +51,7 @@ namespace currency { int epoch = ethash_height_to_epoch(height); const auto& context = progpow::get_global_epoch_context_full(static_cast(epoch)); - auto res_eth = progpow::hash(context, height, *(ethash::hash256*)&block_header_hash, nonce); + auto res_eth = progpow::hash(context, static_cast(height), *(ethash::hash256*)&block_header_hash, nonce); crypto::hash result = currency::null_hash; memcpy(&result.data, &res_eth.final_hash, sizeof(res_eth.final_hash)); return result; diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index cde8316c..043b1cf7 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -58,7 +58,7 @@ using namespace currency; #define BLOCKCHAIN_STORAGE_OPTIONS_ID_CURRENT_BLOCK_CUMUL_SZ_LIMIT 0 #define BLOCKCHAIN_STORAGE_OPTIONS_ID_CURRENT_PRUNED_RS_HEIGHT 1 #define BLOCKCHAIN_STORAGE_OPTIONS_ID_LAST_WORKED_VERSION 2 -#define BLOCKCHAIN_STORAGE_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION 3 //mismatch here means full resync +#define BLOCKCHAIN_STORAGE_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION 3 //DON'T CHANGE THIS, if you need to resync db change BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION #define BLOCKCHAIN_STORAGE_OPTIONS_ID_STORAGE_MINOR_COMPATIBILITY_VERSION 4 //mismatch here means some reinitializations #define TARGETDATA_CACHE_SIZE DIFFICULTY_WINDOW + 10 @@ -68,6 +68,9 @@ using namespace currency; #else #define BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION 18000 #endif +#define BLOCK_POS_STRICT_SEQUENCE_LIMIT 20 + + DISABLE_VS_WARNINGS(4267) @@ -106,7 +109,9 @@ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::share m_current_fee_median(0), m_current_fee_median_effective_index(0), m_is_reorganize_in_process(false), - m_deinit_is_done(false) + m_deinit_is_done(false), + m_cached_next_pow_difficulty(0), + m_cached_next_pos_difficulty(0) { @@ -357,6 +362,16 @@ bool blockchain_storage::set_lost_tx_unmixable() //------------------------------------------------------------------ void blockchain_storage::patch_out_if_needed(txout_to_key& out, const crypto::hash& tx_id, uint64_t n) const { + static crypto::hash tx_id_1 = epee::string_tools::parse_tpod_from_hex_string("c2a2229d614e7c026433efbcfdbd0be1f68d9b419220336df3e2c209f5d57314"); + static crypto::hash tx_id_2 = epee::string_tools::parse_tpod_from_hex_string("647f936c6ffbd136f5c95d9a90ad554bdb4c01541c6eb5755ad40b984d80da67"); + + if (tx_id == tx_id_1 && n == 12) + { + out.mix_attr = CURRENCY_TO_KEY_OUT_FORCED_NO_MIX; + }else if(tx_id == tx_id_2 && n == 5) + { + out.mix_attr = CURRENCY_TO_KEY_OUT_FORCED_NO_MIX; + } } //------------------------------------------------------------------ void blockchain_storage::initialize_db_solo_options_values() @@ -967,12 +982,21 @@ wide_difficulty_type blockchain_storage::get_next_diff_conditional(bool pos) con wide_difficulty_type& dif = pos ? m_cached_next_pos_difficulty : m_cached_next_pow_difficulty; TIME_MEASURE_FINISH_PD(target_calculating_enum_blocks); TIME_MEASURE_START_PD(target_calculating_calc); - dif = next_difficulty(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + if (m_db_blocks.size() > m_core_runtime_config.hard_fork1_starts_after_height) + { + dif = next_difficulty_2(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + } + else + { + dif = next_difficulty_1(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + } + + TIME_MEASURE_FINISH_PD(target_calculating_calc); return dif; } //------------------------------------------------------------------ -wide_difficulty_type blockchain_storage::get_next_diff_conditional2(bool pos, const alt_chain_type& alt_chain, uint64_t split_height) const +wide_difficulty_type blockchain_storage::get_next_diff_conditional2(bool pos, const alt_chain_type& alt_chain, uint64_t split_height, const alt_block_extended_info& abei) const { CRITICAL_REGION_LOCAL(m_read_lock); std::vector timestamps; @@ -995,7 +1019,13 @@ wide_difficulty_type blockchain_storage::get_next_diff_conditional2(bool pos, co return true; }; enum_blockchain(cb, alt_chain, split_height); - return next_difficulty(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + + wide_difficulty_type diff = 0; + if(abei.height > m_core_runtime_config.hard_fork1_starts_after_height) + diff = next_difficulty_2(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + else + diff = next_difficulty_1(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + return diff; } //------------------------------------------------------------------ wide_difficulty_type blockchain_storage::get_cached_next_difficulty(bool pos) const @@ -1054,7 +1084,7 @@ wide_difficulty_type blockchain_storage::get_next_difficulty_for_alternative_cha commulative_difficulties.push_back(m_db_blocks[i]->cumulative_diff_precise); } - return next_difficulty(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET:DIFFICULTY_POW_TARGET); + return next_difficulty_1(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET:DIFFICULTY_POW_TARGET); } //------------------------------------------------------------------ bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height, bool pos) const @@ -1071,9 +1101,29 @@ bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t CHECK_AND_ASSERT_MES(b.miner_tx.vin[1].type() == typeid(txin_to_key), false, "coinstake transaction in the block has the wrong type"); } - CHECK_AND_ASSERT_MES(get_tx_unlock_time(b.miner_tx) == height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW, - false, - "coinbase transaction has wrong unlock time: " << get_tx_unlock_time(b.miner_tx) << ", expected: " << height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + if (height > m_core_runtime_config.hard_fork1_starts_after_height) + { + // new rules that allow different unlock time in coinbase outputs + uint64_t max_unlock_time = 0; + uint64_t min_unlock_time = 0; + bool r = get_tx_max_min_unlock_time(b.miner_tx, max_unlock_time, min_unlock_time); + CHECK_AND_ASSERT_MES(r && min_unlock_time >= height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW, + false, + "coinbase transaction has wrong min_unlock_time: " << min_unlock_time << ", expected: " << height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + } + else + { + //------------------------------------------------------------------ + //bool blockchain_storage:: + // pre-hard fork rules that don't allow different unlock time in coinbase outputs + uint64_t max_unlock_time = 0; + uint64_t min_unlock_time = 0; + bool r = get_tx_max_min_unlock_time(b.miner_tx, max_unlock_time, min_unlock_time); + CHECK_AND_ASSERT_MES(r && max_unlock_time == min_unlock_time && min_unlock_time == height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW, + false, + "coinbase transaction has wrong min_unlock_time: " << min_unlock_time << ", expected: " << height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + } + //check outs overflow if(!check_outs_overflow(b.miner_tx)) @@ -1192,7 +1242,12 @@ bool blockchain_storage::create_block_template(block& b, size_t median_size; boost::multiprecision::uint128_t already_generated_coins; CRITICAL_REGION_BEGIN(m_read_lock); - b.major_version = CURRENT_BLOCK_MAJOR_VERSION; + height = m_db_blocks.size(); + if(height <= m_core_runtime_config.hard_fork1_starts_after_height) + b.major_version = BLOCK_MAJOR_VERSION_INITAL; + else + b.major_version = CURRENT_BLOCK_MAJOR_VERSION; + b.minor_version = CURRENT_BLOCK_MINOR_VERSION; b.prev_id = get_top_block_id(); b.timestamp = m_core_runtime_config.get_core_time(); @@ -1209,7 +1264,7 @@ bool blockchain_storage::create_block_template(block& b, CHECK_AND_ASSERT_MES(diffic, false, "difficulty owverhead."); - height = m_db_blocks.size(); + median_size = m_db_current_block_cumul_sz_limit / 2; already_generated_coins = m_db_blocks.back()->already_generated_coins; @@ -1470,6 +1525,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: alt_block_extended_info abei = AUTO_VAL_INIT(abei); abei.bl = b; + abei.timestamp = m_core_runtime_config.get_core_time(); abei.height = alt_chain.size() ? it_prev->second.height + 1 : *ptr_main_prev + 1; CHECK_AND_ASSERT_MES_CUSTOM(coinbase_height == abei.height, false, bvc.m_verification_failed = true, "block coinbase height doesn't match with altchain height, declined"); uint64_t connection_height = alt_chain.size() ? alt_chain.front()->second.height:abei.height; @@ -1495,8 +1551,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: CHECK_AND_ASSERT_MES_CUSTOM(!(pos_block && abei.height < m_core_runtime_config.pos_minimum_heigh), false, bvc.m_verification_failed = true, "PoS block is not allowed on this height"); - //wide_difficulty_type current_diff = get_next_difficulty_for_alternative_chain(alt_chain, bei, pos_block); - wide_difficulty_type current_diff = get_next_diff_conditional2(pos_block, alt_chain, connection_height); + wide_difficulty_type current_diff = get_next_diff_conditional2(pos_block, alt_chain, connection_height, abei); CHECK_AND_ASSERT_MES_CUSTOM(current_diff, false, bvc.m_verification_failed = true, "!!!!!!! DIFFICULTY OVERHEAD !!!!!!!"); @@ -1555,7 +1610,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: if (abei.height >= m_core_runtime_config.pos_minimum_heigh) cumulative_diff_delta = correct_difficulty_with_sequence_factor(sequence_factor, cumulative_diff_delta); - if (abei.height > BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION && pos_block && sequence_factor > 20) + if (abei.height > BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION && pos_block && sequence_factor > BLOCK_POS_STRICT_SEQUENCE_LIMIT) { LOG_PRINT_RED_L0("Alternative block " << id << " @ " << abei.height << " has too big sequence factor: " << sequence_factor << ", rejected"); bvc.m_verification_failed = true; @@ -1563,9 +1618,15 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: } abei.cumulative_diff_adjusted += cumulative_diff_delta; - - abei.cumulative_diff_precise = get_last_alt_x_block_cumulative_precise_difficulty(alt_chain, abei.height, pos_block); + wide_difficulty_type last_x_cumul_dif_precise_adj = 0; + abei.cumulative_diff_precise = get_last_alt_x_block_cumulative_precise_difficulty(alt_chain, abei.height-1, pos_block, last_x_cumul_dif_precise_adj); abei.cumulative_diff_precise += current_diff; + ////////////////////////////////////////////////////////////////////////// + + wide_difficulty_type diff_precise_adj = correct_difficulty_with_sequence_factor(sequence_factor, current_diff); + abei.cumulative_diff_precise_adjusted = last_x_cumul_dif_precise_adj + diff_precise_adj; + + ////////////////////////////////////////////////////////////////////////// #ifdef _DEBUG auto i_dres = m_alternative_chains.find(id); @@ -1600,7 +1661,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: << ENDL << "HEIGHT " << abei.height << ", difficulty: " << abei.difficulty << ", cumul_diff_precise: " << abei.cumulative_diff_precise << ", cumul_diff_adj: " << abei.cumulative_diff_adjusted << " (current mainchain cumul_diff_adj: " << m_db_blocks.back()->cumulative_diff_adjusted << ", ki lookup total: " << ki_lookup_total <<")" , LOG_LEVEL_0); - if (is_reorganize_required(*m_db_blocks.back(), abei, proof)) + if (is_reorganize_required(*m_db_blocks.back(), alt_chain, proof)) { auto a = epee::misc_utils::create_scope_leave_handler([&]() { m_is_reorganize_in_process = false; }); CHECK_AND_ASSERT_THROW_MES(!m_is_reorganize_in_process, "Detected recursive reorganzie"); @@ -1616,6 +1677,11 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: return r; } bvc.added_to_altchain = true; + + //protect ourself from altchains container flood + if (m_alternative_chains.size() > m_core_runtime_config.max_alt_blocks) + prune_aged_alt_blocks(); + return true; }else { @@ -1641,26 +1707,111 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: CATCH_ENTRY_CUSTOM("blockchain_storage::handle_alternative_block", bvc.m_verification_failed = true, false); } //------------------------------------------------------------------ -bool blockchain_storage::is_reorganize_required(const block_extended_info& main_chain_bei, const block_extended_info& alt_chain_bei, const crypto::hash& proof_alt) +wide_difficulty_type blockchain_storage::get_x_difficulty_after_height(uint64_t height, bool is_pos) { - if (main_chain_bei.cumulative_diff_adjusted < alt_chain_bei.cumulative_diff_adjusted) - return true; - else if (main_chain_bei.cumulative_diff_adjusted > alt_chain_bei.cumulative_diff_adjusted) - return false; - else // main_chain_bei.cumulative_diff_adjusted == alt_chain_bei.cumulative_diff_adjusted + CRITICAL_REGION_LOCAL(m_read_lock); + CHECK_AND_ASSERT_THROW_MES(height < m_db_blocks.size(), "Internal error: condition failed: height (" << height << ") < m_db_blocks.size() " << m_db_blocks.size()); + wide_difficulty_type diff = 0; + for (uint64_t i = height + 1; i != m_db_blocks.size(); i++) { - if (!is_pos_block(main_chain_bei.bl)) - return false; // do not reorganize on the same cummul diff if it's a PoW block + auto bei_ptr = m_db_blocks[i]; + if (is_pos_block(bei_ptr->bl) == is_pos) + { + diff = bei_ptr->difficulty; + break; + } + } + if (diff == 0) + { + //never met x type of block, that meanst that difficulty is current + diff = get_cached_next_difficulty(is_pos); + } + return diff; +} +//------------------------------------------------------------------ +bool blockchain_storage::is_reorganize_required(const block_extended_info& main_chain_bei, const alt_chain_type& alt_chain, const crypto::hash& proof_alt) +{ + //alt_chain - back is latest(top), first - connection with main chain + const block_extended_info& alt_chain_bei = alt_chain.back()->second; + const block_extended_info& connection_point = alt_chain.front()->second; - //in case of simultaneous PoS blocks are happened on the same height (quite common for PoS) - //we also try to weight them to guarantee consensus in network - if (std::memcmp(&main_chain_bei.stake_hash, &proof_alt, sizeof(main_chain_bei.stake_hash)) >= 0) + if (connection_point.height <= m_core_runtime_config.hard_fork1_starts_after_height) + { + //use pre-hard fork, old-style comparing + if (main_chain_bei.cumulative_diff_adjusted < alt_chain_bei.cumulative_diff_adjusted) + return true; + else if (main_chain_bei.cumulative_diff_adjusted > alt_chain_bei.cumulative_diff_adjusted) return false; + else // main_chain_bei.cumulative_diff_adjusted == alt_chain_bei.cumulative_diff_adjusted + { + if (!is_pos_block(main_chain_bei.bl)) + return false; // do not reorganize on the same cummul diff if it's a PoW block + + //in case of simultaneous PoS blocks are happened on the same height (quite common for PoS) + //we also try to weight them to guarantee consensus in network + if (std::memcmp(&main_chain_bei.stake_hash, &proof_alt, sizeof(main_chain_bei.stake_hash)) >= 0) + return false; + + LOG_PRINT_L2("[is_reorganize_required]:TRUE, \"by order of memcmp\" main_stake_hash:" << &main_chain_bei.stake_hash << ", alt_stake_hash" << proof_alt); + return true; + } + } + else if (alt_chain_bei.height > m_core_runtime_config.hard_fork1_starts_after_height) + { + //new rules, applied after HARD_FORK_1 + //to learn this algo please read https://github.com/hyle-team/docs/blob/master/zano/PoS_Analysis_and_improvements_proposal.pdf + + wide_difficulty_type difficulty_pos_at_split_point = get_x_difficulty_after_height(connection_point.height - 1, true); + wide_difficulty_type difficulty_pow_at_split_point = get_x_difficulty_after_height(connection_point.height - 1, false); + + difficulties main_cumul_diff = AUTO_VAL_INIT(main_cumul_diff); + difficulties alt_cumul_diff = AUTO_VAL_INIT(alt_cumul_diff); + //we use get_last_alt_x_block_cumulative_precise_adj_difficulty for getting both alt chain and main chain diff of given block types + + wide_difficulty_type alt_pos_diff_end = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain, alt_chain_bei.height, true); + wide_difficulty_type alt_pos_diff_begin = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), connection_point.height-1, true); + alt_cumul_diff.pos_diff = alt_pos_diff_end - alt_pos_diff_begin; - LOG_PRINT_L2("[is_reorganize_required]:TRUE, \"by order of memcmp\" main_stake_hash:" << &main_chain_bei.stake_hash << ", alt_stake_hash" << proof_alt); - return true; + wide_difficulty_type alt_pow_diff_end = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain, alt_chain_bei.height, false); + wide_difficulty_type alt_pow_diff_begin = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), connection_point.height - 1, false); + alt_cumul_diff.pow_diff = alt_pow_diff_end - alt_pow_diff_begin; + + wide_difficulty_type main_pos_diff_end = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), m_db_blocks.size()-1, true); + wide_difficulty_type main_pos_diff_begin = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), connection_point.height - 1, true); + main_cumul_diff.pos_diff = main_pos_diff_end - main_pos_diff_begin; + + wide_difficulty_type main_pow_diff_end = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), m_db_blocks.size() - 1, false); + wide_difficulty_type main_pow_diff_begin = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), connection_point.height - 1, false); + main_cumul_diff.pow_diff = main_pow_diff_end - main_pow_diff_begin; + + //TODO: measurment of precise cumulative difficult + wide_difficulty_type alt = get_a_to_b_relative_cumulative_difficulty(difficulty_pos_at_split_point, difficulty_pow_at_split_point, alt_cumul_diff, main_cumul_diff); + wide_difficulty_type main = get_a_to_b_relative_cumulative_difficulty(difficulty_pos_at_split_point, difficulty_pow_at_split_point, main_cumul_diff, alt_cumul_diff); + + if (main < alt) + return true; + else if (main > alt) + return false; + else + { + if (!is_pos_block(main_chain_bei.bl)) + return false; // do not reorganize on the same cummul diff if it's a PoW block + + //in case of simultaneous PoS blocks are happened on the same height (quite common for PoS) + //we also try to weight them to guarantee consensus in network + if (std::memcmp(&main_chain_bei.stake_hash, &proof_alt, sizeof(main_chain_bei.stake_hash)) >= 0) + return false; + + LOG_PRINT_L2("[is_reorganize_required]:TRUE, \"by order of memcmp\" main_stake_hash:" << &main_chain_bei.stake_hash << ", alt_stake_hash" << proof_alt); + return true; + } + } + else + { + ASSERT_MES_AND_THROW("Unknown version of block"); } } + //------------------------------------------------------------------ bool blockchain_storage::pre_validate_relayed_block(block& bl, block_verification_context& bvc, const crypto::hash& id)const { @@ -2101,7 +2252,7 @@ bool blockchain_storage::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPU return false; //check if transaction is unlocked - if (!is_tx_spendtime_unlocked(get_tx_unlock_time(tx))) + if (!is_tx_spendtime_unlocked(get_tx_unlock_time(tx, out_ptr->out_no))) return false; //use appropriate mix_attr out @@ -2364,7 +2515,7 @@ bool blockchain_storage::forecast_difficulty(std::vector>& blocks) const +{ + uint64_t count = 0; + bool looking_for_a_pos = true; + for (uint64_t i = m_db_blocks.size() - 1; i != 0; --i) + { + auto block_ptr = m_db_blocks[i]; + if (is_pos_block(block_ptr->bl) == pos_blocks) + { + blocks.push_back(block_ptr); + ++count; + if (count >= n) + break; + } + } +} +//------------------------------------------------------------------ +void blockchain_storage::print_last_n_difficulty_numbers(uint64_t n) const +{ + + std::stringstream ss; + std::list> pos_blocks; + std::list> pow_blocks; + + get_last_n_x_blocks(n, true, pos_blocks); + get_last_n_x_blocks(n, false, pow_blocks); + ss << "PoS blocks difficulty:" << ENDL; + for (auto& bl_ptr : pos_blocks) + { + ss << bl_ptr->difficulty << ENDL; + } + + ss << "PoW blocks difficulty:" << ENDL; + for (auto& bl_ptr : pow_blocks) + { + ss << bl_ptr->difficulty << ENDL; + } + LOG_PRINT_L0("LAST BLOCKS:" << ss.str()); +} +//------------------------------------------------------------------ void blockchain_storage::print_blockchain_outs_stat() const { LOG_ERROR("NOT IMPLEMENTED YET"); @@ -3268,6 +3459,8 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const TIME_MEASURE_START_PD(tx_append_is_expired); CHECK_AND_ASSERT_MES(!is_tx_expired(tx), false, "Transaction can't be added to the blockchain since it's already expired. tx expiration time: " << get_tx_expiration_time(tx) << ", blockchain median time: " << get_tx_expiration_median()); TIME_MEASURE_FINISH_PD_COND(need_to_profile, tx_append_is_expired); + + CHECK_AND_ASSERT_MES(validate_tx_for_hardfork_specific_terms(tx, tx_id, bl_height), false, "tx " << tx_id << ": hardfork-specific validation failed"); TIME_MEASURE_START_PD(tx_process_extra); bool r = process_blockchain_tx_extra(tx); @@ -3280,7 +3473,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const TIME_MEASURE_START_PD(tx_process_inputs); - BOOST_FOREACH(const txin_v& in, tx.vin) + for(const txin_v& in : tx.vin) { if(!boost::apply_visitor(add_transaction_input_visitor(*this, m_db_spent_keys, tx_id, bl_id, bl_height), in)) { @@ -3360,7 +3553,7 @@ bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std:: bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) const { CRITICAL_REGION_LOCAL(m_read_lock); - bool res = check_tx_inputs(tx, tx_prefix_hash, &max_used_block_height); + bool res = check_tx_inputs(tx, tx_prefix_hash, max_used_block_height); if(!res) return false; CHECK_AND_ASSERT_MES(max_used_block_height < m_db_blocks.size(), false, "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_db_blocks.size()); get_block_hash(m_db_blocks[max_used_block_height]->bl, max_used_block_id); @@ -3604,19 +3797,16 @@ bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) const return false; } //------------------------------------------------------------------ -// bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) const -// { -// TIME_MEASURE_START_PD(tx_check_inputs_prefix_hash); -// crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); -// TIME_MEASURE_FINISH_PD(tx_check_inputs_prefix_hash); -// return check_tx_inputs(tx, tx_prefix_hash, pmax_used_block_height); -// } +bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash) const +{ + uint64_t stub = 0; + return check_tx_inputs(tx, tx_prefix_hash, stub); +} //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) const +bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& max_used_block_height) const { size_t sig_index = 0; - if(pmax_used_block_height) - *pmax_used_block_height = 0; + max_used_block_height = 0; std::vector sig_stub; const std::vector* psig = &sig_stub; @@ -3642,7 +3832,8 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::ha return false; } TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_kimage_check); - if (!check_tx_input(tx, sig_index, in_to_key, tx_prefix_hash, *psig, pmax_used_block_height)) + uint64_t max_unlock_time = 0; + if (!check_tx_input(tx, sig_index, in_to_key, tx_prefix_hash, *psig, max_used_block_height, max_unlock_time)) { LOG_ERROR("Failed to validate input #" << sig_index << " tx: " << tx_prefix_hash); return false; @@ -3651,7 +3842,7 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::ha else if (txin.type() == typeid(txin_multisig)) { const txin_multisig& in_ms = boost::get(txin); - if (!check_tx_input(tx, sig_index, in_ms, tx_prefix_hash, *psig, pmax_used_block_height)) + if (!check_tx_input(tx, sig_index, in_ms, tx_prefix_hash, *psig, max_used_block_height)) { LOG_ERROR("Failed to validate multisig input #" << sig_index << " (ms out id: " << in_ms.multisig_out_id << ") in tx: " << tx_prefix_hash); return false; @@ -3682,14 +3873,14 @@ bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) const } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) const +bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height, uint64_t& max_unlock_time) const { CRITICAL_REGION_LOCAL(m_read_lock); //TIME_MEASURE_START_PD(tx_check_inputs_loop_ch_in_get_keys_loop); std::vector output_keys; - if(!get_output_keys_for_input_with_checks(txin, output_keys, pmax_related_block_height)) + if(!get_output_keys_for_input_with_checks(tx, txin, output_keys, max_related_block_height, max_unlock_time)) { LOG_PRINT_L0("Failed to get output keys for input #" << in_index << " (amount = " << print_money(txin.amount) << ", key_offset.size = " << txin.key_offsets.size() << ")"); return false; @@ -3708,7 +3899,7 @@ bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, // 1) source tx unlock time validity // 2) mixin restrictions // 3) general gindex/ref_by_id corectness -bool blockchain_storage::get_output_keys_for_input_with_checks(const txin_to_key& txin, std::vector& output_keys, uint64_t* pmax_related_block_height /* = NULL */) const +bool blockchain_storage::get_output_keys_for_input_with_checks(const transaction& tx, const txin_to_key& txin, std::vector& output_keys, uint64_t& max_related_block_height, uint64_t& max_unlock_time) const { CRITICAL_REGION_LOCAL(m_read_lock); @@ -3716,17 +3907,30 @@ bool blockchain_storage::get_output_keys_for_input_with_checks(const txin_to_key { std::vector& m_results_collector; const blockchain_storage& m_bch; + uint64_t& m_max_unlock_time; outputs_visitor(std::vector& results_collector, - const blockchain_storage& bch) :m_results_collector(results_collector), m_bch(bch) + const blockchain_storage& bch, + uint64_t& max_unlock_time) :m_results_collector(results_collector), m_bch(bch), m_max_unlock_time(max_unlock_time) {} - bool handle_output(const transaction& tx, const tx_out& out) + bool handle_output(const transaction& source_tx, const transaction& validated_tx, const tx_out& out, uint64_t out_i) { //check tx unlock time - if (!m_bch.is_tx_spendtime_unlocked(get_tx_unlock_time(tx))) + uint64_t source_out_unlock_time = get_tx_unlock_time(source_tx, out_i); + //let coinbase sources for PoS block to have locked inputs, the outputs supposed to be locked same way, except the reward + if (is_coinbase(validated_tx) && is_pos_block(validated_tx)) { - LOG_PRINT_L0("One of outputs for one of inputs have wrong tx.unlock_time = " << get_tx_unlock_time(tx)); - return false; + if (source_out_unlock_time > m_max_unlock_time) + m_max_unlock_time = source_out_unlock_time; } + else + { + if (!m_bch.is_tx_spendtime_unlocked(source_out_unlock_time)) + { + LOG_PRINT_L0("One of outputs for one of inputs have wrong tx.unlock_time = " << get_tx_unlock_time(source_tx, out_i)); + return false; + } + } + if(out.target.type() != typeid(txout_to_key)) { @@ -3739,8 +3943,8 @@ bool blockchain_storage::get_output_keys_for_input_with_checks(const txin_to_key } }; - outputs_visitor vi(output_keys, *this); - return scan_outputkeys_for_indexes(txin, vi, pmax_related_block_height); + outputs_visitor vi(output_keys, *this, max_unlock_time); + return scan_outputkeys_for_indexes(tx, txin, vi, max_related_block_height); } //------------------------------------------------------------------ // Note: this function can be used for checking to_key inputs against either main chain or alt chain, that's why it has output_keys_ptrs parameter @@ -3813,7 +4017,7 @@ bool blockchain_storage::check_ms_input(const transaction& tx, size_t in_index, #define LOC_CHK(cond, msg) CHECK_AND_ASSERT_MES(cond, false, "ms input check failed: ms_id: " << txin.multisig_out_id << ", input #" << in_index << " in tx " << tx_prefix_hash << ", refers to ms output #" << out_n << " in source tx " << get_transaction_hash(source_tx) << ENDL << msg) CRITICAL_REGION_LOCAL(m_read_lock); - uint64_t unlock_time = get_tx_unlock_time(source_tx); + uint64_t unlock_time = get_tx_unlock_time(source_tx, out_n); LOC_CHK(is_tx_spendtime_unlocked(unlock_time), "Source transaction is LOCKED! unlock_time: " << unlock_time << ", now is " << m_core_runtime_config.get_core_time() << ", blockchain size is " << get_current_blockchain_size()); LOC_CHK(source_tx.vout.size() > out_n, "internal error: out_n==" << out_n << " is out-of-bounds of source_tx.vout, size=" << source_tx.vout.size()); @@ -3893,7 +4097,7 @@ bool blockchain_storage::check_ms_input(const transaction& tx, size_t in_index, } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) const +bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height) const { CRITICAL_REGION_LOCAL(m_read_lock); @@ -3917,8 +4121,7 @@ bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, if (!check_ms_input(tx, in_index, txin, tx_prefix_hash, sig, source_tx_ptr->tx, n)) return false; - if (pmax_related_block_height != nullptr) - *pmax_related_block_height = source_tx_ptr->m_keeper_block_height; + max_related_block_height = source_tx_ptr->m_keeper_block_height; return true; #undef LOC_CHK @@ -4066,6 +4269,12 @@ bool blockchain_storage::prune_aged_alt_blocks() CRITICAL_REGION_LOCAL1(m_alternative_chains_lock); uint64_t current_height = get_current_blockchain_size(); + size_t count_to_delete = 0; + if(m_alternative_chains.size() > m_core_runtime_config.max_alt_blocks) + count_to_delete = m_alternative_chains.size() - m_core_runtime_config.max_alt_blocks; + + std::map alts_to_delete; + for(auto it = m_alternative_chains.begin(); it != m_alternative_chains.end();) { if (current_height > it->second.height && current_height - it->second.height > CURRENCY_ALT_BLOCK_LIVETIME_COUNT) @@ -4074,9 +4283,28 @@ bool blockchain_storage::prune_aged_alt_blocks() } else { + if (count_to_delete) + { + if (!alts_to_delete.size()) + alts_to_delete[it->second.timestamp] = it; + else + { + if (it->second.timestamp >= alts_to_delete.rbegin()->first) + alts_to_delete[it->second.timestamp] = it; + + if (alts_to_delete.size() > count_to_delete) + alts_to_delete.erase(alts_to_delete.begin()); + } + } + ++it; } } + //now, if there was count_to_delete we should erase most oldest entries of altblocks + for (auto& itd : alts_to_delete) + { + m_alternative_chains.erase(itd.second); + } return true; } @@ -4156,6 +4384,58 @@ void blockchain_storage::get_pos_mining_estimate(uint64_t amount_coins, estimate_result = current_amount; } //------------------------------------------------------------------ +bool blockchain_storage::validate_tx_for_hardfork_specific_terms(const transaction& tx, const crypto::hash& tx_id, uint64_t block_height) const +{ + if (block_height <= m_core_runtime_config.hard_fork1_starts_after_height) + { + // before hardfork 1 + + for (const auto& el : tx.extra) + { + // etc_tx_details_unlock_time2 is not allowed in txs in blocks prior to hardfork 1 + CHECK_AND_ASSERT_MES(el.type() != typeid(etc_tx_details_unlock_time2), false, "tx " << tx_id << " contains etc_tx_details_unlock_time2 which is not allowed on height " << block_height); + } + return true; + } + + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::validate_pos_coinbase_outs_unlock_time(const transaction& miner_tx, uint64_t staked_amount, uint64_t max_unlock_time)const +{ + uint64_t major_unlock_time = get_tx_x_detail(miner_tx); + if (major_unlock_time) + { + //if there was etc_tx_details_unlock_time present in tx, then ignore etc_tx_details_unlock_time2 + if (major_unlock_time < max_unlock_time) + return false; + else + return true; + } + + CHECK_AND_ASSERT_MES(get_block_height(miner_tx) > m_core_runtime_config.hard_fork1_starts_after_height, false, "error in block [" << get_block_height(miner_tx) << "] etc_tx_details_unlock_time2 can exist only after hard fork point : " << m_core_runtime_config.hard_fork1_starts_after_height); + + //etc_tx_details_unlock_time2 can be kept only after hard_fork_1 point + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(miner_tx.extra, ut2); + CHECK_AND_ASSERT_MES(ut2.unlock_time_array.size() == miner_tx.vout.size(), false, "ut2.unlock_time_array.size()<" << ut2.unlock_time_array.size() + << "> != miner_tx.vout.size()<" << miner_tx.vout.size() << ">"); + + uint64_t amount_of_coins_in_unlock_in_range = 0; + + for (uint64_t i = 0; i != miner_tx.vout.size(); i++) + { + if (ut2.unlock_time_array[i] >= max_unlock_time) + amount_of_coins_in_unlock_in_range += miner_tx.vout[i].amount; + } + + if (amount_of_coins_in_unlock_in_range >= staked_amount) + return true; + LOG_ERROR("amount_of_coins_in_unlock_in_range<" << amount_of_coins_in_unlock_in_range << "> is less then staked_amount<" << staked_amount); + return false; + +} +//------------------------------------------------------------------ bool blockchain_storage::validate_pos_block(const block& b, wide_difficulty_type basic_diff, uint64_t& amount, @@ -4185,11 +4465,11 @@ bool blockchain_storage::validate_pos_block(const block& b, //check actual time if it there uint64_t actual_ts = get_actual_timestamp(b); - if ((actual_ts > b.timestamp && actual_ts - b.timestamp > POS_MAC_ACTUAL_TIMESTAMP_TO_MINED) || - (actual_ts < b.timestamp && b.timestamp - actual_ts > POS_MAC_ACTUAL_TIMESTAMP_TO_MINED) + if ((actual_ts > b.timestamp && actual_ts - b.timestamp > POS_MAX_ACTUAL_TIMESTAMP_TO_MINED) || + (actual_ts < b.timestamp && b.timestamp - actual_ts > POS_MAX_ACTUAL_TIMESTAMP_TO_MINED) ) { - LOG_PRINT_L0("PoS block actual timestamp " << actual_ts << " differs from b.timestamp " << b.timestamp << " by " << ((int64_t)actual_ts - (int64_t)b.timestamp) << " s, it's more than allowed " << POS_MAC_ACTUAL_TIMESTAMP_TO_MINED << " s."); + LOG_PRINT_L0("PoS block actual timestamp " << actual_ts << " differs from b.timestamp " << b.timestamp << " by " << ((int64_t)actual_ts - (int64_t)b.timestamp) << " s, it's more than allowed " << POS_MAX_ACTUAL_TIMESTAMP_TO_MINED << " s."); return false; } @@ -4232,8 +4512,23 @@ bool blockchain_storage::validate_pos_block(const block& b, { // Do coinstake input validation for main chain only. // Txs in alternative PoS blocks (including miner_tx) are validated by validate_alt_block_txs() - r = check_tx_input(b.miner_tx, 1, coinstake_in, id, b.miner_tx.signatures[0], &max_related_block_height); + uint64_t max_unlock_time = 0; + r = check_tx_input(b.miner_tx, 1, coinstake_in, id, b.miner_tx.signatures[0], max_related_block_height, max_unlock_time); CHECK_AND_ASSERT_MES(r, false, "Failed to validate coinstake input in miner tx, block_id = " << get_block_hash(b)); + + if (get_block_height(b) > m_core_runtime_config.hard_fork1_starts_after_height) + { + uint64_t last_pow_h = get_last_x_block_height(false); + CHECK_AND_ASSERT_MES(max_related_block_height <= last_pow_h, false, "Failed to failed to validate coinbase in pos block, condition failed: max_related_block_height(" << max_related_block_height << ") < last_pow_h(" << last_pow_h << ")"); + //let's check that coinbase amount and unlock time + r = validate_pos_coinbase_outs_unlock_time(b.miner_tx, coinstake_in.amount, max_unlock_time); + CHECK_AND_ASSERT_MES(r, false, "Failed to validate_pos_coinbase_outs_unlock_time() in miner tx, block_id = " << get_block_hash(b) + << "max_unlock_time=" << max_unlock_time); + } + else + { + CHECK_AND_ASSERT_MES(is_tx_spendtime_unlocked(max_unlock_time), false, "Failed to failed to validate coinbase in pos block, condition failed: is_tx_spendtime_unlocked(max_unlock_time)(" << max_unlock_time << ")"); + } } uint64_t block_height = for_altchain ? split_height + alt_chain.size() : m_db_blocks.size(); @@ -4341,13 +4636,23 @@ uint64_t blockchain_storage::get_last_x_block_height(bool pos) const return 0; } //------------------------------------------------------------------ -wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise_difficulty(alt_chain_type& alt_chain, uint64_t block_height, bool pos) const +wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise_adj_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos) const { - uint64_t main_chain_first_block = block_height - 1; + wide_difficulty_type res = 0; + get_last_alt_x_block_cumulative_precise_difficulty(alt_chain, block_height, pos, res); + return res; +} +//------------------------------------------------------------------ +wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos, wide_difficulty_type& cumulative_diff_precise_adj) const +{ + uint64_t main_chain_first_block = block_height; for (auto it = alt_chain.rbegin(); it != alt_chain.rend(); it++) { if (is_pos_block((*it)->second.bl) == pos) + { + cumulative_diff_precise_adj = (*it)->second.cumulative_diff_precise_adjusted; return (*it)->second.cumulative_diff_precise; + } main_chain_first_block = (*it)->second.height - 1; } @@ -4358,8 +4663,12 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise for (uint64_t i = main_chain_first_block; i != 0; i--) { if (is_pos_block(m_db_blocks[i]->bl) == pos) + { + cumulative_diff_precise_adj = m_db_blocks[i]->cumulative_diff_precise_adjusted; return m_db_blocks[i]->cumulative_diff_precise; + } } + cumulative_diff_precise_adj = 0; return 0; } //------------------------------------------------------------------ @@ -4415,7 +4724,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if (is_pos_bl) { bool r = validate_pos_block(bl, current_diffic, pos_coinstake_amount, this_coin_diff, proof_hash, id, false); - CHECK_AND_ASSERT_MES_CUSTOM(r, false, bvc.m_verification_failed = true, "validate_pos_block failed!!"); + CHECK_AND_ASSERT_MES_CUSTOM(r, false, bvc.m_verification_failed = true, "validate_pos_block failed!!"); } else { @@ -4476,6 +4785,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt size_t tx_processed_count = 0; uint64_t fee_summary = 0; + uint64_t burned_coins = 0; for(const crypto::hash& tx_id : bl.tx_hashes) { @@ -4515,6 +4825,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt return false; } TIME_MEASURE_FINISH_PD(tx_check_inputs_time); + burned_coins += get_burned_amount(tx); TIME_MEASURE_START_PD(tx_prapare_append); uint64_t current_bc_size = get_current_blockchain_size(); @@ -4574,6 +4885,12 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if (is_pos_bl) bei.stake_hash = proof_hash; + + + ////////////////////////////////////////////////////////////////////////// + + //old style cumulative difficulty collecting + //precise difficulty - difficulty used to calculate next difficulty uint64_t last_x_h = get_last_x_block_height(is_pos_bl); if (!last_x_h) @@ -4597,11 +4914,10 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt size_t sequence_factor = get_current_sequence_factor(is_pos_bl); if (bei.height >= m_core_runtime_config.pos_minimum_heigh) cumulative_diff_delta = correct_difficulty_with_sequence_factor(sequence_factor, cumulative_diff_delta); - - if (bei.height > BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION && is_pos_bl && sequence_factor > 20) + + if (bei.height > BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION && is_pos_bl && sequence_factor > BLOCK_POS_STRICT_SEQUENCE_LIMIT) { - LOG_PRINT_L0("Block with id: " << id - << " has too big sequence_factor = " << sequence_factor); + LOG_PRINT_RED_L0("Block " << id << " @ " << bei.height << " has too big sequence factor: " << sequence_factor << ", rejected"); purge_block_data_from_blockchain(bl, tx_processed_count); bvc.m_verification_failed = true; return false; @@ -4609,8 +4925,22 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt bei.cumulative_diff_adjusted += cumulative_diff_delta; + ////////////////////////////////////////////////////////////////////////// + // rebuild cumulative_diff_precise_adjusted for whole period + wide_difficulty_type diff_precise_adj = correct_difficulty_with_sequence_factor(sequence_factor, current_diffic); + bei.cumulative_diff_precise_adjusted = last_x_h ? m_db_blocks[last_x_h]->cumulative_diff_precise_adjusted + diff_precise_adj : diff_precise_adj; + + ////////////////////////////////////////////////////////////////////////// + //etc - bei.already_generated_coins = already_generated_coins + base_reward; + if (already_generated_coins < burned_coins) + { + LOG_ERROR("Condition failed: already_generated_coins(" << already_generated_coins << ") >= burned_coins(" << burned_coins << ")"); + purge_block_data_from_blockchain(bl, tx_processed_count); + bvc.m_verification_failed = true; + return false; + } + bei.already_generated_coins = already_generated_coins - burned_coins + base_reward; auto blocks_index_ptr = m_db_blocks_index.get(id); if (blocks_index_ptr) @@ -4783,13 +5113,26 @@ bool blockchain_storage::update_next_comulative_size_limit() return true; } //------------------------------------------------------------------ -bool blockchain_storage::add_new_block(const block& bl_, block_verification_context& bvc) +bool blockchain_storage::prevalidate_block(const block& bl) +{ + if (bl.major_version == BLOCK_MAJOR_VERSION_INITAL && get_block_height(bl) <= m_core_runtime_config.hard_fork1_starts_after_height) + return true; + if (bl.major_version != CURRENT_BLOCK_MAJOR_VERSION) + { + LOG_ERROR("prevalidation failed for block " << get_block_hash(bl) << ": major block version " << static_cast(bl.major_version) << " is incorrect, " << CURRENT_BLOCK_MAJOR_VERSION << " is expected" << ENDL + << obj_to_json_str(bl)); + return false; + } + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::add_new_block(const block& bl, block_verification_context& bvc) { try { m_db.begin_transaction(); - block bl = bl_; + //block bl = bl_; crypto::hash id = get_block_hash(bl); CRITICAL_REGION_LOCAL(m_tx_pool); //CRITICAL_REGION_LOCAL1(m_read_lock); @@ -4802,7 +5145,18 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont return false; } + if (!prevalidate_block(bl)) + { + LOG_PRINT_RED_L0("block with id = " << id << " failed to prevalidate"); + bvc.m_added_to_main_chain = false; + bvc.m_verification_failed = true; + m_db.commit_transaction(); + return false; + } + + //check that block refers to chain tail + if (!(bl.prev_id == get_top_block_id())) { @@ -5584,6 +5938,8 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha } update_alt_out_indexes_for_tx_in_block(b.miner_tx, abei); + CHECK_AND_ASSERT_MES(validate_tx_for_hardfork_specific_terms(b.miner_tx, null_hash, height), false, "miner tx hardfork-specific validation failed"); + for (auto tx_id : b.tx_hashes) { std::shared_ptr tx_ptr; @@ -5614,6 +5970,8 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha CHECK_AND_ASSERT_MES(false, false, "input #" << n << " has unexpected type (" << tx.vin[n].type().name() << "), tx " << tx_id); } } + + CHECK_AND_ASSERT_MES(validate_tx_for_hardfork_specific_terms(tx, tx_id, height), false, "tx " << tx_id << ": hardfork-specific validation failed"); // Updating abei (and not updating alt_chain) during this cycle is safe because txs in the same block can't reference one another, // so only valid references are either to previous alt blocks (accessed via alt_chain) or to main chain blocks. diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index 8c387253..4d6f7d4a 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -153,6 +153,9 @@ namespace currency // {amount -> pub_keys} map of outputs' pub_keys appeared in this alt block ( index_in_vector == output_gindex - gindex_lookup_table[output_amount] ) std::map > outputs_pub_keys; + + //date added to alt chain storage + uint64_t timestamp; }; typedef std::unordered_map alt_chain_container; //typedef std::list alt_chain_type; @@ -184,6 +187,7 @@ namespace currency //------------- modifying members -------------- bool add_new_block(const block& bl_, block_verification_context& bvc); + bool prevalidate_block(const block& bl); bool clear(); bool reset_and_set_genesis_block(const block& b); //debug function @@ -221,8 +225,11 @@ namespace currency bool have_tx_keyimg_as_spent(const crypto::key_image &key_im, uint64_t before_height = UINT64_MAX) const; std::shared_ptr get_tx(const crypto::hash &id) const; + template - bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL) const ; + bool scan_outputkeys_for_indexes(const transaction &validated_tx, const txin_to_key& tx_in_to_key, visitor_t& vis) { uint64_t stub = 0; return scan_outputkeys_for_indexes(validated_tx, tx_in_to_key, vis, stub); } + template + bool scan_outputkeys_for_indexes(const transaction &validated_tx, const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t& max_related_block_height) const ; uint64_t get_current_blockchain_size() const; uint64_t get_top_block_height() const; @@ -230,7 +237,7 @@ namespace currency crypto::hash get_top_block_id(uint64_t& height) const; bool get_top_block(block& b) const; wide_difficulty_type get_next_diff_conditional(bool pos) const; - wide_difficulty_type get_next_diff_conditional2(bool pos, const alt_chain_type& alt_chain, uint64_t split_height) const; + wide_difficulty_type get_next_diff_conditional2(bool pos, const alt_chain_type& alt_chain, uint64_t split_height, const alt_block_extended_info& abei) const; wide_difficulty_type get_cached_next_difficulty(bool pos) const; typedef bool fill_block_template_func_t(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height); @@ -260,13 +267,14 @@ namespace currency uint64_t get_aliases_count()const; uint64_t get_block_h_older_then(uint64_t timestamp) const; bool validate_tx_service_attachmens_in_services(const tx_service_attachment& a, size_t i, const transaction& tx)const; - bool check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL)const; - bool check_tx_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL)const; - bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL)const; - //bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL)const; - bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id)const; + bool check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height, uint64_t& max_unlock_time)const; + bool check_tx_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height)const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& max_used_block_height)const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash) const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& max_used_block_height, crypto::hash& max_used_block_id)const; bool check_ms_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, const transaction& source_tx, size_t out_n) const; - bool get_output_keys_for_input_with_checks(const txin_to_key& txin, std::vector& output_keys, uint64_t* pmax_related_block_height = NULL) const; + bool validate_tx_for_hardfork_specific_terms(const transaction& tx, const crypto::hash& tx_id, uint64_t block_height) const; + bool get_output_keys_for_input_with_checks(const transaction& tx, const txin_to_key& txin, std::vector& output_keys, uint64_t& max_related_block_height, uint64_t& max_unlock_time) const; bool check_tokey_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, const std::vector& output_keys_ptrs) const; uint64_t get_current_comulative_blocksize_limit()const; uint64_t get_current_hashrate(size_t aprox_count)const; @@ -304,6 +312,7 @@ namespace currency bool build_stake_modifier(stake_modifier_type& sm, const alt_chain_type& alt_chain = alt_chain_type(), uint64_t split_height = 0, crypto::hash *p_last_block_hash = nullptr) const; bool scan_pos(const COMMAND_RPC_SCAN_POS::request& sp, COMMAND_RPC_SCAN_POS::response& rsp)const; + bool validate_pos_coinbase_outs_unlock_time(const transaction& miner_tx, uint64_t staked_amount, uint64_t max_unlock_time)const; bool validate_pos_block(const block& b, const crypto::hash& id, bool for_altchain)const; bool validate_pos_block(const block& b, wide_difficulty_type basic_diff, const crypto::hash& id, bool for_altchain)const; bool validate_pos_block(const block& b, @@ -421,6 +430,7 @@ namespace currency void print_blockchain_outs(const std::string& file) const; void print_blockchain_outs_stat() const; void print_db_cache_perfeormance_data() const; + void print_last_n_difficulty_numbers(uint64_t n) const; bool calc_tx_cummulative_blob(const block& bl)const; bool get_outs_index_stat(outs_index_stat& outs_stat)const; bool print_lookup_key_image(const crypto::key_image& ki) const; @@ -430,6 +440,7 @@ namespace currency bool rebuild_tx_fee_medians(); bool validate_all_aliases_for_new_median_mode(); bool print_tx_outputs_lookup(const crypto::hash& tx_id) const; + uint64_t get_last_x_block_height(bool pos)const; private: //-------------- DB containers -------------- @@ -541,7 +552,8 @@ namespace currency bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); std::string print_alt_chain(alt_chain_type alt_chain); bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); - bool is_reorganize_required(const block_extended_info& main_chain_bei, const block_extended_info& alt_chain_bei, const crypto::hash& proof_alt); + bool is_reorganize_required(const block_extended_info& main_chain_bei, const alt_chain_type& alt_chain, const crypto::hash& proof_alt); + wide_difficulty_type get_x_difficulty_after_height(uint64_t height, bool is_pos); bool purge_keyimage_from_big_heap(const crypto::key_image& ki, const crypto::hash& id); bool purge_altblock_keyimages_from_big_heap(const block& b, const crypto::hash& id); bool append_altblock_keyimages_to_big_heap(const crypto::hash& block_id, const std::set& alt_block_keyimages); @@ -550,9 +562,8 @@ namespace currency bool validate_alt_block_txs(const block& b, const crypto::hash& id, std::set& collected_keyimages, alt_block_extended_info& abei, const alt_chain_type& alt_chain, uint64_t split_height, uint64_t& ki_lookup_time_total) const; bool update_alt_out_indexes_for_tx_in_block(const transaction& tx, alt_block_extended_info& abei)const; bool get_transaction_from_pool_or_db(const crypto::hash& tx_id, std::shared_ptr& tx_ptr, uint64_t min_allowed_block_height = 0) const; - + void get_last_n_x_blocks(uint64_t n, bool pos_blocks, std::list>& blocks) const; bool prevalidate_miner_transaction(const block& b, uint64_t height, bool pos)const; - bool validate_transaction(const block& b, uint64_t height, const transaction& tx)const; bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, uint64_t timestamp); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); @@ -608,8 +619,8 @@ namespace currency //POS wide_difficulty_type get_adjusted_cumulative_difficulty_for_next_pos(wide_difficulty_type next_diff)const; wide_difficulty_type get_adjusted_cumulative_difficulty_for_next_alt_pos(alt_chain_type& alt_chain, uint64_t block_height, wide_difficulty_type next_diff, uint64_t connection_height)const; - uint64_t get_last_x_block_height(bool pos)const; - wide_difficulty_type get_last_alt_x_block_cumulative_precise_difficulty(alt_chain_type& alt_chain, uint64_t block_height, bool pos)const; + wide_difficulty_type get_last_alt_x_block_cumulative_precise_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos, wide_difficulty_type& cumulative_diff_precise_adj)const; + wide_difficulty_type get_last_alt_x_block_cumulative_precise_adj_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos) const; size_t get_current_sequence_factor_for_alt(alt_chain_type& alt_chain, bool pos, uint64_t connection_height)const; }; @@ -648,7 +659,7 @@ namespace currency //------------------------------------------------------------------ //------------------------------------------------------------------ template - bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) const + bool blockchain_storage::scan_outputkeys_for_indexes(const transaction &validated_tx, const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t& max_related_block_height) const { CRITICAL_REGION_LOCAL(m_read_lock); TIME_MEASURE_START_PD(tx_check_inputs_loop_scan_outputkeys_get_item_size); @@ -704,18 +715,16 @@ namespace currency CHECK_AND_ASSERT_MES(mixattr_ok, false, "tx output #" << output_index << " violates mixin restrictions: mix_attr = " << static_cast(outtk.mix_attr) << ", key_offsets.size = " << tx_in_to_key.key_offsets.size()); TIME_MEASURE_START_PD(tx_check_inputs_loop_scan_outputkeys_loop_handle_output); - if (!vis.handle_output(tx_ptr->tx, tx_ptr->tx.vout[n])) + if (!vis.handle_output(tx_ptr->tx, validated_tx, tx_ptr->tx.vout[n], n)) { LOG_PRINT_L0("Failed to handle_output for output id = " << tx_id << ", no " << n); return false; } TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_scan_outputkeys_loop_handle_output); - if (pmax_related_block_height) - { - if (*pmax_related_block_height < tx_ptr->m_keeper_block_height) - *pmax_related_block_height = tx_ptr->m_keeper_block_height; - } + if (max_related_block_height < tx_ptr->m_keeper_block_height) + max_related_block_height = tx_ptr->m_keeper_block_height; + ++output_index; } diff --git a/src/currency_core/blockchain_storage_basic.h b/src/currency_core/blockchain_storage_basic.h index 05709aef..dd3cf354 100644 --- a/src/currency_core/blockchain_storage_basic.h +++ b/src/currency_core/blockchain_storage_basic.h @@ -47,6 +47,7 @@ namespace currency uint64_t block_cumulative_size; wide_difficulty_type cumulative_diff_adjusted; wide_difficulty_type cumulative_diff_precise; + wide_difficulty_type cumulative_diff_precise_adjusted; wide_difficulty_type difficulty; boost::multiprecision::uint128_t already_generated_coins; crypto::hash stake_hash; //TODO: unused field for PoW blocks, subject for refactoring diff --git a/src/currency_core/core_runtime_config.h b/src/currency_core/core_runtime_config.h index f046ce02..8e9d94b0 100644 --- a/src/currency_core/core_runtime_config.h +++ b/src/currency_core/core_runtime_config.h @@ -18,6 +18,8 @@ namespace currency uint64_t pos_minimum_heigh; //height uint64_t tx_pool_min_fee; uint64_t tx_default_fee; + uint64_t hard_fork1_starts_after_height; + uint64_t max_alt_blocks; crypto::public_key alias_validation_pubkey; core_time_func_t get_core_time; @@ -34,6 +36,8 @@ namespace currency pc.pos_minimum_heigh = POS_START_HEIGHT; pc.tx_pool_min_fee = TX_MINIMUM_FEE; pc.tx_default_fee = TX_DEFAULT_FEE; + pc.max_alt_blocks = CURRENCY_ALT_BLOCK_MAX_COUNT; + pc.hard_fork1_starts_after_height = ZANO_HARDFORK_1_AFTER_HEIGHT; pc.get_core_time = &core_runtime_config::_default_core_time_function; bool r = epee::string_tools::hex_to_pod(ALIAS_SHORT_NAMES_VALIDATION_PUB_KEY, pc.alias_validation_pubkey); CHECK_AND_ASSERT_THROW_MES(r, "failed to parse alias_validation_pub_key"); diff --git a/src/currency_core/currency_basic.h b/src/currency_core/currency_basic.h index 25781224..b1f3b9cc 100644 --- a/src/currency_core/currency_basic.h +++ b/src/currency_core/currency_basic.h @@ -336,7 +336,7 @@ namespace currency }; - //number of block (or time), used as a limitation: spend this tx not early then block/time + //number of block (or timestamp if v bigger then CURRENCY_MAX_BLOCK_NUMBER), used as a limitation: spend this tx not early then block/time struct etc_tx_details_unlock_time { uint64_t v; @@ -345,6 +345,16 @@ namespace currency END_SERIALIZE() }; + //number of block (or timestamp if unlock_time_array[i] bigger then CURRENCY_MAX_BLOCK_NUMBER), used as a limitation: spend this tx not early then block/time + //unlock_time_array[i], i - index of output, unlock_time_array.size() == vout.size() + struct etc_tx_details_unlock_time2 + { + std::vector unlock_time_array; + BEGIN_SERIALIZE() + FIELD(unlock_time_array) + END_SERIALIZE() + }; + struct etc_tx_details_expiration_time { uint64_t v; @@ -361,7 +371,7 @@ namespace currency uint64_t v; BEGIN_SERIALIZE() VARINT_FIELD(v) - END_SERIALIZE() + END_SERIALIZE() }; struct etc_tx_details_flags @@ -380,7 +390,7 @@ namespace currency END_SERIALIZE() }; - typedef boost::mpl::vector all_payload_types; + typedef boost::mpl::vector all_payload_types; typedef boost::make_variant_over::type attachment_v; typedef boost::make_variant_over::type extra_v; typedef boost::make_variant_over::type payload_items_v; @@ -553,12 +563,14 @@ namespace currency uint64_t index; crypto::key_image keyimage; uint64_t block_timestamp; + uint64_t stake_unlock_time; //not for serialization uint64_t wallet_index; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) KV_SERIALIZE(index) + KV_SERIALIZE(stake_unlock_time) KV_SERIALIZE(block_timestamp) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(keyimage) END_KV_SERIALIZE_MAP() @@ -613,5 +625,6 @@ SET_VARIANT_TAGS(uint64_t, 26, "uint64_t"); SET_VARIANT_TAGS(currency::etc_tx_time, 27, "etc_tx_time"); SET_VARIANT_TAGS(uint32_t, 28, "uint32_t"); SET_VARIANT_TAGS(currency::tx_receiver, 29, "payer"); +SET_VARIANT_TAGS(currency::etc_tx_details_unlock_time2, 30, "unlock_time2"); #undef SET_VARIANT_TAGS diff --git a/src/currency_core/currency_boost_serialization.h b/src/currency_core/currency_boost_serialization.h index 6802afd4..30e85e92 100644 --- a/src/currency_core/currency_boost_serialization.h +++ b/src/currency_core/currency_boost_serialization.h @@ -190,6 +190,11 @@ namespace boost a & at.v; } template + inline void serialize(Archive &a, currency::etc_tx_details_unlock_time2 &at, const boost::serialization::version_type ver) + { + a & at.unlock_time_array; + } + template inline void serialize(Archive &a, currency::etc_tx_details_expiration_time &at, const boost::serialization::version_type ver) { a & at.v; diff --git a/src/currency_core/currency_config.h b/src/currency_core/currency_config.h index 46182f6b..8aba9113 100644 --- a/src/currency_core/currency_config.h +++ b/src/currency_core/currency_config.h @@ -24,10 +24,6 @@ #define CURRENCY_POS_BLOCK_FUTURE_TIME_LIMIT 60*20 #define BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW 60 - - -// TOTAL_MONEY_SUPPLY - total number coins to be generated -#define TOTAL_MONEY_SUPPLY ((uint64_t)(-1)) #define POS_START_HEIGHT 0 @@ -87,8 +83,10 @@ #define CURRENCY_ALT_BLOCK_LIVETIME_COUNT (CURRENCY_BLOCKS_PER_DAY*7)//one week +#define CURRENCY_ALT_BLOCK_MAX_COUNT 43200 //30 days #define CURRENCY_MEMPOOL_TX_LIVETIME 345600 //seconds, 4 days + #ifndef TESTNET #define P2P_DEFAULT_PORT 11121 #define RPC_DEFAULT_PORT 11211 @@ -128,7 +126,7 @@ //PoS definitions #define POS_SCAN_WINDOW 60*10 //seconds // 10 minutes #define POS_SCAN_STEP 15 //seconds -#define POS_MAC_ACTUAL_TIMESTAMP_TO_MINED (POS_SCAN_WINDOW+100) +#define POS_MAX_ACTUAL_TIMESTAMP_TO_MINED (POS_SCAN_WINDOW+100) #define POS_STARTER_KERNEL_HASH "00000000000000000006382a8d8f94588ce93a1351924f6ccb9e07dd287c6e4b" #define POS_MODFIFIER_INTERVAL 10 @@ -192,10 +190,11 @@ #define GUI_INTERNAL_CONFIG "gui_internal_config.bin" + #define CURRENT_TRANSACTION_CHAIN_ENTRY_ARCHIVE_VER 3 #define CURRENT_BLOCK_EXTENDED_INFO_ARCHIVE_VER 1 -#define BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION CURRENCY_FORMATION_VERSION + 7 +#define BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION CURRENCY_FORMATION_VERSION + 8 #define BLOCKCHAIN_STORAGE_MINOR_COMPATIBILITY_VERSION 1 @@ -203,10 +202,19 @@ #define BC_OFFERS_CURRENCY_MARKET_FILENAME "market.bin" -#define WALLET_FILE_SERIALIZATION_VERSION (CURRENCY_FORMATION_VERSION+64) +#define WALLET_FILE_SERIALIZATION_VERSION (CURRENCY_FORMATION_VERSION+65) #define CURRENT_MEMPOOL_ARCHIVE_VER (CURRENCY_FORMATION_VERSION+31) +//hard forks section +#define BLOCK_MAJOR_VERSION_GENESIS 1 +#define BLOCK_MINOR_VERSION_GENESIS 0 +#define BLOCK_MAJOR_VERSION_INITAL 0 +#ifndef TESTNET +#define ZANO_HARDFORK_1_AFTER_HEIGHT 166440 +#else +#define ZANO_HARDFORK_1_AFTER_HEIGHT 62102 +#endif diff --git a/src/currency_core/currency_format_utils.cpp b/src/currency_core/currency_format_utils.cpp index 9c0eab16..76537043 100644 --- a/src/currency_core/currency_format_utils.cpp +++ b/src/currency_core/currency_format_utils.cpp @@ -107,21 +107,54 @@ namespace currency out_amounts.resize(out_amounts.size() - 1); } + std::vector destinations; for (auto a : out_amounts) { - tx_destination_entry de; + tx_destination_entry de = AUTO_VAL_INIT(de); de.addr.push_back(miner_address); de.amount = a; + if (pe.stake_unlock_time && pe.stake_unlock_time > height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW) + { + //this means that block is creating after hardfork_1 and unlock_time is needed to set for every destination separately + de.unlock_time = height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW; + } destinations.push_back(de); } if (pos) - destinations.push_back(tx_destination_entry(pe.amount, stakeholder_address)); + { + uint64_t stake_lock_time = 0; + if (pe.stake_unlock_time && pe.stake_unlock_time > height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW) + stake_lock_time = pe.stake_unlock_time; + destinations.push_back(tx_destination_entry(pe.amount, stakeholder_address, stake_lock_time)); + } + return construct_miner_tx(height, median_size, already_generated_coins, current_block_size, fee, destinations, tx, extra_nonce, max_outs, pos, pe); } //------------------------------------------------------------------ + bool apply_unlock_time(const std::vector& destinations, transaction& tx) + { + currency::etc_tx_details_unlock_time2 unlock_time2 = AUTO_VAL_INIT(unlock_time2); + unlock_time2.unlock_time_array.resize(destinations.size()); + bool found_unlock_time = false; + for (size_t i = 0; i != unlock_time2.unlock_time_array.size(); i++) + { + if (destinations[i].unlock_time) + { + found_unlock_time = true; + unlock_time2.unlock_time_array[i] = destinations[i].unlock_time; + } + } + if (found_unlock_time) + { + tx.extra.push_back(unlock_time2); + } + + return true; + } + //------------------------------------------------------------------ bool construct_miner_tx(size_t height, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t current_block_size, uint64_t fee, @@ -144,9 +177,12 @@ namespace currency if (!add_tx_extra_userdata(tx, extra_nonce)) return false; + //at this moment we do apply_unlock_time only for coin_base transactions + apply_unlock_time(destinations, tx); //we always add extra_padding with 2 bytes length to make possible for get_block_template to adjust cumulative size tx.extra.push_back(extra_padding()); + txin_gen in; in.height = height; tx.vin.push_back(in); @@ -171,10 +207,15 @@ namespace currency CHECK_AND_ASSERT_MES(r, false, "Failed to contruct miner tx out"); no++; } - + tx.version = CURRENT_TRANSACTION_VERSION; - set_tx_unlock_time(tx, height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + if (!have_type_in_variant_container(tx.extra)) + { + //if stake unlock time was not set, then we can use simple "whole transaction" lock scheme + set_tx_unlock_time(tx, height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + } + return true; } //--------------------------------------------------------------- @@ -891,16 +932,7 @@ namespace currency } return n; } - //--------------------------------------------------------------- - account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations) - { - for (const auto& de : destinations) - { - if (de.addr.size() == 1 && sender_account_keys.m_account_address != de.addr.back()) - return de.addr.back(); // return the first destination address that is non-multisig and not equal to the sender's address - } - return sender_account_keys.m_account_address; // otherwise, fallback to sender's address - } + //--------------------------------------------------------------- bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, @@ -1304,7 +1336,7 @@ namespace currency bool get_inputs_money_amount(const transaction& tx, uint64_t& money) { money = 0; - BOOST_FOREACH(const auto& in, tx.vin) + for(const auto& in : tx.vin) { uint64_t this_amount = get_amount_from_variant(in); if (!this_amount) @@ -1328,7 +1360,7 @@ namespace currency //--------------------------------------------------------------- bool check_inputs_types_supported(const transaction& tx) { - BOOST_FOREACH(const auto& in, tx.vin) + for(const auto& in : tx.vin) { CHECK_AND_ASSERT_MES(in.type() == typeid(txin_to_key) || in.type() == typeid(txin_multisig), false, "wrong variant type: " << in.type().name() << ", expected " << typeid(txin_to_key).name() @@ -1828,9 +1860,7 @@ namespace currency //for future forks std::cout << "Currency name: \t\t" << CURRENCY_NAME << "(" << CURRENCY_NAME_SHORT << ")" << std::endl; - std::cout << "Money supply: \t\t" << print_money(TOTAL_MONEY_SUPPLY) << " coins" - << "(" << print_money(TOTAL_MONEY_SUPPLY) << "), dev bounties is ???" << std::endl; - + std::cout << "Money supply: \t\t " << CURRENCY_BLOCK_REWARD * CURRENCY_BLOCKS_PER_DAY * 365 << " coins per year" << std::endl; std::cout << "PoS block interval: \t" << DIFFICULTY_POS_TARGET << " seconds" << std::endl; std::cout << "PoW block interval: \t" << DIFFICULTY_POW_TARGET << " seconds" << std::endl; std::cout << "Total blocks per day: \t" << CURRENCY_BLOCKS_PER_DAY << " seconds" << std::endl; @@ -1874,8 +1904,8 @@ namespace currency //string_tools::parse_hexstr_to_binbuff(genesis_coinbase_tx_hex, tx_bl); bool r = parse_and_validate_tx_from_blob(tx_bl, bl.miner_tx); CHECK_AND_ASSERT_MES(r, false, "failed to parse coinbase tx from hard coded blob"); - bl.major_version = CURRENT_BLOCK_MAJOR_VERSION; - bl.minor_version = CURRENT_BLOCK_MINOR_VERSION; + bl.major_version = BLOCK_MAJOR_VERSION_GENESIS; + bl.minor_version = BLOCK_MINOR_VERSION_GENESIS; bl.timestamp = 0; bl.nonce = CURRENCY_GENESIS_NONCE; LOG_PRINT_GREEN("Generated genesis: " << get_block_hash(bl), LOG_LEVEL_0); @@ -2081,6 +2111,20 @@ namespace currency return true; } + bool operator()(const etc_tx_details_unlock_time2& ee) + { + tv.type = "unlock_time"; + std::stringstream ss; + ss << "["; + for (auto v : ee.unlock_time_array) + { + ss << " " << v; + } + ss << "]"; + tv.short_view = ss.str(); + + return true; + } bool operator()(const etc_tx_details_expiration_time& ee) { tv.type = "expiration_time"; @@ -2524,16 +2568,6 @@ namespace currency { return epee::string_tools::parse_hexstr_to_binbuff(payment_id_str, payment_id); } - //------------------------------------------------------------------ - bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median) - { - /// tx expiration condition (tx is ok if the following is true) - /// tx_expiration_time - TX_EXPIRATION_MEDIAN_SHIFT > get_last_n_blocks_timestamps_median(TX_EXPIRATION_TIMESTAMP_CHECK_WINDOW) - uint64_t expiration_time = get_tx_expiration_time(tx); - if (expiration_time == 0) - return false; // 0 means it never expires - return expiration_time <= expiration_ts_median + TX_EXPIRATION_MEDIAN_SHIFT; - } //-------------------------------------------------------------------------------- crypto::hash prepare_prefix_hash_for_sign(const transaction& tx, uint64_t in_index, const crypto::hash& tx_id) { @@ -2667,4 +2701,41 @@ namespace currency return false; } + + wide_difficulty_type get_a_to_b_relative_cumulative_difficulty(const wide_difficulty_type& difficulty_pos_at_split_point, + const wide_difficulty_type& difficulty_pow_at_split_point, + const difficulties& a_diff, + const difficulties& b_diff ) + { + static const wide_difficulty_type difficulty_starter = DIFFICULTY_STARTER; + const wide_difficulty_type& a_pos_cumulative_difficulty = a_diff.pos_diff > 0 ? a_diff.pos_diff : difficulty_starter; + const wide_difficulty_type& b_pos_cumulative_difficulty = b_diff.pos_diff > 0 ? b_diff.pos_diff : difficulty_starter; + const wide_difficulty_type& a_pow_cumulative_difficulty = a_diff.pow_diff > 0 ? a_diff.pow_diff : difficulty_starter; + const wide_difficulty_type& b_pow_cumulative_difficulty = b_diff.pow_diff > 0 ? b_diff.pow_diff : difficulty_starter; + + boost::multiprecision::uint1024_t basic_sum = boost::multiprecision::uint1024_t(a_pow_cumulative_difficulty) + (boost::multiprecision::uint1024_t(a_pos_cumulative_difficulty)*difficulty_pow_at_split_point) / difficulty_pos_at_split_point; + boost::multiprecision::uint1024_t res = + (basic_sum * a_pow_cumulative_difficulty * a_pos_cumulative_difficulty) / (boost::multiprecision::uint1024_t(a_pow_cumulative_difficulty)*a_pos_cumulative_difficulty); + + if (res > boost::math::tools::max_value()) + { + ASSERT_MES_AND_THROW("[INTERNAL ERROR]: Failed to get_a_to_b_relative_cumulative_difficulty, res = " << res << ENDL + << ", difficulty_pos_at_split_point: " << difficulty_pos_at_split_point << ENDL + << ", difficulty_pow_at_split_point:" << difficulty_pow_at_split_point << ENDL + << ", a_pos_cumulative_difficulty:" << a_pos_cumulative_difficulty << ENDL + << ", b_pos_cumulative_difficulty:" << b_pos_cumulative_difficulty << ENDL + << ", a_pow_cumulative_difficulty:" << a_pow_cumulative_difficulty << ENDL + << ", b_pow_cumulative_difficulty:" << b_pow_cumulative_difficulty << ENDL + ); + } + TRY_ENTRY(); + wide_difficulty_type short_res = res.convert_to(); + return short_res; + CATCH_ENTRY_WITH_FORWARDING_EXCEPTION(); + } + + + } // namespace currency + + diff --git a/src/currency_core/currency_format_utils.h b/src/currency_core/currency_format_utils.h index 01b7328c..4b1aa928 100644 --- a/src/currency_core/currency_format_utils.h +++ b/src/currency_core/currency_format_utils.h @@ -59,55 +59,7 @@ namespace currency typedef boost::multiprecision::uint128_t uint128_tl; - struct tx_source_entry - { - typedef serializable_pair output_entry; // txout_v is either global output index or ref_by_id; public_key - is output ephemeral pub key - - std::vector outputs; //index + key - uint64_t real_output; //index in outputs vector of real output_entry - crypto::public_key real_out_tx_key; //real output's transaction's public key - size_t real_output_in_tx_index; //index in transaction outputs vector - uint64_t amount; //money - uint64_t transfer_index; //money - crypto::hash multisig_id; //if txin_multisig: multisig output id - size_t ms_sigs_count; //if txin_multisig: must be equal to output's minimum_sigs - size_t ms_keys_count; //if txin_multisig: must be equal to size of output's keys container - bool separately_signed_tx_complete; //for separately signed tx only: denotes the last source entry in complete tx to explicitly mark the final step of tx creation - - bool is_multisig() const { return ms_sigs_count > 0; } - - BEGIN_SERIALIZE_OBJECT() - FIELD(outputs) - FIELD(real_output) - FIELD(real_out_tx_key) - FIELD(real_output_in_tx_index) - FIELD(amount) - FIELD(transfer_index) - FIELD(multisig_id) - FIELD(ms_sigs_count) - FIELD(ms_keys_count) - FIELD(separately_signed_tx_complete) - END_SERIALIZE() - }; - - struct tx_destination_entry - { - uint64_t amount; //money - std::list addr; //destination address, in case of 1 address - txout_to_key, in case of more - txout_multisig - size_t minimum_sigs; // if txout_multisig: minimum signatures that are required to spend this output (minimum_sigs <= addr.size()) IF txout_to_key - not used - uint64_t amount_to_provide; //amount money that provided by initial creator of tx, used with partially created transactions - - tx_destination_entry() : amount(0), minimum_sigs(0), amount_to_provide(0){} - tx_destination_entry(uint64_t a, const account_public_address& ad) : amount(a), addr(1, ad), minimum_sigs(0), amount_to_provide(0){} - tx_destination_entry(uint64_t a, const std::list& addr) : amount(a), addr(addr), minimum_sigs(addr.size()), amount_to_provide(0){} - - BEGIN_SERIALIZE_OBJECT() - FIELD(amount) - FIELD(addr) - FIELD(minimum_sigs) - FIELD(amount_to_provide) - END_SERIALIZE() - }; + struct tx_extra_info { @@ -208,32 +160,6 @@ namespace currency //--------------------------------------------------------------- - template - uint64_t get_tx_x_detail(const transaction& tx) - { - extra_type_t e = AUTO_VAL_INIT(e); - get_type_in_variant_container(tx.extra, e); - return e.v; - } - template - void set_tx_x_detail(transaction& tx, uint64_t v) - { - extra_type_t e = AUTO_VAL_INIT(e); - e.v = v; - update_or_add_field_to_extra(tx.extra, e); - } - - inline uint64_t get_tx_unlock_time(const transaction& tx){ return get_tx_x_detail(tx);} - inline uint64_t get_tx_flags(const transaction& tx){ return get_tx_x_detail(tx); } - inline uint64_t get_tx_expiration_time(const transaction& tx){ return get_tx_x_detail(tx); } - inline void set_tx_unlock_time(transaction& tx, uint64_t v){ set_tx_x_detail(tx, v); } - inline void set_tx_flags(transaction& tx, uint64_t v){ set_tx_x_detail(tx, v); } - inline void set_tx_expiration_time(transaction& tx, uint64_t v){ set_tx_x_detail(tx, v); } - account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations); - - bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median); - - uint64_t get_string_uint64_hash(const std::string& str); bool construct_tx_out(const tx_destination_entry& de, const crypto::secret_key& tx_sec_key, size_t output_index, transaction& tx, std::set& deriv_cache, uint8_t tx_outs_attr = CURRENCY_TO_KEY_OUT_RELAXED); bool validate_alias_name(const std::string& al); @@ -317,6 +243,7 @@ namespace currency bool parse_amount(uint64_t& amount, const std::string& str_amount); + bool unserialize_block_complete_entry(const COMMAND_RPC_GET_BLOCKS_FAST::response& serialized, COMMAND_RPC_GET_BLOCKS_DIRECT::response& unserialized); @@ -670,6 +597,19 @@ namespace currency std::string utf8_to_upper(const std::string& s); std::string utf8_to_lower(const std::string& s); bool utf8_substring_test_case_insensitive(const std::string& match, const std::string& s); // Returns true is 's' contains 'match' (case-insensitive) + + struct difficulties + { + wide_difficulty_type pos_diff; + wide_difficulty_type pow_diff; + }; + + wide_difficulty_type get_a_to_b_relative_cumulative_difficulty(const wide_difficulty_type& difficulty_pos_at_split_point, + const wide_difficulty_type& difficulty_pow_at_split_point, + const difficulties& a_diff, + const difficulties& b_diff + ); + } // namespace currency diff --git a/src/currency_core/currency_format_utils_blocks.cpp b/src/currency_core/currency_format_utils_blocks.cpp index b1ec69b9..30d7f261 100644 --- a/src/currency_core/currency_format_utils_blocks.cpp +++ b/src/currency_core/currency_format_utils_blocks.cpp @@ -6,6 +6,7 @@ #include "currency_format_utils_blocks.h" #include "serialization/serialization.h" +#include "currency_format_utils.h" #include "currency_format_utils_abstract.h" #include "currency_format_utils_transactions.h" namespace currency diff --git a/src/currency_core/currency_format_utils_transactions.cpp b/src/currency_core/currency_format_utils_transactions.cpp index 533313ae..f78ebf16 100644 --- a/src/currency_core/currency_format_utils_transactions.cpp +++ b/src/currency_core/currency_format_utils_transactions.cpp @@ -6,11 +6,127 @@ #include "currency_format_utils_transactions.h" #include "serialization/serialization.h" -#include "currency_format_utils_abstract.h" #include "currency_format_utils.h" +#include "currency_format_utils_abstract.h" namespace currency { + //--------------------------------------------------------------- + account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations) + { + for (const auto& de : destinations) + { + if (de.addr.size() == 1 && sender_account_keys.m_account_address != de.addr.back()) + return de.addr.back(); // return the first destination address that is non-multisig and not equal to the sender's address + } + return sender_account_keys.m_account_address; // otherwise, fallback to sender's address + } + //------------------------------------------------------------------ + bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median) + { + /// tx expiration condition (tx is ok if the following is true) + /// tx_expiration_time - TX_EXPIRATION_MEDIAN_SHIFT > get_last_n_blocks_timestamps_median(TX_EXPIRATION_TIMESTAMP_CHECK_WINDOW) + uint64_t expiration_time = get_tx_expiration_time(tx); + if (expiration_time == 0) + return false; // 0 means it never expires + return expiration_time <= expiration_ts_median + TX_EXPIRATION_MEDIAN_SHIFT; + } + //--------------------------------------------------------------- + uint64_t get_burned_amount(const transaction& tx) + { + uint64_t res = 0; + for (auto& o : tx.vout) + { + if (o.target.type() == typeid(txout_to_key)) + { + if (boost::get(o.target).key == null_pkey) + res += o.amount; + } + } + return res; + } + //--------------------------------------------------------------- + uint64_t get_tx_max_unlock_time(const transaction& tx) + { + // etc_tx_details_unlock_time have priority over etc_tx_details_unlock_time2 + uint64_t v = get_tx_x_detail(tx); + if (v) + return v; + + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(tx.extra, ut2); + if (!ut2.unlock_time_array.size()) + return 0; + + uint64_t max_unlock_time = 0; + CHECK_AND_ASSERT_THROW_MES(ut2.unlock_time_array.size() == tx.vout.size(), "unlock_time_array.size=" << ut2.unlock_time_array.size() + << " is not the same as tx.vout.size =" << tx.vout.size() << " in tx: " << get_transaction_hash(tx)); + for (size_t i = 0; i != tx.vout.size(); i++) + { + if (ut2.unlock_time_array[i] > max_unlock_time) + max_unlock_time = ut2.unlock_time_array[i]; + } + + return max_unlock_time; + } + + //--------------------------------------------------------------- + uint64_t get_tx_unlock_time(const transaction& tx, uint64_t o_i) + { + // etc_tx_details_expiration_time have priority over etc_tx_details_expiration_time2 + uint64_t v = get_tx_x_detail(tx); + if (v) + return v; + + CHECK_AND_ASSERT_THROW_MES(tx.vout.size() > o_i, "tx.vout.size=" << tx.vout.size() + << " is not bigger then o_i=" << o_i << " in tx: " << get_transaction_hash(tx)); + + + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(tx.extra, ut2); + if (!ut2.unlock_time_array.size()) + return 0; + + CHECK_AND_ASSERT_THROW_MES(ut2.unlock_time_array.size() > o_i, "unlock_time_array.size=" << ut2.unlock_time_array.size() + << " is less then o_i=" << o_i << " in tx: " << get_transaction_hash(tx)); + + return ut2.unlock_time_array[o_i]; + } + //--------------------------------------------------------------- + bool get_tx_max_min_unlock_time(const transaction& tx, uint64_t& max_unlock_time, uint64_t& min_unlock_time) + { + max_unlock_time = min_unlock_time = 0; + // etc_tx_details_expiration_time have priority over etc_tx_details_expiration_time2 + uint64_t v = get_tx_x_detail(tx); + if (v) + { + max_unlock_time = min_unlock_time = v; + return true; + } + + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(tx.extra, ut2); + if (!ut2.unlock_time_array.size()) + { + return true; + } + CHECK_AND_ASSERT_THROW_MES(ut2.unlock_time_array.size() == tx.vout.size(), "unlock_time_array.size=" << ut2.unlock_time_array.size() + << " is not equal tx.vout.size()=" << tx.vout.size() << " in tx: " << get_transaction_hash(tx)); + if (ut2.unlock_time_array.size()) + { + max_unlock_time = min_unlock_time = ut2.unlock_time_array[0]; + for (size_t i = 1; i != ut2.unlock_time_array.size(); i++) + { + if (ut2.unlock_time_array[i] > max_unlock_time) + max_unlock_time = ut2.unlock_time_array[i]; + if (ut2.unlock_time_array[i] < min_unlock_time) + min_unlock_time = ut2.unlock_time_array[i]; + } + } + + + return true; + } //--------------------------------------------------------------- void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h) { diff --git a/src/currency_core/currency_format_utils_transactions.h b/src/currency_core/currency_format_utils_transactions.h index a4937fae..9ce1b000 100644 --- a/src/currency_core/currency_format_utils_transactions.h +++ b/src/currency_core/currency_format_utils_transactions.h @@ -9,10 +9,92 @@ #include "crypto/crypto.h" #include "currency_core/currency_basic.h" #include "currency_protocol/blobdatatype.h" +#include "currency_core/account.h" namespace currency { + struct tx_source_entry + { + typedef serializable_pair output_entry; // txout_v is either global output index or ref_by_id; public_key - is output ephemeral pub key + + std::vector outputs; //index + key + uint64_t real_output; //index in outputs vector of real output_entry + crypto::public_key real_out_tx_key; //real output's transaction's public key + size_t real_output_in_tx_index; //index in transaction outputs vector + uint64_t amount; //money + uint64_t transfer_index; //money + crypto::hash multisig_id; //if txin_multisig: multisig output id + size_t ms_sigs_count; //if txin_multisig: must be equal to output's minimum_sigs + size_t ms_keys_count; //if txin_multisig: must be equal to size of output's keys container + bool separately_signed_tx_complete; //for separately signed tx only: denotes the last source entry in complete tx to explicitly mark the final step of tx creation + + bool is_multisig() const { return ms_sigs_count > 0; } + + BEGIN_SERIALIZE_OBJECT() + FIELD(outputs) + FIELD(real_output) + FIELD(real_out_tx_key) + FIELD(real_output_in_tx_index) + FIELD(amount) + FIELD(transfer_index) + FIELD(multisig_id) + FIELD(ms_sigs_count) + FIELD(ms_keys_count) + FIELD(separately_signed_tx_complete) + END_SERIALIZE() + }; + + struct tx_destination_entry + { + uint64_t amount; //money + std::list addr; //destination address, in case of 1 address - txout_to_key, in case of more - txout_multisig + size_t minimum_sigs; // if txout_multisig: minimum signatures that are required to spend this output (minimum_sigs <= addr.size()) IF txout_to_key - not used + uint64_t amount_to_provide; //amount money that provided by initial creator of tx, used with partially created transactions + uint64_t unlock_time; + + tx_destination_entry() : amount(0), minimum_sigs(0), amount_to_provide(0), unlock_time(0){} + tx_destination_entry(uint64_t a, const account_public_address& ad) : amount(a), addr(1, ad), minimum_sigs(0), amount_to_provide(0), unlock_time(0){} + tx_destination_entry(uint64_t a, const account_public_address& ad, uint64_t ut) : amount(a), addr(1, ad), minimum_sigs(0), amount_to_provide(0), unlock_time(ut) {} + tx_destination_entry(uint64_t a, const std::list& addr) : amount(a), addr(addr), minimum_sigs(addr.size()), amount_to_provide(0), unlock_time(0){} + + BEGIN_SERIALIZE_OBJECT() + FIELD(amount) + FIELD(addr) + FIELD(minimum_sigs) + FIELD(amount_to_provide) + FIELD(unlock_time) + END_SERIALIZE() + }; + + template + uint64_t get_tx_x_detail(const transaction& tx) + { + extra_type_t e = AUTO_VAL_INIT(e); + get_type_in_variant_container(tx.extra, e); + return e.v; + } + template + void set_tx_x_detail(transaction& tx, uint64_t v) + { + extra_type_t e = AUTO_VAL_INIT(e); + e.v = v; + update_or_add_field_to_extra(tx.extra, e); + } + + uint64_t get_tx_unlock_time(const transaction& tx, uint64_t o_i); + uint64_t get_tx_max_unlock_time(const transaction& tx); + bool get_tx_max_min_unlock_time(const transaction& tx, uint64_t& max_unlock_time, uint64_t& min_unlock_time); + inline uint64_t get_tx_flags(const transaction& tx) { return get_tx_x_detail(tx); } + inline uint64_t get_tx_expiration_time(const transaction& tx) {return get_tx_x_detail(tx); } + inline void set_tx_unlock_time(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } + inline void set_tx_flags(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } + inline void set_tx_expiration_time(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } + account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations); + + + bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median); + uint64_t get_burned_amount(const transaction& tx); void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h); crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash); diff --git a/src/currency_core/difficulty.cpp b/src/currency_core/difficulty.cpp index d22c934b..18003b62 100644 --- a/src/currency_core/difficulty.cpp +++ b/src/currency_core/difficulty.cpp @@ -179,7 +179,7 @@ namespace currency { return res.convert_to(); } - wide_difficulty_type next_difficulty(vector& timestamps, vector& cumulative_difficulties, size_t target_seconds) + wide_difficulty_type next_difficulty_1(vector& timestamps, vector& cumulative_difficulties, size_t target_seconds) { // timestamps - first is latest, back - is oldest timestamps @@ -220,4 +220,40 @@ namespace currency { } return summ / devider; } + + wide_difficulty_type next_difficulty_2(vector& timestamps, vector& cumulative_difficulties, size_t target_seconds) + { + + // timestamps - first is latest, back - is oldest timestamps + if (timestamps.size() > DIFFICULTY_WINDOW) + { + timestamps.resize(DIFFICULTY_WINDOW); + cumulative_difficulties.resize(DIFFICULTY_WINDOW); + } + + + size_t length = timestamps.size(); + CHECK_AND_ASSERT_MES(length == cumulative_difficulties.size(), 0, "Check \"length == cumulative_difficulties.size()\" failed"); + if (length <= 1) + { + return DIFFICULTY_STARTER; + } + static_assert(DIFFICULTY_WINDOW >= 2, "Window is too small"); + + CHECK_AND_ASSERT_MES(length <= DIFFICULTY_WINDOW, 0, "length <= DIFFICULTY_WINDOW check failed, length=" << length); + + sort(timestamps.begin(), timestamps.end(), std::greater()); + + static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Cut length is too large"); + wide_difficulty_type dif_slow = get_adjustment_for_zone(timestamps, cumulative_difficulties, target_seconds, DIFFICULTY_WINDOW, DIFFICULTY_CUT / 2, DIFFICULTY_CUT / 2); + wide_difficulty_type dif_medium = get_adjustment_for_zone(timestamps, cumulative_difficulties, target_seconds, DIFFICULTY_WINDOW / 3, DIFFICULTY_CUT / 8, DIFFICULTY_CUT / 12); + uint64_t devider = 1; + wide_difficulty_type summ = dif_slow; + if (dif_medium != 0) + { + summ += dif_medium; + ++devider; + } + return summ / devider; + } } diff --git a/src/currency_core/difficulty.h b/src/currency_core/difficulty.h index 529389e3..ab97f142 100644 --- a/src/currency_core/difficulty.h +++ b/src/currency_core/difficulty.h @@ -19,7 +19,8 @@ namespace currency typedef boost::multiprecision::uint128_t wide_difficulty_type; bool check_hash(const crypto::hash &hash, wide_difficulty_type difficulty); - wide_difficulty_type next_difficulty(std::vector& timestamps, std::vector& cumulative_difficulties, size_t target_seconds); + wide_difficulty_type next_difficulty_1(std::vector& timestamps, std::vector& cumulative_difficulties, size_t target_seconds); + wide_difficulty_type next_difficulty_2(std::vector& timestamps, std::vector& cumulative_difficulties, size_t target_seconds); uint64_t difficulty_to_boundary(wide_difficulty_type difficulty); void difficulty_to_boundary_long(wide_difficulty_type difficulty, crypto::hash& result); } diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 65b58cb1..9851a677 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -100,7 +100,7 @@ int main(int argc, char* argv[]) #endif log_space::get_set_log_detalisation_level(true, LOG_LEVEL_2); log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); - log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,wallet"); + log_space::log_singletone::enable_channels("core,currency_protocol,tx_pool,wallet,lmdb"); LOG_PRINT_L0("Starting..."); tools::signal_handler::install_fatal([](int sig_number, void* address) { diff --git a/src/daemon/daemon_commands_handler.h b/src/daemon/daemon_commands_handler.h index ac057703..391f16f4 100644 --- a/src/daemon/daemon_commands_handler.h +++ b/src/daemon/daemon_commands_handler.h @@ -68,6 +68,7 @@ public: m_cmd_binder.set_handler("print_block_from_hex_blob", boost::bind(&daemon_commands_handler::print_block_from_hex_blob, this, _1), "Unserialize block from hex binary data to json-like representation"); m_cmd_binder.set_handler("print_tx_from_hex_blob", boost::bind(&daemon_commands_handler::print_tx_from_hex_blob, this, _1), "Unserialize transaction from hex binary data to json-like representation"); m_cmd_binder.set_handler("print_tx_outputs_usage", boost::bind(&daemon_commands_handler::print_tx_outputs_usage, this, _1), "Analyse if tx outputs for involved in subsequent transactions"); + m_cmd_binder.set_handler("print_difficulties_of_last_n_blocks", boost::bind(&daemon_commands_handler::print_difficulties_of_last_n_blocks, this, _1), "Print difficulties of last n blocks"); } @@ -696,8 +697,26 @@ private: m_srv.get_payload_object().get_core().get_blockchain_storage().print_tx_outputs_lookup(tx_hash); return true; } + //-------------------------------------------------------------------------------- + bool print_difficulties_of_last_n_blocks(const std::vector& args) + { + if (args.empty()) + { + std::cout << "expected: n - number of blocks to read" << std::endl; + return true; + } + const std::string& amount = args.front(); + uint64_t n = 0; + if (!epee::string_tools::get_xtype_from_string(n, amount)) + { + std::cout << "unable to convert to number '" << amount << "'" << std::endl; + return true; + } + m_srv.get_payload_object().get_core().get_blockchain_storage().print_last_n_difficulty_numbers(n); + return true; + } //-------------------------------------------------------------------------------- bool print_pool(const std::vector& args) { diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index d78b0a00..851d5406 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -798,6 +798,7 @@ namespace currency currency::pos_entry pe = AUTO_VAL_INIT(pe); pe.amount = req.pos_amount; pe.index = req.pos_index; + pe.stake_unlock_time = req.stake_unlock_time; //pe.keyimage key image will be set in the wallet //pe.wallet_index is not included in serialization map, TODO: refactoring here diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 01da849a..5e3c19dd 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -778,6 +778,7 @@ namespace currency bool pos_block; //is pos block uint64_t pos_amount; // uint64_t pos_index; // + uint64_t stake_unlock_time; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(extra_text) @@ -786,6 +787,7 @@ namespace currency KV_SERIALIZE(pos_block) KV_SERIALIZE(pos_amount) KV_SERIALIZE(pos_index) + KV_SERIALIZE(stake_unlock_time) END_KV_SERIALIZE_MAP() }; diff --git a/src/stratum/stratum_server.cpp b/src/stratum/stratum_server.cpp index ccb86b27..586d5e09 100644 --- a/src/stratum/stratum_server.cpp +++ b/src/stratum/stratum_server.cpp @@ -80,7 +80,7 @@ namespace #define HR_TO_STREAM_IN_MHS_3P(hr) std::fixed << std::setprecision(3) << hr / 1000000.0 // debug stuff -#define DBG_NETWORK_DIFFICULTY 0 // if non-zero: use this value as net difficulty when checking shares (useful for debugging on testnet, recommended value is 1600000000ull) +#define DBG_NETWORK_DIFFICULTY 0 // if non-zero: use this value as net difficulty when checking shares (useful for debugging on testnet, recommended value is 3000000000ull) #define DBG_CORE_ALWAYS_SYNCRONIZED 0 // if set to 1: allows the server to start even if the core is not syncronized, useful for debugging with --offline-mode #define STRINGIZE_DETAIL(x) #x #define STRINGIZE(x) STRINGIZE_DETAIL(x) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 33c621c7..15a36e4d 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -33,6 +33,35 @@ using namespace currency; ENABLE_CHANNEL_BY_DEFAULT("wallet") namespace tools { + + //--------------------------------------------------------------- + uint64_t wallet2::get_max_unlock_time_from_receive_indices(const currency::transaction& tx, const money_transfer2_details& td) + { + uint64_t max_unlock_time = 0; + // etc_tx_details_expiration_time have priority over etc_tx_details_expiration_time2 + uint64_t major_unlock_time = get_tx_x_detail(tx); + if (major_unlock_time) + return major_unlock_time; + + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(tx.extra, ut2); + if (!ut2.unlock_time_array.size()) + return 0; + + CHECK_AND_ASSERT_THROW_MES(ut2.unlock_time_array.size() == tx.vout.size(), "Internal error: wrong tx transfer details: ut2.unlock_time_array.size()" << ut2.unlock_time_array.size() << " is not equal transaction outputs vector size=" << tx.vout.size()); + + for (auto ri : td.receive_indices) + { + CHECK_AND_ASSERT_THROW_MES(ri < tx.vout.size(), "Internal error: wrong tx transfer details: reciev index=" << ri << " is greater than transaction outputs vector " << tx.vout.size()); + if (tx.vout[ri].target.type() == typeid(currency::txout_to_key)) + { + //update unlock_time if needed + if (ut2.unlock_time_array[ri] > max_unlock_time) + max_unlock_time = ut2.unlock_time_array[ri]; + } + } + return max_unlock_time; + } //---------------------------------------------------------------------------------------------------- void wallet2::fill_transfer_details(const currency::transaction& tx, const tools::money_transfer2_details& td, tools::wallet_rpc::wallet_transfer_info_details& res_td) const { @@ -48,7 +77,9 @@ void wallet2::fill_transfer_details(const currency::transaction& tx, const tools { WLT_CHECK_AND_ASSERT_MES(ri < tx.vout.size(), void(), "Internal error: wrong tx transfer details: reciev index=" << ri << " is greater than transaction outputs vector " << tx.vout.size()); if (tx.vout[ri].target.type() == typeid(currency::txout_to_key)) + { res_td.rcv.push_back(tx.vout[ri].amount); + } } } //---------------------------------------------------------------------------------------------------- @@ -258,8 +289,14 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t i++; } - //check for transaction income + /* + collect unlock_time from every output that transfered coins to this account and use maximum of + all values m_payments entry, use this strict policy is required to protect exchanges from being feeded with + useless outputs + */ + uint64_t max_out_unlock_time = 0; + //check for transaction income crypto::key_derivation derivation = AUTO_VAL_INIT(derivation); r = lookup_acc_outs(m_account.get_keys(), tx, tx_pub_key, outs, tx_money_got_in_outs, derivation); THROW_IF_TRUE_WALLET_EX(!r, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); @@ -355,6 +392,10 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t if (td.m_key_image != currency::null_ki) m_key_images[td.m_key_image] = transfer_index; add_transfer_to_transfers_cache(tx.vout[o].amount, transfer_index); + + if (max_out_unlock_time < get_tx_unlock_time(tx, o)) + max_out_unlock_time = get_tx_unlock_time(tx, o); + WLT_LOG_L0("Received money, transfer #" << transfer_index << ", amount: " << print_money(td.amount()) << ", with tx: " << get_transaction_hash(tx) << ", at height " << height); } else if (tx.vout[o].target.type() == typeid(txout_multisig)) @@ -379,7 +420,7 @@ void wallet2::process_new_transaction(const currency::transaction& tx, uint64_t payment.m_tx_hash = currency::get_transaction_hash(tx); payment.m_amount = received; payment.m_block_height = height; - payment.m_unlock_time = currency::get_tx_unlock_time(tx); + payment.m_unlock_time = max_out_unlock_time; m_payments.emplace(payment_id, payment); WLT_LOG_L2("Payment found, id (hex): " << epee::string_tools::buff_to_hex_nodelimer(payment_id) << ", tx: " << payment.m_tx_hash << ", amount: " << print_money_brief(payment.m_amount)); } @@ -919,9 +960,9 @@ void wallet2::prepare_wti(wallet_rpc::wallet_transfer_info& wti, uint64_t height wti.amount = amount; wti.height = height; fill_transfer_details(tx, td, wti.td); + wti.unlock_time = get_max_unlock_time_from_receive_indices(tx, td); wti.timestamp = timestamp; wti.fee = currency::is_coinbase(tx) ? 0:currency::get_tx_fee(tx); - wti.unlock_time = get_tx_unlock_time(tx); wti.tx_blob_size = static_cast(currency::get_object_blobsize(wti.tx)); wti.tx_hash = currency::get_transaction_hash(tx); wti.is_service = currency::is_service_tx(tx); @@ -1027,6 +1068,8 @@ void wallet2::process_new_blockchain_entry(const currency::block& b, const curre m_blockchain.push_back(bl_id); ++m_local_bc_height; m_last_bc_timestamp = b.timestamp; + if (!is_pos_block(b)) + m_last_pow_block_h = height; m_wcallback->on_new_block(height, b); } @@ -1762,6 +1805,7 @@ bool wallet2::reset_all() m_last_bc_timestamp = 0; m_height_of_start_sync = 0; m_last_sync_percent = 0; + m_last_pow_block_h = 0; return true; } //---------------------------------------------------------------------------------------------------- @@ -2396,17 +2440,22 @@ bool wallet2::get_transfer_address(const std::string& adr_str, currency::account return m_core_proxy->get_transfer_address(adr_str, addr, payment_id); } //---------------------------------------------------------------------------------------------------- -bool wallet2::is_transfer_okay_for_pos(const transfer_details& tr) +bool wallet2::is_transfer_okay_for_pos(const transfer_details& tr, uint64_t& stake_unlock_time) { if (!tr.is_spendable()) return false; //blockchain conditions - if (!is_transfer_unlocked(tr)) + if (!is_transfer_unlocked(tr, true, stake_unlock_time)) return false; + //prevent staking of after-last-pow-coins if (m_blockchain.size() - tr.m_ptx_wallet_info->m_block_height <= m_core_runtime_config.min_coinstake_age) return false; + + if (tr.m_ptx_wallet_info->m_block_height > m_last_pow_block_h) + return false; + return true; } //---------------------------------------------------------------------------------------------------- @@ -2430,13 +2479,15 @@ bool wallet2::get_pos_entries(currency::COMMAND_RPC_SCAN_POS::request& req) for (size_t i = 0; i != m_transfers.size(); i++) { auto& tr = m_transfers[i]; - if (!is_transfer_okay_for_pos(tr)) + uint64_t stake_unlock_time = 0; + if (!is_transfer_okay_for_pos(tr, stake_unlock_time)) continue; currency::pos_entry pe = AUTO_VAL_INIT(pe); pe.amount = tr.amount(); pe.index = tr.m_global_output_index; pe.keyimage = tr.m_key_image; pe.wallet_index = i; + pe.stake_unlock_time = stake_unlock_time; pe.block_timestamp = tr.m_ptx_wallet_info->m_block_timestamp; req.pos_entries.push_back(pe); } @@ -2613,6 +2664,7 @@ bool wallet2::build_minted_block(const currency::COMMAND_RPC_SCAN_POS::request& tmpl_req.pos_amount = req.pos_entries[rsp.index].amount; tmpl_req.pos_index = req.pos_entries[rsp.index].index; tmpl_req.extra_text = m_miner_text_info; + tmpl_req.stake_unlock_time = req.pos_entries[rsp.index].stake_unlock_time; m_core_proxy->call_COMMAND_RPC_GETBLOCKTEMPLATE(tmpl_req, tmpl_rsp); WLT_CHECK_AND_ASSERT_MES(tmpl_rsp.status == CORE_RPC_STATUS_OK, false, "Failed to create block template after kernel hash found!"); @@ -2695,16 +2747,32 @@ currency::core_runtime_config& wallet2::get_core_runtime_config() } //---------------------------------------------------------------------------------------------------- bool wallet2::is_transfer_unlocked(const transfer_details& td) const +{ + uint64_t stub = 0; + return is_transfer_unlocked(td, false, stub); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::is_transfer_unlocked(const transfer_details& td, bool for_pos_mining, uint64_t& stake_lock_time) const { if (td.m_flags&WALLET_TRANSFER_DETAIL_FLAG_BLOCKED) return false; - if (!currency::is_tx_spendtime_unlocked(get_tx_unlock_time(td.m_ptx_wallet_info->m_tx), m_blockchain.size(), m_core_runtime_config.get_core_time())) + if (td.m_ptx_wallet_info->m_block_height + WALLET_DEFAULT_TX_SPENDABLE_AGE > m_blockchain.size()) return false; - if(td.m_ptx_wallet_info->m_block_height + WALLET_DEFAULT_TX_SPENDABLE_AGE > m_blockchain.size()) - return false; + + uint64_t unlock_time = get_tx_unlock_time(td.m_ptx_wallet_info->m_tx, td.m_internal_output_index); + if (for_pos_mining && m_blockchain.size() > m_core_runtime_config.hard_fork1_starts_after_height) + { + //allowed of staking locked coins with + stake_lock_time = unlock_time; + } + else + { + if (!currency::is_tx_spendtime_unlocked(unlock_time, m_blockchain.size(), m_core_runtime_config.get_core_time())) + return false; + } return true; } //---------------------------------------------------------------------------------------------------- diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 97d30549..f4c741c5 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -64,6 +64,8 @@ ENABLE_CHANNEL_BY_DEFAULT("wallet"); #define WLT_CHECK_AND_ASSERT_MES_NO_RET(expr, msg) CHECK_AND_ASSERT_MES_NO_RET(expr, "[W:" << m_log_prefix << "]" << msg) #define WLT_THROW_IF_FALSE_WALLET_INT_ERR_EX(cond, msg) THROW_IF_FALSE_WALLET_INT_ERR_EX(cond, "[W:" << m_log_prefix << "]" << msg) +class test_generator; + namespace tools { #pragma pack(push, 1) @@ -310,7 +312,8 @@ namespace tools m_height_of_start_sync(0), m_last_sync_percent(0), m_do_rise_transfer(false), - m_watch_only(false) + m_watch_only(false), + m_last_pow_block_h(0) {}; public: wallet2() : m_stop(false), @@ -322,7 +325,8 @@ namespace tools m_fake_outputs_count(0), m_do_rise_transfer(false), m_log_prefix("???"), - m_watch_only(false) + m_watch_only(false), + m_last_pow_block_h(0) { m_core_runtime_config = currency::get_default_core_runtime_config(); }; @@ -615,7 +619,7 @@ namespace tools return; } - if (ver < 147) + if (ver < 149) { LOG_PRINT_MAGENTA("Wallet file truncated due to old version", LOG_LEVEL_0); return; @@ -650,6 +654,7 @@ namespace tools a & m_money_expirations; a & m_pending_key_images; a & m_tx_keys; + a & m_last_pow_block_h; } @@ -661,7 +666,7 @@ namespace tools //synchronous version of function bool try_mint_pos(); //for unit tests - friend class test_generator; + friend class ::test_generator; //next functions in public area only because of test_generator //TODO: Need refactoring - remove it back to private zone @@ -680,6 +685,7 @@ namespace tools bool build_minted_block(const currency::COMMAND_RPC_SCAN_POS::request& req, const currency::COMMAND_RPC_SCAN_POS::response& rsp, const currency::account_public_address& miner_address, uint64_t new_block_expected_height = UINT64_MAX); bool reset_history(); bool is_transfer_unlocked(const transfer_details& td) const; + bool is_transfer_unlocked(const transfer_details& td, bool for_pos_mining, uint64_t& stake_lock_time) const; void get_mining_history(wallet_rpc::mining_history& hist); void set_core_runtime_config(const currency::core_runtime_config& pc); currency::core_runtime_config& get_core_runtime_config(); @@ -717,7 +723,7 @@ namespace tools void finalize_transaction(const finalize_tx_param& ftp, currency::transaction& tx, crypto::secret_key& tx_key, bool broadcast_tx); std::string get_log_prefix() const { return m_log_prefix; } - + static uint64_t get_max_unlock_time_from_receive_indices(const currency::transaction& tx, const money_transfer2_details& td); private: void add_transfers_to_expiration_list(const std::vector& selected_transfers, uint64_t expiration, uint64_t change_amount, const crypto::hash& related_tx_id); void remove_transfer_from_expiration_list(uint64_t transfer_index); @@ -765,7 +771,7 @@ private: std::string get_alias_for_address(const std::string& addr); static bool build_kernel(const currency::pos_entry& pe, const currency::stake_modifier_type& stake_modifier, currency::stake_kernel& kernel, uint64_t& coindays_weight, uint64_t timestamp); bool is_connected_to_net(); - bool is_transfer_okay_for_pos(const transfer_details& tr); + bool is_transfer_okay_for_pos(const transfer_details& tr, uint64_t& stake_unlock_time); bool scan_unconfirmed_outdate_tx(); const currency::transaction& get_transaction_by_id(const crypto::hash& tx_hash); void rise_on_transfer2(const wallet_rpc::wallet_transfer_info& wti); @@ -812,6 +818,7 @@ private: const std::vector& decrypted_items, crypto::hash& ms_id, bc_services::contract_private_details& cpd, const currency::transaction& proposal_template_tx); + void fill_transfer_details(const currency::transaction& tx, const tools::money_transfer2_details& td, tools::wallet_rpc::wallet_transfer_info_details& res_td) const; void print_source_entry(const currency::tx_source_entry& src) const; @@ -862,6 +869,7 @@ private: std::shared_ptr m_wcallback; uint64_t m_height_of_start_sync; uint64_t m_last_sync_percent; + uint64_t m_last_pow_block_h; currency::core_runtime_config m_core_runtime_config; escrow_contracts_container m_contracts; std::list m_money_expirations; @@ -871,6 +879,8 @@ private: uint64_t m_fake_outputs_count; std::string m_miner_text_info; + //this needed to access wallets state in coretests, for creating abnormal blocks and tranmsactions + friend class test_generator; }; // class wallet2 @@ -947,10 +957,11 @@ namespace boost a & x.contract; a & x.selected_indicies; a & x.srv_attachments; + a & x.unlock_time; //do not store this items in the file since it's quite easy to restore it from original tx if (Archive::is_loading::value) { - x.unlock_time = currency::get_tx_unlock_time(x.tx); + x.is_service = currency::is_service_tx(x.tx); x.is_mixing = currency::is_mixin_tx(x.tx); x.is_mining = currency::is_coinbase(x.tx); diff --git a/src/wallet/wallet2_escrow.cpp b/src/wallet/wallet2_escrow.cpp index 4fb9db35..9d264508 100644 --- a/src/wallet/wallet2_escrow.cpp +++ b/src/wallet/wallet2_escrow.cpp @@ -26,7 +26,7 @@ bool wallet2::validate_escrow_proposal(const wallet_rpc::wallet_transfer_info& w // I. validate escrow proposal tx const transaction& escrow_proposal_tx = wti.tx; - uint64_t escrow_proposal_tx_unlock_time = get_tx_unlock_time(escrow_proposal_tx); + uint64_t escrow_proposal_tx_unlock_time = get_tx_max_unlock_time(escrow_proposal_tx); LOC_CHK(escrow_proposal_tx_unlock_time == 0, "proposal tx unlock time is non-zero: " << escrow_proposal_tx_unlock_time); uint64_t escrow_proposal_expiration_time = get_tx_expiration_time(escrow_proposal_tx); @@ -66,7 +66,7 @@ bool wallet2::validate_escrow_proposal(const wallet_rpc::wallet_transfer_info& w uint64_t template_expiration_time = get_tx_expiration_time(prop.tx_template); LOC_CHK(template_expiration_time != 0, "template has no expiration time"); - uint64_t template_unlock_time = get_tx_unlock_time(prop.tx_template); + uint64_t template_unlock_time = get_tx_max_unlock_time(prop.tx_template); LOC_CHK(template_unlock_time == 0, "template has non-zero unlock time: " << template_unlock_time); // (3/5) outputs @@ -156,7 +156,7 @@ bool wallet2::validate_escrow_release(const transaction& tx, bool release_type_n uint64_t expiration_time = get_tx_expiration_time(tx); LOC_CHK(expiration_time == 0, "tx has non-zero expiration time: " << expiration_time); - uint64_t unlock_time = get_tx_unlock_time(tx); + uint64_t unlock_time = get_tx_max_unlock_time(tx); LOC_CHK(unlock_time == 0, "tx has non-zero unlock time: " << unlock_time); tx_service_attachment tsa = AUTO_VAL_INIT(tsa); @@ -285,7 +285,7 @@ bool wallet2::validate_escrow_contract(const wallet_rpc::wallet_transfer_info& w uint64_t tx_expiration_time = get_tx_expiration_time(wti.tx); LOC_CHK(tx_expiration_time != 0, "no or zero expiration time specified"); - uint64_t tx_unlock_time = get_tx_unlock_time(wti.tx); + uint64_t tx_unlock_time = get_tx_max_unlock_time(wti.tx); LOC_CHK(tx_unlock_time == 0, "non-zero unlock time: " << tx_unlock_time); #undef LOC_CHK @@ -341,7 +341,7 @@ bool wallet2::validate_escrow_cancel_release(const currency::transaction& tx, co uint64_t expiration_time = get_tx_expiration_time(tx); LOC_CHK(expiration_time != 0, "tx has zero or not specified expiration time"); - uint64_t unlock_time = get_tx_unlock_time(tx); + uint64_t unlock_time = get_tx_max_unlock_time(tx); LOC_CHK(unlock_time == 0, "tx has non-zero unlock time: " << unlock_time); tx_service_attachment tsa = AUTO_VAL_INIT(tsa); @@ -430,7 +430,7 @@ bool wallet2::validate_escrow_cancel_proposal(const wallet_rpc::wallet_transfer_ uint64_t flags = get_tx_flags(wti.tx); LOC_CHK(flags == 0, "invalid tx flags: " << flags); - uint64_t unlock_time = get_tx_unlock_time(cancellation_request_tx); + uint64_t unlock_time = get_tx_max_unlock_time(cancellation_request_tx); LOC_CHK(unlock_time == 0, "invalid unlock time: " << unlock_time); uint64_t expiration_time = get_tx_expiration_time(cancellation_request_tx); diff --git a/tests/core_tests/block_validation.cpp b/tests/core_tests/block_validation.cpp index d01b65c8..9e65c8ec 100644 --- a/tests/core_tests/block_validation.cpp +++ b/tests/core_tests/block_validation.cpp @@ -21,7 +21,7 @@ namespace for (size_t i = 0; i < new_block_count; ++i) { block blk_next; - wide_difficulty_type diffic = next_difficulty(timestamps, cummulative_difficulties, DIFFICULTY_POW_TARGET); + wide_difficulty_type diffic = next_difficulty_1(timestamps, cummulative_difficulties, DIFFICULTY_POW_TARGET); if (!generator.construct_block_manually(blk_next, blk_prev, miner_account, test_generator::bf_timestamp | test_generator::bf_diffic, 0, 0, blk_prev.timestamp, crypto::hash(), diffic)) return false; @@ -152,7 +152,7 @@ bool gen_block_invalid_nonce::generate(std::vector& events) co return false; // Create invalid nonce - wide_difficulty_type diffic = next_difficulty(timestamps, commulative_difficulties, DIFFICULTY_POW_TARGET); + wide_difficulty_type diffic = next_difficulty_1(timestamps, commulative_difficulties, DIFFICULTY_POW_TARGET); CHECK_AND_ASSERT_MES(diffic > 1, false, "diffic > 1 validation failed"); const block& blk_last = boost::get(events.back()); uint64_t timestamp = blk_last.timestamp; @@ -193,7 +193,7 @@ bool gen_block_unlock_time_is_low::generate(std::vector& event BLOCK_VALIDATION_INIT_GENERATE(); MAKE_MINER_TX_MANUALLY(miner_tx, blk_0); - currency::set_tx_unlock_time(miner_tx, currency::get_tx_unlock_time(miner_tx) - 1); + currency::set_tx_unlock_time(miner_tx, currency::get_tx_max_unlock_time(miner_tx) - 1); block blk_1; generator.construct_block_manually(blk_1, blk_0, miner_account, test_generator::bf_miner_tx, 0, 0, 0, crypto::hash(), 0, miner_tx); @@ -209,7 +209,7 @@ bool gen_block_unlock_time_is_high::generate(std::vector& even BLOCK_VALIDATION_INIT_GENERATE(); MAKE_MINER_TX_MANUALLY(miner_tx, blk_0); - set_tx_unlock_time(miner_tx, get_tx_unlock_time(miner_tx) + 1); + set_tx_unlock_time(miner_tx, get_tx_max_unlock_time(miner_tx) + 1); block blk_1; generator.construct_block_manually(blk_1, blk_0, miner_account, test_generator::bf_miner_tx, 0, 0, 0, crypto::hash(), 0, miner_tx); diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index 0faf29d9..430f2d6f 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -46,10 +46,19 @@ crypto::signature create_invalid_signature() const crypto::signature invalid_signature = create_invalid_signature(); test_generator::test_generator() - : m_wallet_test_core_proxy(new wallet_test_core_proxy()) + : m_wallet_test_core_proxy(new wallet_test_core_proxy()), + m_do_pos_to_low_timestamp(false), + m_ignore_last_pow_in_wallets(false), + m_last_found_timestamp(0), + m_hardfork_after_heigh(CURRENCY_MAX_BLOCK_NUMBER) { } +void test_generator::set_hardfork_height(uint64_t h) +{ + m_hardfork_after_heigh = h; +} + void test_generator::get_block_chain(std::vector& blockchain, const crypto::hash& head, size_t n) const { crypto::hash curr = head; @@ -142,6 +151,7 @@ void test_generator::add_block(const currency::block& blk, get_block_reward(is_pos_block(blk), misc_utils::median(block_sizes), block_size, already_generated_coins, block_reward, currency::get_block_height(blk)); m_blocks_info[get_block_hash(blk)] = block_info(blk, already_generated_coins + block_reward, block_size, cum_diff, tx_list, ks_hash); + LOG_PRINT_MAGENTA("ADDED_BLOCK[" << get_block_hash(blk) << "][" << (is_pos_block(blk)? "PoS":"PoW") <<"][" << get_block_height(blk) << "][cumul_diff:" << cum_diff << "]", LOG_LEVEL_0); } void test_generator::add_block_info(const block_info& bi) @@ -191,7 +201,11 @@ bool test_generator::construct_block(currency::block& blk, const std::list& tx_list, const std::list& coin_stake_sources)//in case of PoS block { - blk.major_version = CURRENT_BLOCK_MAJOR_VERSION; + if (height > m_hardfork_after_heigh) + blk.major_version = CURRENT_BLOCK_MAJOR_VERSION; + else + blk.major_version = BLOCK_MAJOR_VERSION_INITAL; + blk.minor_version = CURRENT_BLOCK_MINOR_VERSION; blk.timestamp = timestamp; blk.prev_id = prev_id; @@ -305,11 +319,13 @@ bool test_generator::construct_block(currency::block& blk, CHECK_AND_ASSERT_MES(r, false, "Failed to find_kernel_and_sign()"); } + uint64_t last_x = get_last_block_of_type(is_pos_block(blk), blocks); + add_block(blk, txs_size, block_sizes, already_generated_coins, - blocks.size() ? blocks.back()->cumul_difficulty + a_diffic: a_diffic, + last_x ? blocks[last_x]->cumul_difficulty + a_diffic: a_diffic, tx_list, kernerl_hash); @@ -382,6 +398,7 @@ bool test_generator::build_wallets(const blockchain_vector& blocks, currency::core_runtime_config pc = cc; pc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; pc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; + pc.hard_fork1_starts_after_height = m_hardfork_after_heigh; wallets.back()->set_core_runtime_config(pc); } @@ -466,33 +483,52 @@ bool test_generator::find_kernel(const std::list& accs, uint64_t& found_timestamp, crypto::hash& found_kh) { + bool is_after_hardfork = blck_chain.size() > m_hardfork_after_heigh ? true : false; uint64_t median_timestamp = get_timestamps_median(blck_chain); wide_difficulty_type basic_diff = 0; - uint64_t i_last_pow_block = get_last_block_of_type(false, blck_chain); - uint64_t expected_avr_timestamp = blck_chain[i_last_pow_block]->b.timestamp + (blck_chain.size() - i_last_pow_block - 1)*DIFFICULTY_POW_TARGET; - uint64_t starter_timestamp = expected_avr_timestamp - POS_SCAN_WINDOW; + uint64_t i_last_pos_block = get_last_block_of_type(true, blck_chain); + + uint64_t last_pos_block_timestamp = 0; + if(i_last_pos_block) + last_pos_block_timestamp = blck_chain[i_last_pos_block]->b.timestamp; + else + last_pos_block_timestamp = blck_chain.back()->b.timestamp - DIFFICULTY_POS_TARGET/2; + + uint64_t starter_timestamp = last_pos_block_timestamp + DIFFICULTY_POS_TARGET; + if (starter_timestamp < median_timestamp) starter_timestamp = median_timestamp; - + m_last_found_timestamp = 0; basic_diff = get_difficulty_for_next_block(blck_chain, false); - - //lets try to find block - for (auto& w : wallets) + if (basic_diff < 10) { - currency::COMMAND_RPC_SCAN_POS::request scan_pos_entries; - bool r = w->get_pos_entries(scan_pos_entries); - CHECK_AND_ASSERT_THROW_MES(r, "Failed to get_pos_entries"); + starter_timestamp -= 90; + } + if (m_do_pos_to_low_timestamp) + starter_timestamp += 60; - for (size_t i = 0; i != scan_pos_entries.pos_entries.size(); i++) + //adjust timestamp starting from timestamp%POS_SCAN_STEP = 0 + //starter_timestamp = starter_timestamp - POS_SCAN_WINDOW; + starter_timestamp = POS_SCAN_STEP - (starter_timestamp%POS_SCAN_STEP) + starter_timestamp; + + for (uint64_t ts = starter_timestamp; ts < starter_timestamp + POS_SCAN_WINDOW/2; ts += POS_SCAN_STEP) + { + //lets try to find block + for (auto& w : wallets) { - //adjust timestamp starting from timestamp%POS_SCAN_STEP = 0 - starter_timestamp = starter_timestamp - POS_SCAN_WINDOW; - starter_timestamp = POS_SCAN_STEP - (starter_timestamp%POS_SCAN_STEP) + starter_timestamp; + //set m_last_pow_block_h to big value, to let wallet to use any available outputs, including the those which is not behind last pow block + if (m_ignore_last_pow_in_wallets) + w->m_last_pow_block_h = CURRENCY_MAX_BLOCK_NUMBER; - for (uint64_t ts = starter_timestamp; ts < expected_avr_timestamp + POS_SCAN_WINDOW; ts += POS_SCAN_STEP) + currency::COMMAND_RPC_SCAN_POS::request scan_pos_entries; + bool r = w->get_pos_entries(scan_pos_entries); + CHECK_AND_ASSERT_THROW_MES(r, "Failed to get_pos_entries"); + + for (size_t i = 0; i != scan_pos_entries.pos_entries.size(); i++) { + stake_kernel sk = AUTO_VAL_INIT(sk); uint64_t coindays_weight = 0; build_kernel(scan_pos_entries.pos_entries[i].amount, @@ -509,10 +545,23 @@ bool test_generator::find_kernel(const std::list& accs, continue; else { + if (m_do_pos_to_low_timestamp) + { + if (!m_last_found_timestamp) + { + m_last_found_timestamp = ts; + continue; + } + + if(m_last_found_timestamp >= ts) + continue; + } + //found kernel LOG_PRINT_GREEN("Found kernel: amount=" << print_money(scan_pos_entries.pos_entries[i].amount) << ", index=" << scan_pos_entries.pos_entries[i].index - << ", key_image" << scan_pos_entries.pos_entries[i].keyimage, LOG_LEVEL_0); + << ", key_image" << scan_pos_entries.pos_entries[i].keyimage + << ", diff: " << this_coin_diff, LOG_LEVEL_0); pe = scan_pos_entries.pos_entries[i]; found_wallet_index = i; found_kh = kernel_hash; @@ -673,7 +722,7 @@ currency::wide_difficulty_type test_generator::get_difficulty_for_next_block(con timestamps.push_back(blocks[i]->b.timestamp); commulative_difficulties.push_back(blocks[i]->cumul_difficulty); } - return next_difficulty(timestamps, commulative_difficulties, pow ? DIFFICULTY_POW_TARGET : DIFFICULTY_POS_TARGET); + return next_difficulty_1(timestamps, commulative_difficulties, pow ? DIFFICULTY_POW_TARGET : DIFFICULTY_POS_TARGET); } currency::wide_difficulty_type test_generator::get_cumul_difficulty_for_next_block(const crypto::hash& head_id, bool pow) const @@ -743,10 +792,19 @@ bool test_generator::construct_block(const std::vector& events const currency::block& blk_prev, const currency::account_base& miner_acc, const std::list& tx_list, - const std::list& coin_stake_sources - ) + const std::list& coin_stake_sources) { - uint64_t height = boost::get(blk_prev.miner_tx.vin.front()).height + 1; + return construct_block(0, events, blk, blk_prev, miner_acc, tx_list, coin_stake_sources); +} +bool test_generator::construct_block(int64_t manual_timestamp_adjustment, + const std::vector& events, + currency::block& blk, + const currency::block& blk_prev, + const currency::account_base& miner_acc, + const std::list& tx_list, + const std::list& coin_stake_sources) +{ + uint64_t height = boost::get(blk_prev.miner_tx.vin[0]).height + 1; crypto::hash prev_id = get_block_hash(blk_prev); // Keep push difficulty little up to be sure about PoW hash success std::vector blockchain; @@ -769,6 +827,7 @@ bool test_generator::construct_block(const std::vector& events block& prev_same_type = blockchain[prev_i]; timestamp = adjust_timestamp_finished ? prev_same_type.timestamp + diff_target : prev_same_type.timestamp + diff_target - diff_up_timestamp_delta; + timestamp = timestamp + manual_timestamp_adjustment; uint64_t already_generated_coins = get_already_generated_coins(prev_id); std::vector block_sizes; @@ -786,7 +845,7 @@ bool test_generator::construct_block(const std::vector& events size_t txs_sizes/* = 0*/) { size_t height = get_block_height(prev_block) + 1; - blk.major_version = actual_params & bf_major_ver ? major_ver : CURRENT_BLOCK_MAJOR_VERSION; + blk.major_version = actual_params & bf_major_ver ? major_ver : BLOCK_MAJOR_VERSION_INITAL; blk.minor_version = actual_params & bf_minor_ver ? minor_ver : CURRENT_BLOCK_MINOR_VERSION; blk.timestamp = actual_params & bf_timestamp ? timestamp : (height > 10 ? prev_block.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN: prev_block.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN-POW_DIFF_UP_TIMESTAMP_DELTA); // Keep difficulty unchanged blk.prev_id = actual_params & bf_prev_id ? prev_id : get_block_hash(prev_block); @@ -1197,10 +1256,10 @@ bool fill_tx_sources(std::vector& sources, const std: continue; if (check_for_unlocktime) { - if (currency::get_tx_unlock_time(*oi.p_tx) < CURRENCY_MAX_BLOCK_NUMBER) + if (currency::get_tx_max_unlock_time(*oi.p_tx) < CURRENCY_MAX_BLOCK_NUMBER) { //interpret as block index - if (currency::get_tx_unlock_time(*oi.p_tx) > blockchain.size()) + if (currency::get_tx_max_unlock_time(*oi.p_tx) > blockchain.size()) continue; } else @@ -1445,14 +1504,14 @@ bool construct_tx_to_key(const std::vector& events, if (!fill_tx_sources(sources, events, blk_head, from.get_keys(), spending_amount, nmix, check_for_spends, check_for_unlocktime, use_ref_by_id)) return false; - int64_t change = get_sources_total_amount(sources); + boost::multiprecision::int128_t change = get_sources_total_amount(sources); change -= spending_amount; if (change < 0) return false; // should never happen if fill_tx_sources succeded if (change == 0) return construct_tx(from.get_keys(), sources, destinations, extr, att, tx, sk, 0, mix_attr); std::vector local_dst = destinations; - local_dst.push_back(tx_destination_entry(change, from.get_public_address())); + local_dst.push_back(tx_destination_entry(change.convert_to(), from.get_public_address())); return construct_tx(from.get_keys(), sources, local_dst, extr, att, tx, sk, 0, mix_attr); } diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index 07b3bf30..27e1deec 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -247,7 +247,7 @@ struct offers_count_param size_t offers_count_raw; }; -class test_chain_unit_enchanced : public test_chain_unit_base +class test_chain_unit_enchanced : virtual public test_chain_unit_base { public: test_chain_unit_enchanced(); @@ -345,7 +345,7 @@ public: currency::wide_difficulty_type cumul_difficulty; std::vector m_transactions; crypto::hash ks_hash; - }; + }; // amount vec_ind, tx_index, out index in tx typedef std::map > > outputs_index; @@ -473,6 +473,7 @@ public: bool construct_genesis_block(currency::block& blk, const currency::account_base& miner_acc, uint64_t timestamp); + bool construct_block(const std::vector& events, currency::block& blk, const currency::block& blk_prev, @@ -480,6 +481,14 @@ public: const std::list& tx_list = std::list(), const std::list& coin_stake_sources = std::list() //in case of PoS block ); + bool construct_block(int64_t manual_timestamp_adjustment, + const std::vector& events, + currency::block& blk, + const currency::block& blk_prev, + const currency::account_base& miner_acc, + const std::list& tx_list = std::list(), + const std::list& coin_stake_sources = std::list() //in case of PoS block + ); bool construct_block_manually(currency::block& blk, const currency::block& prev_block, @@ -500,8 +509,17 @@ public: static const test_gentime_settings& get_test_gentime_settings() { return m_test_gentime_settings; } static void set_test_gentime_settings(const test_gentime_settings& s) { m_test_gentime_settings = s; } static void set_test_gentime_settings_default() { m_test_gentime_settings = m_test_gentime_settings_default; } + void set_pos_to_low_timestamp(bool do_pos_to_low_timestamp) { m_do_pos_to_low_timestamp = do_pos_to_low_timestamp; } + void set_ignore_last_pow_in_wallets(bool ignore_last_pow_in_wallets) { m_ignore_last_pow_in_wallets = ignore_last_pow_in_wallets; } + void set_hardfork_height(uint64_t h); private: + bool m_do_pos_to_low_timestamp; + bool m_ignore_last_pow_in_wallets; + uint64_t m_last_found_timestamp; + + uint64_t m_hardfork_after_heigh; + std::unordered_map m_blocks_info; static test_gentime_settings m_test_gentime_settings; static test_gentime_settings m_test_gentime_settings_default; @@ -956,12 +974,30 @@ void append_vector_by_another_vector(U& dst, const V& src) VEC_EVENTS.push_back(BLK_NAME); \ PRINT_EVENT_NO(VEC_EVENTS); + + +#define MAKE_NEXT_BLOCK_TIMESTAMP_ADJUSTMENT(ADJ, VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC) \ + currency::block BLK_NAME = AUTO_VAL_INIT(BLK_NAME); \ + generator.construct_block(ADJ, VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC); \ + VEC_EVENTS.push_back(BLK_NAME); \ + PRINT_EVENT_NO(VEC_EVENTS); + #define MAKE_NEXT_POS_BLOCK(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, MINERS_ACC_LIST) \ currency::block BLK_NAME = AUTO_VAL_INIT(BLK_NAME); \ generator.construct_block(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, std::list(), MINERS_ACC_LIST); \ VEC_EVENTS.push_back(BLK_NAME); \ PRINT_EVENT_NO(VEC_EVENTS) +#define MAKE_NEXT_POS_BLOCK_TX1(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, MINERS_ACC_LIST, TX_1) \ + currency::block BLK_NAME = AUTO_VAL_INIT(BLK_NAME); \ + { \ + std::listtx_list; \ + tx_list.push_back(TX_1); \ + generator.construct_block(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, tx_list, MINERS_ACC_LIST); \ + } \ + VEC_EVENTS.push_back(BLK_NAME); \ + PRINT_EVENT_NO(VEC_EVENTS) + #define MAKE_NEXT_BLOCK_NO_ADD(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC) \ currency::block BLK_NAME = AUTO_VAL_INIT(BLK_NAME); \ generator.construct_block(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC); \ @@ -982,6 +1018,7 @@ void append_vector_by_another_vector(U& dst, const V& src) VEC_EVENTS.push_back(BLK_NAME); \ PRINT_EVENT_NO(VEC_EVENTS) + #define MAKE_NEXT_BLOCK_TX_LIST(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST) \ currency::block BLK_NAME = AUTO_VAL_INIT(BLK_NAME); \ generator.construct_block(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST); \ diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index fbace5cd..641f1908 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -936,6 +936,17 @@ int main(int argc, char* argv[]) GENERATE_AND_PLAY(gen_uint_overflow_1); GENERATE_AND_PLAY(gen_uint_overflow_2); + + // Hardfok1 tests + GENERATE_AND_PLAY(before_hard_fork_1_cumulative_difficulty); + GENERATE_AND_PLAY(inthe_middle_hard_fork_1_cumulative_difficulty); + GENERATE_AND_PLAY(after_hard_fork_1_cumulative_difficulty); + GENERATE_AND_PLAY(hard_fork_1_locked_mining_test); + GENERATE_AND_PLAY(hard_fork_1_bad_pos_source); + GENERATE_AND_PLAY(hard_fork_1_unlock_time_2_in_normal_tx); + GENERATE_AND_PLAY(hard_fork_1_unlock_time_2_in_coinbase); + GENERATE_AND_PLAY(hard_fork_1_chain_switch_pow_only); + GENERATE_AND_PLAY(hard_fork_1_checkpoint_basic_test); //GENERATE_AND_PLAY(gen_block_reward); */ diff --git a/tests/core_tests/chaingen_tests_list.h b/tests/core_tests/chaingen_tests_list.h index 5eaa024e..635e1cd7 100644 --- a/tests/core_tests/chaingen_tests_list.h +++ b/tests/core_tests/chaingen_tests_list.h @@ -33,3 +33,7 @@ #include "escrow_wallet_altchain_test.h" #include "misc_tests.h" #include "emission_test.h" +#include "hard_fork_1_locked_pos_test.h" +#include "hard_fork_1_consensus_test.h" +#include "hard_fork_1_bad_pos_source.h" +#include "hard_fork_1.h" diff --git a/tests/core_tests/checkpoints_tests.h b/tests/core_tests/checkpoints_tests.h index 43950877..64e5b69d 100644 --- a/tests/core_tests/checkpoints_tests.h +++ b/tests/core_tests/checkpoints_tests.h @@ -7,7 +7,7 @@ #include "chaingen.h" #include "random_helper.h" -struct checkpoints_test : public test_chain_unit_enchanced +struct checkpoints_test : virtual public test_chain_unit_enchanced { checkpoints_test(); diff --git a/tests/core_tests/hard_fork_1.cpp b/tests/core_tests/hard_fork_1.cpp new file mode 100644 index 00000000..41f561e7 --- /dev/null +++ b/tests/core_tests/hard_fork_1.cpp @@ -0,0 +1,423 @@ +// Copyright (c) 2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "chaingen.h" +#include "hard_fork_1.h" +#include "pos_block_builder.h" +//#include "tx_builder.h" +//#include "random_helper.h" + +using namespace currency; + +hard_fork_1_base_test::hard_fork_1_base_test(size_t hardfork_height) + : m_hardfork_height(hardfork_height) +{ + REGISTER_CALLBACK_METHOD(hard_fork_1_base_test, configure_core); +} + +bool hard_fork_1_base_test::configure_core(currency::core& c, size_t ev_index, const std::vector& events) +{ + currency::core_runtime_config pc = c.get_blockchain_storage().get_core_runtime_config(); + pc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; + pc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; + pc.hard_fork1_starts_after_height = m_hardfork_height; + c.get_blockchain_storage().set_core_runtime_config(pc); + return true; +} + +//------------------------------------------------------------------------------ + + +hard_fork_1_unlock_time_2_in_normal_tx::hard_fork_1_unlock_time_2_in_normal_tx() + : hard_fork_1_base_test(12) +{ + REGISTER_CALLBACK_METHOD(hard_fork_1_unlock_time_2_in_normal_tx, configure_core); +} + +bool hard_fork_1_unlock_time_2_in_normal_tx::generate(std::vector& events) const +{ + // Test idea: make sure etc_tx_details_unlock_time2 can be used in normal (non-coinbase) tx + // only after hardfork 1 + + bool r = false; + GENERATE_ACCOUNT(miner_acc); + GENERATE_ACCOUNT(alice_acc); + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + generator.set_hardfork_height(m_hardfork_height); + DO_CALLBACK(events, "configure_core"); + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + + // + // before hardfork 1 + // + + std::vector sources; + std::vector destinations; + CHECK_AND_ASSERT_MES(fill_tx_sources_and_destinations(events, blk_0r, miner_acc, alice_acc, MK_TEST_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations), false, ""); + + // set unlock_time_2 + std::vector extra; + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + ut2.unlock_time_array.resize(destinations.size()); + ut2.unlock_time_array[0] = 1; // not zero, unlocked from block 1 + extra.push_back(ut2); + + transaction tx_0 = AUTO_VAL_INIT(tx_0); + crypto::secret_key tx_sec_key; + r = construct_tx(miner_acc.get_keys(), sources, destinations, extra, empty_attachment, tx_0, tx_sec_key, 0 /* unlock time 1 is zero and thus will not be set */); + CHECK_AND_ASSERT_MES(r, false, "construct_tx failed"); + // tx_0 should be accepted + events.push_back(tx_0); + + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); + DO_CALLBACK(events, "mark_invalid_block"); + MAKE_NEXT_BLOCK_TX1(events, blk_1_bad, blk_0r, miner_acc, tx_0); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); + DO_CALLBACK(events, "clear_tx_pool"); + + // make another tx with the same inputs and extra (tx_0 was rejected so inputs can be reused) + transaction tx_0a = AUTO_VAL_INIT(tx_0a); + r = construct_tx(miner_acc.get_keys(), sources, destinations, extra, empty_attachment, tx_0a, tx_sec_key, 0 /* unlock time 1 is zero and thus will not be set */); + CHECK_AND_ASSERT_MES(r, false, "construct_tx failed"); + // tx_0a should be accepted as well + events.push_back(tx_0a); + // make an alternative block with it and make sure it is rejected + + MAKE_NEXT_BLOCK(events, blk_1, blk_0r, miner_acc); + + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); + DO_CALLBACK(events, "mark_invalid_block"); + MAKE_NEXT_BLOCK_TX1(events, blk_1_alt_bad, blk_0r, miner_acc, tx_0a); // this alt block should be rejected because of tx_0a + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); + DO_CALLBACK(events, "clear_tx_pool"); + + + // okay, go for a hardfork + + MAKE_NEXT_BLOCK(events, blk_2, blk_1, miner_acc); // hardfork should happen here + MAKE_NEXT_BLOCK(events, blk_3, blk_2, miner_acc); + // make sure hardfork went okay + CHECK_AND_ASSERT_MES(blk_2.major_version != CURRENT_BLOCK_MAJOR_VERSION && blk_3.major_version == CURRENT_BLOCK_MAJOR_VERSION, false, "hardfork did not happen as expected"); + + + // + // after hardfork 1 + // + + sources.clear(); + destinations.clear(); + CHECK_AND_ASSERT_MES(fill_tx_sources_and_destinations(events, blk_3, miner_acc, alice_acc, MK_TEST_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations), false, ""); + + // set unlock_time_2 + extra.clear(); + ut2 = AUTO_VAL_INIT(ut2); + ut2.unlock_time_array.resize(destinations.size()); + ut2.unlock_time_array[0] = 1; // not zero, unlocked from block 1 + extra.push_back(ut2); + + transaction tx_1 = AUTO_VAL_INIT(tx_1); + r = construct_tx(miner_acc.get_keys(), sources, destinations, extra, empty_attachment, tx_1, tx_sec_key, 0 /* unlock time 1 is zero and not set */); + CHECK_AND_ASSERT_MES(r, false, "construct_tx failed"); + events.push_back(tx_1); + + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); + MAKE_NEXT_BLOCK_TX1(events, blk_4, blk_3, miner_acc, tx_1); // block with tx_1 should be accepted + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(0)); + + // do the same check for alt block + sources.clear(); + destinations.clear(); + CHECK_AND_ASSERT_MES(fill_tx_sources_and_destinations(events, blk_4, miner_acc, alice_acc, MK_TEST_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations), false, ""); + extra.clear(); + ut2 = AUTO_VAL_INIT(ut2); + ut2.unlock_time_array.resize(destinations.size()); + ut2.unlock_time_array[0] = 1; // not zero, unlocked from block 1 + extra.push_back(ut2); + + transaction tx_1a = AUTO_VAL_INIT(tx_1a); + r = construct_tx(miner_acc.get_keys(), sources, destinations, extra, empty_attachment, tx_1a, tx_sec_key, 0 /* unlock time 1 is zero and not set */); + CHECK_AND_ASSERT_MES(r, false, "construct_tx failed"); + events.push_back(tx_1a); + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); + + MAKE_NEXT_BLOCK(events, blk_5, blk_4, miner_acc); + + MAKE_NEXT_BLOCK_TX1(events, blk_5a, blk_4, miner_acc, tx_1a); // alt block with tx_1a should be accepted + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); // tx is still in the pool + + // switch chains + MAKE_NEXT_BLOCK(events, blk_6a, blk_5a, miner_acc); + MAKE_NEXT_BLOCK(events, blk_7a, blk_6a, miner_acc); + + // make sure switching really happened + DO_CALLBACK_PARAMS(events, "check_top_block", params_top_block(blk_7a)); + + // and tx_1a has gone + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(0)); + + return true; +} + +//------------------------------------------------------------------------------ + +hard_fork_1_unlock_time_2_in_coinbase::hard_fork_1_unlock_time_2_in_coinbase() + : hard_fork_1_base_test(3) +{ +} + +bool hard_fork_1_unlock_time_2_in_coinbase::generate(std::vector& events) const +{ + // Test idea: make sure etc_tx_details_unlock_time2 can be used in-coinbase txs + // only after hardfork 1 + + bool r = false; + GENERATE_ACCOUNT(miner_acc); + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + generator.set_hardfork_height(m_hardfork_height); + + DO_CALLBACK(events, "configure_core"); + + + // before hardfork 1 + + MAKE_NEXT_BLOCK(events, blk_1, blk_0, miner_acc); + events.pop_back(); // remove blk_1 + + // remove etc_tx_details_unlock_time entries from miner_tx + blk_1.miner_tx.extra.erase(std::remove_if(blk_1.miner_tx.extra.begin(), blk_1.miner_tx.extra.end(), [](extra_v& extra_element) { return extra_element.type() == typeid(etc_tx_details_unlock_time); }), blk_1.miner_tx.extra.end()); + + // add etc_tx_details_unlock_time2 entry + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + ut2.unlock_time_array.resize(blk_1.miner_tx.vout.size()); + ut2.unlock_time_array[0] = get_block_height(blk_1) + CURRENCY_MINED_MONEY_UNLOCK_WINDOW; + blk_1.miner_tx.extra.push_back(ut2); + + DO_CALLBACK(events, "mark_invalid_block"); + // add blk_1 with modified miner tx + events.push_back(blk_1); + generator.add_block_info(blk_1, std::list()); // add modified block info + + MAKE_NEXT_BLOCK(events, blk_1a, blk_0, miner_acc); + MAKE_NEXT_BLOCK(events, blk_2, blk_1a, miner_acc); + MAKE_NEXT_BLOCK(events, blk_3, blk_2, miner_acc); // hardfork should happen here + MAKE_NEXT_BLOCK(events, blk_4, blk_3, miner_acc); + // make sure hardfork went okay + CHECK_AND_ASSERT_MES(blk_3.major_version != CURRENT_BLOCK_MAJOR_VERSION && blk_4.major_version == CURRENT_BLOCK_MAJOR_VERSION, false, "hardfork did not happen as expected"); + + + // after hardfork 1 + + MAKE_NEXT_BLOCK(events, blk_5, blk_4, miner_acc); + wide_difficulty_type diff = generator.get_block_difficulty(get_block_hash(blk_5)); // remember block difficulty for nonce searching after modification + events.pop_back(); + + // remove etc_tx_details_unlock_time entries from miner_tx + blk_5.miner_tx.extra.erase(std::remove_if(blk_5.miner_tx.extra.begin(), blk_5.miner_tx.extra.end(), [](extra_v& extra_element) { return extra_element.type() == typeid(etc_tx_details_unlock_time); }), blk_5.miner_tx.extra.end()); + + // add etc_tx_details_unlock_time2 entry + ut2 = AUTO_VAL_INIT(ut2); + ut2.unlock_time_array.resize(blk_5.miner_tx.vout.size()); + ut2.unlock_time_array[0] = get_block_height(blk_5) + CURRENCY_MINED_MONEY_UNLOCK_WINDOW; + blk_5.miner_tx.extra.push_back(ut2); + miner::find_nonce_for_given_block(blk_5, diff, get_block_height(blk_5)); + + // add blk_5 with modified miner tx + events.push_back(blk_5); + generator.add_block_info(blk_5, std::list()); // add modified block info + + MAKE_NEXT_BLOCK(events, blk_6, blk_5, miner_acc); + + return true; +} + +//------------------------------------------------------------------------------ + +hard_fork_1_chain_switch_pow_only::hard_fork_1_chain_switch_pow_only() + : hard_fork_1_base_test(13) +{ +} + +bool hard_fork_1_chain_switch_pow_only::generate(std::vector& events) const +{ + // Test idea: make sure chain switches without PoS before and after hardfork + + bool r = false; + GENERATE_ACCOUNT(miner_acc); + GENERATE_ACCOUNT(alice_acc); + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + generator.set_hardfork_height(m_hardfork_height); + DO_CALLBACK(events, "configure_core"); + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + + // + // before hardfork 1 + // + + MAKE_NEXT_BLOCK(events, blk_1, blk_0r, miner_acc); + MAKE_NEXT_BLOCK(events, blk_1a, blk_0r, miner_acc); + MAKE_NEXT_BLOCK(events, blk_2a, blk_1a, miner_acc); + + // make sure switch happened + DO_CALLBACK_PARAMS(events, "check_top_block", params_top_block(blk_2a)); + + MAKE_NEXT_BLOCK(events, blk_3a, blk_2a, miner_acc); // hardfork should happen here + MAKE_NEXT_BLOCK(events, blk_4a, blk_3a, miner_acc); + // make sure hardfork went okay + CHECK_AND_ASSERT_MES(blk_3a.major_version != CURRENT_BLOCK_MAJOR_VERSION && blk_4a.major_version == CURRENT_BLOCK_MAJOR_VERSION, false, "hardfork did not happen as expected"); + + + // + // after hardfork 1 + // + + MAKE_NEXT_BLOCK(events, blk_5a, blk_4a, miner_acc); + MAKE_NEXT_BLOCK(events, blk_5b, blk_4a, miner_acc); // alternative chain B + // switch chains + MAKE_NEXT_BLOCK(events, blk_6b, blk_5b, miner_acc); + MAKE_NEXT_BLOCK(events, blk_7b, blk_6b, miner_acc); + + // make sure switch happened + DO_CALLBACK_PARAMS(events, "check_top_block", params_top_block(blk_7b)); + + return true; +} + +//------------------------------------------------------------------------------ + +hard_fork_1_checkpoint_basic_test::hard_fork_1_checkpoint_basic_test() + : hard_fork_1_base_test(13) + , checkpoints_test() +{ +} + +bool hard_fork_1_checkpoint_basic_test::generate(std::vector& events) const +{ + bool r = false; + GENERATE_ACCOUNT(miner_acc); + GENERATE_ACCOUNT(alice_acc); + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + generator.set_hardfork_height(m_hardfork_height); + DO_CALLBACK(events, "configure_core"); + DO_CALLBACK_PARAMS(events, "set_checkpoint", params_checkpoint(CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 2)); + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + + // + // before hardfork 1 + // + + std::vector sources; + std::vector destinations; + CHECK_AND_ASSERT_MES(fill_tx_sources_and_destinations(events, blk_0r, miner_acc, alice_acc, MK_TEST_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations), false, ""); + + // set unlock_time_2, should be rejected before hardfork 1 + std::vector extra; + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + ut2.unlock_time_array.resize(destinations.size()); + ut2.unlock_time_array[0] = 1; // not zero, unlocked from block 1 + extra.push_back(ut2); + transaction tx_0 = AUTO_VAL_INIT(tx_0); + crypto::secret_key tx_sec_key; + r = construct_tx(miner_acc.get_keys(), sources, destinations, extra, empty_attachment, tx_0, tx_sec_key, 0 /* unlock time 1 is zero and thus will not be set */); + CHECK_AND_ASSERT_MES(r, false, "construct_tx failed"); + // tx_0 should be accepted + events.push_back(tx_0); + + DO_CALLBACK(events, "mark_invalid_block"); + MAKE_NEXT_BLOCK_TX1(events, blk_1_bad, blk_0r, miner_acc, tx_0); // should be rejected because of tx_0 + DO_CALLBACK_PARAMS(events, "check_tx_pool_count", static_cast(1)); + DO_CALLBACK(events, "clear_tx_pool"); + + MAKE_NEXT_BLOCK(events, blk_1, blk_0r, miner_acc); + MAKE_NEXT_BLOCK(events, blk_2, blk_1, miner_acc); // <-- checkpoint + + MAKE_NEXT_BLOCK(events, blk_3, blk_2, miner_acc); // <-- hard fork + MAKE_NEXT_BLOCK(events, blk_4, blk_3, miner_acc); + // make sure hardfork went okay + CHECK_AND_ASSERT_MES(blk_3.major_version != CURRENT_BLOCK_MAJOR_VERSION && blk_4.major_version == CURRENT_BLOCK_MAJOR_VERSION, false, "hardfork did not happen as expected"); + + return true; +} + +//------------------------------------------------------------------------------ + +hard_fork_1_pos_and_locked_coins::hard_fork_1_pos_and_locked_coins() + : hard_fork_1_base_test(13) // hardfork height + , m_unique_amount(TESTS_DEFAULT_FEE * 9) +{ + REGISTER_CALLBACK_METHOD(hard_fork_1_pos_and_locked_coins, check_outputs_with_unique_amount); +} + +bool hard_fork_1_pos_and_locked_coins::generate(std::vector& events) const +{ + bool r = false; + GENERATE_ACCOUNT(miner_acc); + GENERATE_ACCOUNT(alice_acc); + MAKE_GENESIS_BLOCK(events, blk_0, miner_acc, test_core_time::get_time()); + generator.set_hardfork_height(m_hardfork_height); + DO_CALLBACK(events, "configure_core"); + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 3); + + DO_CALLBACK_PARAMS(events, "check_outputs_with_unique_amount", static_cast(0)); + + // create few locked outputs in the blockchain with unique amount + std::vector extra; + etc_tx_details_unlock_time ut = AUTO_VAL_INIT(ut); + ut.v = 100; // locked until block 100 + extra.push_back(ut); + + std::vector destinations; + for (size_t i = 0; i < 5; ++i) + destinations.push_back(tx_destination_entry(m_unique_amount, alice_acc.get_public_address())); + + transaction tx_0 = AUTO_VAL_INIT(tx_0); + r = construct_tx_to_key(events, tx_0, blk_0r, miner_acc, destinations, TESTS_DEFAULT_FEE, 0, 0, extra); + CHECK_AND_ASSERT_MES(r, false, "construct_tx failed"); + events.push_back(tx_0); + + MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, miner_acc, tx_0); + + DO_CALLBACK_PARAMS(events, "check_outputs_with_unique_amount", static_cast(5)); + + + block blk_0a; + { + crypto::hash prev_id = get_block_hash(blk_0); + size_t height = get_block_height(blk_0) + 1; + currency::wide_difficulty_type diff = generator.get_difficulty_for_next_block(prev_id, false); + + const transaction& stake = blk_0.miner_tx; + crypto::public_key stake_tx_pub_key = get_tx_pub_key_from_extra(stake); + size_t stake_output_idx = 0; + size_t stake_output_gidx = 0; + uint64_t stake_output_amount = stake.vout[stake_output_idx].amount; + crypto::key_image stake_output_key_image; + keypair kp; + generate_key_image_helper(miner_acc.get_keys(), stake_tx_pub_key, stake_output_idx, kp, stake_output_key_image); + crypto::public_key stake_output_pubkey = boost::get(stake.vout[stake_output_idx].target).key; + + pos_block_builder pb; + pb.step1_init_header(height, prev_id); + pb.step2_set_txs(std::vector()); + pb.step3_build_stake_kernel(stake_output_amount, stake_output_gidx, stake_output_key_image, diff, prev_id, null_hash, blk_0r.timestamp); + pb.step4_generate_coinbase_tx(generator.get_timestamps_median(prev_id), generator.get_already_generated_coins(blk_0r), miner_acc.get_public_address()); + pb.step5_sign(stake_tx_pub_key, stake_output_idx, stake_output_pubkey, miner_acc); + blk_0a = pb.m_block; + } + + return true; +} + +bool hard_fork_1_pos_and_locked_coins::check_outputs_with_unique_amount(currency::core& c, size_t ev_index, const std::vector& events) +{ + size_t expected_outputs_count = 0; + const std::string& params = boost::get(events[ev_index]).callback_params; + CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(params, expected_outputs_count), false, "hex_to_pod failed, params = " << params); + + std::list pub_keys; + bool r = c.get_outs(m_unique_amount, pub_keys); + + CHECK_AND_ASSERT_MES(r && pub_keys.size() == expected_outputs_count, false, "amount " << print_money_brief(m_unique_amount) << ": " << pub_keys.size() << " != " << expected_outputs_count); + + return true; +} diff --git a/tests/core_tests/hard_fork_1.h b/tests/core_tests/hard_fork_1.h new file mode 100644 index 00000000..261dcd8b --- /dev/null +++ b/tests/core_tests/hard_fork_1.h @@ -0,0 +1,50 @@ +// Copyright (c) 2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#pragma once + +#include "chaingen.h" +//#include "wallet_tests_basic.h" +#include "checkpoints_tests.h" + +struct hard_fork_1_base_test : virtual public test_chain_unit_enchanced +{ + hard_fork_1_base_test(size_t hardfork_height); + bool configure_core(currency::core& c, size_t ev_index, const std::vector& events); + + size_t m_hardfork_height; +}; + +struct hard_fork_1_unlock_time_2_in_normal_tx : public hard_fork_1_base_test +{ + hard_fork_1_unlock_time_2_in_normal_tx(); + bool generate(std::vector& events) const; +}; + +struct hard_fork_1_unlock_time_2_in_coinbase : public hard_fork_1_base_test +{ + hard_fork_1_unlock_time_2_in_coinbase(); + bool generate(std::vector& events) const; +}; + +struct hard_fork_1_chain_switch_pow_only : public hard_fork_1_base_test +{ + hard_fork_1_chain_switch_pow_only(); + bool generate(std::vector& events) const; +}; + +struct hard_fork_1_checkpoint_basic_test : public hard_fork_1_base_test, public checkpoints_test +{ + hard_fork_1_checkpoint_basic_test(); + bool generate(std::vector& events) const; +}; + +struct hard_fork_1_pos_and_locked_coins : public hard_fork_1_base_test +{ + hard_fork_1_pos_and_locked_coins(); + bool generate(std::vector& events) const; + + bool check_outputs_with_unique_amount(currency::core& c, size_t ev_index, const std::vector& events); + + uint64_t m_unique_amount; +}; diff --git a/tests/core_tests/hard_fork_1_bad_pos_source.cpp b/tests/core_tests/hard_fork_1_bad_pos_source.cpp new file mode 100644 index 00000000..1fd43a0c --- /dev/null +++ b/tests/core_tests/hard_fork_1_bad_pos_source.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "chaingen.h" +#include "pos_validation.h" +#include "tx_builder.h" +#include "hard_fork_1_bad_pos_source.h" +#include "random_helper.h" + +using namespace epee; +using namespace crypto; +using namespace currency; + +hard_fork_1_bad_pos_source::hard_fork_1_bad_pos_source() +{ + REGISTER_CALLBACK_METHOD(hard_fork_1_bad_pos_source, c1); + REGISTER_CALLBACK_METHOD(hard_fork_1_bad_pos_source, configure_core); +} + +bool hard_fork_1_bad_pos_source::generate(std::vector& events) const +{ + random_state_test_restorer::reset_random(); + + GENERATE_ACCOUNT(preminer_acc); + GENERATE_ACCOUNT(miner_acc); + GENERATE_ACCOUNT(pos_miner_acc_before_pow); + GENERATE_ACCOUNT(pos_miner_acc_after_pow); + m_accounts.push_back(miner_acc); + m_accounts.push_back(pos_miner_acc_before_pow); + m_accounts.push_back(pos_miner_acc_after_pow); + std::list miner_acc_lst(1, miner_acc); + + MAKE_GENESIS_BLOCK(events, blk_0, preminer_acc, 1564434616); + generator.set_hardfork_height(get_hardfork_height()); + + DO_CALLBACK(events, "configure_core"); + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 5); + generator.set_pos_to_low_timestamp(true); + MAKE_TX(events, tx_1, miner_acc, pos_miner_acc_before_pow, 1000000000000, blk_0r); + MAKE_NEXT_BLOCK_TX1(events, blk_pow_tx1, blk_0r, miner_acc, tx_1); + MAKE_TX(events, tx_2, preminer_acc, pos_miner_acc_after_pow, 1000000000000, blk_pow_tx1); + MAKE_NEXT_POS_BLOCK_TX1(events, blk_pos_tx2, blk_pow_tx1, miner_acc, miner_acc_lst, tx_2); + + block last_block = blk_pos_tx2; + for (size_t i = 0; i != CURRENCY_MINED_MONEY_UNLOCK_WINDOW; i++) + { + MAKE_NEXT_POS_BLOCK(events, next_blk_pos, last_block, miner_acc, miner_acc_lst); + events.push_back(event_core_time(next_blk_pos.timestamp - 10)); + last_block = next_blk_pos; + } + + std::list accounts_before_pow; + accounts_before_pow.push_back(pos_miner_acc_before_pow); + //let's try to mint PoW block from account + MAKE_NEXT_POS_BLOCK(events, next_blk_pos2, last_block, miner_acc, accounts_before_pow); + + std::list accounts_after_pow; + accounts_after_pow.push_back(pos_miner_acc_after_pow); + //let's try to mint PoW block from account + generator.set_ignore_last_pow_in_wallets(true); + DO_CALLBACK(events, "mark_invalid_block"); + MAKE_NEXT_POS_BLOCK(events, next_blk_pos3, next_blk_pos2, miner_acc, accounts_after_pow); + + DO_CALLBACK(events, "c1"); + return true; +} + +bool hard_fork_1_bad_pos_source::configure_core(currency::core& c, size_t ev_index, const std::vector& events) +{ + currency::core_runtime_config pc = c.get_blockchain_storage().get_core_runtime_config(); + pc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; //four blocks + pc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; //four blocks + pc.hard_fork1_starts_after_height = get_hardfork_height(); + + c.get_blockchain_storage().set_core_runtime_config(pc); + return true; +} + +bool hard_fork_1_bad_pos_source::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + + uint64_t h = c.get_blockchain_storage().get_current_blockchain_size(); + CHECK_AND_ASSERT_MES(h == 29, false, "locked pos block is not accepted"); + return true; +} + +uint64_t hard_fork_1_bad_pos_source::get_hardfork_height()const +{ + return 10; +} + diff --git a/tests/core_tests/hard_fork_1_bad_pos_source.h b/tests/core_tests/hard_fork_1_bad_pos_source.h new file mode 100644 index 00000000..e59adfa3 --- /dev/null +++ b/tests/core_tests/hard_fork_1_bad_pos_source.h @@ -0,0 +1,18 @@ +// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#include "chaingen.h" +#include "wallet_tests_basic.h" + + +struct hard_fork_1_bad_pos_source : public wallet_test +{ + hard_fork_1_bad_pos_source(); + bool generate(std::vector& events) const; + bool configure_core(currency::core& c, size_t ev_index, const std::vector& events); + virtual bool c1(currency::core& c, size_t ev_index, const std::vector& events); + virtual uint64_t get_hardfork_height()const; +}; diff --git a/tests/core_tests/hard_fork_1_consensus_test.cpp b/tests/core_tests/hard_fork_1_consensus_test.cpp new file mode 100644 index 00000000..60bafdb7 --- /dev/null +++ b/tests/core_tests/hard_fork_1_consensus_test.cpp @@ -0,0 +1,119 @@ +// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "chaingen.h" +#include "pos_validation.h" +#include "tx_builder.h" +#include "hard_fork_1_consensus_test.h" +#include "random_helper.h" + +using namespace epee; +using namespace crypto; +using namespace currency; + +hard_fork_1_cumulative_difficulty_base::hard_fork_1_cumulative_difficulty_base() +{ + REGISTER_CALLBACK_METHOD(hard_fork_1_cumulative_difficulty_base, c1); + REGISTER_CALLBACK_METHOD(hard_fork_1_cumulative_difficulty_base, configure_core); +} + +bool hard_fork_1_cumulative_difficulty_base::generate(std::vector& events) const +{ + random_state_test_restorer::reset_random(); + + GENERATE_ACCOUNT(preminer_acc); + GENERATE_ACCOUNT(miner_acc); + m_accounts.push_back(miner_acc); + std::list miner_acc_lst(1, miner_acc); + + MAKE_GENESIS_BLOCK(events, blk_0, preminer_acc, 1564434616); + generator.set_hardfork_height(get_hardfork_height()); + + DO_CALLBACK(events, "configure_core"); + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 3); + block last_block = blk_0r; + for (size_t i = 0; i != 20; i++) + { + MAKE_NEXT_POS_BLOCK(events, next_blk_pos, last_block, miner_acc, miner_acc_lst); + MAKE_NEXT_BLOCK(events, next_blk_pow, next_blk_pos, miner_acc); + events.push_back(event_core_time(next_blk_pow.timestamp - 10)); + + last_block = next_blk_pow; + } + + generator.set_pos_to_low_timestamp(true); + last_block = blk_0r; + for (size_t i = 0; i != 20; i++) + { + MAKE_NEXT_POS_BLOCK(events, next_blk_pos, last_block, miner_acc, miner_acc_lst); + MAKE_NEXT_BLOCK_TIMESTAMP_ADJUSTMENT(-14, events, next_blk_pow, next_blk_pos, miner_acc); + last_block = next_blk_pow; + } + + + DO_CALLBACK(events, "c1"); + return true; +} + +bool hard_fork_1_cumulative_difficulty_base::configure_core(currency::core& c, size_t ev_index, const std::vector& events) +{ + currency::core_runtime_config pc = c.get_blockchain_storage().get_core_runtime_config(); + pc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; //four blocks + pc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; //four blocks + pc.hard_fork1_starts_after_height = get_hardfork_height(); + + c.get_blockchain_storage().set_core_runtime_config(pc); + return true; +} + +bool before_hard_fork_1_cumulative_difficulty::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + + std::shared_ptr last_pow_block = c.get_blockchain_storage().get_last_block_of_type(false); + std::shared_ptr last_pos_block = c.get_blockchain_storage().get_last_block_of_type(true); + CHECK_AND_ASSERT_MES(last_pow_block->cumulative_diff_precise == 184, false, "Incorrect condition failed: last_pow_block->cumulative_diff_precise == 184"); + CHECK_AND_ASSERT_MES(last_pos_block->cumulative_diff_precise == 20, false, "Incorrect condition failed: last_pos_block->cumulative_diff_precise == 20"); + // + return true; +} + +uint64_t before_hard_fork_1_cumulative_difficulty::get_hardfork_height()const +{ + return 10000; //just big number which is obviously bigger then test blockchain +} + + + +bool inthe_middle_hard_fork_1_cumulative_difficulty::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + + std::shared_ptr last_pow_block = c.get_blockchain_storage().get_last_block_of_type(false); + std::shared_ptr last_pos_block = c.get_blockchain_storage().get_last_block_of_type(true); + CHECK_AND_ASSERT_MES(last_pow_block->cumulative_diff_precise == 184, false, "Incorrect condition failed: last_pow_block->cumulative_diff_precise == 184"); + CHECK_AND_ASSERT_MES(last_pos_block->cumulative_diff_precise == 20, false, "Incorrect condition failed: last_pos_block->cumulative_diff_precise == 20"); + // + return true; +} + +uint64_t inthe_middle_hard_fork_1_cumulative_difficulty::get_hardfork_height()const +{ + return 15; +} +bool after_hard_fork_1_cumulative_difficulty::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + + std::shared_ptr last_pow_block = c.get_blockchain_storage().get_last_block_of_type(false); + std::shared_ptr last_pos_block = c.get_blockchain_storage().get_last_block_of_type(true); + CHECK_AND_ASSERT_MES(last_pow_block->cumulative_diff_precise == 172, false, "Incorrect condition failed: last_pow_block->cumulative_diff_precise == 184"); + CHECK_AND_ASSERT_MES(last_pos_block->cumulative_diff_precise == 199, false, "Incorrect condition failed: last_pos_block->cumulative_diff_precise == 20"); + // + return true; +} + +uint64_t after_hard_fork_1_cumulative_difficulty::get_hardfork_height()const +{ + return 12; +} + diff --git a/tests/core_tests/hard_fork_1_consensus_test.h b/tests/core_tests/hard_fork_1_consensus_test.h new file mode 100644 index 00000000..6898e46c --- /dev/null +++ b/tests/core_tests/hard_fork_1_consensus_test.h @@ -0,0 +1,40 @@ +// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#include "chaingen.h" +#include "wallet_tests_basic.h" + + +struct hard_fork_1_cumulative_difficulty_base : public wallet_test +{ + hard_fork_1_cumulative_difficulty_base(); + bool generate(std::vector& events) const; + bool configure_core(currency::core& c, size_t ev_index, const std::vector& events); + virtual bool c1(currency::core& c, size_t ev_index, const std::vector& events)= 0; + virtual uint64_t get_hardfork_height()const =0; +}; + +// this test check if code still work same way as it supposed to work before hard fork point +struct before_hard_fork_1_cumulative_difficulty : public hard_fork_1_cumulative_difficulty_base +{ + bool c1(currency::core& c, size_t ev_index, const std::vector& events); + virtual uint64_t get_hardfork_height()const; +}; + +// this test check if code still work same way as it supposed to work is split happened before hard fork point but finished after hard fork point +struct inthe_middle_hard_fork_1_cumulative_difficulty : public hard_fork_1_cumulative_difficulty_base +{ + bool c1(currency::core& c, size_t ev_index, const std::vector& events); + virtual uint64_t get_hardfork_height()const; +}; + +// this test check if code follow the new consensus algorithm and prefer balanced branch of the blockchain tree +struct after_hard_fork_1_cumulative_difficulty : public hard_fork_1_cumulative_difficulty_base +{ + bool c1(currency::core& c, size_t ev_index, const std::vector& events); + virtual uint64_t get_hardfork_height()const; +}; + diff --git a/tests/core_tests/hard_fork_1_locked_pos_test.cpp b/tests/core_tests/hard_fork_1_locked_pos_test.cpp new file mode 100644 index 00000000..3c96e07c --- /dev/null +++ b/tests/core_tests/hard_fork_1_locked_pos_test.cpp @@ -0,0 +1,97 @@ +// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "chaingen.h" +#include "pos_validation.h" +#include "tx_builder.h" +#include "hard_fork_1_locked_pos_test.h" +#include "random_helper.h" + +using namespace epee; +using namespace crypto; +using namespace currency; + +hard_fork_1_locked_mining_test::hard_fork_1_locked_mining_test() +{ + REGISTER_CALLBACK_METHOD(hard_fork_1_locked_mining_test, c1); + REGISTER_CALLBACK_METHOD(hard_fork_1_locked_mining_test, configure_core); +} + +bool hard_fork_1_locked_mining_test::generate(std::vector& events) const +{ + random_state_test_restorer::reset_random(); + + GENERATE_ACCOUNT(preminer_acc); + GENERATE_ACCOUNT(miner_acc); + GENERATE_ACCOUNT(pos_miner_acc); + m_accounts.push_back(miner_acc); + m_accounts.push_back(pos_miner_acc); + std::list miner_acc_lst(1, miner_acc); + + MAKE_GENESIS_BLOCK(events, blk_0, preminer_acc, 1564434616); + generator.set_hardfork_height(get_hardfork_height()); + + DO_CALLBACK(events, "configure_core"); + REWIND_BLOCKS_N_WITH_TIME(events, blk_0r, blk_0, miner_acc, CURRENCY_MINED_MONEY_UNLOCK_WINDOW + 3); + + //construc tx that locks transaction for some period of time + // make a couple of huge txs + bool r = false; + std::vector extra; + + std::vector sources_1; + r = fill_tx_sources(sources_1, events, blk_0r, miner_acc.get_keys(), 2000000000000+TESTS_DEFAULT_FEE, 0); + CHECK_AND_ASSERT_MES(r, false, "fill_tx_sources failed"); + std::vector destinations({ tx_destination_entry(2010000000000, pos_miner_acc.get_public_address()) }); + crypto::secret_key stub; + transaction tx_1 = AUTO_VAL_INIT(tx_1); + r = construct_tx(miner_acc.get_keys(), sources_1, destinations, extra, empty_attachment, tx_1, stub, get_block_height(blk_0r)+2000); + CHECK_AND_ASSERT_MES(r, false, "construct_tx failed"); + events.push_back(tx_1); // push it to the pool + + MAKE_NEXT_BLOCK_TX1(events, blk_0r_tx, blk_0r, miner_acc, tx_1); + + block last_block = blk_0r_tx; + for (size_t i = 0; i != CURRENCY_MINED_MONEY_UNLOCK_WINDOW; i++) + { + MAKE_NEXT_POS_BLOCK(events, next_blk_pos, last_block, miner_acc, miner_acc_lst); + MAKE_NEXT_BLOCK(events, next_blk_pow, next_blk_pos, miner_acc); + events.push_back(event_core_time(next_blk_pow.timestamp - 10)); + last_block = next_blk_pow; + } + + std::list accounts_2; + accounts_2.push_back(pos_miner_acc); + //let's try to mint PoS block from locked account + MAKE_NEXT_POS_BLOCK(events, next_blk_pos, last_block, pos_miner_acc, accounts_2); + + DO_CALLBACK(events, "c1"); + return true; +} + +bool hard_fork_1_locked_mining_test::configure_core(currency::core& c, size_t ev_index, const std::vector& events) +{ + currency::core_runtime_config pc = c.get_blockchain_storage().get_core_runtime_config(); + pc.min_coinstake_age = TESTS_POS_CONFIG_MIN_COINSTAKE_AGE; //four blocks + pc.pos_minimum_heigh = TESTS_POS_CONFIG_POS_MINIMUM_HEIGH; //four blocks + pc.hard_fork1_starts_after_height = get_hardfork_height(); + + c.get_blockchain_storage().set_core_runtime_config(pc); + return true; +} + +bool hard_fork_1_locked_mining_test::c1(currency::core& c, size_t ev_index, const std::vector& events) +{ + + uint64_t h = c.get_blockchain_storage().get_current_blockchain_size(); + CHECK_AND_ASSERT_MES(h == 36, false, "locked pos block is not accepted"); + return true; +} + +uint64_t hard_fork_1_locked_mining_test::get_hardfork_height()const +{ + return 10; +} + diff --git a/tests/core_tests/hard_fork_1_locked_pos_test.h b/tests/core_tests/hard_fork_1_locked_pos_test.h new file mode 100644 index 00000000..34fda63d --- /dev/null +++ b/tests/core_tests/hard_fork_1_locked_pos_test.h @@ -0,0 +1,18 @@ +// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#include "chaingen.h" +#include "wallet_tests_basic.h" + + +struct hard_fork_1_locked_mining_test : public wallet_test +{ + hard_fork_1_locked_mining_test(); + bool generate(std::vector& events) const; + bool configure_core(currency::core& c, size_t ev_index, const std::vector& events); + virtual bool c1(currency::core& c, size_t ev_index, const std::vector& events); + virtual uint64_t get_hardfork_height()const; +}; diff --git a/tests/core_tests/integer_overflow.cpp b/tests/core_tests/integer_overflow.cpp index cf71c954..014d7302 100644 --- a/tests/core_tests/integer_overflow.cpp +++ b/tests/core_tests/integer_overflow.cpp @@ -8,6 +8,10 @@ #include "integer_overflow.h" +// TOTAL_MONEY_SUPPLY - total number coins to be generated +#define TX_MAX_TRANSFER_AMOUNT ((uint64_t)(-1)) + + using namespace epee; using namespace currency; @@ -83,7 +87,7 @@ bool gen_uint_overflow_1::generate(std::vector& events) const // Problem 1. Miner tx output overflow MAKE_MINER_TX_MANUALLY(miner_tx_0, blk_0); - split_miner_tx_outs(miner_tx_0, TOTAL_MONEY_SUPPLY); + split_miner_tx_outs(miner_tx_0, TX_MAX_TRANSFER_AMOUNT); block blk_1; if (!generator.construct_block_manually(blk_1, blk_0, miner_account, test_generator::bf_miner_tx, 0, 0, 0, crypto::hash(), 0, miner_tx_0)) return false; @@ -91,23 +95,23 @@ bool gen_uint_overflow_1::generate(std::vector& events) const // Problem 1. Miner tx outputs overflow MAKE_MINER_TX_MANUALLY(miner_tx_1, blk_1); - split_miner_tx_outs(miner_tx_1, TOTAL_MONEY_SUPPLY); + split_miner_tx_outs(miner_tx_1, TX_MAX_TRANSFER_AMOUNT); block blk_2; if (!generator.construct_block_manually(blk_2, blk_1, miner_account, test_generator::bf_miner_tx, 0, 0, 0, crypto::hash(), 0, miner_tx_1)) return false; events.push_back(blk_2); REWIND_BLOCKS(events, blk_2r, blk_2, miner_account); - MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, TOTAL_MONEY_SUPPLY, blk_2r); - MAKE_TX_LIST(events, txs_0, miner_account, bob_account, TOTAL_MONEY_SUPPLY, blk_2r); + MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, TX_MAX_TRANSFER_AMOUNT, blk_2r); + MAKE_TX_LIST(events, txs_0, miner_account, bob_account, TX_MAX_TRANSFER_AMOUNT, blk_2r); MAKE_NEXT_BLOCK_TX_LIST(events, blk_3, blk_2r, miner_account, txs_0); REWIND_BLOCKS(events, blk_3r, blk_3, miner_account); // Problem 2. total_fee overflow, block_reward overflow std::list txs_1; // Create txs with huge fee - txs_1.push_back(construct_tx_with_fee(events, blk_3, bob_account, alice_account, MK_TEST_COINS(1), TOTAL_MONEY_SUPPLY - MK_TEST_COINS(1))); - txs_1.push_back(construct_tx_with_fee(events, blk_3, bob_account, alice_account, MK_TEST_COINS(1), TOTAL_MONEY_SUPPLY - MK_TEST_COINS(1))); + txs_1.push_back(construct_tx_with_fee(events, blk_3, bob_account, alice_account, MK_TEST_COINS(1), TX_MAX_TRANSFER_AMOUNT - MK_TEST_COINS(1))); + txs_1.push_back(construct_tx_with_fee(events, blk_3, bob_account, alice_account, MK_TEST_COINS(1), TX_MAX_TRANSFER_AMOUNT - MK_TEST_COINS(1))); MAKE_NEXT_BLOCK_TX_LIST(events, blk_4, blk_3r, miner_account, txs_1); return true; @@ -143,10 +147,10 @@ bool gen_uint_overflow_2::generate(std::vector& events) const std::vector destinations; const account_public_address& bob_addr = bob_account.get_keys().m_account_address; - destinations.push_back(tx_destination_entry(TOTAL_MONEY_SUPPLY, bob_addr)); - destinations.push_back(tx_destination_entry(TOTAL_MONEY_SUPPLY - 1, bob_addr)); + destinations.push_back(tx_destination_entry(TX_MAX_TRANSFER_AMOUNT, bob_addr)); + destinations.push_back(tx_destination_entry(TX_MAX_TRANSFER_AMOUNT - 1, bob_addr)); // sources.front().amount = destinations[0].amount + destinations[2].amount + destinations[3].amount + TESTS_DEFAULT_FEE - destinations.push_back(tx_destination_entry(sources.front().amount - TOTAL_MONEY_SUPPLY - TOTAL_MONEY_SUPPLY + 1 - TESTS_DEFAULT_FEE, bob_addr)); + destinations.push_back(tx_destination_entry(sources.front().amount - TX_MAX_TRANSFER_AMOUNT - TX_MAX_TRANSFER_AMOUNT + 1 - TESTS_DEFAULT_FEE, bob_addr)); currency::transaction tx_1; std::vector attachments; @@ -162,7 +166,7 @@ bool gen_uint_overflow_2::generate(std::vector& events) const for (size_t i = 0; i < tx_1.vout.size(); ++i) { auto& tx_1_out = tx_1.vout[i]; - if (tx_1_out.amount < TOTAL_MONEY_SUPPLY - 1) + if (tx_1_out.amount < TX_MAX_TRANSFER_AMOUNT - 1) continue; append_tx_source_entry(sources, tx_1, i); @@ -171,7 +175,7 @@ bool gen_uint_overflow_2::generate(std::vector& events) const destinations.clear(); currency::tx_destination_entry de; de.addr.push_back(alice_account.get_keys().m_account_address); - de.amount = TOTAL_MONEY_SUPPLY - TESTS_DEFAULT_FEE; + de.amount = TX_MAX_TRANSFER_AMOUNT - TESTS_DEFAULT_FEE; destinations.push_back(de); destinations.push_back(de); diff --git a/tests/core_tests/multisig_wallet_tests.cpp b/tests/core_tests/multisig_wallet_tests.cpp index 3bd63fd5..1b677ecf 100644 --- a/tests/core_tests/multisig_wallet_tests.cpp +++ b/tests/core_tests/multisig_wallet_tests.cpp @@ -1194,7 +1194,7 @@ bool multisig_and_unlock_time::generate(std::vector& events) c transaction tx_1 = AUTO_VAL_INIT(tx_1); r = construct_tx(miner_acc.get_keys(), sources, destinations, empty_attachment, tx_1, unlock_time, CURRENCY_TO_KEY_OUT_RELAXED, true); CHECK_AND_ASSERT_MES(r, false, "construct_tx failed"); - CHECK_AND_ASSERT_MES(get_tx_unlock_time(tx_1) == unlock_time, false, "Unlock time was not correctly set"); + CHECK_AND_ASSERT_MES(get_tx_max_unlock_time(tx_1) == unlock_time, false, "Unlock time was not correctly set"); events.push_back(tx_1); MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, miner_acc, tx_1); diff --git a/tests/core_tests/pos_block_builder.cpp b/tests/core_tests/pos_block_builder.cpp index 95b281a9..71819a64 100644 --- a/tests/core_tests/pos_block_builder.cpp +++ b/tests/core_tests/pos_block_builder.cpp @@ -24,7 +24,7 @@ void pos_block_builder::step1_init_header(size_t block_height, crypto::hash& pre { CHECK_AND_ASSERT_THROW_MES(m_step == 0, "pos_block_builder: incorrect step sequence"); m_block.minor_version = CURRENT_BLOCK_MINOR_VERSION; - m_block.major_version = CURRENT_BLOCK_MAJOR_VERSION; + m_block.major_version = BLOCK_MAJOR_VERSION_INITAL; m_block.timestamp = 0; // to be set at step 3 m_block.prev_id = prev_block_hash; m_block.flags = CURRENCY_BLOCK_FLAG_POS_BLOCK; diff --git a/tests/core_tests/wallet_tests.cpp b/tests/core_tests/wallet_tests.cpp index e4af7dee..511250fa 100644 --- a/tests/core_tests/wallet_tests.cpp +++ b/tests/core_tests/wallet_tests.cpp @@ -48,15 +48,15 @@ bool determine_tx_real_inputs(currency::core& c, const currency::transaction& tx , m_found(false) {} - bool handle_output(const currency::transaction& tx, const currency::tx_out& out) + bool handle_output(const transaction& source_tx, const transaction& validated_tx, const tx_out& out, uint64_t out_i) { CHECK_AND_ASSERT_MES(!m_found, false, "Internal error: m_found is true but the visitor is still being applied"); - auto it = std::find(tx.vout.begin(), tx.vout.end(), out); - if (it == tx.vout.end()) + auto it = std::find(validated_tx.vout.begin(), validated_tx.vout.end(), out); + if (it == validated_tx.vout.end()) return false; - size_t output_tx_index = it - tx.vout.begin(); + size_t output_tx_index = it - validated_tx.vout.begin(); - crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); + crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(validated_tx); crypto::key_derivation derivation; bool r = generate_key_derivation(tx_pub_key, m_keys.m_view_secret_key, derivation); CHECK_AND_ASSERT_MES(r, false, "generate_key_derivation failed"); @@ -96,7 +96,7 @@ bool determine_tx_real_inputs(currency::core& c, const currency::transaction& tx continue; } local_visitor vis(keys, in.k_image); - bool r = c.get_blockchain_storage().scan_outputkeys_for_indexes(in, vis); + bool r = c.get_blockchain_storage().scan_outputkeys_for_indexes(tx, in, vis); CHECK_AND_ASSERT_MES(r || vis.m_found, false, "scan_outputkeys_for_indexes failed"); if (!vis.m_found) return false; diff --git a/tests/functional_tests/core_concurrency_test.cpp b/tests/functional_tests/core_concurrency_test.cpp index 810c135e..6da67324 100644 --- a/tests/functional_tests/core_concurrency_test.cpp +++ b/tests/functional_tests/core_concurrency_test.cpp @@ -46,7 +46,7 @@ static const std::vector empty_attachment; bool create_block_template_manually(const currency::block& prev_block, boost::multiprecision::uint128_t already_generated_coins, const std::vector& txs, const currency::account_public_address& miner_addr, currency::block& result) { result.flags = 0; - result.major_version = CURRENT_BLOCK_MAJOR_VERSION; + result.major_version = BLOCK_MAJOR_VERSION_INITAL; result.minor_version = CURRENT_BLOCK_MINOR_VERSION; result.nonce = 0; result.prev_id = get_block_hash(prev_block); diff --git a/tests/functional_tests/difficulty_analysis.cpp b/tests/functional_tests/difficulty_analysis.cpp index be994253..0af4d4d8 100644 --- a/tests/functional_tests/difficulty_analysis.cpp +++ b/tests/functional_tests/difficulty_analysis.cpp @@ -313,7 +313,7 @@ void run_emulation(const std::string& path) PERFORME_SIMULATION_FOR_FUNCTION(bbr_next_difficulty_configurable, BBR_DIFFICULTY_WINDOW, BBR_DIFFICULTY_CUT, BBR_DIFFICULTY_CUT); PERFORME_SIMULATION_FOR_FUNCTION(bbr_next_difficulty_configurable, 500, 60, 60); PERFORME_SIMULATION_FOR_FUNCTION(bbr_next_difficulty_configurable, 300, 60, 60); - PERFORME_SIMULATION_FOR_FUNCTION_NO_WINDOW(currency::next_difficulty); + PERFORME_SIMULATION_FOR_FUNCTION_NO_WINDOW(currency::next_difficulty_1); print_blocks(result_blocks, path + "result.txt"); LOG_PRINT_L0("Done"); diff --git a/tests/unit_tests/lmdb_tests.cpp b/tests/unit_tests/lmdb_tests.cpp new file mode 100644 index 00000000..10dc1229 --- /dev/null +++ b/tests/unit_tests/lmdb_tests.cpp @@ -0,0 +1,104 @@ +// Copyright (c) 2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include +#include "crypto/crypto.h" +#include "gtest/gtest.h" +#include "common/db_backend_lmdb.h" +#include "common/db_abstract_accessor.h" + +using namespace tools; + +namespace lmdb_test +{ + + TEST(lmdb, 2gb_test) + { + bool r = false; + epee::shared_recursive_mutex rw_lock; + + epee::log_space::log_singletone::enable_channels("lmdb"); + + static const uint64_t buffer_size = 64 * 1024; // 64 KB + static const uint64_t db_total_size = static_cast(2.1 * 1024 * 1024 * 1024); // 2.1 GB -- a bit more than 2GB to test 2GB boundary + static const std::string db_file_path = "2gb_lmdb_test"; + + std::shared_ptr lmdb_ptr = std::make_shared(); + db::basic_db_accessor bdba(lmdb_ptr, rw_lock); + + // + // write data + // + + r = bdba.open(db_file_path, CACHE_SIZE); + ASSERT_TRUE(r); + + db::container_handle h; + r = lmdb_ptr->open_container("c1", h); + ASSERT_TRUE(r); + + ASSERT_TRUE(lmdb_ptr->begin_transaction()); + ASSERT_TRUE(lmdb_ptr->clear(h)); + + std::vector buffer; + buffer.resize(buffer_size); + crypto::generate_random_bytes(buffer_size, buffer.data()); + + uint64_t total_data = 0; + for (uint64_t key = 0; key < db_total_size / buffer_size; ++key) + { + r = lmdb_ptr->set(h, (char*)&key, sizeof key, reinterpret_cast(buffer.data()), buffer_size); + ASSERT_TRUE(r); + total_data += buffer_size; + + if (key % 1024 == 0) + { + ASSERT_TRUE(lmdb_ptr->commit_transaction()); + ASSERT_TRUE(lmdb_ptr->resize_if_needed()); + ASSERT_TRUE(lmdb_ptr->begin_transaction()); + std::cout << total_data / 1024 / 1024 << " MB written to DB" << ENDL; + } + } + + ASSERT_TRUE(lmdb_ptr->commit_transaction()); + + r = bdba.close(); + ASSERT_TRUE(r); + + + // + // read data and check + // + + r = bdba.open(db_file_path); + ASSERT_TRUE(r); + r = lmdb_ptr->open_container("c1", h); + ASSERT_TRUE(r); + + ASSERT_TRUE(lmdb_ptr->begin_transaction()); + + std::string out_buffer; + total_data = 0; + for (uint64_t key = 0; key < db_total_size / buffer_size; ++key) + { + r = lmdb_ptr->get(h, (char*)&key, sizeof key, out_buffer); + ASSERT_TRUE(r); + ASSERT_EQ(buffer_size, out_buffer.size()); + + ASSERT_TRUE(0 == memcmp(buffer.data(), out_buffer.c_str(), buffer_size)); + + total_data += buffer_size; + if (key % 1024 == 0) + std::cout << total_data / 1024 / 1024 << " MB read from DB" << ENDL; + } + + ASSERT_TRUE(lmdb_ptr->commit_transaction()); + + r = bdba.close(); + ASSERT_TRUE(r); + + boost::filesystem::remove_all(db_file_path); + + } + +} diff --git a/tests/unit_tests/main.cpp b/tests/unit_tests/main.cpp index c6780fa8..9b62eef2 100644 --- a/tests/unit_tests/main.cpp +++ b/tests/unit_tests/main.cpp @@ -16,5 +16,6 @@ int main(int argc, char** argv) ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + int ret = RUN_ALL_TESTS(); + return ret; }