From f046b1490d19be00297f82e7504ec3ab1aeb97d7 Mon Sep 17 00:00:00 2001 From: "crypro.zoidberg" Date: Wed, 2 Jun 2021 05:53:19 +0300 Subject: [PATCH] updated to latest codebase --- src/common/atomics_boost_serialization.h | 32 + src/common/boost_serialization_helper.h | 18 +- src/common/callstack_helper.cpp | 184 + src/common/callstack_helper.h | 24 + src/common/command_line.cpp | 23 +- src/common/command_line.h | 35 +- src/common/config_encrypt_helper.h | 66 + src/common/crypto_stream_operators.cpp | 18 + src/common/crypto_stream_operators.h | 32 +- src/common/db_abstract_accessor.h | 108 +- src/common/db_backend_base.h | 4 +- src/common/db_backend_lmdb.cpp | 61 +- src/common/db_backend_lmdb.h | 37 +- src/common/db_backend_mdbx.cpp | 410 + src/common/db_backend_mdbx.h | 67 + src/common/db_backend_selector.cpp | 132 + src/common/db_backend_selector.h | 43 + src/common/encryption_filter.cpp | 7 + src/common/encryption_filter.h | 244 + src/common/error_codes.h | 42 + src/common/int-util.h | 18 +- src/common/make_hashable.h | 20 +- src/common/miniupnp_helper.h | 68 +- src/common/mnemonic-encoding.cpp | 13 +- src/common/mnemonic-encoding.h | 3 + src/common/ntp.cpp | 265 + src/common/ntp.h | 18 + src/common/pod_array_file_container.h | 127 + src/common/pre_download.h | 226 + src/common/util.cpp | 187 +- src/common/util.h | 77 +- src/contrib/db/CMakeLists.txt | 25 +- src/contrib/db/liblmdb/CMakeLists.txt | 2 +- src/contrib/db/liblmdb/mdb.c | 4 +- src/contrib/db/libmdbx/.circleci/config.yml | 20 + src/contrib/db/libmdbx/.clang-format | 3 + src/contrib/db/libmdbx/.gitignore | 35 + src/contrib/db/libmdbx/.travis.yml | 61 + src/contrib/db/libmdbx/AUTHORS | 32 + .../db/libmdbx/CMakeLists.dist-minimal | 192 + src/contrib/db/libmdbx/CMakeLists.txt | 342 + src/contrib/db/libmdbx/COPYRIGHT | 22 + src/contrib/db/libmdbx/GNUmakefile | 362 + src/contrib/db/libmdbx/LICENSE | 47 + src/contrib/db/libmdbx/README.md | 591 + src/contrib/db/libmdbx/appveyor.yml | 99 + src/contrib/db/libmdbx/cmake/compiler.cmake | 666 + src/contrib/db/libmdbx/cmake/profile.cmake | 45 + src/contrib/db/libmdbx/cmake/utils.cmake | 183 + src/contrib/db/libmdbx/example/CMakeLists.txt | 6 + src/contrib/db/libmdbx/example/README.md | 1 + src/contrib/db/libmdbx/example/example-mdbx.c | 112 + src/contrib/db/libmdbx/example/sample-bdb.txt | 77 + src/contrib/db/libmdbx/mdbx.h | 3497 ++++ .../db/libmdbx/packages/rpm/CMakeLists.txt | 184 + src/contrib/db/libmdbx/packages/rpm/build.sh | 18 + .../db/libmdbx/packages/rpm/package.sh | 25 + src/contrib/db/libmdbx/src/CMakeLists.txt | 225 + src/contrib/db/libmdbx/src/alloy.c | 26 + .../db/libmdbx/src/elements/config.h.in | 56 + src/contrib/db/libmdbx/src/elements/core.c | 16709 ++++++++++++++++ src/contrib/db/libmdbx/src/elements/defs.h | 450 + .../db/libmdbx/src/elements/internals.h | 1367 ++ .../db/libmdbx/src/elements/lck-posix.c | 551 + .../db/libmdbx/src/elements/lck-windows.c | 777 + src/contrib/db/libmdbx/src/elements/ntdll.def | 1244 ++ src/contrib/db/libmdbx/src/elements/osal.c | 1899 ++ src/contrib/db/libmdbx/src/elements/osal.h | 959 + .../db/libmdbx/src/elements/version.c.in | 46 + src/contrib/db/libmdbx/src/man1/mdbx_chk.1 | 87 + src/contrib/db/libmdbx/src/man1/mdbx_copy.1 | 60 + src/contrib/db/libmdbx/src/man1/mdbx_dump.1 | 80 + src/contrib/db/libmdbx/src/man1/mdbx_load.1 | 89 + src/contrib/db/libmdbx/src/man1/mdbx_stat.1 | 69 + .../db/libmdbx/src/tools/CMakeLists.txt | 42 + src/contrib/db/libmdbx/src/tools/mdbx_chk.c | 1430 ++ src/contrib/db/libmdbx/src/tools/mdbx_copy.c | 130 + src/contrib/db/libmdbx/src/tools/mdbx_dump.c | 352 + src/contrib/db/libmdbx/src/tools/mdbx_load.c | 567 + src/contrib/db/libmdbx/src/tools/mdbx_stat.c | 436 + src/contrib/db/libmdbx/src/tools/wingetopt.c | 95 + src/contrib/db/libmdbx/src/tools/wingetopt.h | 30 + src/contrib/db/libmdbx/test/CMakeLists.txt | 53 + src/contrib/db/libmdbx/test/append.cc | 164 + src/contrib/db/libmdbx/test/base.h | 116 + src/contrib/db/libmdbx/test/cases.cc | 99 + src/contrib/db/libmdbx/test/chrono.cc | 136 + src/contrib/db/libmdbx/test/chrono.h | 99 + src/contrib/db/libmdbx/test/config.cc | 602 + src/contrib/db/libmdbx/test/config.h | 326 + src/contrib/db/libmdbx/test/copy.cc | 26 + src/contrib/db/libmdbx/test/darwin/LICENSE | 24 + src/contrib/db/libmdbx/test/darwin/README.md | 8 + .../db/libmdbx/test/darwin/pthread_barrier.c | 110 + .../db/libmdbx/test/darwin/pthread_barrier.h | 83 + src/contrib/db/libmdbx/test/dead.cc | 35 + src/contrib/db/libmdbx/test/hill.cc | 409 + src/contrib/db/libmdbx/test/jitter.cc | 59 + src/contrib/db/libmdbx/test/keygen.cc | 275 + src/contrib/db/libmdbx/test/keygen.h | 130 + src/contrib/db/libmdbx/test/log.cc | 371 + src/contrib/db/libmdbx/test/log.h | 104 + .../db/libmdbx/test/long_stochastic.sh | 220 + src/contrib/db/libmdbx/test/main.cc | 617 + src/contrib/db/libmdbx/test/nested.cc | 284 + src/contrib/db/libmdbx/test/osal-unix.cc | 368 + src/contrib/db/libmdbx/test/osal-windows.cc | 461 + src/contrib/db/libmdbx/test/osal.h | 49 + .../db/libmdbx/test/pcrf/CMakeLists.txt | 5 + src/contrib/db/libmdbx/test/pcrf/README.md | 2 + src/contrib/db/libmdbx/test/pcrf/pcrf_test.c | 413 + src/contrib/db/libmdbx/test/test.cc | 736 + src/contrib/db/libmdbx/test/test.h | 314 + src/contrib/db/libmdbx/test/try.cc | 20 + src/contrib/db/libmdbx/test/ttl.cc | 172 + src/contrib/db/libmdbx/test/utils.cc | 370 + src/contrib/db/libmdbx/test/utils.h | 362 + .../db/libmdbx/test/valgrind_suppress.txt | 24 + .../eos/portable_archive_exception.hpp | 13 +- .../eos/portable_iarchive.hpp | 20 +- .../eos/portable_oarchive.hpp | 25 +- src/contrib/epee/include/console_handler.h | 6 +- src/contrib/epee/include/file_io_utils.h | 137 +- src/contrib/epee/include/gzip_encoding.h | 469 +- src/contrib/epee/include/math_helper.h | 6 +- src/contrib/epee/include/misc_helpers.h | 116 + src/contrib/epee/include/misc_language.h | 74 +- src/contrib/epee/include/misc_log_ex.h | 513 +- src/contrib/epee/include/misc_os_dependent.h | 35 +- .../epee/include/net/abstract_tcp_server2.h | 421 +- .../epee/include/net/abstract_tcp_server2.inl | 1506 +- src/contrib/epee/include/net/http_base.h | 6 +- src/contrib/epee/include/net/http_client.h | 197 +- .../include/net/http_protocol_handler.inl | 4 +- .../include/net/http_server_handlers_map2.h | 6 + .../net/levin_protocol_handler_async.h | 27 +- src/contrib/epee/include/net/net_helper.h | 9 +- src/contrib/epee/include/net/net_utils_base.h | 26 +- src/contrib/epee/include/profile_tools.h | 31 +- src/contrib/epee/include/reg_exp_definer.h | 11 +- .../include/serialization/keyvalue_helpers.h | 31 +- .../serialization/keyvalue_hexemizer.h | 41 + .../serialization/keyvalue_serialization.h | 19 +- src/contrib/epee/include/singleton.h | 4 +- src/contrib/epee/include/static_helpers.h | 101 + .../include/storages/levin_abstract_invoke2.h | 9 +- .../epee/include/storages/portable_storage.h | 6 +- .../include/storages/portable_storage_base.h | 3 + .../portable_storage_template_helper.h | 10 +- .../storages/portable_storage_to_bin.h | 8 +- .../storages/portable_storage_to_json.h | 78 +- .../portable_storage_val_converters.h | 16 +- src/contrib/epee/include/string_coding.h | 6 +- src/contrib/epee/include/string_tools.h | 64 +- src/contrib/epee/include/sync_locked_object.h | 9 + src/contrib/epee/include/syncobj.h | 76 +- src/contrib/epee/include/time_helper.h | 16 +- src/contrib/epee/include/warnings.h | 16 +- src/contrib/epee/include/zlib_helper.h | 13 +- .../ethereum/libethash/bit_manipulation.h | 3 +- src/contrib/ethereum/libethash/ethash.cpp | 35 +- .../ethereum/libethash/ethash/ethash.hpp | 33 +- src/contrib/ethereum/libethash/managed.cpp | 4 +- src/contrib/miniupnp/.travis.yml | 49 + src/contrib/miniupnp/CODE_OF_CONDUCT | 1 + src/contrib/miniupnp/CONTRIBUTING | 11 + src/contrib/miniupnp/LICENSE | 26 + src/contrib/miniupnp/README | 61 + src/contrib/miniupnp/appveyor.yml | 21 + src/contrib/miniupnp/minissdpd/.gitignore | 9 + src/contrib/miniupnp/minissdpd/Changelog.txt | 158 + src/contrib/miniupnp/minissdpd/LICENSE | 26 + src/contrib/miniupnp/minissdpd/Makefile | 127 + src/contrib/miniupnp/minissdpd/README | 70 + src/contrib/miniupnp/minissdpd/README.fr | 49 + src/contrib/miniupnp/minissdpd/VERSION | 1 + src/contrib/miniupnp/minissdpd/asyncsendto.c | 348 + src/contrib/miniupnp/minissdpd/asyncsendto.h | 51 + src/contrib/miniupnp/minissdpd/codelength.h | 54 + src/contrib/miniupnp/minissdpd/config.h | 33 + src/contrib/miniupnp/minissdpd/daemonize.c | 129 + src/contrib/miniupnp/minissdpd/daemonize.h | 35 + src/contrib/miniupnp/minissdpd/getifaddr.c | 261 + src/contrib/miniupnp/minissdpd/getifaddr.h | 32 + src/contrib/miniupnp/minissdpd/getroute.c | 282 + src/contrib/miniupnp/minissdpd/getroute.h | 17 + src/contrib/miniupnp/minissdpd/ifacewatch.c | 345 + src/contrib/miniupnp/minissdpd/ifacewatch.h | 17 + src/contrib/miniupnp/minissdpd/listifaces.c | 120 + src/contrib/miniupnp/minissdpd/minissdpd.1 | 43 + src/contrib/miniupnp/minissdpd/minissdpd.c | 1646 ++ .../minissdpd/minissdpd.init.d.script | 39 + .../miniupnp/minissdpd/minissdpdtypes.h | 28 + .../miniupnp/minissdpd/openssdpsocket.c | 233 + .../miniupnp/minissdpd/openssdpsocket.h | 33 + .../miniupnp/minissdpd/printresponse.c | 90 + .../miniupnp/minissdpd/printresponse.h | 12 + .../miniupnp/minissdpd/showminissdpdnotif.c | 85 + .../miniupnp/minissdpd/testcodelength.c | 31 + .../miniupnp/minissdpd/testminissdpd.c | 196 + .../miniupnp/minissdpd/testminissdpd.sh | 18 + .../miniupnp/minissdpd/testminissdpdnotif.sh | 26 + src/contrib/miniupnp/minissdpd/upnputils.c | 172 + src/contrib/miniupnp/minissdpd/upnputils.h | 53 + src/contrib/miniupnp/miniupnp.podspec | 54 + .../miniupnp/miniupnpc-async/.gitignore | 3 + .../miniupnp/miniupnpc-async/Changelog.txt | 4 + src/contrib/miniupnp/miniupnpc-async/Makefile | 115 + src/contrib/miniupnp/miniupnpc-async/README | 10 + src/contrib/miniupnp/miniupnpc-async/config.h | 5 + .../miniupnp/miniupnpc-async/declspec.h | 15 + .../miniupnp/miniupnpc-async/igd_desc_parse.c | 123 + .../miniupnp/miniupnpc-async/igd_desc_parse.h | 49 + .../miniupnpc-async/miniupnpc-async.c | 1044 + .../miniupnpc-async/miniupnpc-async.h | 118 + .../miniupnp/miniupnpc-async/minixml.c | 229 + .../miniupnp/miniupnpc-async/minixml.h | 37 + .../miniupnp/miniupnpc-async/parsessdpreply.c | 80 + .../miniupnp/miniupnpc-async/parsessdpreply.h | 32 + .../miniupnp/miniupnpc-async/testasync.c | 153 + .../miniupnp/miniupnpc-async/upnpreplyparse.c | 197 + .../miniupnp/miniupnpc-async/upnpreplyparse.h | 63 + .../miniupnp/miniupnpc-async/upnputils.c | 87 + .../miniupnp/miniupnpc-async/upnputils.h | 27 + .../miniupnp/miniupnpc-libevent/.gitignore | 3 + .../miniupnp/miniupnpc-libevent/Makefile | 69 + .../miniupnp/miniupnpc-libevent/README | 13 + .../miniupnp/miniupnpc-libevent/declspec.h | 15 + .../miniupnpc-libevent/igd_desc_parse.c | 123 + .../miniupnpc-libevent/igd_desc_parse.h | 49 + .../miniupnpc-libevent/miniupnpc-libevent.c | 1026 + .../miniupnpc-libevent/miniupnpc-libevent.h | 132 + .../miniupnp/miniupnpc-libevent/minixml.c | 229 + .../miniupnp/miniupnpc-libevent/minixml.h | 37 + .../miniupnpc-libevent/upnpc-libevent.c | 252 + .../miniupnpc-libevent/upnpreplyparse.c | 197 + .../miniupnpc-libevent/upnpreplyparse.h | 63 + src/contrib/miniupnp/miniupnpc/.gitignore | 37 + src/contrib/miniupnp/miniupnpc/CMakeLists.txt | 190 + src/contrib/miniupnp/miniupnpc/Changelog.txt | 711 + src/contrib/miniupnp/miniupnpc/LICENSE | 27 + src/contrib/miniupnp/miniupnpc/MANIFEST.in | 9 + src/contrib/miniupnp/miniupnpc/Makefile | 404 + src/contrib/miniupnp/miniupnpc/Makefile.mingw | 121 + src/contrib/miniupnp/miniupnpc/README | 63 + src/contrib/miniupnp/miniupnpc/VERSION | 1 + .../miniupnp/miniupnpc/apiversions.txt | 178 + src/contrib/miniupnp/miniupnpc/codelength.h | 54 + .../miniupnp/miniupnpc/connecthostport.c | 264 + .../miniupnp/miniupnpc/connecthostport.h | 20 + src/contrib/miniupnp/miniupnpc/external-ip.sh | 4 + .../miniupnp/miniupnpc/igd_desc_parse.c | 123 + .../miniupnp/miniupnpc/igd_desc_parse.h | 49 + .../miniupnp/miniupnpc/java/.gitignore | 2 + .../miniupnpc/java/JavaBridgeTest.java | 97 + .../miniupnp/miniupnpc/java/testjava.bat | 8 + .../miniupnp/miniupnpc/java/testjava.sh | 8 + src/contrib/miniupnp/miniupnpc/listdevices.c | 197 + .../miniupnp/miniupnpc/man3/miniupnpc.3 | 55 + .../miniupnp/miniupnpc/mingw32make.bat | 8 + .../miniupnp/miniupnpc/minihttptestserver.c | 675 + src/contrib/miniupnp/miniupnpc/minisoap.c | 124 + src/contrib/miniupnp/miniupnpc/minisoap.h | 17 + src/contrib/miniupnp/miniupnpc/minissdpc.c | 888 + src/contrib/miniupnp/miniupnpc/minissdpc.h | 58 + src/contrib/miniupnp/miniupnpc/miniupnpc.c | 727 + src/contrib/miniupnp/miniupnpc/miniupnpc.def | 45 + src/contrib/miniupnp/miniupnpc/miniupnpc.h | 153 + .../miniupnp/miniupnpc/miniupnpc_declspec.h | 21 + .../miniupnp/miniupnpc/miniupnpc_socketdef.h | 37 + .../miniupnp/miniupnpc/miniupnpcmodule.c | 703 + .../miniupnpc/miniupnpcstrings.h.cmake | 15 + .../miniupnp/miniupnpc/miniupnpcstrings.h.in | 23 + .../miniupnp/miniupnpc/miniupnpctypes.h | 19 + src/contrib/miniupnp/miniupnpc/miniwget.c | 662 + src/contrib/miniupnp/miniupnpc/miniwget.h | 27 + .../miniupnp/miniupnpc/miniwget_private.h | 15 + src/contrib/miniupnp/miniupnpc/minixml.c | 231 + src/contrib/miniupnp/miniupnpc/minixml.h | 37 + src/contrib/miniupnp/miniupnpc/minixmlvalid.c | 163 + .../miniupnp/miniupnpc/msvc/.gitignore | 8 + .../miniupnpc/msvc/genminiupnpcstrings.vbs | 53 + .../miniupnp/miniupnpc/msvc/miniupnpc.sln | 29 + .../miniupnp/miniupnpc/msvc/miniupnpc.vcproj | 283 + .../miniupnp/miniupnpc/msvc/miniupnpc.vcxproj | 117 + .../miniupnpc/msvc/miniupnpc.vcxproj.filters | 108 + .../miniupnpc/msvc/miniupnpc_vs2010.sln | 26 + .../miniupnpc/msvc/miniupnpc_vs2010.vcxproj | 118 + .../msvc/miniupnpc_vs2010.vcxproj.filters | 111 + .../miniupnpc/msvc/miniupnpc_vs2015.sln | 28 + .../miniupnpc/msvc/upnpc-static.vcproj | 195 + .../miniupnpc/msvc/upnpc-static.vcxproj | 103 + .../msvc/upnpc-static.vcxproj.filters | 22 + .../msvc/upnpc-static_vs2010.vcxproj | 101 + .../miniupnp/miniupnpc/portlistingparse.c | 172 + .../miniupnp/miniupnpc/portlistingparse.h | 65 + .../miniupnp/miniupnpc/pymoduletest.py | 88 + .../miniupnp/miniupnpc/pymoduletest3.py | 52 + src/contrib/miniupnp/miniupnpc/receivedata.c | 99 + src/contrib/miniupnp/miniupnpc/receivedata.h | 21 + src/contrib/miniupnp/miniupnpc/setup.py | 35 + .../miniupnp/miniupnpc/setupmingw32.py | 28 + .../testdesc/linksys_WAG200G_desc.values | 14 + .../testdesc/linksys_WAG200G_desc.xml | 110 + .../testdesc/new_LiveBox_desc.values | 20 + .../miniupnpc/testdesc/new_LiveBox_desc.xml | 90 + .../miniupnp/miniupnpc/testigddescparse.c | 187 + src/contrib/miniupnp/miniupnpc/testminiwget.c | 56 + .../miniupnp/miniupnpc/testminiwget.sh | 111 + src/contrib/miniupnp/miniupnpc/testminixml.c | 89 + .../miniupnp/miniupnpc/testportlistingparse.c | 151 + .../DeletePortMapping.namevalue | 3 + .../testreplyparse/DeletePortMapping.xml | 6 + .../GetExternalIPAddress.namevalue | 2 + .../testreplyparse/GetExternalIPAddress.xml | 2 + .../GetSpecificPortMappingEntryReq.namevalue | 3 + .../GetSpecificPortMappingEntryReq.xml | 3 + .../GetSpecificPortMappingEntryResp.namevalue | 5 + .../GetSpecificPortMappingEntryResp.xml | 2 + .../SetDefaultConnectionService.namevalue | 1 + .../SetDefaultConnectionService.xml | 1 + .../miniupnpc/testreplyparse/readme.txt | 7 + src/contrib/miniupnp/miniupnpc/testupnpigd.py | 88 + .../miniupnp/miniupnpc/testupnpreplyparse.c | 115 + .../miniupnp/miniupnpc/testupnpreplyparse.sh | 14 + .../miniupnpc/updateminiupnpcstrings.sh | 53 + src/contrib/miniupnp/miniupnpc/upnpc.c | 861 + src/contrib/miniupnp/miniupnpc/upnpcommands.c | 1241 ++ src/contrib/miniupnp/miniupnpc/upnpcommands.h | 348 + src/contrib/miniupnp/miniupnpc/upnpdev.c | 23 + src/contrib/miniupnp/miniupnpc/upnpdev.h | 36 + src/contrib/miniupnp/miniupnpc/upnperrors.c | 107 + src/contrib/miniupnp/miniupnpc/upnperrors.h | 26 + .../miniupnp/miniupnpc/upnpreplyparse.c | 196 + .../miniupnp/miniupnpc/upnpreplyparse.h | 63 + .../miniupnpc/wingenminiupnpcstrings.c | 83 + src/contrib/miniupnp/miniupnpd/.gitignore | 26 + src/contrib/miniupnp/miniupnpd/Changelog.txt | 1270 ++ src/contrib/miniupnp/miniupnpd/INSTALL | 195 + src/contrib/miniupnp/miniupnpd/LICENSE | 27 + src/contrib/miniupnp/miniupnpd/Makefile | 274 + src/contrib/miniupnp/miniupnpd/Makefile.linux | 343 + .../miniupnp/miniupnpd/Makefile.linux_nft | 231 + .../miniupnp/miniupnpd/Makefile.macosx | 132 + src/contrib/miniupnp/miniupnpd/Makefile.sunos | 202 + src/contrib/miniupnp/miniupnpd/README | 37 + src/contrib/miniupnp/miniupnpd/TODO | 24 + src/contrib/miniupnp/miniupnpd/VERSION | 1 + src/contrib/miniupnp/miniupnpd/asyncsendto.c | 348 + src/contrib/miniupnp/miniupnpd/asyncsendto.h | 51 + src/contrib/miniupnp/miniupnpd/bsd/Makefile | 20 + .../miniupnp/miniupnpd/bsd/getifstats.c | 77 + src/contrib/miniupnp/miniupnpd/bsd/getroute.c | 145 + .../miniupnp/miniupnpd/bsd/ifacewatcher.c | 125 + .../miniupnp/miniupnpd/bsd/testgetifstats.c | 32 + .../miniupnp/miniupnpd/bsd/testifacewatcher.c | 32 + src/contrib/miniupnp/miniupnpd/codelength.h | 54 + src/contrib/miniupnp/miniupnpd/commonrdr.h | 67 + src/contrib/miniupnp/miniupnpd/daemonize.c | 129 + src/contrib/miniupnp/miniupnpd/daemonize.h | 35 + src/contrib/miniupnp/miniupnpd/genconfig.sh | 634 + .../miniupnp/miniupnpd/getconnstatus.c | 74 + .../miniupnp/miniupnpd/getconnstatus.h | 30 + src/contrib/miniupnp/miniupnpd/getifaddr.c | 261 + src/contrib/miniupnp/miniupnpd/getifaddr.h | 32 + src/contrib/miniupnp/miniupnpd/getifstats.h | 25 + src/contrib/miniupnp/miniupnpd/getroute.h | 17 + src/contrib/miniupnp/miniupnpd/ifacewatcher.h | 48 + src/contrib/miniupnp/miniupnpd/ipf/Makefile | 16 + src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.c | 831 + src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.h | 55 + .../miniupnp/miniupnpd/ipf/testipfrdr.c | 74 + src/contrib/miniupnp/miniupnpd/ipfw/Makefile | 17 + src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.c | 107 + src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.h | 29 + src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.c | 548 + src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.h | 79 + .../miniupnp/miniupnpd/ipfw/testipfwrdr.c | 87 + .../miniupnp/miniupnpd/linux/getifstats.c | 135 + .../miniupnp/miniupnpd/linux/getroute.c | 197 + .../miniupnp/miniupnpd/linux/ifacewatcher.c | 350 + .../miniupnpd/linux/miniupnpd.init.d.script | 57 + src/contrib/miniupnp/miniupnpd/mac/Makefile | 14 + .../miniupnp/miniupnpd/mac/getifstats.c | 94 + .../mac/org.tuxfamily.miniupnpd.plist.before | 16 + .../miniupnp/miniupnpd/mac/testgetifstats.c | 29 + src/contrib/miniupnp/miniupnpd/macros.h | 47 + src/contrib/miniupnp/miniupnpd/minissdp.c | 1566 ++ src/contrib/miniupnp/miniupnpd/minissdp.h | 58 + src/contrib/miniupnp/miniupnpd/miniupnpd.8 | 89 + src/contrib/miniupnp/miniupnpd/miniupnpd.c | 2741 +++ src/contrib/miniupnp/miniupnpd/miniupnpd.conf | 152 + .../miniupnpd/miniupnpd.rc.once.d.script | 11 + src/contrib/miniupnp/miniupnpd/miniupnpdctl.c | 83 + .../miniupnp/miniupnpd/miniupnpdctl.txt | 18 + .../miniupnp/miniupnpd/miniupnpdpath.h | 49 + .../miniupnp/miniupnpd/miniupnpdtypes.h | 30 + src/contrib/miniupnp/miniupnpd/minixml.c | 231 + src/contrib/miniupnp/miniupnpd/minixml.h | 37 + src/contrib/miniupnp/miniupnpd/natpmp.c | 486 + src/contrib/miniupnp/miniupnpd/natpmp.h | 35 + .../miniupnp/miniupnpd/netfilter/Makefile | 79 + .../miniupnpd/netfilter/ip6tables_display.sh | 9 + .../miniupnpd/netfilter/ip6tables_flush.sh | 8 + .../miniupnpd/netfilter/ip6tables_init.sh | 22 + .../netfilter/ip6tables_removeall.sh | 16 + .../miniupnpd/netfilter/iptables_display.sh | 14 + .../netfilter/iptables_display_miniupnpd.sh | 10 + .../miniupnpd/netfilter/iptables_flush.sh | 10 + .../miniupnpd/netfilter/iptables_init.sh | 37 + .../miniupnpd/netfilter/iptables_removeall.sh | 46 + .../miniupnp/miniupnpd/netfilter/iptcrdr.c | 1948 ++ .../miniupnp/miniupnpd/netfilter/iptcrdr.h | 65 + .../miniupnp/miniupnpd/netfilter/iptpinhole.c | 484 + .../miniupnp/miniupnpd/netfilter/iptpinhole.h | 43 + .../netfilter/miniupnpd_functions.sh | 64 + .../miniupnp/miniupnpd/netfilter/nfct_get.c | 258 + .../miniupnpd/netfilter/test_nfct_get.c | 50 + .../miniupnpd/netfilter/testiptcrdr.c | 91 + .../miniupnpd/netfilter/testiptcrdr_dscp.c | 73 + .../miniupnpd/netfilter/testiptcrdr_peer.c | 74 + .../miniupnpd/netfilter/testiptpinhole.c | 27 + .../miniupnpd/netfilter/tiny_nf_nat.h | 37 + .../miniupnp/miniupnpd/netfilter_nft/Makefile | 33 + .../miniupnpd/netfilter_nft/README.md | 21 + .../miniupnpd/netfilter_nft/nfct_get.c | 258 + .../miniupnpd/netfilter_nft/nftnlrdr.c | 499 + .../miniupnpd/netfilter_nft/nftnlrdr.h | 84 + .../miniupnpd/netfilter_nft/nftnlrdr_misc.c | 1145 ++ .../miniupnpd/netfilter_nft/nftnlrdr_misc.h | 91 + .../netfilter_nft/scripts/nft_delete_chain.sh | 5 + .../netfilter_nft/scripts/nft_flush.sh | 5 + .../netfilter_nft/scripts/nft_init.sh | 47 + .../netfilter_nft/scripts/nft_removeall.sh | 5 + .../miniupnpd/netfilter_nft/test_nfct_get.c | 50 + .../miniupnpd/netfilter_nft/testnftnlrdr.c | 91 + .../miniupnpd/netfilter_nft/tiny_nf_nat.h | 37 + src/contrib/miniupnp/miniupnpd/options.c | 323 + src/contrib/miniupnp/miniupnpd/options.h | 98 + .../miniupnp/miniupnpd/pcp_msg_struct.h | 319 + src/contrib/miniupnp/miniupnpd/pcplearndscp.c | 298 + src/contrib/miniupnp/miniupnpd/pcplearndscp.h | 51 + src/contrib/miniupnp/miniupnpd/pcpserver.c | 1700 ++ src/contrib/miniupnp/miniupnpd/pcpserver.h | 65 + src/contrib/miniupnp/miniupnpd/pf/Makefile | 25 + src/contrib/miniupnp/miniupnpd/pf/obsdrdr.c | 1160 ++ src/contrib/miniupnp/miniupnpd/pf/obsdrdr.h | 70 + src/contrib/miniupnp/miniupnpd/pf/pfpinhole.c | 445 + src/contrib/miniupnp/miniupnpd/pf/pfpinhole.h | 41 + .../miniupnp/miniupnpd/pf/testobsdrdr.c | 147 + .../miniupnp/miniupnpd/pf/testpfpinhole.c | 105 + src/contrib/miniupnp/miniupnpd/portinuse.c | 409 + src/contrib/miniupnp/miniupnpd/portinuse.h | 23 + .../miniupnp/miniupnpd/solaris/getifstats.c | 97 + .../miniupnp/miniupnpd/testasyncsendto.c | 129 + .../miniupnp/miniupnpd/testgetifaddr.c | 54 + .../miniupnp/miniupnpd/testgetifaddr.sh | 29 + .../miniupnp/miniupnpd/testgetifstats.c | 41 + src/contrib/miniupnp/miniupnpd/testgetroute.c | 101 + src/contrib/miniupnp/miniupnpd/testminissdp.c | 82 + .../miniupnp/miniupnpd/testportinuse.c | 54 + .../miniupnp/miniupnpd/testssdppktgen.c | 100 + .../miniupnp/miniupnpd/testupnpdescgen.c | 267 + .../miniupnp/miniupnpd/testupnppermissions.c | 62 + .../miniupnp/miniupnpd/testupnppermissions.sh | 56 + src/contrib/miniupnp/miniupnpd/upnpdescgen.c | 1362 ++ src/contrib/miniupnp/miniupnpd/upnpdescgen.h | 102 + .../miniupnp/miniupnpd/upnpdescstrings.h | 41 + src/contrib/miniupnp/miniupnpd/upnpevents.c | 670 + src/contrib/miniupnp/miniupnpd/upnpevents.h | 52 + .../miniupnp/miniupnpd/upnpglobalvars.c | 170 + .../miniupnp/miniupnpd/upnpglobalvars.h | 168 + src/contrib/miniupnp/miniupnpd/upnphttp.c | 1307 ++ src/contrib/miniupnp/miniupnpd/upnphttp.h | 165 + .../miniupnp/miniupnpd/upnppermissions.c | 264 + .../miniupnp/miniupnpd/upnppermissions.h | 55 + src/contrib/miniupnp/miniupnpd/upnppinhole.c | 545 + src/contrib/miniupnp/miniupnpd/upnppinhole.h | 89 + src/contrib/miniupnp/miniupnpd/upnpredirect.c | 782 + src/contrib/miniupnp/miniupnpd/upnpredirect.h | 123 + .../miniupnp/miniupnpd/upnpreplyparse.c | 196 + .../miniupnp/miniupnpd/upnpreplyparse.h | 63 + src/contrib/miniupnp/miniupnpd/upnpsoap.c | 2322 +++ src/contrib/miniupnp/miniupnpd/upnpsoap.h | 23 + src/contrib/miniupnp/miniupnpd/upnpurns.h | 30 + src/contrib/miniupnp/miniupnpd/upnputils.c | 238 + src/contrib/miniupnp/miniupnpd/upnputils.h | 74 + src/contrib/zlib/CMakeLists.txt | 40 +- src/contrib/zlib/test/infcover.c | 2 +- src/contrib/zlib/test/minigzip.c | 2 +- src/crypto/RIPEMD160.c | 287 + src/crypto/RIPEMD160.h | 150 + src/crypto/RIPEMD160_helper.cpp | 67 + src/crypto/RIPEMD160_helper.h | 27 + src/crypto/bitcoin/byteswap.h | 59 + src/crypto/bitcoin/common.h | 110 + src/crypto/bitcoin/cpuid.h | 24 + src/crypto/bitcoin/endian.h | 241 + src/crypto/bitcoin/sha256.cpp | 719 + src/crypto/bitcoin/sha256.h | 41 + src/crypto/bitcoin/sha256_helper.h | 27 + src/crypto/blake2-impl.h | 160 + src/crypto/blake2.h | 200 + src/crypto/blake2b-ref.c | 379 + src/crypto/chacha8.h | 15 +- src/crypto/chacha8_stream.c | 115 + src/crypto/chacha8_stream.h | 289 + src/crypto/crypto-ops.c | 1461 +- src/crypto/crypto-ops.h | 28 +- src/crypto/crypto-sugar.cpp | 26 + src/crypto/crypto-sugar.h | 904 + src/crypto/crypto.cpp | 125 +- src/crypto/crypto.h | 88 +- src/crypto/ecrypt-config.h | 272 + src/crypto/ecrypt-machine.h | 46 + src/crypto/ecrypt-portable.h | 303 + src/crypto/ecrypt-sync.h | 258 + src/crypto/hash-ops.h | 6 +- src/crypto/hash.c | 9 +- src/crypto/hash.h | 12 +- src/crypto/random.c | 59 +- src/crypto/random.h | 29 +- src/currency_core/account.cpp | 291 +- src/currency_core/account.h | 53 +- .../account_boost_serialization.h | 6 +- src/currency_core/alias_helper.h | 2 +- src/currency_core/basic_kv_structs.h | 27 + src/currency_core/basic_pow_helpers.cpp | 44 +- .../bc_attachments_service_manager.h | 4 +- src/currency_core/bc_escrow_service.h | 19 +- src/currency_core/bc_offers_service.cpp | 2 + src/currency_core/blockchain_storage.cpp | 2150 +- src/currency_core/blockchain_storage.h | 223 +- src/currency_core/blockchain_storage_basic.h | 33 +- .../blockchain_storage_boost_serialization.h | 19 +- src/currency_core/checkpoints_create.h | 18 +- src/currency_core/core_runtime_config.h | 27 +- src/currency_core/currency_basic.h | 264 +- .../currency_boost_serialization.h | 92 +- src/currency_core/currency_config.h | 106 +- src/currency_core/currency_core.cpp | 248 +- src/currency_core/currency_core.h | 23 +- src/currency_core/currency_format_utils.cpp | 916 +- src/currency_core/currency_format_utils.h | 272 +- .../currency_format_utils_abstract.h | 55 +- .../currency_format_utils_blocks.cpp | 1 + .../currency_format_utils_transactions.cpp | 143 +- .../currency_format_utils_transactions.h | 108 +- src/currency_core/difficulty.cpp | 50 +- src/currency_core/difficulty.h | 3 +- src/currency_core/genesis.cpp | 10 +- src/currency_core/genesis.h | 13 +- src/currency_core/genesis_acc.cpp | 30 +- src/currency_core/genesis_acc.h | 70 +- src/currency_core/miner.cpp | 7 +- src/currency_core/miner.h | 3 +- src/currency_core/offers_service_basics.h | 67 +- src/currency_core/offers_services_helpers.cpp | 26 +- src/currency_core/offers_services_helpers.h | 71 +- src/currency_core/tx_pool.cpp | 482 +- src/currency_core/tx_pool.h | 57 +- src/currency_core/tx_semantic_validation.cpp | 100 + src/currency_core/tx_semantic_validation.h | 16 + src/currency_core/verification_context.h | 8 +- .../currency_protocol_defs.h | 9 +- .../currency_protocol_handler.h | 52 +- .../currency_protocol_handler.inl | 273 +- .../currency_protocol_handler_common.h | 16 + src/p2p/net_node.h | 37 +- src/p2p/net_node.inl | 113 +- src/p2p/net_peerlist.h | 4 +- src/rpc/core_rpc_server.cpp | 323 +- src/rpc/core_rpc_server.h | 10 +- src/rpc/core_rpc_server_commands_defs.h | 92 +- src/serialization/binary_archive.h | 4 +- src/serialization/json_archive.h | 3 +- src/serialization/serialization.h | 29 +- src/serialization/stl_containers.h | 2 +- 578 files changed, 116275 insertions(+), 3706 deletions(-) create mode 100644 src/common/atomics_boost_serialization.h create mode 100644 src/common/callstack_helper.cpp create mode 100644 src/common/callstack_helper.h create mode 100644 src/common/config_encrypt_helper.h create mode 100644 src/common/crypto_stream_operators.cpp create mode 100644 src/common/db_backend_mdbx.cpp create mode 100644 src/common/db_backend_mdbx.h create mode 100644 src/common/db_backend_selector.cpp create mode 100644 src/common/db_backend_selector.h create mode 100644 src/common/encryption_filter.cpp create mode 100644 src/common/encryption_filter.h create mode 100644 src/common/error_codes.h create mode 100644 src/common/ntp.cpp create mode 100644 src/common/ntp.h create mode 100644 src/common/pod_array_file_container.h create mode 100644 src/common/pre_download.h create mode 100644 src/contrib/db/libmdbx/.circleci/config.yml create mode 100644 src/contrib/db/libmdbx/.clang-format create mode 100644 src/contrib/db/libmdbx/.gitignore create mode 100644 src/contrib/db/libmdbx/.travis.yml create mode 100644 src/contrib/db/libmdbx/AUTHORS create mode 100644 src/contrib/db/libmdbx/CMakeLists.dist-minimal create mode 100644 src/contrib/db/libmdbx/CMakeLists.txt create mode 100644 src/contrib/db/libmdbx/COPYRIGHT create mode 100644 src/contrib/db/libmdbx/GNUmakefile create mode 100644 src/contrib/db/libmdbx/LICENSE create mode 100644 src/contrib/db/libmdbx/README.md create mode 100644 src/contrib/db/libmdbx/appveyor.yml create mode 100644 src/contrib/db/libmdbx/cmake/compiler.cmake create mode 100644 src/contrib/db/libmdbx/cmake/profile.cmake create mode 100644 src/contrib/db/libmdbx/cmake/utils.cmake create mode 100644 src/contrib/db/libmdbx/example/CMakeLists.txt create mode 100644 src/contrib/db/libmdbx/example/README.md create mode 100644 src/contrib/db/libmdbx/example/example-mdbx.c create mode 100644 src/contrib/db/libmdbx/example/sample-bdb.txt create mode 100644 src/contrib/db/libmdbx/mdbx.h create mode 100644 src/contrib/db/libmdbx/packages/rpm/CMakeLists.txt create mode 100644 src/contrib/db/libmdbx/packages/rpm/build.sh create mode 100644 src/contrib/db/libmdbx/packages/rpm/package.sh create mode 100644 src/contrib/db/libmdbx/src/CMakeLists.txt create mode 100644 src/contrib/db/libmdbx/src/alloy.c create mode 100644 src/contrib/db/libmdbx/src/elements/config.h.in create mode 100644 src/contrib/db/libmdbx/src/elements/core.c create mode 100644 src/contrib/db/libmdbx/src/elements/defs.h create mode 100644 src/contrib/db/libmdbx/src/elements/internals.h create mode 100644 src/contrib/db/libmdbx/src/elements/lck-posix.c create mode 100644 src/contrib/db/libmdbx/src/elements/lck-windows.c create mode 100644 src/contrib/db/libmdbx/src/elements/ntdll.def create mode 100644 src/contrib/db/libmdbx/src/elements/osal.c create mode 100644 src/contrib/db/libmdbx/src/elements/osal.h create mode 100644 src/contrib/db/libmdbx/src/elements/version.c.in create mode 100644 src/contrib/db/libmdbx/src/man1/mdbx_chk.1 create mode 100644 src/contrib/db/libmdbx/src/man1/mdbx_copy.1 create mode 100644 src/contrib/db/libmdbx/src/man1/mdbx_dump.1 create mode 100644 src/contrib/db/libmdbx/src/man1/mdbx_load.1 create mode 100644 src/contrib/db/libmdbx/src/man1/mdbx_stat.1 create mode 100644 src/contrib/db/libmdbx/src/tools/CMakeLists.txt create mode 100644 src/contrib/db/libmdbx/src/tools/mdbx_chk.c create mode 100644 src/contrib/db/libmdbx/src/tools/mdbx_copy.c create mode 100644 src/contrib/db/libmdbx/src/tools/mdbx_dump.c create mode 100644 src/contrib/db/libmdbx/src/tools/mdbx_load.c create mode 100644 src/contrib/db/libmdbx/src/tools/mdbx_stat.c create mode 100644 src/contrib/db/libmdbx/src/tools/wingetopt.c create mode 100644 src/contrib/db/libmdbx/src/tools/wingetopt.h create mode 100644 src/contrib/db/libmdbx/test/CMakeLists.txt create mode 100644 src/contrib/db/libmdbx/test/append.cc create mode 100644 src/contrib/db/libmdbx/test/base.h create mode 100644 src/contrib/db/libmdbx/test/cases.cc create mode 100644 src/contrib/db/libmdbx/test/chrono.cc create mode 100644 src/contrib/db/libmdbx/test/chrono.h create mode 100644 src/contrib/db/libmdbx/test/config.cc create mode 100644 src/contrib/db/libmdbx/test/config.h create mode 100644 src/contrib/db/libmdbx/test/copy.cc create mode 100644 src/contrib/db/libmdbx/test/darwin/LICENSE create mode 100644 src/contrib/db/libmdbx/test/darwin/README.md create mode 100644 src/contrib/db/libmdbx/test/darwin/pthread_barrier.c create mode 100644 src/contrib/db/libmdbx/test/darwin/pthread_barrier.h create mode 100644 src/contrib/db/libmdbx/test/dead.cc create mode 100644 src/contrib/db/libmdbx/test/hill.cc create mode 100644 src/contrib/db/libmdbx/test/jitter.cc create mode 100644 src/contrib/db/libmdbx/test/keygen.cc create mode 100644 src/contrib/db/libmdbx/test/keygen.h create mode 100644 src/contrib/db/libmdbx/test/log.cc create mode 100644 src/contrib/db/libmdbx/test/log.h create mode 100644 src/contrib/db/libmdbx/test/long_stochastic.sh create mode 100644 src/contrib/db/libmdbx/test/main.cc create mode 100644 src/contrib/db/libmdbx/test/nested.cc create mode 100644 src/contrib/db/libmdbx/test/osal-unix.cc create mode 100644 src/contrib/db/libmdbx/test/osal-windows.cc create mode 100644 src/contrib/db/libmdbx/test/osal.h create mode 100644 src/contrib/db/libmdbx/test/pcrf/CMakeLists.txt create mode 100644 src/contrib/db/libmdbx/test/pcrf/README.md create mode 100644 src/contrib/db/libmdbx/test/pcrf/pcrf_test.c create mode 100644 src/contrib/db/libmdbx/test/test.cc create mode 100644 src/contrib/db/libmdbx/test/test.h create mode 100644 src/contrib/db/libmdbx/test/try.cc create mode 100644 src/contrib/db/libmdbx/test/ttl.cc create mode 100644 src/contrib/db/libmdbx/test/utils.cc create mode 100644 src/contrib/db/libmdbx/test/utils.h create mode 100644 src/contrib/db/libmdbx/test/valgrind_suppress.txt create mode 100644 src/contrib/epee/include/misc_helpers.h create mode 100644 src/contrib/epee/include/serialization/keyvalue_hexemizer.h create mode 100644 src/contrib/epee/include/static_helpers.h create mode 100644 src/contrib/miniupnp/.travis.yml create mode 100644 src/contrib/miniupnp/CODE_OF_CONDUCT create mode 100644 src/contrib/miniupnp/CONTRIBUTING create mode 100644 src/contrib/miniupnp/LICENSE create mode 100644 src/contrib/miniupnp/README create mode 100644 src/contrib/miniupnp/appveyor.yml create mode 100644 src/contrib/miniupnp/minissdpd/.gitignore create mode 100644 src/contrib/miniupnp/minissdpd/Changelog.txt create mode 100644 src/contrib/miniupnp/minissdpd/LICENSE create mode 100644 src/contrib/miniupnp/minissdpd/Makefile create mode 100644 src/contrib/miniupnp/minissdpd/README create mode 100644 src/contrib/miniupnp/minissdpd/README.fr create mode 100644 src/contrib/miniupnp/minissdpd/VERSION create mode 100644 src/contrib/miniupnp/minissdpd/asyncsendto.c create mode 100644 src/contrib/miniupnp/minissdpd/asyncsendto.h create mode 100644 src/contrib/miniupnp/minissdpd/codelength.h create mode 100644 src/contrib/miniupnp/minissdpd/config.h create mode 100644 src/contrib/miniupnp/minissdpd/daemonize.c create mode 100644 src/contrib/miniupnp/minissdpd/daemonize.h create mode 100644 src/contrib/miniupnp/minissdpd/getifaddr.c create mode 100644 src/contrib/miniupnp/minissdpd/getifaddr.h create mode 100644 src/contrib/miniupnp/minissdpd/getroute.c create mode 100644 src/contrib/miniupnp/minissdpd/getroute.h create mode 100644 src/contrib/miniupnp/minissdpd/ifacewatch.c create mode 100644 src/contrib/miniupnp/minissdpd/ifacewatch.h create mode 100644 src/contrib/miniupnp/minissdpd/listifaces.c create mode 100644 src/contrib/miniupnp/minissdpd/minissdpd.1 create mode 100644 src/contrib/miniupnp/minissdpd/minissdpd.c create mode 100644 src/contrib/miniupnp/minissdpd/minissdpd.init.d.script create mode 100644 src/contrib/miniupnp/minissdpd/minissdpdtypes.h create mode 100644 src/contrib/miniupnp/minissdpd/openssdpsocket.c create mode 100644 src/contrib/miniupnp/minissdpd/openssdpsocket.h create mode 100644 src/contrib/miniupnp/minissdpd/printresponse.c create mode 100644 src/contrib/miniupnp/minissdpd/printresponse.h create mode 100644 src/contrib/miniupnp/minissdpd/showminissdpdnotif.c create mode 100644 src/contrib/miniupnp/minissdpd/testcodelength.c create mode 100644 src/contrib/miniupnp/minissdpd/testminissdpd.c create mode 100644 src/contrib/miniupnp/minissdpd/testminissdpd.sh create mode 100644 src/contrib/miniupnp/minissdpd/testminissdpdnotif.sh create mode 100644 src/contrib/miniupnp/minissdpd/upnputils.c create mode 100644 src/contrib/miniupnp/minissdpd/upnputils.h create mode 100644 src/contrib/miniupnp/miniupnp.podspec create mode 100644 src/contrib/miniupnp/miniupnpc-async/.gitignore create mode 100644 src/contrib/miniupnp/miniupnpc-async/Changelog.txt create mode 100644 src/contrib/miniupnp/miniupnpc-async/Makefile create mode 100644 src/contrib/miniupnp/miniupnpc-async/README create mode 100644 src/contrib/miniupnp/miniupnpc-async/config.h create mode 100644 src/contrib/miniupnp/miniupnpc-async/declspec.h create mode 100644 src/contrib/miniupnp/miniupnpc-async/igd_desc_parse.c create mode 100644 src/contrib/miniupnp/miniupnpc-async/igd_desc_parse.h create mode 100644 src/contrib/miniupnp/miniupnpc-async/miniupnpc-async.c create mode 100644 src/contrib/miniupnp/miniupnpc-async/miniupnpc-async.h create mode 100644 src/contrib/miniupnp/miniupnpc-async/minixml.c create mode 100644 src/contrib/miniupnp/miniupnpc-async/minixml.h create mode 100644 src/contrib/miniupnp/miniupnpc-async/parsessdpreply.c create mode 100644 src/contrib/miniupnp/miniupnpc-async/parsessdpreply.h create mode 100644 src/contrib/miniupnp/miniupnpc-async/testasync.c create mode 100644 src/contrib/miniupnp/miniupnpc-async/upnpreplyparse.c create mode 100644 src/contrib/miniupnp/miniupnpc-async/upnpreplyparse.h create mode 100644 src/contrib/miniupnp/miniupnpc-async/upnputils.c create mode 100644 src/contrib/miniupnp/miniupnpc-async/upnputils.h create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/.gitignore create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/Makefile create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/README create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/declspec.h create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/igd_desc_parse.c create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/igd_desc_parse.h create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/miniupnpc-libevent.c create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/miniupnpc-libevent.h create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/minixml.c create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/minixml.h create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/upnpc-libevent.c create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/upnpreplyparse.c create mode 100644 src/contrib/miniupnp/miniupnpc-libevent/upnpreplyparse.h create mode 100644 src/contrib/miniupnp/miniupnpc/.gitignore create mode 100644 src/contrib/miniupnp/miniupnpc/CMakeLists.txt create mode 100644 src/contrib/miniupnp/miniupnpc/Changelog.txt create mode 100644 src/contrib/miniupnp/miniupnpc/LICENSE create mode 100644 src/contrib/miniupnp/miniupnpc/MANIFEST.in create mode 100644 src/contrib/miniupnp/miniupnpc/Makefile create mode 100644 src/contrib/miniupnp/miniupnpc/Makefile.mingw create mode 100644 src/contrib/miniupnp/miniupnpc/README create mode 100644 src/contrib/miniupnp/miniupnpc/VERSION create mode 100644 src/contrib/miniupnp/miniupnpc/apiversions.txt create mode 100644 src/contrib/miniupnp/miniupnpc/codelength.h create mode 100644 src/contrib/miniupnp/miniupnpc/connecthostport.c create mode 100644 src/contrib/miniupnp/miniupnpc/connecthostport.h create mode 100644 src/contrib/miniupnp/miniupnpc/external-ip.sh create mode 100644 src/contrib/miniupnp/miniupnpc/igd_desc_parse.c create mode 100644 src/contrib/miniupnp/miniupnpc/igd_desc_parse.h create mode 100644 src/contrib/miniupnp/miniupnpc/java/.gitignore create mode 100644 src/contrib/miniupnp/miniupnpc/java/JavaBridgeTest.java create mode 100644 src/contrib/miniupnp/miniupnpc/java/testjava.bat create mode 100644 src/contrib/miniupnp/miniupnpc/java/testjava.sh create mode 100644 src/contrib/miniupnp/miniupnpc/listdevices.c create mode 100644 src/contrib/miniupnp/miniupnpc/man3/miniupnpc.3 create mode 100644 src/contrib/miniupnp/miniupnpc/mingw32make.bat create mode 100644 src/contrib/miniupnp/miniupnpc/minihttptestserver.c create mode 100644 src/contrib/miniupnp/miniupnpc/minisoap.c create mode 100644 src/contrib/miniupnp/miniupnpc/minisoap.h create mode 100644 src/contrib/miniupnp/miniupnpc/minissdpc.c create mode 100644 src/contrib/miniupnp/miniupnpc/minissdpc.h create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpc.c create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpc.def create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpc.h create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpc_declspec.h create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpc_socketdef.h create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpcmodule.c create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpcstrings.h.cmake create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpcstrings.h.in create mode 100644 src/contrib/miniupnp/miniupnpc/miniupnpctypes.h create mode 100644 src/contrib/miniupnp/miniupnpc/miniwget.c create mode 100644 src/contrib/miniupnp/miniupnpc/miniwget.h create mode 100644 src/contrib/miniupnp/miniupnpc/miniwget_private.h create mode 100644 src/contrib/miniupnp/miniupnpc/minixml.c create mode 100644 src/contrib/miniupnp/miniupnpc/minixml.h create mode 100644 src/contrib/miniupnp/miniupnpc/minixmlvalid.c create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/.gitignore create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/genminiupnpcstrings.vbs create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.sln create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcproj create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcxproj create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcxproj.filters create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.sln create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.vcxproj create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.vcxproj.filters create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2015.sln create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcproj create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcxproj create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcxproj.filters create mode 100644 src/contrib/miniupnp/miniupnpc/msvc/upnpc-static_vs2010.vcxproj create mode 100644 src/contrib/miniupnp/miniupnpc/portlistingparse.c create mode 100644 src/contrib/miniupnp/miniupnpc/portlistingparse.h create mode 100644 src/contrib/miniupnp/miniupnpc/pymoduletest.py create mode 100644 src/contrib/miniupnp/miniupnpc/pymoduletest3.py create mode 100644 src/contrib/miniupnp/miniupnpc/receivedata.c create mode 100644 src/contrib/miniupnp/miniupnpc/receivedata.h create mode 100644 src/contrib/miniupnp/miniupnpc/setup.py create mode 100644 src/contrib/miniupnp/miniupnpc/setupmingw32.py create mode 100644 src/contrib/miniupnp/miniupnpc/testdesc/linksys_WAG200G_desc.values create mode 100644 src/contrib/miniupnp/miniupnpc/testdesc/linksys_WAG200G_desc.xml create mode 100644 src/contrib/miniupnp/miniupnpc/testdesc/new_LiveBox_desc.values create mode 100644 src/contrib/miniupnp/miniupnpc/testdesc/new_LiveBox_desc.xml create mode 100644 src/contrib/miniupnp/miniupnpc/testigddescparse.c create mode 100644 src/contrib/miniupnp/miniupnpc/testminiwget.c create mode 100644 src/contrib/miniupnp/miniupnpc/testminiwget.sh create mode 100644 src/contrib/miniupnp/miniupnpc/testminixml.c create mode 100644 src/contrib/miniupnp/miniupnpc/testportlistingparse.c create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/DeletePortMapping.namevalue create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/DeletePortMapping.xml create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/GetExternalIPAddress.namevalue create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/GetExternalIPAddress.xml create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryReq.namevalue create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryReq.xml create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryResp.namevalue create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryResp.xml create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/SetDefaultConnectionService.namevalue create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/SetDefaultConnectionService.xml create mode 100644 src/contrib/miniupnp/miniupnpc/testreplyparse/readme.txt create mode 100644 src/contrib/miniupnp/miniupnpc/testupnpigd.py create mode 100644 src/contrib/miniupnp/miniupnpc/testupnpreplyparse.c create mode 100644 src/contrib/miniupnp/miniupnpc/testupnpreplyparse.sh create mode 100644 src/contrib/miniupnp/miniupnpc/updateminiupnpcstrings.sh create mode 100644 src/contrib/miniupnp/miniupnpc/upnpc.c create mode 100644 src/contrib/miniupnp/miniupnpc/upnpcommands.c create mode 100644 src/contrib/miniupnp/miniupnpc/upnpcommands.h create mode 100644 src/contrib/miniupnp/miniupnpc/upnpdev.c create mode 100644 src/contrib/miniupnp/miniupnpc/upnpdev.h create mode 100644 src/contrib/miniupnp/miniupnpc/upnperrors.c create mode 100644 src/contrib/miniupnp/miniupnpc/upnperrors.h create mode 100644 src/contrib/miniupnp/miniupnpc/upnpreplyparse.c create mode 100644 src/contrib/miniupnp/miniupnpc/upnpreplyparse.h create mode 100644 src/contrib/miniupnp/miniupnpc/wingenminiupnpcstrings.c create mode 100644 src/contrib/miniupnp/miniupnpd/.gitignore create mode 100644 src/contrib/miniupnp/miniupnpd/Changelog.txt create mode 100644 src/contrib/miniupnp/miniupnpd/INSTALL create mode 100644 src/contrib/miniupnp/miniupnpd/LICENSE create mode 100644 src/contrib/miniupnp/miniupnpd/Makefile create mode 100644 src/contrib/miniupnp/miniupnpd/Makefile.linux create mode 100644 src/contrib/miniupnp/miniupnpd/Makefile.linux_nft create mode 100644 src/contrib/miniupnp/miniupnpd/Makefile.macosx create mode 100644 src/contrib/miniupnp/miniupnpd/Makefile.sunos create mode 100644 src/contrib/miniupnp/miniupnpd/README create mode 100644 src/contrib/miniupnp/miniupnpd/TODO create mode 100644 src/contrib/miniupnp/miniupnpd/VERSION create mode 100644 src/contrib/miniupnp/miniupnpd/asyncsendto.c create mode 100644 src/contrib/miniupnp/miniupnpd/asyncsendto.h create mode 100644 src/contrib/miniupnp/miniupnpd/bsd/Makefile create mode 100644 src/contrib/miniupnp/miniupnpd/bsd/getifstats.c create mode 100644 src/contrib/miniupnp/miniupnpd/bsd/getroute.c create mode 100644 src/contrib/miniupnp/miniupnpd/bsd/ifacewatcher.c create mode 100644 src/contrib/miniupnp/miniupnpd/bsd/testgetifstats.c create mode 100644 src/contrib/miniupnp/miniupnpd/bsd/testifacewatcher.c create mode 100644 src/contrib/miniupnp/miniupnpd/codelength.h create mode 100644 src/contrib/miniupnp/miniupnpd/commonrdr.h create mode 100644 src/contrib/miniupnp/miniupnpd/daemonize.c create mode 100644 src/contrib/miniupnp/miniupnpd/daemonize.h create mode 100644 src/contrib/miniupnp/miniupnpd/genconfig.sh create mode 100644 src/contrib/miniupnp/miniupnpd/getconnstatus.c create mode 100644 src/contrib/miniupnp/miniupnpd/getconnstatus.h create mode 100644 src/contrib/miniupnp/miniupnpd/getifaddr.c create mode 100644 src/contrib/miniupnp/miniupnpd/getifaddr.h create mode 100644 src/contrib/miniupnp/miniupnpd/getifstats.h create mode 100644 src/contrib/miniupnp/miniupnpd/getroute.h create mode 100644 src/contrib/miniupnp/miniupnpd/ifacewatcher.h create mode 100644 src/contrib/miniupnp/miniupnpd/ipf/Makefile create mode 100644 src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.h create mode 100644 src/contrib/miniupnp/miniupnpd/ipf/testipfrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/ipfw/Makefile create mode 100644 src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.c create mode 100644 src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.h create mode 100644 src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.h create mode 100644 src/contrib/miniupnp/miniupnpd/ipfw/testipfwrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/linux/getifstats.c create mode 100644 src/contrib/miniupnp/miniupnpd/linux/getroute.c create mode 100644 src/contrib/miniupnp/miniupnpd/linux/ifacewatcher.c create mode 100644 src/contrib/miniupnp/miniupnpd/linux/miniupnpd.init.d.script create mode 100644 src/contrib/miniupnp/miniupnpd/mac/Makefile create mode 100644 src/contrib/miniupnp/miniupnpd/mac/getifstats.c create mode 100644 src/contrib/miniupnp/miniupnpd/mac/org.tuxfamily.miniupnpd.plist.before create mode 100644 src/contrib/miniupnp/miniupnpd/mac/testgetifstats.c create mode 100644 src/contrib/miniupnp/miniupnpd/macros.h create mode 100644 src/contrib/miniupnp/miniupnpd/minissdp.c create mode 100644 src/contrib/miniupnp/miniupnpd/minissdp.h create mode 100644 src/contrib/miniupnp/miniupnpd/miniupnpd.8 create mode 100644 src/contrib/miniupnp/miniupnpd/miniupnpd.c create mode 100644 src/contrib/miniupnp/miniupnpd/miniupnpd.conf create mode 100644 src/contrib/miniupnp/miniupnpd/miniupnpd.rc.once.d.script create mode 100644 src/contrib/miniupnp/miniupnpd/miniupnpdctl.c create mode 100644 src/contrib/miniupnp/miniupnpd/miniupnpdctl.txt create mode 100644 src/contrib/miniupnp/miniupnpd/miniupnpdpath.h create mode 100644 src/contrib/miniupnp/miniupnpd/miniupnpdtypes.h create mode 100644 src/contrib/miniupnp/miniupnpd/minixml.c create mode 100644 src/contrib/miniupnp/miniupnpd/minixml.h create mode 100644 src/contrib/miniupnp/miniupnpd/natpmp.c create mode 100644 src/contrib/miniupnp/miniupnpd/natpmp.h create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/Makefile create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_display.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_flush.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_init.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_removeall.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptables_display.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptables_display_miniupnpd.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptables_flush.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptables_init.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptables_removeall.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptcrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptcrdr.h create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptpinhole.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/iptpinhole.h create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/miniupnpd_functions.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/nfct_get.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/test_nfct_get.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr_dscp.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr_peer.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/testiptpinhole.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter/tiny_nf_nat.h create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/Makefile create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/README.md create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/nfct_get.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr.h create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr_misc.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr_misc.h create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_delete_chain.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_flush.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_init.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_removeall.sh create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/test_nfct_get.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/testnftnlrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/netfilter_nft/tiny_nf_nat.h create mode 100644 src/contrib/miniupnp/miniupnpd/options.c create mode 100644 src/contrib/miniupnp/miniupnpd/options.h create mode 100644 src/contrib/miniupnp/miniupnpd/pcp_msg_struct.h create mode 100644 src/contrib/miniupnp/miniupnpd/pcplearndscp.c create mode 100644 src/contrib/miniupnp/miniupnpd/pcplearndscp.h create mode 100644 src/contrib/miniupnp/miniupnpd/pcpserver.c create mode 100644 src/contrib/miniupnp/miniupnpd/pcpserver.h create mode 100644 src/contrib/miniupnp/miniupnpd/pf/Makefile create mode 100644 src/contrib/miniupnp/miniupnpd/pf/obsdrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/pf/obsdrdr.h create mode 100644 src/contrib/miniupnp/miniupnpd/pf/pfpinhole.c create mode 100644 src/contrib/miniupnp/miniupnpd/pf/pfpinhole.h create mode 100644 src/contrib/miniupnp/miniupnpd/pf/testobsdrdr.c create mode 100644 src/contrib/miniupnp/miniupnpd/pf/testpfpinhole.c create mode 100644 src/contrib/miniupnp/miniupnpd/portinuse.c create mode 100644 src/contrib/miniupnp/miniupnpd/portinuse.h create mode 100644 src/contrib/miniupnp/miniupnpd/solaris/getifstats.c create mode 100644 src/contrib/miniupnp/miniupnpd/testasyncsendto.c create mode 100644 src/contrib/miniupnp/miniupnpd/testgetifaddr.c create mode 100644 src/contrib/miniupnp/miniupnpd/testgetifaddr.sh create mode 100644 src/contrib/miniupnp/miniupnpd/testgetifstats.c create mode 100644 src/contrib/miniupnp/miniupnpd/testgetroute.c create mode 100644 src/contrib/miniupnp/miniupnpd/testminissdp.c create mode 100644 src/contrib/miniupnp/miniupnpd/testportinuse.c create mode 100644 src/contrib/miniupnp/miniupnpd/testssdppktgen.c create mode 100644 src/contrib/miniupnp/miniupnpd/testupnpdescgen.c create mode 100644 src/contrib/miniupnp/miniupnpd/testupnppermissions.c create mode 100644 src/contrib/miniupnp/miniupnpd/testupnppermissions.sh create mode 100644 src/contrib/miniupnp/miniupnpd/upnpdescgen.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnpdescgen.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnpdescstrings.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnpevents.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnpevents.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnpglobalvars.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnpglobalvars.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnphttp.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnphttp.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnppermissions.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnppermissions.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnppinhole.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnppinhole.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnpredirect.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnpredirect.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnpreplyparse.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnpreplyparse.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnpsoap.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnpsoap.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnpurns.h create mode 100644 src/contrib/miniupnp/miniupnpd/upnputils.c create mode 100644 src/contrib/miniupnp/miniupnpd/upnputils.h create mode 100644 src/crypto/RIPEMD160.c create mode 100644 src/crypto/RIPEMD160.h create mode 100644 src/crypto/RIPEMD160_helper.cpp create mode 100644 src/crypto/RIPEMD160_helper.h create mode 100644 src/crypto/bitcoin/byteswap.h create mode 100644 src/crypto/bitcoin/common.h create mode 100644 src/crypto/bitcoin/cpuid.h create mode 100644 src/crypto/bitcoin/endian.h create mode 100644 src/crypto/bitcoin/sha256.cpp create mode 100644 src/crypto/bitcoin/sha256.h create mode 100644 src/crypto/bitcoin/sha256_helper.h create mode 100644 src/crypto/blake2-impl.h create mode 100644 src/crypto/blake2.h create mode 100644 src/crypto/blake2b-ref.c create mode 100644 src/crypto/chacha8_stream.c create mode 100644 src/crypto/chacha8_stream.h create mode 100644 src/crypto/crypto-sugar.cpp create mode 100644 src/crypto/crypto-sugar.h create mode 100644 src/crypto/ecrypt-config.h create mode 100644 src/crypto/ecrypt-machine.h create mode 100644 src/crypto/ecrypt-portable.h create mode 100644 src/crypto/ecrypt-sync.h create mode 100644 src/currency_core/basic_kv_structs.h create mode 100644 src/currency_core/tx_semantic_validation.cpp create mode 100644 src/currency_core/tx_semantic_validation.h diff --git a/src/common/atomics_boost_serialization.h b/src/common/atomics_boost_serialization.h new file mode 100644 index 0000000..e9aa979 --- /dev/null +++ b/src/common/atomics_boost_serialization.h @@ -0,0 +1,32 @@ +// Copyright (c) 2014-2018 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include + +namespace boost +{ + namespace serialization + { + template + inline void save(Archive &a, const std::atomic &x, const boost::serialization::version_type ver) + { + a << x.load(); + } + + template + inline void load(Archive &a, std::atomic &x, const boost::serialization::version_type ver) + { + value_t s = AUTO_VAL_INIT(s); + a >> s; + x.store(s); + } + template + inline void serialize(Archive &a, std::atomic &x, const boost::serialization::version_type ver) + { + split_free(a, x, ver); + } + } +} diff --git a/src/common/boost_serialization_helper.h b/src/common/boost_serialization_helper.h index 8ca8989..2ba8a52 100644 --- a/src/common/boost_serialization_helper.h +++ b/src/common/boost_serialization_helper.h @@ -21,8 +21,8 @@ namespace tools bool serialize_obj_to_file(t_object& obj, const std::string& file_path) { TRY_ENTRY(); - std::ofstream data_file; - data_file.open( file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); + boost::filesystem::ofstream data_file; + data_file.open( epee::string_encoding::utf8_to_wstring(file_path) , std::ios_base::binary | std::ios_base::out| std::ios::trunc); if(data_file.fail()) return false; @@ -30,7 +30,7 @@ namespace tools a << obj; return !data_file.fail(); - CATCH_ENTRY_L0("serialize_obj_to_file", false); + CATCH_ENTRY_L0("serialize_obj_to_file: could not serialize into " << file_path, false); } @@ -56,7 +56,7 @@ namespace tools a << obj; return !stream.fail(); - CATCH_ENTRY_L0("serialize_obj_to_file", false); + CATCH_ENTRY_L0("portble_serialize_obj_to_stream", false); } template @@ -64,15 +64,15 @@ namespace tools { TRY_ENTRY(); - std::ifstream data_file; - data_file.open( file_path, std::ios_base::binary | std::ios_base::in); + boost::filesystem::ifstream data_file; + data_file.open( epee::string_encoding::utf8_to_wstring(file_path), std::ios_base::binary | std::ios_base::in); if(data_file.fail()) return false; boost::archive::binary_iarchive a(data_file); a >> obj; return !data_file.fail(); - CATCH_ENTRY_L0("unserialize_obj_from_file", false); + CATCH_ENTRY_L0("unserialize_obj_from_file: could not load " << file_path, false); } template @@ -85,7 +85,7 @@ namespace tools a >> obj; return !ss.fail(); - CATCH_ENTRY_L0("unserialize_obj_from_obj", false); + CATCH_ENTRY_L0("unserialize_obj_from_buff", false); } @@ -98,6 +98,6 @@ namespace tools a >> obj; return !stream.fail(); - CATCH_ENTRY_L0("unserialize_obj_from_file", false); + CATCH_ENTRY_L0("portable_unserialize_obj_from_stream", false); } } diff --git a/src/common/callstack_helper.cpp b/src/common/callstack_helper.cpp new file mode 100644 index 0000000..74bcc55 --- /dev/null +++ b/src/common/callstack_helper.cpp @@ -0,0 +1,184 @@ +// Copyright (c) 2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(WIN32) + +#define WIN32_LEAN_AND_MEAN 1 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#pragma comment(lib, "psapi.lib") +#pragma comment(lib, "dbghelp.lib") + +#pragma pack( push, before_imagehlp, 8 ) +#include +#pragma pack( pop, before_imagehlp ) + +#include "include_base_utils.h" +#include "callstack_helper.h" + +namespace +{ + struct module_data + { + std::string image_name; + std::string module_name; + void *base_address; + DWORD load_size; + }; + + + class get_mod_info + { + HANDLE process; + static const int buffer_length = 4096; + public: + get_mod_info(HANDLE h) : process(h) {} + + module_data operator()(HMODULE module) + { + module_data ret; + char temp[buffer_length]; + MODULEINFO mi; + + GetModuleInformation(process, module, &mi, sizeof(mi)); + ret.base_address = mi.lpBaseOfDll; + ret.load_size = mi.SizeOfImage; + + GetModuleFileNameEx(process, module, temp, sizeof(temp)); + ret.image_name = temp; + GetModuleBaseName(process, module, temp, sizeof(temp)); + ret.module_name = temp; + std::vector img(ret.image_name.begin(), ret.image_name.end()); + std::vector mod(ret.module_name.begin(), ret.module_name.end()); + SymLoadModule64(process, 0, &img[0], &mod[0], (DWORD64)ret.base_address, ret.load_size); + return ret; + } + }; + + std::string get_symbol_undecorated_name(HANDLE process, DWORD64 address, std::stringstream& ss) + { + SYMBOL_INFO* sym; + static const int max_name_len = 1024; + std::vector sym_buffer(sizeof(SYMBOL_INFO) + max_name_len, '\0'); + sym = (SYMBOL_INFO *)sym_buffer.data(); + sym->SizeOfStruct = sizeof(SYMBOL_INFO); + sym->MaxNameLen = max_name_len; + + DWORD64 displacement; + if (!SymFromAddr(process, address, &displacement, sym)) + return std::string("SymFromAddr failed1: ") + epee::string_tools::num_to_string_fast(GetLastError()); + + if (*sym->Name == '\0') + return std::string("SymFromAddr failed2: ") + epee::string_tools::num_to_string_fast(GetLastError()); + + /* + ss << " SizeOfStruct : " << sym->SizeOfStruct << ENDL; + ss << " TypeIndex : " << sym->TypeIndex << ENDL; // Type Index of symbol + ss << " Index : " << sym->Index << ENDL; + ss << " Size : " << sym->Size << ENDL; + ss << " ModBase : " << sym->ModBase << ENDL; // Base Address of module comtaining this symbol + ss << " Flags : " << sym->Flags << ENDL; + ss << " Value : " << sym->Value << ENDL; // Value of symbol, ValuePresent should be 1 + ss << " Address : " << sym->Address << ENDL; // Address of symbol including base address of module + ss << " Register : " << sym->Register << ENDL; // register holding value or pointer to value + ss << " Scope : " << sym->Scope << ENDL; // scope of the symbol + ss << " Tag : " << sym->Tag << ENDL; // pdb classification + ss << " NameLen : " << sym->NameLen << ENDL; // Actual length of name + ss << " MaxNameLen : " << sym->MaxNameLen << ENDL; + ss << " Name[1] : " << &sym->Name << ENDL; // Name of symbol + */ + + std::string und_name(max_name_len, '\0'); + DWORD chars_written = UnDecorateSymbolName(sym->Name, &und_name.front(), max_name_len, UNDNAME_COMPLETE); + und_name.resize(chars_written); + return und_name; + } + +} // namespace + +namespace tools +{ + + std::string get_callstack_win_x64() + { + // @TODO@ + // static epee::static_helpers::wrapper cs; + static std::recursive_mutex cs; + std::lock_guard lock(cs); + + HANDLE h_process = GetCurrentProcess(); + HANDLE h_thread = GetCurrentThread(); + + PCSTR user_search_path = NULL; // may be path to a pdb? + if (!SymInitialize(h_process, user_search_path, false)) + return ""; + + DWORD sym_options = SymGetOptions(); + sym_options |= SYMOPT_LOAD_LINES | SYMOPT_UNDNAME; + SymSetOptions(sym_options); + + DWORD cb_needed; + std::vector module_handles(1); + EnumProcessModules(h_process, &module_handles[0], static_cast(module_handles.size() * sizeof(HMODULE)), &cb_needed); + module_handles.resize(cb_needed / sizeof(HMODULE)); + EnumProcessModules(h_process, &module_handles[0], static_cast(module_handles.size() * sizeof(HMODULE)), &cb_needed); + + std::vector modules; + std::transform(module_handles.begin(), module_handles.end(), std::back_inserter(modules), get_mod_info(h_process)); + void *base = modules[0].base_address; + + CONTEXT context; + memset(&context, 0, sizeof context); + RtlCaptureContext( &context ); + + STACKFRAME64 frame; + memset(&frame, 0, sizeof frame); + frame.AddrPC.Offset = context.Rip; + frame.AddrPC.Mode = AddrModeFlat; + frame.AddrStack.Offset = context.Rsp; + frame.AddrStack.Mode = AddrModeFlat; + frame.AddrFrame.Offset = context.Rbp; + frame.AddrFrame.Mode = AddrModeFlat; + + IMAGEHLP_LINE64 line = { 0 }; + line.SizeOfStruct = sizeof line; + IMAGE_NT_HEADERS *image_nt_header = ImageNtHeader(base); + + std::stringstream ss; + ss << ENDL; + // ss << "main module loaded at 0x" << std::hex << std::setw(16) << std::setfill('0') << base << std::dec << " from " << modules[0].image_name << ENDL; + for (size_t n = 0; n < 250; ++n) + { + if (!StackWalk64(image_nt_header->FileHeader.Machine, h_process, h_thread, &frame, &context, NULL, SymFunctionTableAccess64, SymGetModuleBase64, NULL)) + break; + if (frame.AddrReturn.Offset == 0) + break; + + std::string fnName = get_symbol_undecorated_name(h_process, frame.AddrPC.Offset, ss); + ss << "0x" << std::setw(16) << std::setfill('0') << std::hex << frame.AddrPC.Offset << " " << std::dec << fnName; + DWORD offset_from_line = 0; + if (SymGetLineFromAddr64(h_process, frame.AddrPC.Offset, &offset_from_line, &line)) + ss << "+" << offset_from_line << " " << line.FileName << "(" << line.LineNumber << ")"; + + for (auto el : modules) + { + if ((DWORD64)el.base_address <= frame.AddrPC.Offset && frame.AddrPC.Offset < (DWORD64)el.base_address + (DWORD64)el.load_size) + { + ss << " : " << el.module_name << " @ 0x" << std::setw(0) << std::hex << (DWORD64)el.base_address << ENDL; + break; + } + } + + } + SymCleanup(h_process); + + return ss.str(); + } + +} // namespace tools + +#endif // #if defined(WIN32) diff --git a/src/common/callstack_helper.h b/src/common/callstack_helper.h new file mode 100644 index 0000000..3853d14 --- /dev/null +++ b/src/common/callstack_helper.h @@ -0,0 +1,24 @@ +// Copyright (c) 2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#pragma once +#include + +namespace tools +{ +#if defined(WIN32) + extern std::string get_callstack_win_x64(); +#endif + + inline std::string get_callstack() + { +#if defined(__GNUC__) + return epee::misc_utils::print_trace_default(); +#elif defined(WIN32) + return get_callstack_win_x64(); +#else + return ""; +#endif + } + +} // namespace tools diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index c1b90b7..92a59b6 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying @@ -12,14 +12,31 @@ namespace command_line { const arg_descriptor arg_help = {"help", "Produce help message"}; const arg_descriptor arg_version = {"version", "Output version information"}; - const arg_descriptor arg_data_dir = {"data-dir", "Specify data directory"}; + const arg_descriptor arg_data_dir = {"data-dir", "Specify data directory", ""}; + + const arg_descriptor arg_stop_after_height = { "stop-after-height", "If specified, the daemon will stop immediately after a block with the given height is added", 0 }; const arg_descriptor arg_config_file = { "config-file", "Specify configuration file", std::string(CURRENCY_NAME_SHORT ".conf") }; const arg_descriptor arg_os_version = { "os-version", "" }; + const arg_descriptor arg_log_dir = { "log-dir", "", "", true}; - const arg_descriptor arg_log_level = { "log-level", "", LOG_LEVEL_0, true}; + const arg_descriptor arg_log_file = { "log-file", "", "" }; + const arg_descriptor arg_log_level = { "log-level", "", LOG_LEVEL_0, true }; + const arg_descriptor arg_console = { "no-console", "Disable daemon console commands" }; const arg_descriptor arg_show_details = { "currency-details", "Display currency details" }; const arg_descriptor arg_show_rpc_autodoc = { "show_rpc_autodoc", "Display rpc auto-generated documentation template" }; + const arg_descriptor arg_disable_upnp = { "disable-upnp", "Disable UPnP (enhances local network privacy)", false, true }; + + const arg_descriptor arg_disable_stop_if_time_out_of_sync = { "disable-stop-if-time-out-of-sync", "Do not stop the daemon if serious time synchronization problem is detected", false, true }; + const arg_descriptor arg_disable_stop_on_low_free_space = { "disable-stop-on-low-free-space", "Do not stop the daemon if free space at data dir is critically low", false, true }; + const arg_descriptor arg_enable_offers_service = { "enable-offers-service", "Enables marketplace feature", false, false}; + const arg_descriptor arg_db_engine = { "db-engine", "Specify database engine for storage. May be \"lmdb\"(default) or \"mdbx\"", ARG_DB_ENGINE_LMDB, false }; + + const arg_descriptor arg_no_predownload = { "no-predownload", "Do not pre-download blockchain database", }; + const arg_descriptor arg_force_predownload = { "force-predownload", "Pre-download blockchain database regardless of it's status", }; + const arg_descriptor arg_validate_predownload = { "validate-predownload", "Paranoid mode, re-validate each block from pre-downloaded database and rebuild own database", }; + const arg_descriptor arg_predownload_link = { "predownload-link", "Override url for blockchain database pre-downloading", "", true }; + } diff --git a/src/common/command_line.h b/src/common/command_line.h index 152c923..f19a04a 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -23,6 +23,25 @@ namespace command_line struct arg_descriptor { typedef T value_type; + arg_descriptor(const char* _name, const char* _description): + name(_name), + description(_description), + not_use_default(true), + default_value(T()) + {} + arg_descriptor(const char* _name, const char* _description, const T& default_val) : + name(_name), + description(_description), + not_use_default(false), + default_value(default_val) + {} + arg_descriptor(const char* _name, const char* _description, const T& default_val, bool not_use_default) : + name(_name), + description(_description), + default_value(default_val), + not_use_default(not_use_default) + {} + const char* name; const char* description; @@ -122,7 +141,7 @@ namespace command_line boost::program_options::basic_parsed_options parse_command_line(int argc, const charT* const argv[], const boost::program_options::options_description& desc, bool allow_unregistered = false) { - auto parser = boost::program_options::command_line_parser(argc, argv); + auto parser = boost::program_options::basic_command_line_parser(argc, argv); parser.options(desc); if (allow_unregistered) { @@ -172,15 +191,29 @@ namespace command_line return get_arg(vm, arg); } +#define ARG_DB_ENGINE_LMDB "lmdb" +#define ARG_DB_ENGINE_MDBX "mdbx" + extern const arg_descriptor arg_help; extern const arg_descriptor arg_version; extern const arg_descriptor arg_data_dir; + extern const arg_descriptor arg_stop_after_height; extern const arg_descriptor arg_config_file; extern const arg_descriptor arg_os_version; extern const arg_descriptor arg_log_dir; + extern const arg_descriptor arg_log_file; extern const arg_descriptor arg_log_level; extern const arg_descriptor arg_console; extern const arg_descriptor arg_show_details; extern const arg_descriptor arg_show_rpc_autodoc; + extern const arg_descriptor arg_disable_upnp; + extern const arg_descriptor arg_disable_stop_if_time_out_of_sync; + extern const arg_descriptor arg_disable_stop_on_low_free_space; + extern const arg_descriptor arg_enable_offers_service; + extern const arg_descriptor arg_db_engine; + extern const arg_descriptor arg_no_predownload; + extern const arg_descriptor arg_force_predownload; + extern const arg_descriptor arg_validate_predownload; + extern const arg_descriptor arg_predownload_link; } diff --git a/src/common/config_encrypt_helper.h b/src/common/config_encrypt_helper.h new file mode 100644 index 0000000..c2208c7 --- /dev/null +++ b/src/common/config_encrypt_helper.h @@ -0,0 +1,66 @@ +// Copyright (c) 2014-2020 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include "wallet/view_iface.h" + + +namespace tools +{ + +#pragma pack(push, 1) + struct app_data_file_binary_header + { + uint64_t m_signature; + uint64_t m_cb_body; + }; +#pragma pack (pop) + + inline + std::string load_encrypted_file(const std::string& path, const std::string& key, std::string& body, uint64_t signature) + { + std::string app_data_buff; + bool r = epee::file_io_utils::load_file_to_string(path, app_data_buff); + if (!r) + { + return API_RETURN_CODE_NOT_FOUND; + } + + if (app_data_buff.size() < sizeof(app_data_file_binary_header)) + { + LOG_ERROR("app_data_buff.size()(" << app_data_buff.size() << ") < sizeof(app_data_file_binary_header) (" << sizeof(app_data_file_binary_header) << ") check failed while loading from " << path); + return API_RETURN_CODE_INVALID_FILE; + } + + crypto::chacha_crypt(app_data_buff, key); + + const app_data_file_binary_header* phdr = reinterpret_cast(app_data_buff.data()); + if (phdr->m_signature != signature) + { + return API_RETURN_CODE_WRONG_PASSWORD; + } + body = app_data_buff.substr(sizeof(app_data_file_binary_header)).c_str(); + return API_RETURN_CODE_OK; + } + inline + std::string store_encrypted_file(const std::string& path, const std::string& key, const std::string& body, uint64_t signature) + { + std::string buff(sizeof(app_data_file_binary_header), 0); + app_data_file_binary_header* phdr = (app_data_file_binary_header*)buff.data(); + phdr->m_signature = signature; + phdr->m_cb_body = 0; // for future use + + buff.append(body); + crypto::chacha_crypt(buff, key); + + bool r = epee::file_io_utils::save_string_to_file(path, buff); + if (r) + return API_RETURN_CODE_OK; + else + return API_RETURN_CODE_FAIL; + } + +} \ No newline at end of file diff --git a/src/common/crypto_stream_operators.cpp b/src/common/crypto_stream_operators.cpp new file mode 100644 index 0000000..cc87dd3 --- /dev/null +++ b/src/common/crypto_stream_operators.cpp @@ -0,0 +1,18 @@ +// Copyright (c) 2018-2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "crypto_stream_operators.h" + +bool parse_hash256(const std::string str_hash, crypto::hash& hash) +{ + std::string buf; + bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf); + if (!res || buf.size() != sizeof(crypto::hash)) + { + std::cout << "invalid hash format: <" << str_hash << '>' << std::endl; + return false; + } + + buf.copy(reinterpret_cast(&hash), sizeof(crypto::hash)); + return true; +} diff --git a/src/common/crypto_stream_operators.h b/src/common/crypto_stream_operators.h index 8ae7f53..0201603 100644 --- a/src/common/crypto_stream_operators.h +++ b/src/common/crypto_stream_operators.h @@ -8,29 +8,37 @@ #include "include_base_utils.h" #include "crypto/crypto.h" #include "crypto/hash.h" -//------ +#include "crypto/RIPEMD160_helper.h" + bool parse_hash256(const std::string str_hash, crypto::hash& hash); + template -std::ostream &print256(std::ostream &o, const T &v) { +std::ostream &print_t(std::ostream &o, const T &v) +{ return o << "<" << epee::string_tools::pod_to_hex(v) << ">"; } + template -std::ostream &print16(std::ostream &o, const T &v) { +std::ostream &print16(std::ostream &o, const T &v) +{ return o << "<" << epee::string_tools::pod_to_hex(v).substr(0, 5) << "..>"; } template -std::string print16(const T &v) { +std::string print16(const T &v) +{ return std::string("<") + epee::string_tools::pod_to_hex(v).substr(0, 5) + "..>"; } -namespace crypto { - inline std::ostream &operator <<(std::ostream &o, const crypto::public_key &v) { return print256(o, v); } - inline std::ostream &operator <<(std::ostream &o, const crypto::secret_key &v) { return print256(o, v); } - inline std::ostream &operator <<(std::ostream &o, const crypto::key_derivation &v) { return print256(o, v); } - inline std::ostream &operator <<(std::ostream &o, const crypto::key_image &v) { return print256(o, v); } - inline std::ostream &operator <<(std::ostream &o, const crypto::signature &v) { return print256(o, v); } - inline std::ostream &operator <<(std::ostream &o, const crypto::hash &v) { return print256(o, v); } -} \ No newline at end of file +namespace crypto +{ + inline std::ostream &operator <<(std::ostream &o, const crypto::public_key &v) { return print_t(o, v); } + inline std::ostream &operator <<(std::ostream &o, const crypto::secret_key &v) { return print_t(o, v); } + inline std::ostream &operator <<(std::ostream &o, const crypto::key_derivation &v) { return print_t(o, v); } + inline std::ostream &operator <<(std::ostream &o, const crypto::key_image &v) { return print_t(o, v); } + inline std::ostream &operator <<(std::ostream &o, const crypto::signature &v) { return print_t(o, v); } + inline std::ostream &operator <<(std::ostream &o, const crypto::hash &v) { return print_t(o, v); } + inline std::ostream &operator <<(std::ostream &o, const crypto::hash160 &v) { return print_t(o, v); } +} // namespace crypto diff --git a/src/common/db_abstract_accessor.h b/src/common/db_abstract_accessor.h index 6112629..71be14f 100644 --- a/src/common/db_abstract_accessor.h +++ b/src/common/db_abstract_accessor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Boolberry developers // Distributed under the MIT/X11 software license, see the accompanying @@ -89,13 +89,16 @@ namespace tools mutable performance_data m_gperformance_data; mutable std::unordered_map m_performance_data_map; public: - basic_db_accessor(std::shared_ptr backend, epee::shared_recursive_mutex& rwlock) :m_backend(backend), m_rwlock(rwlock), m_is_open(false) - {} + basic_db_accessor(std::shared_ptr backend, epee::shared_recursive_mutex& rwlock) + : m_backend(backend), m_rwlock(rwlock), m_is_open(false) + { + } + ~basic_db_accessor() { close(); } - + void reset_backend(std::shared_ptr backend) { m_backend = backend; } performance_data& get_performance_data_for_handle(container_handle h) const { return m_performance_data_map[h]; } performance_data& get_performance_data_global() const { return m_gperformance_data; } @@ -239,8 +242,11 @@ namespace tools bool close() { m_is_open = false; + if (!m_backend) + return true; return m_backend->close(); } + bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) { bool r = m_backend->open(path, cache_sz); @@ -513,7 +519,9 @@ namespace tools ~basic_key_value_accessor() { + TRY_ENTRY(); bdb.unbind_parent_container(this); + CATCH_ALL_DO_NOTHING(); } virtual bool on_write_transaction_begin() @@ -562,11 +570,29 @@ namespace tools m_set_profiler.m_name = container_name + ":set"; m_explicit_get_profiler.m_name = container_name + ":explicit_get"; m_explicit_set_profiler.m_name = container_name + ":explicit_set"; - m_commit_profiler.m_name = container_name + ":commit";; + m_commit_profiler.m_name = container_name + ":commit"; #endif return bdb.get_backend()->open_container(container_name, m_h); } + bool deinit() + { +#ifdef ENABLE_PROFILING + m_get_profiler.m_name = ""; + m_set_profiler.m_name = ""; + m_explicit_get_profiler.m_name = ""; + m_explicit_set_profiler.m_name = ""; + m_commit_profiler.m_name = ""; +#endif + bool r = true; + if (m_h) + r = bdb.get_backend()->close_container(m_h); + m_h = AUTO_VAL_INIT(m_h); + size_cache = 0; + size_cache_valid = false; + return r; + } + template void enumerate_keys(t_cb cb) const { @@ -644,28 +670,27 @@ namespace tools { return bdb.size(m_h); } - size_t clear() + + bool clear() { - bdb.clear(m_h); + bool result = bdb.clear(m_h); m_isolation.isolated_write_access([&](){ size_cache_valid = false; return true; }); - return true; + return result; } bool erase_validate(const t_key& k) { - auto res_ptr = this->get(k); - bdb.erase(m_h, k); + bool result = bdb.erase(m_h, k); m_isolation.isolated_write_access([&](){ size_cache_valid = false; return true; }); - return static_cast(res_ptr); + return result; } - void erase(const t_key& k) { bdb.erase(m_h, k); @@ -725,7 +750,11 @@ namespace tools } ~cached_key_value_accessor() { - m_cache.clear(); //will clear cache isolated + NESTED_TRY_ENTRY(); + + m_cache.clear(); //will clear cache isolated + + NESTED_CATCH_ENTRY(__func__); } void clear_cache() const @@ -842,13 +871,10 @@ namespace tools operator t_value() const { - static_assert(std::is_pod::value, "t_value must be a POD type."); + std::shared_ptr value_ptr = m_accessor.template explicit_get >(m_key); + if (value_ptr.get()) + return *value_ptr.get(); - std::shared_ptr vptr = m_accessor.template explicit_get >(m_key); - if (vptr.get()) - { - return *vptr.get(); - } return AUTO_VAL_INIT(t_value()); } }; @@ -890,28 +916,48 @@ namespace tools { typedef basic_key_value_accessor, t_value, is_t_access_strategy> basic_accessor_type; -// template solo_db_value, uint64_t, basic_accessor_type> - get_counter_accessor(const t_key& container_id) + get_counter_accessor(const t_key& container_id) { - static_assert(std::is_pod::value, "t_pod_key must be a POD type."); composite_key cc = { container_id, const_counter_suffix}; return solo_db_value, uint64_t, basic_accessor_type >(cc, *this); } -// template const solo_db_value, uint64_t, basic_accessor_type > - get_counter_accessor(const t_key& container_id) const + get_counter_accessor(const t_key& container_id) const { + static_assert(std::is_pod::value, "t_pod_key must be a POD type."); + composite_key cc = { container_id, const_counter_suffix }; - static_assert(std::is_pod::value, "t_pod_key must be a POD type."); - composite_key cc = { container_id, const_counter_suffix }; + return solo_db_value, uint64_t, basic_accessor_type >(cc, const_cast(static_cast(*this))); + } - return solo_db_value, uint64_t, basic_accessor_type >(cc, const_cast(static_cast(*this))); + template + struct subitems_visitor : public i_db_callback + { + subitems_visitor(callback_t cb) + : m_callback(cb) + {} + + virtual bool on_enum_item(uint64_t i, const void* key_data, uint64_t key_size, const void* value_data, uint64_t value_size) override + { + if (key_size != sizeof(composite_key)) + return true; // skip solo values containing items size + + composite_key key = AUTO_VAL_INIT(key); + key_from_ptr(key, key_data, key_size); + + t_value value = AUTO_VAL_INIT(value); + access_strategy_selector::from_buff_to_obj(value_data, value_size, value); + + return m_callback(i, key.container_id, key.sufix, value); } + callback_t m_callback; + }; + public: basic_key_to_array_accessor(basic_db_accessor& db) : basic_key_value_accessor, t_value, is_t_access_strategy>(db) {} @@ -968,6 +1014,14 @@ namespace tools counter = 0; } + + template + void enumerate_subitems(callback_t callback) const + { + subitems_visitor visitor(callback); + basic_accessor_type::bdb.get_backend()->enumerate(basic_accessor_type::m_h, &visitor); + } + }; /************************************************************************/ diff --git a/src/common/db_backend_base.h b/src/common/db_backend_base.h index db566d8..01c87e1 100644 --- a/src/common/db_backend_base.h +++ b/src/common/db_backend_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -38,6 +38,7 @@ namespace tools virtual void abort_transaction()=0; virtual bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) = 0; virtual bool open_container(const std::string& name, container_handle& h)=0; + virtual bool close_container(container_handle& h) = 0; virtual bool erase(container_handle h, const char* k, size_t s) = 0; virtual uint64_t size(container_handle h) = 0; virtual bool get(container_handle h, const char* k, size_t s, std::string& res_buff) = 0; @@ -45,6 +46,7 @@ namespace tools virtual bool clear(container_handle h) = 0; virtual bool enumerate(container_handle h, i_db_callback* pcb)=0; virtual bool get_stat_info(stat_info& si) = 0; + virtual const char* name()=0; virtual ~i_db_backend(){}; }; } diff --git a/src/common/db_backend_lmdb.cpp b/src/common/db_backend_lmdb.cpp index 29870ee..6dfb1ee 100644 --- a/src/common/db_backend_lmdb.cpp +++ b/src/common/db_backend_lmdb.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,10 +7,13 @@ #include "misc_language.h" #include "string_coding.h" #include "profile_tools.h" +#include "util.h" #define BUF_SIZE 1024 -#define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); +#define CHECK_AND_ASSERT_MESS_LMDB_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(rc == MDB_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); +#define CHECK_AND_ASSERT_THROW_MESS_LMDB_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(rc == MDB_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); +#define ASSERT_MES_AND_THROW_LMDB(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdb_strerror(rc) << ", [message]: " << mess); #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "lmdb" @@ -26,7 +29,11 @@ namespace tools } lmdb_db_backend::~lmdb_db_backend() { - close(); + NESTED_TRY_ENTRY(); + + close(); + + NESTED_CATCH_ENTRY(__func__); } bool lmdb_db_backend::open(const std::string& path_, uint64_t cache_sz) @@ -42,9 +49,7 @@ namespace tools CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_set_mapsize"); m_path = path_; -#ifdef WIN32 - m_path = epee::string_encoding::convert_ansii_to_utf8(m_path); -#endif + CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path); res = mdb_env_open(m_penv, m_path.c_str(), MDB_NORDAHEAD , 0644); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_env_open, m_path=" << m_path); @@ -54,7 +59,6 @@ namespace tools bool lmdb_db_backend::open_container(const std::string& name, container_handle& h) { - MDB_dbi dbi = AUTO_VAL_INIT(dbi); begin_transaction(); int res = mdb_dbi_open(get_current_tx(), name.c_str(), MDB_CREATE, &dbi); @@ -64,6 +68,18 @@ namespace tools return true; } + bool lmdb_db_backend::close_container(container_handle& h) + { + static const container_handle null_handle = AUTO_VAL_INIT(null_handle); + CHECK_AND_ASSERT_MES(h != null_handle, false, "close_container is called for null container handle"); + MDB_dbi dbi = static_cast(h); + begin_transaction(); + mdb_dbi_close(m_penv, dbi); + commit_transaction(); + h = null_handle; + return true; + } + bool lmdb_db_backend::close() { { @@ -104,8 +120,12 @@ namespace tools transactions_list& rtxlist = m_txs[std::this_thread::get_id()]; MDB_txn* pparent_tx = nullptr; MDB_txn* p_new_tx = nullptr; + bool parent_read_only = false; if (rtxlist.size()) + { pparent_tx = rtxlist.back().ptx; + parent_read_only = rtxlist.back().read_only; + } if (pparent_tx && read_only) @@ -119,9 +139,20 @@ namespace tools if (read_only) flags += MDB_RDONLY; + //don't use parent tx in write transactions if parent tx was read-only (restriction in lmdb) + //see "Nested transactions: Max 1 child, write txns only, no writemap" + if (pparent_tx && parent_read_only) + pparent_tx = nullptr; + CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed"); res = mdb_txn_begin(m_penv, pparent_tx, flags, &p_new_tx); - CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_txn_begin"); + if(res != MDB_SUCCESS) + { + //Important: if mdb_txn_begin is failed need to unlock previously locked mutex + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + //throw exception to avoid regular code execution + ASSERT_MES_AND_THROW_LMDB(res, "Unable to mdb_txn_begin"); + } rtxlist.push_back(tx_entry()); rtxlist.back().count = read_only ? 1 : 0; @@ -295,13 +326,13 @@ namespace tools PROFILE_FUNC("lmdb_db_backend::set"); int res = 0; MDB_val key = AUTO_VAL_INIT(key); - MDB_val data = AUTO_VAL_INIT(data); + MDB_val data[2] = {}; // mdb_put may access data[1] if some flags are set, this may trigger static code analizers, so here we allocate two elements to avoid it key.mv_data = (void*)k; key.mv_size = ks; - data.mv_data = (void*)v; - data.mv_size = vs; + data[0].mv_data = (void*)v; + data[0].mv_size = vs; - res = mdb_put(get_current_tx(), static_cast(h), &key, &data, 0); + res = mdb_put(get_current_tx(), static_cast(h), &key, data, 0); CHECK_AND_ASSERT_MESS_LMDB_DB(res, false, "Unable to mdb_put"); return true; } @@ -359,8 +390,12 @@ namespace tools } return true; } + const char* lmdb_db_backend::name() + { + return "lmdb"; + } } } #undef LOG_DEFAULT_CHANNEL -#define LOG_DEFAULT_CHANNEL NULL \ No newline at end of file +#define LOG_DEFAULT_CHANNEL NULL diff --git a/src/common/db_backend_lmdb.h b/src/common/db_backend_lmdb.h index f41f3e3..ef56c34 100644 --- a/src/common/db_backend_lmdb.h +++ b/src/common/db_backend_lmdb.h @@ -1,11 +1,11 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once -#include +#include #include "include_base_utils.h" @@ -17,9 +17,6 @@ namespace tools { namespace db { - - - class lmdb_db_backend : public i_db_backend { @@ -44,23 +41,25 @@ namespace tools ~lmdb_db_backend(); //----------------- i_db_backend ----------------------------------------------------- - bool close(); - bool begin_transaction(bool read_only = false); - bool commit_transaction(); - void abort_transaction(); - bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE); - bool open_container(const std::string& name, container_handle& h); - bool erase(container_handle h, const char* k, size_t s); - bool get(container_handle h, const char* k, size_t s, std::string& res_buff); - bool clear(container_handle h); - uint64_t size(container_handle h); - bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs); - bool enumerate(container_handle h, i_db_callback* pcb); - bool get_stat_info(tools::db::stat_info& si); + bool close() override; + bool begin_transaction(bool read_only = false) override; + bool commit_transaction() override; + void abort_transaction() override; + bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) override; + bool open_container(const std::string& name, container_handle& h) override; + bool close_container(container_handle& h) override; + bool erase(container_handle h, const char* k, size_t s) override; + bool get(container_handle h, const char* k, size_t s, std::string& res_buff) override; + bool clear(container_handle h) override; + uint64_t size(container_handle h) override; + bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs) override; + bool enumerate(container_handle h, i_db_callback* pcb) override; + bool get_stat_info(tools::db::stat_info& si) override; + const char* name() override; //------------------------------------------------------------------------------------- bool have_tx(); MDB_txn* get_current_tx(); }; } -} \ No newline at end of file +} diff --git a/src/common/db_backend_mdbx.cpp b/src/common/db_backend_mdbx.cpp new file mode 100644 index 0000000..09670a8 --- /dev/null +++ b/src/common/db_backend_mdbx.cpp @@ -0,0 +1,410 @@ +// Copyright (c) 2014-2019 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#ifdef ENABLED_ENGINE_MDBX +#include "db_backend_mdbx.h" +#include "misc_language.h" +#include "string_coding.h" +#include "profile_tools.h" +#include "util.h" + + +#define BUF_SIZE 1024 + +#define CHECK_AND_ASSERT_MESS_MDBX_DB(rc, ret, mess) CHECK_AND_ASSERT_MES(res == MDBX_SUCCESS, ret, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); +#define CHECK_AND_ASSERT_THROW_MESS_MDBX_DB(rc, mess) CHECK_AND_ASSERT_THROW_MES(res == MDBX_SUCCESS, "[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); +#define ASSERT_MES_AND_THROW_MDBX(rc, mess) ASSERT_MES_AND_THROW("[DB ERROR]:(" << rc << ")" << mdbx_strerror(rc) << ", [message]: " << mess); + +#undef LOG_DEFAULT_CHANNEL +#define LOG_DEFAULT_CHANNEL "mdbx" +// 'mdbx' channel is disabled by default + +namespace tools +{ + namespace db + { + mdbx_db_backend::mdbx_db_backend() : m_penv(AUTO_VAL_INIT(m_penv)) + { + + } + mdbx_db_backend::~mdbx_db_backend() + { + NESTED_TRY_ENTRY(); + + close(); + + NESTED_CATCH_ENTRY(__func__); + } + + bool mdbx_db_backend::open(const std::string& path_, uint64_t cache_sz) + { + int res = 0; + res = mdbx_env_create(&m_penv); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_create"); + + res = mdbx_env_set_maxdbs(m_penv, 15); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_maxdbs"); + + intptr_t size_lower = 0; + intptr_t size_now = -1; //don't change current database size + intptr_t size_upper = 0x10000000000; //don't set db file size limit + intptr_t growth_step = 0x40000000; //increment step 1GB + intptr_t shrink_threshold = -1; + intptr_t pagesize = 0x00001000; //4kb + res = mdbx_env_set_geometry(m_penv, size_lower, size_now, size_upper, growth_step, shrink_threshold, pagesize); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_set_mapsize"); + + m_path = path_; + CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_path), false, "create_directories_if_necessary failed: " << m_path); + + res = mdbx_env_open(m_penv, m_path.c_str(), MDBX_NORDAHEAD , 0644); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_env_open, m_path=" << m_path); + + return true; + } + + bool mdbx_db_backend::open_container(const std::string& name, container_handle& h) + { + MDBX_dbi dbi = AUTO_VAL_INIT(dbi); + begin_transaction(); + int res = mdbx_dbi_open(get_current_tx(), name.c_str(), MDBX_CREATE, &dbi); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_dbi_open with container name: " << name); + commit_transaction(); + h = static_cast(dbi); + return true; + } + + bool mdbx_db_backend::close_container(container_handle& h) + { + static const container_handle null_handle = AUTO_VAL_INIT(null_handle); + CHECK_AND_ASSERT_MES(h != null_handle, false, "close_container is called for null container handle"); + + MDBX_dbi dbi = static_cast(h); + begin_transaction(); + mdbx_dbi_close(m_penv, dbi); + commit_transaction(); + h = null_handle; + return true; + } + + bool mdbx_db_backend::close() + { + { + std::lock_guard lock(m_cs); + for (auto& tx_thread : m_txs) + { + for (auto txe : tx_thread.second) + { + int res = mdbx_txn_commit(txe.ptx); + if (res != MDBX_SUCCESS) + { + LOG_ERROR("[DB ERROR]: On close tranactions: " << mdbx_strerror(res)); + } + } + } + + m_txs.clear(); + } + if (m_penv) + { + mdbx_env_close(m_penv); + m_penv = nullptr; + } + return true; + } + + bool mdbx_db_backend::begin_transaction(bool read_only) + { + if (!read_only) + { + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE LOCKED", LOG_LEVEL_3); + CRITICAL_SECTION_LOCK(m_write_exclusive_lock); + } + PROFILE_FUNC("mdbx_db_backend::begin_transaction"); + { + std::lock_guard lock(m_cs); + CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed"); + transactions_list& rtxlist = m_txs[std::this_thread::get_id()]; + MDBX_txn* pparent_tx = nullptr; + MDBX_txn* p_new_tx = nullptr; + bool parent_read_only = false; + if (rtxlist.size()) + { + pparent_tx = rtxlist.back().ptx; + parent_read_only = rtxlist.back().read_only; + } + + + if (pparent_tx && read_only) + { + ++rtxlist.back().count; + } + else + { + int res = 0; + unsigned int flags = 0; + if (read_only) + flags += MDBX_RDONLY; + + //don't use parent tx in write transactions if parent tx was read-only (restriction in mdbx) + //see "Nested transactions: Max 1 child, write txns only, no writemap" + if (pparent_tx && parent_read_only) + pparent_tx = nullptr; + + CHECK_AND_ASSERT_THROW_MES(m_penv, "m_penv==null, db closed"); + res = mdbx_txn_begin(m_penv, pparent_tx, flags, &p_new_tx); + if(res != MDBX_SUCCESS) + { + //Important: if mdbx_txn_begin is failed need to unlock previously locked mutex + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + //throw exception to avoid regular code execution + ASSERT_MES_AND_THROW_MDBX(res, "Unable to mdbx_txn_begin"); + } + + rtxlist.push_back(tx_entry()); + rtxlist.back().count = read_only ? 1 : 0; + rtxlist.back().ptx = p_new_tx; + rtxlist.back().read_only = read_only; + } + } + + + LOG_PRINT_L4("[DB] Transaction started"); + return true; + } + + MDBX_txn* mdbx_db_backend::get_current_tx() + { + std::lock_guard lock(m_cs); + auto& rtxlist = m_txs[std::this_thread::get_id()]; + CHECK_AND_ASSERT_MES(rtxlist.size(), nullptr, "Unable to find active tx for thread " << std::this_thread::get_id()); + return rtxlist.back().ptx; + } + + bool mdbx_db_backend::pop_tx_entry(tx_entry& txe) + { + std::lock_guard lock(m_cs); + auto it = m_txs.find(std::this_thread::get_id()); + CHECK_AND_ASSERT_MES(it != m_txs.end(), false, "[DB] Unable to find id cor current thread"); + CHECK_AND_ASSERT_MES(it->second.size(), false, "[DB] No active tx for current thread"); + + txe = it->second.back(); + + if (it->second.back().read_only && it->second.back().count == 0) + { + LOG_ERROR("Internal db tx state error: read_only and count readers == 0"); + } + + if ((it->second.back().read_only && it->second.back().count < 2) || (!it->second.back().read_only && it->second.back().count < 1)) + { + it->second.pop_back(); + if (!it->second.size()) + m_txs.erase(it); + } + else + { + --it->second.back().count; + } + return true; + } + + bool mdbx_db_backend::commit_transaction() + { + PROFILE_FUNC("mdbx_db_backend::commit_transaction"); + { + tx_entry txe = AUTO_VAL_INIT(txe); + bool r = pop_tx_entry(txe); + CHECK_AND_ASSERT_MES(r, false, "Unable to pop_tx_entry"); + + if (txe.count == 0 || (txe.read_only && txe.count == 1)) + { + int res = 0; + res = mdbx_txn_commit(txe.ptx); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_txn_commit (error " << res << ")"); + if (!txe.read_only && !txe.count) + { + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED", LOG_LEVEL_3); + } + } + } + LOG_PRINT_L4("[DB] Transaction committed"); + return true; + } + + void mdbx_db_backend::abort_transaction() + { + { + tx_entry txe = AUTO_VAL_INIT(txe); + bool r = pop_tx_entry(txe); + CHECK_AND_ASSERT_MES(r, void(), "Unable to pop_tx_entry"); + if (txe.count == 0 || (txe.read_only && txe.count == 1)) + { + mdbx_txn_abort(txe.ptx); + if (!txe.read_only && !txe.count) + { + CRITICAL_SECTION_UNLOCK(m_write_exclusive_lock); + LOG_PRINT_CYAN("[DB " << m_path << "] WRITE UNLOCKED(ABORTED)", LOG_LEVEL_3); + } + } + + + } + LOG_PRINT_L4("[DB] Transaction aborted"); + } + + bool mdbx_db_backend::erase(container_handle h, const char* k, size_t ks) + { + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + key.iov_base = (void*)k; + key.iov_len = ks; + + res = mdbx_del(get_current_tx(), static_cast(h), &key, nullptr); + if (res == MDBX_NOTFOUND) + return false; + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_del"); + return true; + } + + bool mdbx_db_backend::have_tx() + { + std::lock_guard lock(m_cs); + auto it = m_txs.find(std::this_thread::get_id()); + if (it == m_txs.end()) + return false; + return it->second.size() ? true : false; + } + + bool mdbx_db_backend::get(container_handle h, const char* k, size_t ks, std::string& res_buff) + { + PROFILE_FUNC("mdbx_db_backend::get"); + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data = AUTO_VAL_INIT(data); + key.iov_base = (void*)k; + key.iov_len = ks; + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + + res = mdbx_get(get_current_tx(), static_cast(h), &key, &data); + + if (need_to_commit) + commit_transaction(); + + if (res == MDBX_NOTFOUND) + return false; + + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_get, h: " << h << ", ks: " << ks); + res_buff.assign((const char*)data.iov_base, data.iov_len); + return true; + } + + bool mdbx_db_backend::clear(container_handle h) + { + int res = mdbx_drop(get_current_tx(), static_cast(h), 0); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_drop"); + return true; + } + + uint64_t mdbx_db_backend::size(container_handle h) + { + PROFILE_FUNC("mdbx_db_backend::size"); + MDBX_stat container_stat = AUTO_VAL_INIT(container_stat); + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + int res = mdbx_dbi_stat(get_current_tx(), static_cast(h), &container_stat, sizeof(MDBX_stat)); + if (need_to_commit) + commit_transaction(); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_stat"); + return container_stat.ms_entries; + } + + bool mdbx_db_backend::set(container_handle h, const char* k, size_t ks, const char* v, size_t vs) + { + PROFILE_FUNC("mdbx_db_backend::set"); + int res = 0; + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data[2] = {}; // mdbx_put may access data[1] if some flags are set, this may trigger static code analizers, so here we allocate two elements to avoid it + key.iov_base = (void*)k; + key.iov_len = ks; + data[0].iov_base = (void*)v; + data[0].iov_len = vs; + + res = mdbx_put(get_current_tx(), static_cast(h), &key, data, 0); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_put"); + return true; + } + bool mdbx_db_backend::enumerate(container_handle h, i_db_callback* pcb) + { + CHECK_AND_ASSERT_MES(pcb, false, "null capback ptr passed to enumerate"); + MDBX_val key = AUTO_VAL_INIT(key); + MDBX_val data = AUTO_VAL_INIT(data); + + bool need_to_commit = false; + if (!have_tx()) + { + need_to_commit = true; + begin_transaction(true); + } + MDBX_cursor* cursor_ptr = nullptr; + int res = mdbx_cursor_open(get_current_tx(), static_cast(h), &cursor_ptr); + CHECK_AND_ASSERT_MESS_MDBX_DB(res, false, "Unable to mdbx_cursor_open"); + CHECK_AND_ASSERT_MES(cursor_ptr, false, "cursor_ptr is null after mdbx_cursor_open"); + + uint64_t count = 0; + do + { + int res = mdbx_cursor_get(cursor_ptr, &key, &data, MDBX_NEXT); + if (res == MDBX_NOTFOUND) + break; + if (!pcb->on_enum_item(count, key.iov_base, key.iov_len, data.iov_base, data.iov_len)) + break; + count++; + } while (cursor_ptr); + + mdbx_cursor_close(cursor_ptr); + if (need_to_commit) + commit_transaction(); + return true; + } + + bool mdbx_db_backend::get_stat_info(tools::db::stat_info& si) + { + si = AUTO_VAL_INIT_T(tools::db::stat_info); + + MDBX_envinfo ei = AUTO_VAL_INIT(ei); + mdbx_env_info(m_penv, &ei, sizeof(MDBX_envinfo)); + si.map_size = ei.mi_mapsize; + + std::lock_guard lock(m_cs); + for (auto& e : m_txs) + { + for (auto& pr : e.second) + { + ++si.tx_count; + if(!pr.read_only) + ++si.write_tx_count; + } + } + return true; + } + const char* mdbx_db_backend::name() + { + return "mdbx"; + } + } +} + +#undef LOG_DEFAULT_CHANNEL +#define LOG_DEFAULT_CHANNEL NULL +#endif \ No newline at end of file diff --git a/src/common/db_backend_mdbx.h b/src/common/db_backend_mdbx.h new file mode 100644 index 0000000..af5363f --- /dev/null +++ b/src/common/db_backend_mdbx.h @@ -0,0 +1,67 @@ +// Copyright (c) 2014-2019 Zano Project +// Copyright (c) 2014-2018 The Louisdor Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#ifdef ENABLED_ENGINE_MDBX +#include + +#include "include_base_utils.h" + +#include "db_backend_base.h" +#include "db/libmdbx/mdbx.h" + + +namespace tools +{ + namespace db + { + class mdbx_db_backend : public i_db_backend + { + + struct tx_entry + { + MDBX_txn* ptx; + bool read_only; // needed for thread-top transaction, for figure out if we need to unlock exclusive access + size_t count; //count of read-only nested emulated transactions + }; + typedef std::list transactions_list; + + + std::string m_path; + MDBX_env *m_penv; + + boost::recursive_mutex m_cs; + boost::recursive_mutex m_write_exclusive_lock; + std::map m_txs; // size_t -> count of nested read_only transactions + bool pop_tx_entry(tx_entry& txe); + public: + mdbx_db_backend(); + ~mdbx_db_backend(); + + //----------------- i_db_backend ----------------------------------------------------- + bool close() override; + bool begin_transaction(bool read_only = false) override; + bool commit_transaction() override; + void abort_transaction() override; + bool open(const std::string& path, uint64_t cache_sz = CACHE_SIZE) override; + bool open_container(const std::string& name, container_handle& h) override; + bool close_container(container_handle& h) override; + bool erase(container_handle h, const char* k, size_t s) override; + bool get(container_handle h, const char* k, size_t s, std::string& res_buff) override; + bool clear(container_handle h) override; + uint64_t size(container_handle h) override; + bool set(container_handle h, const char* k, size_t s, const char* v, size_t vs) override; + bool enumerate(container_handle h, i_db_callback* pcb) override; + bool get_stat_info(tools::db::stat_info& si) override; + const char* name() override; + //------------------------------------------------------------------------------------- + bool have_tx(); + MDBX_txn* get_current_tx(); + + }; + + } +} +#endif \ No newline at end of file diff --git a/src/common/db_backend_selector.cpp b/src/common/db_backend_selector.cpp new file mode 100644 index 0000000..9c7917c --- /dev/null +++ b/src/common/db_backend_selector.cpp @@ -0,0 +1,132 @@ +// Copyright (c) 2014-2020 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "db_backend_selector.h" +#include "currency_core/currency_config.h" +#include "command_line.h" +#include "db_backend_lmdb.h" +#include "db_backend_mdbx.h" + +#define LMDB_MAIN_FILE_NAME "data.mdb" +#define MDBX_MAIN_FILE_NAME "mdbx.dat" + +namespace tools +{ +namespace db +{ + + db_backend_selector::db_backend_selector() + : m_engine_type(db_none) + { + } + + void db_backend_selector::init_options(boost::program_options::options_description& desc) + { + command_line::add_arg(desc, command_line::arg_db_engine); + } + + bool db_backend_selector::init(const boost::program_options::variables_map& vm) + { + try + { + m_config_folder = command_line::get_arg(vm, command_line::arg_data_dir); + + if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_LMDB) + { + m_engine_type = db_lmdb; + } + else if (command_line::get_arg(vm, command_line::arg_db_engine) == ARG_DB_ENGINE_MDBX) + { +#ifdef ENABLED_ENGINE_MDBX + m_engine_type = db_mdbx; +#else + LOG_PRINT_L0(" DB ENGINE: " << ARG_DB_ENGINE_MDBX << " is not suported by this build(see DISABLE_MDBX cmake option), STOPPING"); +#endif + } + else + { + LOG_PRINT_RED_L0("UNKNOWN DB ENGINE: " << command_line::get_arg(vm, command_line::arg_db_engine) << ", STOPPING"); + } + } + catch (std::exception& e) + { + LOG_ERROR("internal error: db_backend_selector::init failed on command-line parsing, exception: " << e.what()); + return false; + } + + if (m_engine_type == db_none) + return false; + + return true; + } + + std::string db_backend_selector::get_db_folder_path() const + { + //CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited"); + return m_config_folder + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX; + } + + std::string db_backend_selector::get_temp_db_folder_path() const + { + //CHECK_AND_ASSERT_THROW_MES(m_engine_type != db_none, "db_backend_selector was no inited"); + return get_temp_config_folder() + ("/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX; + } + + std::string db_backend_selector::get_pool_db_folder_path() const + { + return m_config_folder + ("/" CURRENCY_POOLDATA_FOLDERNAME_PREFIX) + get_engine_name() + CURRENCY_POOLDATA_FOLDERNAME_SUFFIX; + } + + std::string db_backend_selector::get_db_main_file_name() const + { + switch (m_engine_type) + { + case db_lmdb: + return LMDB_MAIN_FILE_NAME; + case db_mdbx: + return MDBX_MAIN_FILE_NAME; + default: + return ""; + } + } + + std::string db_backend_selector::get_engine_name() const + { + switch (m_engine_type) + { + case db_lmdb: + return "lmdb"; + case db_mdbx: + return "mdbx"; + default: + return "unknown"; + } + } + + std::shared_ptr db_backend_selector::create_backend() + { + switch (m_engine_type) + { + case db_lmdb: + return std::shared_ptr(new tools::db::lmdb_db_backend); + + case db_mdbx: + return std::shared_ptr(new tools::db::mdbx_db_backend); + + default: + LOG_ERROR("db_backend_selector was no inited"); + return nullptr; + } + } + + std::string db_backend_selector::get_temp_config_folder() const + { + return m_config_folder + "_TEMP"; + } + + + + +} // namespace db +} // namespace tools diff --git a/src/common/db_backend_selector.h b/src/common/db_backend_selector.h new file mode 100644 index 0000000..b9d811b --- /dev/null +++ b/src/common/db_backend_selector.h @@ -0,0 +1,43 @@ +// Copyright (c) 2014-2020 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include "misc_language.h" +#include "db_backend_base.h" + +namespace tools +{ + namespace db + { + enum db_engine_type { db_none = 0, db_lmdb, db_mdbx }; + + class db_backend_selector + { + public: + db_backend_selector(); + + static void init_options(boost::program_options::options_description& desc); + bool init(const boost::program_options::variables_map& vm); + + std::string get_db_folder_path() const; + std::string get_db_main_file_name() const; + db_engine_type get_engine_type() const { return m_engine_type; } + std::string get_engine_name() const; + std::string get_config_folder() const { return m_config_folder; } + std::string get_temp_config_folder() const; + std::string get_temp_db_folder_path() const; + + std::string get_pool_db_folder_path() const; + + std::shared_ptr create_backend(); + + private: + db_engine_type m_engine_type; + std::string m_config_folder; + }; + + } // namespace db +} // namespace tools diff --git a/src/common/encryption_filter.cpp b/src/common/encryption_filter.cpp new file mode 100644 index 0000000..8861fa8 --- /dev/null +++ b/src/common/encryption_filter.cpp @@ -0,0 +1,7 @@ +// Copyright (c) 2014-2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + + +#include "encryption_filter.h" +#include "crypto/chacha8_stream.h" \ No newline at end of file diff --git a/src/common/encryption_filter.h b/src/common/encryption_filter.h new file mode 100644 index 0000000..854dc7a --- /dev/null +++ b/src/common/encryption_filter.h @@ -0,0 +1,244 @@ +// Copyright (c) 2014-2018 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include +#include // sink_tag +#include // boost::iostreams::write, boost::iostreams::read +#include "include_base_utils.h" +#include "crypto/chacha8.h" +#include "crypto/chacha8_stream.h" + + +namespace tools +{ + + /************************************************************************/ + /* */ + /************************************************************************/ + + class encrypt_chacha_processer_base + { + public: + typedef char char_type; + //typedef boost::iostreams::multichar_output_filter_tag category; + //typedef boost::iostreams::flushable_tag category; + static const uint32_t block_size = ECRYPT_BLOCKLENGTH; + + encrypt_chacha_processer_base(std::string const &pass, const crypto::chacha8_iv& iv) :m_iv(iv), m_ctx(AUTO_VAL_INIT(m_ctx)) + { + crypto::generate_chacha8_key(pass, m_key); + ECRYPT_keysetup(&m_ctx, &m_key.data[0], sizeof(m_key.data) * 8, sizeof(m_iv.data) * 8); + ECRYPT_ivsetup(&m_ctx, &m_iv.data[0]); + } + ~encrypt_chacha_processer_base() + { + + } + + template + std::streamsize process(char_type const * const buf, std::streamsize const n, cb_handler cb) const + { + if (n == 0) + return n; + if (n%ECRYPT_BLOCKLENGTH == 0 && m_buff.empty()) + { + std::vector buff(n); + ECRYPT_encrypt_blocks(&m_ctx, (u8*)buf, (u8*)&buff[0], (u32)(n / ECRYPT_BLOCKLENGTH)); + cb(&buff[0], n); + //m_underlying_stream.write(&buff[0], n); + return n; + } + else + { + m_buff.append(buf, n); + size_t encr_count = m_buff.size() - m_buff.size() % ECRYPT_BLOCKLENGTH; + if (!encr_count) + return n; + std::vector buff(encr_count); + ECRYPT_encrypt_blocks(&m_ctx, (u8*)m_buff.data(), (u8*)&buff[0], (u32)(m_buff.size() / ECRYPT_BLOCKLENGTH)); + //m_underlying_stream.write(&buff[0], encr_count); + cb(&buff[0], encr_count); + m_buff.erase(0, encr_count); + return encr_count; + } + + } + + template + bool flush(cb_handler cb) + { + if (m_buff.empty()) + return true; + + std::vector buff(m_buff.size()); + ECRYPT_encrypt_bytes(&m_ctx, (u8*)m_buff.data(), (u8*)&buff[0], (u32)m_buff.size()); + cb(&buff[0], m_buff.size()); + //m_underlying_stream.write(&buff[0], m_buff.size()); + return true; + } + + private: + const crypto::chacha8_iv& m_iv; + mutable ECRYPT_ctx m_ctx; + //std::ostream &m_underlying_stream; + crypto::chacha8_key m_key; + mutable std::string m_buff; + }; + + /************************************************************************/ + /* */ + /************************************************************************/ + + + class encrypt_chacha_out_filter : public encrypt_chacha_processer_base + { + public: + typedef char char_type; + struct category : + public boost::iostreams::multichar_output_filter_tag, + public boost::iostreams::flushable_tag + { }; + + encrypt_chacha_out_filter(std::string const &pass, const crypto::chacha8_iv& iv) :encrypt_chacha_processer_base(pass, iv) + { + } + ~encrypt_chacha_out_filter() + { + } + + template + std::streamsize write(t_sink& snk, char_type const * const buf, std::streamsize const n) const + { + return encrypt_chacha_processer_base::process(buf, n, [&](char_type const * const buf_lambda, std::streamsize const n_lambda) { + boost::iostreams::write(snk, &buf_lambda[0], n_lambda); + }); + } + + template + bool flush(Sink& snk) + { + + encrypt_chacha_processer_base::flush([&](char_type const * const buf_lambda, std::streamsize const n_lambda) { + boost::iostreams::write(snk, &buf_lambda[0], n_lambda); + }); + return true; + } + + private: + }; + + + /************************************************************************/ + /* */ + /************************************************************************/ + + class encrypt_chacha_in_filter : public encrypt_chacha_processer_base + { + public: + typedef char char_type; + struct category : //public boost::iostreams::seekable_device_tag, + public boost::iostreams::multichar_input_filter_tag, + public boost::iostreams::flushable_tag + //public boost::iostreams::seekable_filter_tag + { }; + encrypt_chacha_in_filter(std::string const &pass, const crypto::chacha8_iv& iv) :encrypt_chacha_processer_base(pass, iv), m_was_eof(false) + { + } + ~encrypt_chacha_in_filter() + { + } + + template + std::streamsize read(Source& src, char* s, std::streamsize n) + { + if (m_buff.size() >= static_cast(n)) + { + return withdraw_to_read_buff(s, n); + } + if (m_was_eof && m_buff.empty()) + { + return -1; + } + + std::streamsize size_to_read_for_decrypt = (n - m_buff.size()); + size_to_read_for_decrypt += size_to_read_for_decrypt % encrypt_chacha_processer_base::block_size; + size_t offset_in_buff = m_buff.size(); + m_buff.resize(m_buff.size() + size_to_read_for_decrypt); + + std::streamsize result = boost::iostreams::read(src, (char*)&m_buff.data()[offset_in_buff], size_to_read_for_decrypt); + if (result == size_to_read_for_decrypt) + { + //regular read proocess, readed data enought to get decrypteds + encrypt_chacha_processer_base::process(&m_buff.data()[offset_in_buff], size_to_read_for_decrypt, [&](char_type const* const buf_lambda, std::streamsize const n_lambda) + { + CHECK_AND_ASSERT_THROW_MES(n_lambda == size_to_read_for_decrypt, "Error in decrypt: check n_lambda == size_to_read_for_decrypt failed"); + std::memcpy((char*)&m_buff.data()[offset_in_buff], buf_lambda, n_lambda); + }); + return withdraw_to_read_buff(s, n); + } + else + { + //been read some size_but it's basically might be eof + if (!m_was_eof) + { + size_t offset_before_flush = offset_in_buff; + if (result != -1) + { + //eof + encrypt_chacha_processer_base::process(&m_buff.data()[offset_in_buff], result, [&](char_type const* const buf_lambda, std::streamsize const n_lambda) { + std::memcpy((char*)&m_buff.data()[offset_in_buff], buf_lambda, n_lambda); + offset_before_flush = offset_in_buff + n_lambda; + }); + } + + encrypt_chacha_processer_base::flush([&](char_type const* const buf_lambda, std::streamsize const n_lambda) { + if (n_lambda + offset_before_flush > m_buff.size()) + { + m_buff.resize(n_lambda + offset_before_flush); + } + std::memcpy((char*)&m_buff.data()[offset_before_flush], buf_lambda, n_lambda); + m_buff.resize(offset_before_flush + n_lambda); + }); + + //just to make sure that it's over + std::string buff_stub(10, ' '); + std::streamsize r = boost::iostreams::read(src, (char*)&buff_stub.data()[0], 10); + CHECK_AND_ASSERT_THROW_MES(r == -1, "expected EOF"); + m_was_eof = true; + return withdraw_to_read_buff(s, n); + } + else + { + return -1; + } + } + } + + + template + bool flush(Sink& snk) + { + + return true; + } + + private: + + std::streamsize withdraw_to_read_buff(char* s, std::streamsize n) + { + + size_t copy_size = m_buff.size() > static_cast(n) ? static_cast(n) : m_buff.size(); + std::memcpy(s, m_buff.data(), copy_size); + m_buff.erase(0, copy_size); + return copy_size; + } + + std::string m_buff; + bool m_was_eof; + }; +} \ No newline at end of file diff --git a/src/common/error_codes.h b/src/common/error_codes.h new file mode 100644 index 0000000..648bd6f --- /dev/null +++ b/src/common/error_codes.h @@ -0,0 +1,42 @@ +// Copyright (c) 2014-2018 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + + +#define API_RETURN_CODE_OK BASIC_RESPONSE_STATUS_OK +#define API_RETURN_CODE_FAIL BASIC_RESPONSE_STATUS_FAILED +#define API_RETURN_CODE_NOT_FOUND BASIC_RESPONSE_STATUS_NOT_FOUND +#define API_RETURN_CODE_ACCESS_DENIED "ACCESS_DENIED" +#define API_RETURN_CODE_INTERNAL_ERROR "INTERNAL_ERROR" +#define API_RETURN_CODE_NOT_ENOUGH_MONEY "NOT_ENOUGH_MONEY" +#define API_RETURN_CODE_NOT_ENOUGH_OUTPUTS_FOR_MIXING "NOT_ENOUGH_OUTPUTS_FOR_MIXING" +#define API_RETURN_CODE_INTERNAL_ERROR_QUE_FULL "INTERNAL_ERROR_QUE_FULL" +#define API_RETURN_CODE_BAD_ARG "BAD_ARG" +#define API_RETURN_CODE_BAD_ARG_EMPTY_DESTINATIONS "BAD_ARG_EMPTY_DESTINATIONS" +#define API_RETURN_CODE_BAD_ARG_WRONG_FEE "BAD_ARG_WRONG_FEE" +#define API_RETURN_CODE_BAD_ARG_INVALID_ADDRESS "BAD_ARG_INVALID_ADDRESS" +#define API_RETURN_CODE_BAD_ARG_WRONG_AMOUNT "BAD_ARG_WRONG_AMOUNT" +#define API_RETURN_CODE_BAD_ARG_WRONG_PAYMENT_ID "BAD_ARG_WRONG_PAYMENT_ID" +#define API_RETURN_CODE_WRONG_PASSWORD "WRONG_PASSWORD" +#define API_RETURN_CODE_WALLET_WRONG_ID "WALLET_WRONG_ID" +#define API_RETURN_CODE_WALLET_WATCH_ONLY_NOT_SUPPORTED "WALLET_WATCH_ONLY_NOT_SUPPORTED" +#define API_RETURN_CODE_WALLET_AUDITABLE_NOT_SUPPORTED "WALLET_AUDITABLE_NOT_SUPPORTED" +#define API_RETURN_CODE_FILE_NOT_FOUND "FILE_NOT_FOUND" +#define API_RETURN_CODE_ALREADY_EXISTS "ALREADY_EXISTS" +#define API_RETURN_CODE_CANCELED "CANCELED" +#define API_RETURN_CODE_FILE_RESTORED "FILE_RESTORED" +#define API_RETURN_CODE_TRUE "TRUE" +#define API_RETURN_CODE_FALSE "FALSE" +#define API_RETURN_CODE_CORE_BUSY "CORE_BUSY" +#define API_RETURN_CODE_OVERFLOW "OVERFLOW" +#define API_RETURN_CODE_BUSY "BUSY" +#define API_RETURN_CODE_INVALID_FILE "INVALID_FILE" +#define API_RETURN_CODE_WRONG_SEED "WRONG_SEED" +#define API_RETURN_CODE_GENESIS_MISMATCH "GENESIS_MISMATCH" +#define API_RETURN_CODE_DISCONNECTED "DISCONNECTED" +#define API_RETURN_CODE_UNINITIALIZED "UNINITIALIZED" +#define API_RETURN_CODE_TX_IS_TOO_BIG "TX_IS_TOO_BIG" +#define API_RETURN_CODE_TX_REJECTED "TX_REJECTED" +#define API_RETURN_CODE_HTLC_ORIGIN_HASH_MISSMATCHED "HTLC_ORIGIN_HASH_MISSMATCHED" diff --git a/src/common/int-util.h b/src/common/int-util.h index 9eff50b..ddf06df 100644 --- a/src/common/int-util.h +++ b/src/common/int-util.h @@ -10,12 +10,22 @@ #include #include #include -#include #if defined(_MSC_VER) #include #include +#if !defined(__ORDER_LITTLE_ENDIAN__) +#define __ORDER_LITTLE_ENDIAN__ 1011012001 +#endif + +#if !defined(__BYTE_ORDER__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ +#endif + +#if !defined(__ORDER_BIG_ENDIAN__) +#define __ORDER_BIG_ENDIAN__ 0 +#endif static inline uint32_t rol32(uint32_t x, int r) { static_assert(sizeof(uint32_t) == sizeof(unsigned int), "this code assumes 32-bit integers"); @@ -194,11 +204,11 @@ static inline void memcpy_swap64(void *dst, const void *src, size_t n) { } } -#if !defined(BYTE_ORDER) || !defined(LITTLE_ENDIAN) || !defined(BIG_ENDIAN) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not enabled"); #endif -#if BYTE_ORDER == LITTLE_ENDIAN +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define SWAP32LE IDENT32 #define SWAP32BE SWAP32 #define swap32le ident32 @@ -217,7 +227,7 @@ static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not e #define memcpy_swap64be memcpy_swap64 #endif -#if BYTE_ORDER == BIG_ENDIAN +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ #define SWAP32BE IDENT32 #define SWAP32LE SWAP32 #define swap32be ident32 diff --git a/src/common/make_hashable.h b/src/common/make_hashable.h index af18f47..c22bfa4 100644 --- a/src/common/make_hashable.h +++ b/src/common/make_hashable.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2020 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Copyright (c) 2012-2013 The Boolberry developers @@ -37,3 +37,21 @@ namespace std { \ } \ }; \ } + +namespace std +{ + + // this allows using std::pair<> as a key in unordered std containers + template + struct hash> + { + size_t operator()(const pair& p) const + { + auto hash1 = hash{}(p.first); + auto hash2 = hash{}(p.second); + return hash1 ^ hash2; + } + }; + +} // namespace std + diff --git a/src/common/miniupnp_helper.h b/src/common/miniupnp_helper.h index 2b2fd09..be7c688 100644 --- a/src/common/miniupnp_helper.h +++ b/src/common/miniupnp_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Boolberry developers // Distributed under the MIT/X11 software license, see the accompanying @@ -11,9 +11,9 @@ #include #include "include_base_utils.h" extern "C" { -#include "miniupnpc/miniupnpc.h" -#include "miniupnpc/upnpcommands.h" -#include "miniupnpc/upnperrors.h" +#include "miniupnp/miniupnpc/miniupnpc.h" +#include "miniupnp/miniupnpc/upnpcommands.h" +#include "miniupnp/miniupnpc/upnperrors.h" } #include "misc_language.h" @@ -30,26 +30,37 @@ namespace tools char m_lanaddr[64]; int m_IGD; boost::thread m_mapper_thread; + boost::thread m_initializer_thread; uint32_t m_external_port; + uint32_t m_internal_port; + uint32_t m_period_ms; public: miniupnp_helper():m_devlist(nullptr), m_urls(AUTO_VAL_INIT(m_urls)), m_data(AUTO_VAL_INIT(m_data)), - m_IGD(0) + m_IGD(0), + m_external_port(0), + m_internal_port(0), + m_period_ms(0) { m_lanaddr[0] = 0; } ~miniupnp_helper() { + NESTED_TRY_ENTRY(); + deinit(); + + NESTED_CATCH_ENTRY(__func__); } bool start_regular_mapping(uint32_t internal_port, uint32_t external_port, uint32_t period_ms) { m_external_port = external_port; + m_internal_port = internal_port; + m_period_ms = period_ms; if(!init()) return false; - m_mapper_thread = boost::thread([=](){run_port_mapping_loop(internal_port, external_port, period_ms);}); return true; } @@ -72,27 +83,42 @@ namespace tools bool init() { - deinit(); + m_initializer_thread = boost::thread([=]() + { + deinit(); - int error = 0; - m_devlist = upnpDiscover(2000, nullptr, nullptr, 0, 0, &error); - if(error) - { - LOG_PRINT_L0("Failed to call upnpDiscover"); - return false; - } - - m_IGD = UPNP_GetValidIGD(m_devlist, &m_urls, &m_data, m_lanaddr, sizeof(m_lanaddr)); - if(m_IGD != 1) - { - LOG_PRINT_L2("IGD not found"); - return false; - } + int error = 0; + m_devlist = upnpDiscover(2000, nullptr, nullptr, 0, 0, 2, &error); + if (error) + { + LOG_PRINT_L0("Failed to call upnpDiscover"); + return false; + } + + m_IGD = UPNP_GetValidIGD(m_devlist, &m_urls, &m_data, m_lanaddr, sizeof(m_lanaddr)); + if (m_IGD != 1) + { + LOG_PRINT_L2("IGD not found"); + return false; + } + + m_mapper_thread = boost::thread([=]() {run_port_mapping_loop(m_internal_port, m_external_port, m_period_ms); }); + return true; + }); return true; } bool deinit() { + if(m_initializer_thread.get_id() != boost::this_thread::get_id()) + { + if (m_initializer_thread.joinable()) + { + m_initializer_thread.interrupt(); + m_initializer_thread.join(); + } + } + stop_mapping(); if(m_devlist) diff --git a/src/common/mnemonic-encoding.cpp b/src/common/mnemonic-encoding.cpp index e5f4664..c2e3e30 100644 --- a/src/common/mnemonic-encoding.cpp +++ b/src/common/mnemonic-encoding.cpp @@ -48,8 +48,6 @@ namespace tools { using namespace std; - const int NUMWORDS = 1626; - const map wordsMap = { {"like", 0}, {"just", 1}, @@ -3358,7 +3356,7 @@ namespace tools throw runtime_error("Invalid binary data size for mnemonic encoding"); // 4 bytes -> 3 words. 8 digits base 16 -> 3 digits base 1626 string res; - for (unsigned int i=0; i < binary.size() / 4; i++, res += ' ') + for (unsigned int i=0; i < binary.size() / 4; i++) { const uint32_t* val = reinterpret_cast(&binary[i * 4]); @@ -3369,8 +3367,9 @@ namespace tools res += wordsArray[w1] + " "; res += wordsArray[w2] + " "; - res += wordsArray[w3]; + res += wordsArray[w3] + " "; } + res.erase(--res.end()); // remove trailing space return res; } std::string word_by_num(uint32_t n) @@ -3380,6 +3379,12 @@ namespace tools return wordsArray[n]; } + bool valid_word(const std::string& w) + { + auto it = wordsMap.find(w); + return it != wordsMap.end(); + } + uint64_t num_by_word(const std::string& w) { auto it = wordsMap.find(w); diff --git a/src/common/mnemonic-encoding.h b/src/common/mnemonic-encoding.h index 409a935..4ce6b9b 100644 --- a/src/common/mnemonic-encoding.h +++ b/src/common/mnemonic-encoding.h @@ -40,9 +40,12 @@ namespace tools { namespace mnemonic_encoding { + constexpr int NUMWORDS = 1626; + std::vector text2binary(const std::string& text); std::string binary2text(const std::vector& binary); std::string word_by_num(uint32_t n); uint64_t num_by_word(const std::string& w); + bool valid_word(const std::string& w); } } diff --git a/src/common/ntp.cpp b/src/common/ntp.cpp new file mode 100644 index 0000000..fdbc2a5 --- /dev/null +++ b/src/common/ntp.cpp @@ -0,0 +1,265 @@ +// Copyright (c) 2019 Zano Project + +// Note: class udp_blocking_client is a slightly modified version of an example +// taken from https://www.boost.org/doc/libs/1_53_0/doc/html/boost_asio/example/timeouts/blocking_udp_client.cpp +// +// Copyright (c) 2003-2012 Christopher M. Kohlhoff (chris at kohlhoff dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ntp.h" + +using boost::asio::deadline_timer; +using boost::asio::ip::udp; + +//---------------------------------------------------------------------- + +// +// This class manages socket timeouts by applying the concept of a deadline. +// Each asynchronous operation is given a deadline by which it must complete. +// Deadlines are enforced by an "actor" that persists for the lifetime of the +// client object: +// +// +----------------+ +// | | +// | check_deadline |<---+ +// | | | +// +----------------+ | async_wait() +// | | +// +---------+ +// +// If the actor determines that the deadline has expired, any outstanding +// socket operations are cancelled. The socket operations themselves are +// implemented as transient actors: +// +// +---------------+ +// | | +// | receive | +// | | +// +---------------+ +// | +// async_- | +----------------+ +// receive() | | | +// +--->| handle_receive | +// | | +// +----------------+ +// +// The client object runs the io_service to block thread execution until the +// actor completes. +// +namespace +{ +class udp_blocking_client +{ +public: + udp_blocking_client(const udp::endpoint& listen_endpoint, udp::socket& socket, boost::asio::io_service& io_service) + : socket_(socket), + io_service_(io_service), + deadline_(io_service) + { + // No deadline is required until the first socket operation is started. We + // set the deadline to positive infinity so that the actor takes no action + // until a specific deadline is set. + deadline_.expires_at(boost::posix_time::pos_infin); + + // Start the persistent actor that checks for deadline expiry. + check_deadline(); + } + + std::size_t receive(const boost::asio::mutable_buffer& buffer, + boost::posix_time::time_duration timeout, boost::system::error_code& ec) + { + // Set a deadline for the asynchronous operation. + deadline_.expires_from_now(timeout); + + // Set up the variables that receive the result of the asynchronous + // operation. The error code is set to would_block to signal that the + // operation is incomplete. Asio guarantees that its asynchronous + // operations will never fail with would_block, so any other value in + // ec indicates completion. + ec = boost::asio::error::would_block; + std::size_t length = 0; + + // Start the asynchronous operation itself. The handle_receive function + // used as a callback will update the ec and length variables. + socket_.async_receive(boost::asio::buffer(buffer), + boost::bind(&udp_blocking_client::handle_receive, _1, _2, &ec, &length)); + + // Block until the asynchronous operation has completed. + do io_service_.run_one(); while (ec == boost::asio::error::would_block); + + return length; + } + +private: + void check_deadline() + { + // Check whether the deadline has passed. We compare the deadline against + // the current time since a new asynchronous operation may have moved the + // deadline before this actor had a chance to run. + if (deadline_.expires_at() <= deadline_timer::traits_type::now()) + { + // The deadline has passed. The outstanding asynchronous operation needs + // to be cancelled so that the blocked receive() function will return. + // + // Please note that cancel() has portability issues on some versions of + // Microsoft Windows, and it may be necessary to use close() instead. + // Consult the documentation for cancel() for further information. + socket_.cancel(); + + // There is no longer an active deadline. The expiry is set to positive + // infinity so that the actor takes no action until a new deadline is set. + deadline_.expires_at(boost::posix_time::pos_infin); + } + + // Put the actor back to sleep. + deadline_.async_wait(boost::bind(&udp_blocking_client::check_deadline, this)); + } + + static void handle_receive( + const boost::system::error_code& ec, std::size_t length, + boost::system::error_code* out_ec, std::size_t* out_length) + { + *out_ec = ec; + *out_length = length; + } + +private: + boost::asio::io_service& io_service_; + udp::socket& socket_; + deadline_timer deadline_; +}; + + +#pragma pack(push, 1) +struct ntp_packet +{ + + uint8_t li_vn_mode; // Eight bits. li, vn, and mode. + // li. Two bits. Leap indicator. + // vn. Three bits. Version number of the protocol. + // mode. Three bits. Client will pick mode 3 for client. + + uint8_t stratum; // Eight bits. Stratum level of the local clock. + uint8_t poll; // Eight bits. Maximum interval between successive messages. + uint8_t precision; // Eight bits. Precision of the local clock. + + uint32_t rootDelay; // 32 bits. Total round trip delay time. + uint32_t rootDispersion; // 32 bits. Max error aloud from primary clock source. + uint32_t refId; // 32 bits. Reference clock identifier. + + uint32_t refTm_s; // 32 bits. Reference time-stamp seconds. + uint32_t refTm_f; // 32 bits. Reference time-stamp fraction of a second. + + uint64_t orig_tm; // 64 bits. Originate time-stamp (set by client) + + uint32_t rxTm_s; // 32 bits. Received time-stamp seconds. + uint32_t rxTm_f; // 32 bits. Received time-stamp fraction of a second. + + uint32_t txTm_s; // 32 bits and the most important field the client cares about. Transmit time-stamp seconds. + uint32_t txTm_f; // 32 bits. Transmit time-stamp fraction of a second. + +}; // Total: 384 bits or 48 bytes. +#pragma pack(pop) + +static_assert(sizeof(ntp_packet) == 48, "ntp_packet has invalid size"); + +} // namespace + + +namespace tools +{ + int64_t get_ntp_time(const std::string& host_name, size_t timeout_sec) + { + try + { + boost::asio::io_service io_service; + boost::asio::ip::udp::resolver resolver(io_service); + boost::asio::ip::udp::resolver::query query(boost::asio::ip::udp::v4(), host_name, "ntp"); + boost::asio::ip::udp::endpoint receiver_endpoint = *resolver.resolve(query); + boost::asio::ip::udp::socket socket(io_service); + socket.open(boost::asio::ip::udp::v4()); + + ntp_packet packet_sent = AUTO_VAL_INIT(packet_sent); + packet_sent.li_vn_mode = 0x1b; + auto packet_sent_time = std::chrono::high_resolution_clock::now(); + socket.send_to(boost::asio::buffer(&packet_sent, sizeof packet_sent), receiver_endpoint); + + ntp_packet packet_received = AUTO_VAL_INIT(packet_received); + boost::asio::ip::udp::endpoint sender_endpoint; + + udp_blocking_client ubc(sender_endpoint, socket, io_service); + boost::system::error_code ec; + ubc.receive(boost::asio::buffer(&packet_received, sizeof packet_received), boost::posix_time::seconds(static_cast(timeout_sec)), ec); + if (ec) + { + LOG_PRINT_L3("NTP: get_ntp_time(" << host_name << "): boost error: " << ec.message()); + return 0; + } + + auto packet_received_time = std::chrono::high_resolution_clock::now(); + int64_t roundtrip_mcs = std::chrono::duration_cast(packet_received_time - packet_sent_time).count(); + uint64_t roundtrip_s = roundtrip_mcs > 2000000 ? roundtrip_mcs / 2000000 : 0; + + time_t ntp_time = ntohl(packet_received.txTm_s); + if (ntp_time <= 2208988800U) + { + LOG_PRINT_L3("NTP: get_ntp_time(" << host_name << "): wrong txTm_s: " << packet_received.txTm_s); + return 0; + } + // LOG_PRINT_L2("NTP: get_ntp_time(" << host_name << "): RAW time received: " << ntp_time << ", refTm_s: " << ntohl(packet_received.refTm_s) << ", stratum: " << packet_received.stratum << ", round: " << roundtrip_mcs); + ntp_time -= 2208988800U; // Unix time starts from 01/01/1970 == 2208988800U + ntp_time += roundtrip_s; + return ntp_time; + } + catch (const std::exception& e) + { + LOG_PRINT_L2("NTP: get_ntp_time(" << host_name << "): exception: " << e.what()); + return 0; + } + catch (...) + { + LOG_PRINT_L2("NTP: get_ntp_time(" << host_name << "): unknown exception"); + return 0; + } + } + + + #define TIME_SYNC_NTP_SERVERS "time1.google.com", "time2.google.com", "time3.google.com", "time4.google.com" /* , "pool.ntp.org" -- we need to request a vender zone before using this pool */ + #define TIME_SYNC_NTP_TIMEOUT_SEC 5 + #define TIME_SYNC_NTP_ATTEMPTS_COUNT 3 // max number of attempts when getting time from NTP server + + int64_t get_ntp_time() + { + static const std::vector ntp_servers { TIME_SYNC_NTP_SERVERS }; + + for (size_t att = 0; att < TIME_SYNC_NTP_ATTEMPTS_COUNT; ++att) + { + const std::string& ntp_server = ntp_servers[att % ntp_servers.size()]; + LOG_PRINT_L3("NTP: trying to get time from " << ntp_server); + int64_t time = get_ntp_time(ntp_server, TIME_SYNC_NTP_TIMEOUT_SEC); + if (time > 0) + { + LOG_PRINT_L3("NTP: " << ntp_server << " responded with " << time << " (" << epee::misc_utils::get_time_str_v2(time) << ")"); + return time; + } + } + + LOG_PRINT_RED("NTP: unable to get time from NTP with " << TIME_SYNC_NTP_ATTEMPTS_COUNT << " trials", LOG_LEVEL_2); + return 0; // smth went wrong + } + + + +} // namespace tools diff --git a/src/common/ntp.h b/src/common/ntp.h new file mode 100644 index 0000000..4e453b3 --- /dev/null +++ b/src/common/ntp.h @@ -0,0 +1,18 @@ +// Copyright (c) 2019 Zano Project + +#pragma once +#include +#include + +namespace tools +{ + + // requests current time via NTP from 'host_hame' using 'timeout_sec' + // may return zero -- means error + int64_t get_ntp_time(const std::string& host_name, size_t timeout_sec = 5); + + // request time via predefined NTP servers + // may return zero -- mean error + int64_t get_ntp_time(); + +} // namespace tools diff --git a/src/common/pod_array_file_container.h b/src/common/pod_array_file_container.h new file mode 100644 index 0000000..7c168d7 --- /dev/null +++ b/src/common/pod_array_file_container.h @@ -0,0 +1,127 @@ +// Copyright (c) 2012-2013 The Cryptonote developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#include +#include +#include + +namespace tools +{ + + template + class pod_array_file_container + { + public: + pod_array_file_container() + {} + + ~pod_array_file_container() + { + close(); + } + + bool open(const std::wstring& filename, bool create_if_not_exist, bool* p_corrupted = nullptr, std::string* p_reason = nullptr) + { + if (!create_if_not_exist && !boost::filesystem::exists(filename)) + { + if (p_reason) + *p_reason = "file doest not exist"; + return false; + } + + m_stream.open(filename, std::ios::binary | std::ios::app | std::ios::in); + if (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit) + { + if (p_reason) + *p_reason = "file could not be opened"; + return false; + } + + if (p_corrupted) + *p_corrupted = false; + + size_t file_size = size_bytes(); + if (file_size % sizeof(pod_t) != 0) + { + // currupted + if (p_corrupted) + *p_corrupted = true; + + size_t corrected_size = file_size - file_size % sizeof(pod_t); + + // truncate to nearest item boundary + close(); + boost::filesystem::resize_file(filename, corrected_size); + m_stream.open(filename, std::ios::binary | std::ios::app | std::ios::in); + if ((m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit) || + size_bytes() != corrected_size) + { + if (p_reason) + *p_reason = "truncation failed"; + return false; + } + + if (p_reason) + *p_reason = std::string("file was corrupted, truncated: ") + epee::string_tools::num_to_string_fast(file_size) + " -> " + epee::string_tools::num_to_string_fast(corrected_size); + } + + return true; + } + + void close() + { + m_stream.close(); + } + + bool push_back(const pod_t& item) + { + if (!m_stream.is_open() || (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit)) + return false; + + m_stream.seekp(0, std::ios_base::end); + m_stream.write(reinterpret_cast(&item), sizeof item); + + if (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit) + return false; + + m_stream.flush(); + + return true; + } + + bool get_item(size_t index, pod_t& result) const + { + if (!m_stream.is_open() || (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit)) + return false; + + size_t offset = index * sizeof result; + m_stream.seekg(offset); + if (m_stream.rdstate() != std::ios::goodbit) + return false; + + m_stream.read(reinterpret_cast(&result), sizeof result); + + return m_stream.gcount() == sizeof result; + } + + size_t size_bytes() const + { + if (!m_stream.is_open() || (m_stream.rdstate() != std::ios::goodbit && m_stream.rdstate() != std::ios::eofbit)) + return 0; + + m_stream.seekg(0, std::ios_base::end); + return m_stream.tellg(); + } + + size_t size() const + { + return size_bytes() / sizeof(pod_t); + } + + private: + mutable boost::filesystem::fstream m_stream; + }; + +} // namespace tools diff --git a/src/common/pre_download.h b/src/common/pre_download.h new file mode 100644 index 0000000..6ea723b --- /dev/null +++ b/src/common/pre_download.h @@ -0,0 +1,226 @@ +// Copyright (c) 2020 Zano Project +// Copyright (c) 2012-2018 The Boolberry developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#pragma once + +#include +#include "net/http_client.h" +#include "db_backend_selector.h" +#include "crypto/crypto.h" +#include "currency_core/currency_core.h" + +namespace tools +{ + struct pre_download_entry + { + const char* url; + const char* hash; + uint64_t packed_size; + uint64_t unpacked_size; + }; + +#ifndef TESTNET + static constexpr pre_download_entry c_pre_download_mdbx = { "http://95.217.43.225/pre-download/zano_mdbx_95_1000000.pak", "6b0bbba85bc420eaae5ec68373e528f70bffaa17fb111c796e951d06ad71e4fe", 1104150892, 2147450880 }; + static constexpr pre_download_entry c_pre_download_lmdb = { "http://95.217.43.225/pre-download/zano_lmdb_95_1000000.pak", "b4d45c727dbf1b92671f9fd1a9624e79019e890bd3d33cb71e011ab4bcb0d21e", 1450748151, 2114449408 }; +#else + static constexpr pre_download_entry c_pre_download_mdbx = { "", "", 0, 0 }; + static constexpr pre_download_entry c_pre_download_lmdb = { "", "", 0, 0 }; +#endif + + static constexpr uint64_t pre_download_min_size_difference = 512 * 1024 * 1024; // minimum difference in size between local DB and the downloadable one to start downloading + + template + bool process_predownload(const boost::program_options::variables_map& vm, callback_t cb_should_stop) + { + tools::db::db_backend_selector dbbs; + bool r = dbbs.init(vm); + CHECK_AND_ASSERT_MES(r, false, "db_backend_selector failed to initialize"); + + std::string config_folder = dbbs.get_config_folder(); + std::string working_folder = dbbs.get_db_folder_path(); + std::string db_main_file_path = working_folder + "/" + dbbs.get_db_main_file_name(); + + pre_download_entry pre_download = dbbs.get_engine_type() == db::db_lmdb ? c_pre_download_lmdb : c_pre_download_mdbx; + + // override pre-download link if necessary + std::string url = pre_download.url; + if (command_line::has_arg(vm, command_line::arg_predownload_link)) + url = command_line::get_arg(vm, command_line::arg_predownload_link); + + boost::system::error_code ec; + uint64_t sz = boost::filesystem::file_size(db_main_file_path, ec); + if (pre_download.unpacked_size == 0 || !(ec || (pre_download.unpacked_size > sz && pre_download.unpacked_size - sz > pre_download_min_size_difference) || command_line::has_arg(vm, command_line::arg_force_predownload)) ) + { + LOG_PRINT_MAGENTA("Pre-downloading not needed (db file size = " << sz << ")", LOG_LEVEL_0); + return true; + } + + // okay, let's download + + + std::string downloading_file_path = db_main_file_path + ".download"; + + LOG_PRINT_MAGENTA("Trying to download blockchain database from " << url << " ...", LOG_LEVEL_0); + epee::net_utils::http::interruptible_http_client cl; + + crypto::stream_cn_hash hash_stream; + auto last_update = std::chrono::system_clock::now(); + + auto cb = [&hash_stream, &last_update, &cb_should_stop](const std::string& buff, uint64_t total_bytes, uint64_t received_bytes) + { + if (cb_should_stop(total_bytes, received_bytes)) + { + LOG_PRINT_MAGENTA(ENDL << "Interrupting download", LOG_LEVEL_0); + return false; + } + + hash_stream.update(buff.data(), buff.size()); + + auto dif = std::chrono::system_clock::now() - last_update; + if (dif >= std::chrono::milliseconds(300)) + { + boost::io::ios_flags_saver ifs(std::cout); + std::cout << "Received " << received_bytes / 1048576 << " of " << total_bytes / 1048576 << " MiB ( " << std::fixed << std::setprecision(1) << 100.0 * received_bytes / total_bytes << " %)\r"; + last_update = std::chrono::system_clock::now(); + } + + return true; + }; + + tools::create_directories_if_necessary(working_folder); + r = cl.download_and_unzip(cb, downloading_file_path, url, 5000 /* timout */, "GET", std::string(), 30 /* fails count */); + if (!r) + { + LOG_PRINT_RED("Download failed", LOG_LEVEL_0); + return false; + } + + crypto::hash data_hash = hash_stream.calculate_hash(); + if (epee::string_tools::pod_to_hex(data_hash) != pre_download.hash) + { + LOG_ERROR("hash missmatch in downloaded file, got: " << epee::string_tools::pod_to_hex(data_hash) << ", expected: " << pre_download.hash); + return false; + } + + LOG_PRINT_GREEN("Download succeeded, hash " << pre_download.hash << " is correct" , LOG_LEVEL_0); + + if (!command_line::has_arg(vm, command_line::arg_validate_predownload)) + { + boost::filesystem::remove(db_main_file_path, ec); + if (ec) + { + LOG_ERROR("Failed to remove " << db_main_file_path); + return false; + } + LOG_PRINT_L1("Removed " << db_main_file_path); + + boost::filesystem::rename(downloading_file_path, db_main_file_path, ec); + if (ec) + { + LOG_ERROR("Failed to rename " << downloading_file_path << " -> " << db_main_file_path); + return false; + } + LOG_PRINT_L1("Renamed " << downloading_file_path << " -> " << db_main_file_path); + + LOG_PRINT_GREEN("Blockchain successfully replaced with the new pre-downloaded data file", LOG_LEVEL_0); + return true; + } + + // + // paranoid mode + // move downloaded blockchain into a temporary folder + // + LOG_PRINT_MAGENTA(ENDL << "Downloaded blockchain database is about to be validated and added to the local database block-by-block" << ENDL, LOG_LEVEL_0); + std::string path_to_temp_datafolder = dbbs.get_temp_config_folder(); + std::string path_to_temp_blockchain = dbbs.get_temp_db_folder_path(); + std::string path_to_temp_blockchain_file = path_to_temp_blockchain + "/" + dbbs.get_db_main_file_name(); + + tools::create_directories_if_necessary(path_to_temp_blockchain); + boost::filesystem::rename(downloading_file_path, path_to_temp_blockchain_file, ec); + if (ec) + { + LOG_ERROR("Rename failed: " << downloading_file_path << " -> " << path_to_temp_blockchain_file); + return false; + } + + // remove old blockchain database from disk + boost::filesystem::remove_all(working_folder, ec); + if (ec) + { + LOG_ERROR("Failed to remove all from " << working_folder); + return false; + } + + std::string pool_db_path = dbbs.get_pool_db_folder_path(); + boost::filesystem::remove_all(pool_db_path, ec); + if (ec) + { + LOG_ERROR("Failed to remove all from " << pool_db_path); + return false; + } + + // source core + currency::core source_core(nullptr); + boost::program_options::variables_map source_core_vm; + source_core_vm.insert(std::make_pair("data-dir", boost::program_options::variable_value(path_to_temp_datafolder, false))); + source_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false))); + //source_core_vm.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false))); + + r = source_core.init(source_core_vm); + CHECK_AND_ASSERT_MES(r, false, "Failed to init source core"); + + // target core + currency::core target_core(nullptr); + boost::program_options::variables_map target_core_vm(vm); + target_core_vm.insert(std::make_pair("db-engine", boost::program_options::variable_value(dbbs.get_engine_name(), false))); + //vm_with_fast_sync.insert(std::make_pair("db-sync-mode", boost::program_options::variable_value(std::string("fast"), false))); + + r = target_core.init(target_core_vm); + CHECK_AND_ASSERT_MES(r, false, "Failed to init target core"); + + CHECK_AND_ASSERT_MES(target_core.get_top_block_height() == 0, false, "Target blockchain initialized not empty"); + uint64_t total_blocks = source_core.get_current_blockchain_size(); + + LOG_PRINT_GREEN("Manually processing blocks from 1 to " << total_blocks << "...", LOG_LEVEL_0); + + for (uint64_t i = 1; i != total_blocks; i++) + { + std::list blocks; + std::list txs; + bool r = source_core.get_blocks(i, 1, blocks, txs); + CHECK_AND_ASSERT_MES(r && blocks.size()==1, false, "Failed to get block " << i << " from core"); + currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + crypto::hash tx_hash = AUTO_VAL_INIT(tx_hash); + for (auto& tx : txs) + { + r = target_core.handle_incoming_tx(tx, tvc, true /* kept_by_block */); + CHECK_AND_ASSERT_MES(r && tvc.m_added_to_pool == true, false, "Failed to add a tx from block " << i << " from core"); + } + currency::block_verification_context bvc = AUTO_VAL_INIT(bvc); + r = target_core.handle_incoming_block(*blocks.begin(), bvc); + CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain == true, false, "Failed to add block " << i << " to core"); + if (!(i % 100)) + std::cout << "Block " << i << "(" << (i * 100) / total_blocks << "%) \r"; + + if (cb_should_stop(total_blocks, i)) + { + LOG_PRINT_MAGENTA(ENDL << "Interrupting updating db...", LOG_LEVEL_0); + return false; + } + } + + LOG_PRINT_GREEN("Processing finished, " << total_blocks << " successfully added.", LOG_LEVEL_0); + target_core.deinit(); + source_core.deinit(); + + boost::filesystem::remove_all(path_to_temp_datafolder, ec); + if (ec) + { + LOG_ERROR("Failed to remove all from " << path_to_temp_datafolder); + } + + return true; + } +} + diff --git a/src/common/util.cpp b/src/common/util.cpp index b219831..f5d4baa 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -10,6 +10,7 @@ using namespace epee; #include "util.h" #include "currency_core/currency_config.h" +#include "version.h" #ifdef WIN32 #include @@ -21,6 +22,9 @@ using namespace epee; #include #endif +#include + +#include "string_coding.h" namespace tools { @@ -361,7 +365,69 @@ namespace tools return pszOS; } } -#else + + void signal_handler::GenerateCrashDump(EXCEPTION_POINTERS *pep /* = NULL*/) + { + SYSTEMTIME sysTime = { 0 }; + GetSystemTime(&sysTime); + // get the computer name + char compName[MAX_COMPUTERNAME_LENGTH + 1] = { 0 }; + DWORD compNameLen = ARRAYSIZE(compName); + GetComputerNameA(compName, &compNameLen); + // build the filename: APPNAME_COMPUTERNAME_DATE_TIME.DMP + char path[MAX_PATH*10] = { 0 }; + std::string folder = epee::log_space::log_singletone::get_default_log_folder(); + sprintf_s(path, ARRAYSIZE(path),"%s\\crashdump_" PROJECT_VERSION_LONG "_%s_%04u-%02u-%02u_%02u-%02u-%02u.dmp", + folder.c_str(), compName, sysTime.wYear, sysTime.wMonth, sysTime.wDay, + sysTime.wHour, sysTime.wMinute, sysTime.wSecond); + + HANDLE hFile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, + 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + + if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE)) + { + // Create the minidump + MINIDUMP_EXCEPTION_INFORMATION mdei; + + mdei.ThreadId = GetCurrentThreadId(); + mdei.ExceptionPointers = pep; + mdei.ClientPointers = FALSE; + + MINIDUMP_CALLBACK_INFORMATION mci; + + mci.CallbackRoutine = (MINIDUMP_CALLBACK_ROUTINE)MyMiniDumpCallback; + mci.CallbackParam = 0; + + MINIDUMP_TYPE mdt = (MINIDUMP_TYPE)(MiniDumpWithPrivateReadWriteMemory | + MiniDumpWithDataSegs | + MiniDumpWithHandleData | + MiniDumpWithFullMemoryInfo | + MiniDumpWithThreadInfo | + MiniDumpWithUnloadedModules); + + BOOL rv = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), + hFile, mdt, (pep != 0) ? &mdei : 0, 0, &mci); + + if (!rv) + { + LOG_ERROR("Minidump file create FAILED(error " << GetLastError() << ") on path: " << path); + } + else + { + LOG_PRINT_L0("Minidump file created on path: " << path); + } + // Close the file + CloseHandle(hFile); + } + else + { + LOG_ERROR("Minidump FAILED to create file (error " << GetLastError() << ") on path: " << path); + } + } + + + +#else // ifdef WIN32 std::string get_nix_version_display_string() { utsname un; @@ -386,18 +452,22 @@ std::string get_nix_version_display_string() #ifdef WIN32 - std::string get_special_folder_path(int nfolder, bool iscreate) + std::wstring get_special_folder_path_w(int nfolder, bool iscreate) { - namespace fs = boost::filesystem; - char psz_path[MAX_PATH] = ""; + wchar_t psz_path[MAX_PATH] = L""; - if(SHGetSpecialFolderPathA(NULL, psz_path, nfolder, iscreate)) + if (SHGetSpecialFolderPathW(NULL, psz_path, nfolder, iscreate)) { return psz_path; } - LOG_ERROR("SHGetSpecialFolderPathA() failed, could not obtain requested path."); - return ""; + LOG_ERROR("SHGetSpecialFolderPathW(" << nfolder << ", " << iscreate << ") failed, could not obtain requested path."); + return L""; + } + + std::string get_special_folder_path_utf8(int nfolder, bool iscreate) + { + return epee::string_encoding::wstring_to_utf8(get_special_folder_path_w(nfolder, iscreate)); } #endif @@ -412,9 +482,9 @@ std::string get_nix_version_display_string() #ifdef WIN32 // Windows #ifdef _M_X64 - config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CURRENCY_NAME_SHORT; + config_folder = get_special_folder_path_utf8(CSIDL_APPDATA, true) + "/" + CURRENCY_NAME_SHORT; #else - config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CURRENCY_NAME_SHORT + "-x86"; + config_folder = get_special_folder_path_utf8(CSIDL_APPDATA, true) + "/" + CURRENCY_NAME_SHORT + "-x86"; #endif #else std::string pathRet; @@ -454,7 +524,7 @@ std::string get_nix_version_display_string() std::string wallets_dir; #ifdef WIN32 // Windows - wallets_dir = get_special_folder_path(CSIDL_PERSONAL, true) + "/" + CURRENCY_NAME_BASE; + wallets_dir = get_special_folder_path_utf8(CSIDL_PERSONAL, true) + "/" + CURRENCY_NAME_BASE; #else std::string pathRet; char* pszHome = getenv("HOME"); @@ -489,7 +559,7 @@ std::string get_nix_version_display_string() { namespace fs = boost::filesystem; boost::system::error_code ec; - fs::path fs_path(path); + fs::path fs_path = epee::string_encoding::utf8_to_wstring(path); if (fs::is_directory(fs_path, ec)) { return true; @@ -588,4 +658,97 @@ std::string get_nix_version_display_string() return static_cast(in.tellg()); } -} + bool check_remote_client_version(const std::string& client_ver) + { + std::string v = client_ver.substr(0, client_ver.find('[')); // remove commit id + v = v.substr(0, v.rfind('.')); // remove build number + + int v_major = 0, v_minor = 0, v_revision = 0; + + size_t dot_pos = v.find('.'); + if (dot_pos == std::string::npos || !epee::string_tools::string_to_num_fast(v.substr(0, dot_pos), v_major)) + return false; + + v = v.substr(dot_pos + 1); + dot_pos = v.find('.'); + if (!epee::string_tools::string_to_num_fast(v.substr(0, dot_pos), v_minor)) + return false; + + if (dot_pos != std::string::npos) + { + // revision + v = v.substr(dot_pos + 1); + if (!epee::string_tools::string_to_num_fast(v, v_revision)) + return false; + } + + // got v_major, v_minor, v_revision + + // allow 1.1.x and greater + + if (v_major < 1) + return false; + + if (v_major == 1 && v_minor < 1) + return false; + + return true; + } + + //this code was taken from https://stackoverflow.com/a/8594696/5566653 + //credits goes to @nijansen: https://stackoverflow.com/users/1056003/nijansen + bool copy_dir( boost::filesystem::path const & source, boost::filesystem::path const & destination) + { + namespace fs = boost::filesystem; + try + { + // Check whether the function call is valid + if (!fs::exists(source) ||!fs::is_directory(source)) + { + LOG_ERROR("Source directory " << source.string() << " does not exist or is not a directory."); + return false; + } + if (!fs::exists(destination)) + { + if (!fs::create_directory(destination)) + { + LOG_ERROR("Unable to create destination directory" << destination.string()); + return false; + } + } + // Create the destination directory + } + catch (fs::filesystem_error const & e) + { + LOG_ERROR("Exception: " << e.what()); + return false; + } + // Iterate through the source directory + for (fs::directory_iterator file(source); file != fs::directory_iterator(); ++file) + { + try + { + fs::path current(file->path()); + if (fs::is_directory(current)) + { + // Found directory: Recursion + if (!copy_dir(current, destination / current.filename())) + { + return false; + } + } + else + { + // Found file: Copy + fs::copy_file( current, destination / current.filename()); + } + } + catch (fs::filesystem_error const & e) + { + LOG_ERROR("Exception: " << e.what()); + } + } + return true; + } + +} // namespace tools diff --git a/src/common/util.h b/src/common/util.h index 11374f8..0c5b0bb 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -17,11 +17,18 @@ #include "crypto/hash.h" #include "misc_language.h" #include "p2p/p2p_protocol_defs.h" +#include "ntp.h" #if defined(WIN32) #include #endif +#ifdef NDEBUG + #define BUILD_TYPE "Release" +#else + #define BUILD_TYPE "Debug" +#endif + namespace tools { std::string get_host_computer_name(); @@ -29,6 +36,9 @@ namespace tools std::string get_default_user_dir(); std::string get_current_username(); std::string get_os_version_string(); + bool copy_dir(boost::filesystem::path const & source, boost::filesystem::path const & destination); + bool check_remote_client_version(const std::string& client_ver); + bool create_directories_if_necessary(const std::string& path); std::error_code replace_file(const std::string& replacement_name, const std::string& replaced_name); @@ -231,67 +241,7 @@ namespace tools } - static void GenerateCrashDump(EXCEPTION_POINTERS *pep = NULL) - - { - SYSTEMTIME sysTime = { 0 }; - GetSystemTime(&sysTime); - // get the computer name - char compName[MAX_COMPUTERNAME_LENGTH + 1] = { 0 }; - DWORD compNameLen = ARRAYSIZE(compName); - GetComputerNameA(compName, &compNameLen); - // build the filename: APPNAME_COMPUTERNAME_DATE_TIME.DMP - char path[MAX_PATH*10] = { 0 }; - std::string folder = epee::log_space::log_singletone::get_default_log_folder(); - sprintf_s(path, ARRAYSIZE(path),"%s\\crashdump_%s_%04u-%02u-%02u_%02u-%02u-%02u.dmp", - folder.c_str(), compName, sysTime.wYear, sysTime.wMonth, sysTime.wDay, - sysTime.wHour, sysTime.wMinute, sysTime.wSecond); - - HANDLE hFile = CreateFileA(path, GENERIC_READ | GENERIC_WRITE, - 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - - if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE)) - { - // Create the minidump - MINIDUMP_EXCEPTION_INFORMATION mdei; - - mdei.ThreadId = GetCurrentThreadId(); - mdei.ExceptionPointers = pep; - mdei.ClientPointers = FALSE; - - MINIDUMP_CALLBACK_INFORMATION mci; - - mci.CallbackRoutine = (MINIDUMP_CALLBACK_ROUTINE)MyMiniDumpCallback; - mci.CallbackParam = 0; - - MINIDUMP_TYPE mdt = (MINIDUMP_TYPE)(MiniDumpWithPrivateReadWriteMemory | - MiniDumpWithDataSegs | - MiniDumpWithHandleData | - MiniDumpWithFullMemoryInfo | - MiniDumpWithThreadInfo | - MiniDumpWithUnloadedModules); - - BOOL rv = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), - hFile, mdt, (pep != 0) ? &mdei : 0, 0, &mci); - - if (!rv) - { - LOG_ERROR("Minidump file create FAILED(error " << GetLastError() << ") on path: " << path); - } - else - { - LOG_PRINT_L0("Minidump file created on path: " << path); - } - // Close the file - CloseHandle(hFile); - } - else - { - LOG_ERROR("Minidump FAILED to create file (error " << GetLastError() << ") on path: " << path); - } - } - - + static void GenerateCrashDump(EXCEPTION_POINTERS *pep = NULL); static LONG WINAPI win_unhandled_exception_handler(_In_ struct _EXCEPTION_POINTERS *ep) { @@ -316,14 +266,14 @@ namespace tools static void handle_signal() { - static std::mutex m_mutex; + static epee::static_helpers::wrapper m_mutex; std::unique_lock lock(m_mutex); m_handler(); } static void handle_fatal_signal(int sig_number, void* address) { - static std::mutex m_mutex_fatal; + static epee::static_helpers::wrapper m_mutex_fatal; std::unique_lock lock(m_mutex_fatal); m_fatal_handler(sig_number, address); uninstall_fatal(); @@ -333,4 +283,5 @@ namespace tools static std::function m_handler; static std::function m_fatal_handler; }; + } diff --git a/src/contrib/db/CMakeLists.txt b/src/contrib/db/CMakeLists.txt index eb5c841..6f2213b 100644 --- a/src/contrib/db/CMakeLists.txt +++ b/src/contrib/db/CMakeLists.txt @@ -1,7 +1,20 @@ -add_subdirectory(liblmdb) -if(MSVC) - target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172) -else() - # Warnings as used by LMDB itself (LMDB_0.9.23) - target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized) + + +if(CMAKE_SYSTEM_NAME STREQUAL "iOS" OR CMAKE_SYSTEM_NAME STREQUAL "Android") + message("excluded db support for IOS build") + return() endif() + + + message("DB ENGINE: lmdb") + add_subdirectory(liblmdb) + if(MSVC) + target_compile_options(lmdb PRIVATE /wd4996 /wd4503 /wd4345 /wd4267 /wd4244 /wd4146 /wd4333 /wd4172) + else() + # Warnings as used by LMDB itself (LMDB_0.9.23) + target_compile_options(lmdb PRIVATE -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized) + endif() + if(NOT DISABLE_MDBX) + message("DB ENGINE: mdbx") + add_subdirectory(libmdbx) + endif() diff --git a/src/contrib/db/liblmdb/CMakeLists.txt b/src/contrib/db/liblmdb/CMakeLists.txt index 4a60e07..dc49182 100644 --- a/src/contrib/db/liblmdb/CMakeLists.txt +++ b/src/contrib/db/liblmdb/CMakeLists.txt @@ -4,7 +4,7 @@ set (lmdb_sources mdb.c midl.c) include_directories("${CMAKE_CURRENT_SOURCE_DIR}") if(NOT MSVC) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-missing-braces -Wno-aggregate-return") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-missing-braces -Wno-aggregate-return -Wno-discarded-qualifiers -Wno-unused-but-set-variable -Wno-implicit-fallthrough -Wno-maybe-uninitialized ") endif() if(FREEBSD) add_definitions(-DMDB_DSYNC=O_SYNC) diff --git a/src/contrib/db/liblmdb/mdb.c b/src/contrib/db/liblmdb/mdb.c index ca9f3b1..17762e8 100644 --- a/src/contrib/db/liblmdb/mdb.c +++ b/src/contrib/db/liblmdb/mdb.c @@ -132,8 +132,10 @@ extern int cacheflush(char *addr, int nbytes, int cache); #ifdef _WIN32 typedef int64_t off64_t; -#else +#elif __APPLE__ typedef off_t off64_t; +#else + #endif diff --git a/src/contrib/db/libmdbx/.circleci/config.yml b/src/contrib/db/libmdbx/.circleci/config.yml new file mode 100644 index 0000000..91e11a4 --- /dev/null +++ b/src/contrib/db/libmdbx/.circleci/config.yml @@ -0,0 +1,20 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/buildpack-deps:artful + environment: + - TESTDB: /tmp/test.db + - TESTLOG: /tmp/test.log + steps: + - checkout + - run: make all + - run: ulimit -c unlimited && make check + - run: + command: | + mkdir -p /tmp/artifacts + mv -t /tmp/artifacts $TESTLOG $TESTDB core.* + when: on_fail + - store_artifacts: + path: /tmp/artifacts + destination: test-artifacts diff --git a/src/contrib/db/libmdbx/.clang-format b/src/contrib/db/libmdbx/.clang-format new file mode 100644 index 0000000..6c59ef3 --- /dev/null +++ b/src/contrib/db/libmdbx/.clang-format @@ -0,0 +1,3 @@ +BasedOnStyle: LLVM +Standard: Cpp11 +ReflowComments: true diff --git a/src/contrib/db/libmdbx/.gitignore b/src/contrib/db/libmdbx/.gitignore new file mode 100644 index 0000000..0496b62 --- /dev/null +++ b/src/contrib/db/libmdbx/.gitignore @@ -0,0 +1,35 @@ +*.[ao] +*.bak +*.exe +*.gcda +*.gcno +*.gcov +*.lo +*.orig +*.rej +*.so +*[~#] +.idea +.le.ini +.vs/ +cmake-build-* +@* +core +mdbx_example +libmdbx.creator.user +mdbx_chk +mdbx_copy +mdbx_dump +mdbx_load +mdbx_stat +mdbx_test +test.log +test/tmp.db +test/tmp.db-lck +tmp.db +tmp.db-lck +valgrind.* +src/elements/version.c +src/elements/config.h +dist/ +*.tar* diff --git a/src/contrib/db/libmdbx/.travis.yml b/src/contrib/db/libmdbx/.travis.yml new file mode 100644 index 0000000..7b468ad --- /dev/null +++ b/src/contrib/db/libmdbx/.travis.yml @@ -0,0 +1,61 @@ +language: c cpp + +matrix: + include: + - os: linux + dist: precise + env: CC=cc CXX=c++ + - os: linux + dist: trusty + compiler: clang + env: CC=clang CXX=clang++ + - os: linux + dist: xenial + compiler: gcc + env: CC=gcc CXX=g++ + - os: linux + dist: bionic + compiler: clang + env: CC=clang CXX=clang++ + - os: osx + osx_image: xcode11 + env: CC=cc CXX=c++ + - os: osx + osx_image: xcode9.4 + env: CC=cc CXX=c++ + +script: > + if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then + git fetch --unshallow --tags --prune && + git submodule foreach --recursive git fetch --unshallow --tags --prune && + (if which clang-format-6.0 > /dev/null && make reformat && [[ -n $(git diff) ]]; + then + echo "You must run 'make reformat' before submitting a pull request"; + echo ""; + git diff; + exit -1; + fi) && + make --keep-going all && MALLOC_CHECK_=7 MALLOC_PERTURB_=42 make --keep-going check + else + [ ! -s cov-int/scm_log.txt ] || cat cov-int/scm_log.txt; + fi && sleep 3 + +env: + global: + - secure: "M+W+heGGyRQJoBq2W0uqWVrpL4KBXmL0MFL7FSs7f9vmAaDyEgziUXeZRj3GOKzW4kTef3LpIeiu9SmvqSMoQivGGiomZShqPVl045o/OUgRCAT7Al1RLzEZ0efSHpIPf0PZ6byEf6GR2ML76OfuL6JxTVdnz8iVyO2sgLE1HbX1VeB+wgd/jfMeOBhCCXskfK6MLyZihfMYsiYZYSaV98ZDhDLSlzuuRIgzb0bMi8aL6AErs0WLW0NelRBeHkKPYfAUc85pdQHscgrJw6Rh/zT6+8BQ/q5f4IgWhiu4xoRg3Ngl7SNoedRQh93ADM3UG2iGl6HDFpVORaXcFWKAtuYY+kHQ0HB84BRYpQmeBuXNpltsfxQ3d1Q3u0RlE45zRvmr2+X1mFnkcNUAWISLPbsOUlriDQM8irGwRpho77/uYnRC00bJsHW//s6+uPf9zrAw1nI4f0y3PAWukGF/xs6HAI3FZPsuSSnx18Tj3Opgbc9Spop+V3hkhdiJoPGpNKTkFX4ZRXfkPgoRVJmtp4PpbpH0Ps/mCriKjMEfGGi0HcVCi0pEGLXiecdqJ5KPg5+22zNycEujQBJcNTKd9shN+R3glrbmhAxTEzGdGwxXXJ2ybwJ2PWJLMYZ7g98nLyX+uQPaA3BlsbYJHNeS5283/9pJsd9DzfHKsN2nFSc=" + +before_install: + - echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- + - ${CC} --version + - ${CXX} --version + +addons: + coverity_scan: + project: + name: "ReOpen/libmdbx" + version: 0.1 + description: "Build submitted via Travis CI" + notification_email: leo@yuriev.ru + build_command_prepend: "git fetch --unshallow --tags --prune && make dist" + build_command: "make MDBX_OPTIONS=-DMDBX_DEBUG=2 -C dist all" + branch_pattern: coverity_scan diff --git a/src/contrib/db/libmdbx/AUTHORS b/src/contrib/db/libmdbx/AUTHORS new file mode 100644 index 0000000..10910e5 --- /dev/null +++ b/src/contrib/db/libmdbx/AUTHORS @@ -0,0 +1,32 @@ +Contributors +============ + +Alexey Naumov +Chris Mikkelson +Claude Brisson +David Barbour +David Wilson +dreamsxin +Hallvard Furuseth , +Heiko Becker +Howard Chu , +Ignacio Casal Quinteiro +James Rouzier +Jean-Christophe DUBOIS +John Hewson +Klaus Malorny +Kurt Zeilenga +Leonid Yuriev , +Lorenz Bauer +Luke Yeager +Martin Hedenfalk +Ondrej Kuznik +Orivej Desh +Oskari Timperi +Pavel Medvedev +Philipp Storz +Quanah Gibson-Mount +Salvador Ortiz +Sebastien Launay +Vladimir Romanov +Zano Foundation diff --git a/src/contrib/db/libmdbx/CMakeLists.dist-minimal b/src/contrib/db/libmdbx/CMakeLists.dist-minimal new file mode 100644 index 0000000..1f138af --- /dev/null +++ b/src/contrib/db/libmdbx/CMakeLists.dist-minimal @@ -0,0 +1,192 @@ +## +## This is the minimal template for CMakeList.txt which could be used +## to build libmdbx from the "amalgamated form" of libmdbx's source code. +## +## The amalgamated form is intended to embedding libmdbx in other projects +## in cases when using as git-submodule is not acceptable or inconveniently. +## +## The amalgamated form could be generated from full git repository +## on Linux just by `make dist`. +## + +## +## Copyright 2019 Leonid Yuriev +## and other libmdbx authors: please see AUTHORS file. +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted only as authorized by the OpenLDAP +## Public License. +## +## A copy of this license is available in the file LICENSE in the +## top-level directory of the distribution or, alternatively, at +## . +## + +## +## libmdbx = { Revised and extended descendant of Symas LMDB. } +## Please see README.md at https://github.com/leo-yuriev/libmdbx +## +## Libmdbx is superior to LMDB in terms of features and reliability, +## not inferior in performance. libmdbx works on Linux, FreeBSD, MacOS X +## and other systems compliant with POSIX.1-2008, but also support Windows +## as a complementary platform. +## +## The next version is under active non-public development and will be +## released as MithrilDB and libmithrildb for libraries & packages. +## Admittedly mythical Mithril is resembling silver but being stronger and +## lighter than steel. Therefore MithrilDB is rightly relevant name. +## +## MithrilDB will be radically different from libmdbx by the new database +## format and API based on C++17, as well as the Apache 2.0 License. +## The goal of this revolution is to provide a clearer and robust API, +## add more features and new valuable properties of database. +## +## The Future will (be) Positive. Ð’ÑÑ‘ будет хорошо. +## + +cmake_minimum_required(VERSION 3.8.2) +cmake_policy(PUSH) +cmake_policy(VERSION 3.8.2) +if(NOT CMAKE_VERSION VERSION_LESS 3.9) + cmake_policy(SET CMP0069 NEW) +endif() +if(NOT CMAKE_VERSION VERSION_LESS 3.12) + cmake_policy(SET CMP0075 NEW) +endif() +if(NOT CMAKE_VERSION VERSION_LESS 3.13) + cmake_policy(SET CMP0077 NEW) +endif() + +if(DEFINED PROJECT_NAME) + set(SUBPROJECT ON) + set(NOT_SUBPROJECT OFF) +else() + set(SUBPROJECT OFF) + set(NOT_SUBPROJECT ON) + project(libmdbx C CXX) +endif() + +find_package(Threads REQUIRED) + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) +endif() + +list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11) +if(NOT HAS_C11 LESS 0) + set(MDBX_C_STANDARD 11) +else() + set(MDBX_C_STANDARD 99) +endif() +message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx") + +# not supported by this (minimal) script +add_definitions(-DMDBX_AVOID_CRT=0) + +# provide build timestamp +string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC) +add_definitions(-DMDBX_BUILD_TIMESTAMP="${MDBX_BUILD_TIMESTAMP}") + +# provide compiler info +execute_process(COMMAND sh -c "${CMAKE_C_COMPILER} --version | head -1" + OUTPUT_VARIABLE MDBX_BUILD_COMPILER + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + RESULT_VARIABLE rc) +if(rc OR NOT MDBX_BUILD_COMPILER) + string(STRIP "${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION}" MDBX_BUILD_COMPILER) +endif() +add_definitions(-DMDBX_BUILD_COMPILER="${MDBX_BUILD_COMPILER}") + +# provide cpu/arch-system pair +if(CMAKE_C_COMPILER_TARGET) + set(MDBX_BUILD_TARGET "${CMAKE_C_COMPILER_TARGET}") +elseif(CMAKE_C_PLATFORM_ID AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME) + string(STRIP "${CMAKE_C_PLATFORM_ID}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +elseif(CMAKE_LIBRARY_ARCHITECTURE) + string(STRIP "${CMAKE_LIBRARY_ARCHITECTURE}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +elseif(CMAKE_GENERATOR_PLATFORM AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME) + string(STRIP "${CMAKE_GENERATOR_PLATFORM}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +elseif(CMAKE_SYSTEM_ARCH) + string(STRIP "${CMAKE_SYSTEM_ARCH}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +else() + string(STRIP "${CMAKE_SYSTEM_PROCESSOR}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +endif() +add_definitions(-DMDBX_BUILD_TARGET="${MDBX_BUILD_TARGET}") + +# provide build target-config +if(CMAKE_CONFIGURATION_TYPES) + add_definitions(-DMDBX_BUILD_CONFIG="$") +else() + add_definitions(-DMDBX_BUILD_CONFIG="${CMAKE_BUILD_TYPE}") +endif() + +# provide build cflags +set(MDBX_BUILD_FLAGS "") +list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS}) +list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES}) +if(CMAKE_CONFIGURATION_TYPES) + add_definitions(-DMDBX_BUILD_FLAGS_CONFIG="$<$:${CMAKE_C_FLAGS_DEBUG} ${CMAKE_C_DEFINES_DEBUG}>$<$:${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_DEFINES_RELEASE}>$<$:${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_DEFINES_RELWITHDEBINFO}>$<$:${CMAKE_C_FLAGS_MINSIZEREL} ${CMAKE_C_DEFINES_MINSIZEREL}>") +else() + string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPERCASE) + list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}}) + list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES_${CMAKE_BUILD_TYPE_UPPERCASE}}) +endif() +list(REMOVE_DUPLICATES MDBX_BUILD_FLAGS) +string(REPLACE ";" " " MDBX_BUILD_FLAGS "${MDBX_BUILD_FLAGS}") +add_definitions(-DMDBX_BUILD_FLAGS="${MDBX_BUILD_FLAGS}") + +# shared library +if(NOT DEFINED MDBX_BUILD_SHARED_LIBRARY) + if(DEFINED BUILD_SHARED_LIBS) + option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS}) + else() + option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ON) + endif() +endif() +if(MDBX_BUILD_SHARED_LIBRARY) + add_library(mdbx SHARED mdbx.c mdbx.h) + set_target_properties(mdbx PROPERTIES + C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON + PUBLIC_HEADER mdbx.h) + target_compile_definitions(mdbx PRIVATE LIBMDBX_EXPORTS INTERFACE LIBMDBX_IMPORTS) + if(DEFINED INTERPROCEDURAL_OPTIMIZATION) + set_target_properties(mdbx PROPERTIES + INTERPROCEDURAL_OPTIMIZATION $) + endif() + target_link_libraries(mdbx PRIVATE ${CMAKE_THREAD_LIBS_INIT}) + if(WIN32) + target_link_libraries(mdbx PRIVATE ntdll.lib) + endif() +endif() + +# static library used for tools, to avoid rpath/dll-path troubles +add_library(mdbx-static STATIC EXCLUDE_FROM_ALL mdbx.c mdbx.h) +set_target_properties(mdbx-static PROPERTIES + C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON + PUBLIC_HEADER mdbx.h) +target_link_libraries(mdbx-static INTERFACE ${CMAKE_THREAD_LIBS_INIT}) +if(DEFINED INTERPROCEDURAL_OPTIMIZATION) + set_target_properties(mdbx-static PROPERTIES + INTERPROCEDURAL_OPTIMIZATION $) +endif() +if(WIN32) + target_link_libraries(mdbx-static INTERFACE ntdll.lib) +endif() + +# mdbx-tools +foreach(TOOL mdbx_chk mdbx_copy mdbx_stat mdbx_dump mdbx_load) + add_executable(${TOOL} ${TOOL}.c) + set_target_properties(${TOOL} PROPERTIES + C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON) + if(DEFINED INTERPROCEDURAL_OPTIMIZATION) + set_target_properties(${TOOL} PROPERTIES + INTERPROCEDURAL_OPTIMIZATION $) + endif() + target_link_libraries(${TOOL} mdbx-static) +endforeach() + +cmake_policy(POP) diff --git a/src/contrib/db/libmdbx/CMakeLists.txt b/src/contrib/db/libmdbx/CMakeLists.txt new file mode 100644 index 0000000..75e9b3b --- /dev/null +++ b/src/contrib/db/libmdbx/CMakeLists.txt @@ -0,0 +1,342 @@ +## +## Copyright 2019 Leonid Yuriev +## and other libmdbx authors: please see AUTHORS file. +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted only as authorized by the OpenLDAP +## Public License. +## +## A copy of this license is available in the file LICENSE in the +## top-level directory of the distribution or, alternatively, at +## . +## + +## +## libmdbx = { Revised and extended descendant of Symas LMDB. } +## Please see README.md at https://github.com/leo-yuriev/libmdbx +## +## Libmdbx is superior to LMDB in terms of features and reliability, +## not inferior in performance. libmdbx works on Linux, FreeBSD, MacOS X +## and other systems compliant with POSIX.1-2008, but also support Windows +## as a complementary platform. +## +## The next version is under active non-public development and will be +## released as MithrilDB and libmithrildb for libraries & packages. +## Admittedly mythical Mithril is resembling silver but being stronger and +## lighter than steel. Therefore MithrilDB is rightly relevant name. +## +## MithrilDB will be radically different from libmdbx by the new database +## format and API based on C++17, as well as the Apache 2.0 License. +## The goal of this revolution is to provide a clearer and robust API, +## add more features and new valuable properties of database. +## +## The Future will (be) Positive. Ð’ÑÑ‘ будет хорошо. +## + +cmake_minimum_required(VERSION 3.8.2) +cmake_policy(PUSH) +cmake_policy(VERSION 3.8.2) +if(NOT CMAKE_VERSION VERSION_LESS 3.13) + cmake_policy(SET CMP0077 NEW) +endif() +if(NOT CMAKE_VERSION VERSION_LESS 3.12) + cmake_policy(SET CMP0075 NEW) +endif() +if(NOT CMAKE_VERSION VERSION_LESS 3.9) + cmake_policy(SET CMP0069 NEW) + include(CheckIPOSupported) + check_ipo_supported(RESULT CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE) +else() + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE FALSE) +endif() + +if(DEFINED PROJECT_NAME) + set(SUBPROJECT ON) + set(NOT_SUBPROJECT OFF) + if(NOT DEFINED BUILD_TESTING) + set(BUILD_TESTING OFF) + endif() +else() + set(SUBPROJECT OFF) + set(NOT_SUBPROJECT ON) + project(libmdbx C CXX) + if(NOT DEFINED BUILD_TESTING) + set(BUILD_TESTING ON) + endif() +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE MinSizeRel CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) +endif() + +macro(add_mdbx_option NAME DESCRIPTION DEFAULT) + list(APPEND MDBX_BUILD_OPTIONS ${NAME}) + if(NOT ${DEFAULT} STREQUAL "AUTO") + option(${NAME} "${DESCRIPTION}" ${DEFAULT}) + endif() +endmacro() + +# only for compatibility testing +# set(CMAKE_CXX_STANDARD 14) + +if(NOT "$ENV{TEAMCITY_PROCESS_FLOW_ID}" STREQUAL "") + set(CI TEAMCITY) + message(STATUS "TeamCity CI") +elseif(NOT "$ENV{TRAVIS}" STREQUAL "") + set(CI TRAVIS) + message(STATUS "Travis CI") +elseif(NOT "$ENV{CIRCLECI}" STREQUAL "") + set(CI CIRCLE) + message(STATUS "Circle CI") +elseif(NOT "$ENV{APPVEYOR}" STREQUAL "") + set(CI APPVEYOR) + message(STATUS "AppVeyor CI") +elseif(NOT "$ENV{CI}" STREQUAL "") + set(CI "$ENV{CI}") + message(STATUS "Other CI (${CI})") +else() + message(STATUS "Assume No any CI environment") + unset(CI) +endif() + +# output all mdbx-related targets in single directory +if(NOT DEFINED MDBX_OUTPUT_DIR) + set(MDBX_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) +endif() +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR}) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR}) +set(CMAKE_PDB_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR}) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${MDBX_OUTPUT_DIR}) + +include(CheckLibraryExists) +include(CheckIncludeFiles) +include(CheckCCompilerFlag) +include(CheckSymbolExists) +include(CheckCSourceRuns) +include(CheckCXXSourceRuns) +include(CheckCSourceCompiles) +include(CheckCXXSourceCompiles) +include(TestBigEndian) +include(CheckFunctionExists) +include(FindPackageMessage) +include(CheckStructHasMember) +include(CMakeDependentOption) +include(GNUInstallDirs) + +if(CMAKE_C_COMPILER_ID STREQUAL "MSVC" AND MSVC_VERSION LESS 1900) + message(SEND_ERROR "MSVC compiler ${MSVC_VERSION} is too old for building MDBX." + " At least 'Microsoft Visual Studio 2015' is required.") +endif() + +# Set default build type to Release. This is to ease a User's life. +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) +endif() +string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPERCASE) + +include(cmake/utils.cmake) +include(cmake/compiler.cmake) +include(cmake/profile.cmake) + +find_program(ECHO echo) +find_program(CAT cat) +find_program(GIT git) +find_program(LD ld) + +# CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H) +# CHECK_INCLUDE_FILES(sys/uio.h HAVE_SYS_UIO_H) +# CHECK_INCLUDE_FILES(sys/stat.h HAVE_SYS_STAT_H) + +CHECK_FUNCTION_EXISTS(pow NOT_NEED_LIBM) +if(NOT_NEED_LIBM) + set(LIB_MATH "") +else() + set(CMAKE_REQUIRED_LIBRARIES m) + CHECK_FUNCTION_EXISTS(pow HAVE_LIBM) + if(HAVE_LIBM) + set(LIB_MATH m) + else() + message(FATAL_ERROR "No libm found for math support") + endif() +endif() + +find_package(Threads REQUIRED) + +if(SUBPROJECT) + if(NOT DEFINED BUILD_SHARED_LIBS) + option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" OFF) + endif() + if(NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE) + option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independed (PIC)" ON) + endif() +else() + option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" ON) + option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independed (PIC)" ON) + if (CC_HAS_ARCH_NATIVE) + option(BUILD_FOR_NATIVE_CPU "Generate code for the compiling machine CPU" OFF) + endif() + + if(CMAKE_CONFIGURATION_TYPES OR NOT CMAKE_BUILD_TYPE_UPPERCASE STREQUAL "DEBUG") + set(INTERPROCEDURAL_OPTIMIZATION_DEFAULT ON) + else() + set(INTERPROCEDURAL_OPTIMIZATION_DEFAULT OFF) + endif() + + if(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE + OR GCC_LTO_AVAILABLE OR MSVC_LTO_AVAILABLE OR CLANG_LTO_AVAILABLE) + option(INTERPROCEDURAL_OPTIMIZATION "Enable interprocedural/LTO optimization" ${INTERPROCEDURAL_OPTIMIZATION_DEFAULT}) + endif() + + if(INTERPROCEDURAL_OPTIMIZATION) + if(GCC_LTO_AVAILABLE) + set(LTO_ENABLED TRUE) + set(CMAKE_AR ${CMAKE_GCC_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE) + set(CMAKE_NM ${CMAKE_GCC_NM} CACHE PATH "Path to nm program with LTO-plugin" FORCE) + set(CMAKE_RANLIB ${CMAKE_GCC_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) + message(STATUS "MDBX indulge Link-Time Optimization by GCC") + elseif(CLANG_LTO_AVAILABLE) + set(LTO_ENABLED TRUE) + set(CMAKE_AR ${CMAKE_CLANG_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE) + set(CMAKE_NM ${CMAKE_CLANG_NM} CACHE PATH "Path to nm program with LTO-plugin" FORCE) + set(CMAKE_RANLIB ${CMAKE_CLANG_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) + message(STATUS "MDBX indulge Link-Time Optimization by CLANG") + elseif(MSVC_LTO_AVAILABLE) + set(LTO_ENABLED TRUE) + message(STATUS "MDBX indulge Link-Time Optimization by MSVC") + elseif(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE) + message(STATUS "MDBX indulge Interprocedural Optimization by CMake") + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) + set(LTO_ENABLED TRUE) + else() + message(WARNING "Unable to engage interprocedural/LTO optimization.") + endif() + else() + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION FALSE) + set(LTO_ENABLED FALSE) + endif() + + find_program(VALGRIND valgrind) + if(VALGRIND) + # LY: cmake is ugly and nasty. + # - therefore memcheck-options should be defined before including ctest; + # - otherwise ctest may ignore it. + set(MEMORYCHECK_SUPPRESSIONS_FILE + "${PROJECT_SOURCE_DIR}/test/valgrind_suppress.txt" + CACHE FILEPATH "Suppressions file for Valgrind" FORCE) + set(MEMORYCHECK_COMMAND_OPTIONS + "--trace-children=yes --leak-check=full --track-origins=yes --error-exitcode=42 --error-markers=@ --errors-for-leak-kinds=definite --fair-sched=yes --suppressions=${MEMORYCHECK_SUPPRESSIONS_FILE}" + CACHE STRING "Valgrind options" FORCE) + set(VALGRIND_COMMAND_OPTIONS "${MEMORYCHECK_COMMAND_OPTIONS}" CACHE STRING "Valgrind options" FORCE) + endif() + + # + # Enable 'make tags' target. + find_program(CTAGS ctags) + if(CTAGS) + add_custom_target(tags COMMAND ${CTAGS} -R -f tags + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + add_custom_target(ctags DEPENDS tags) + endif(CTAGS) + + # + # Enable 'make reformat' target. + find_program(CLANG_FORMAT + NAMES clang-format-6.0 clang-format-5.0 clang-format-4.0 + clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format) + if(CLANG_FORMAT AND UNIX) + add_custom_target(reformat + VERBATIM + COMMAND + git ls-files | + grep -E \\.\(c|cxx|cc|cpp|h|hxx|hpp\)\(\\.in\)?\$ | + xargs ${CLANG_FORMAT} -i --style=file + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + endif() + + if(NOT "${PROJECT_BINARY_DIR}" STREQUAL "${PROJECT_SOURCE_DIR}") + add_custom_target(distclean) + add_custom_command(TARGET distclean + COMMAND ${CMAKE_COMMAND} -E remove_directory "${PROJECT_BINARY_DIR}" + COMMENT "Removing the build directory and its content") + elseif(IS_DIRECTORY .git AND GIT) + add_custom_target(distclean) + add_custom_command(TARGET distclean + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${GIT} submodule foreach --recursive git clean -f -X -d + COMMAND ${GIT} clean -f -X -d + COMMENT "Removing all build files from the source directory") + endif() + + setup_compile_flags() +endif(SUBPROJECT) + +list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11) +if(NOT HAS_C11 LESS 0) + set(MDBX_C_STANDARD 11) +else() + set(MDBX_C_STANDARD 99) +endif() +message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx") + +############################################################################## +############################################################################## +# +# #### ##### ##### # #### # # #### +# # # # # # # # # ## # # +# # # # # # # # # # # # #### +# # # ##### # # # # # # # # +# # # # # # # # # ## # # +# #### # # # #### # # #### +# + +set(MDBX_BUILD_OPTIONS ENABLE_ASAN MDBX_USE_VALGRIND ENABLE_GPROF ENABLE_GCOV) +add_mdbx_option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS}) +add_mdbx_option(MDBX_ALLOY_BUILD "Build MDBX library as single object file" ON) +add_mdbx_option(MDBX_TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON) +add_mdbx_option(MDBX_TXN_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO) +mark_as_advanced(MDBX_TXN_CHECKPID) +if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + add_mdbx_option(MDBX_DISABLE_GNU_SOURCE "Don't use nonstandard GNU/Linux extension functions" OFF) + mark_as_advanced(MDBX_DISABLE_GNU_SOURCE) +endif() +if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + add_mdbx_option(MDBX_OSX_SPEED_INSTEADOF_DURABILITY "Disable use fcntl(F_FULLFSYNC) in favor of speed" OFF) + mark_as_advanced(MDBX_OSX_SPEED_INSTEADOF_DURABILITY) +endif() +if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + add_mdbx_option(MDBX_AVOID_CRT "Avoid dependence from MSVC CRT" ${NOT_SUBPROJECT}) + if(NOT MDBX_BUILD_SHARED_LIBRARY) + add_mdbx_option(MDBX_CONFIG_MANUAL_TLS_CALLBACK + "Provide mdbx_dll_handler() for manual initialization" OFF) + mark_as_advanced(MDBX_CONFIG_MANUAL_TLS_CALLBACK) + endif() +else() + add_mdbx_option(MDBX_USE_ROBUST "Use POSIX.1-2008 robust mutexes" AUTO) + mark_as_advanced(MDBX_USE_ROBUST) + add_mdbx_option(MDBX_USE_OFDLOCKS "Use Open file description locks (aka OFD locks, non-POSIX)" AUTO) + mark_as_advanced(MDBX_USE_OFDLOCKS) +endif() +option(MDBX_ENABLE_TESTS "Build MDBX tests." ${BUILD_TESTING}) + +################################################################################ +################################################################################ + +add_subdirectory(src) +if(MDBX_ENABLE_TESTS) + add_subdirectory(test) +endif() + +set(PACKAGE "libmdbx") +set(CPACK_PACKAGE_VERSION_MAJOR ${MDBX_VERSION_MAJOR}) +set(CPACK_PACKAGE_VERSION_MINOR ${MDBX_VERSION_MINOR}) +set(CPACK_PACKAGE_VERSION_PATCH ${MDBX_VERSION_RELEASE}) +set(CPACK_PACKAGE_VERSION_COMMIT ${MDBX_VERSION_REVISION}) +set(PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${CPACK_PACKAGE_VERSION_COMMIT}") +message(STATUS "libmdbx package version is ${PACKAGE_VERSION}") + +cmake_policy(POP) diff --git a/src/contrib/db/libmdbx/COPYRIGHT b/src/contrib/db/libmdbx/COPYRIGHT new file mode 100644 index 0000000..46e0961 --- /dev/null +++ b/src/contrib/db/libmdbx/COPYRIGHT @@ -0,0 +1,22 @@ +Copyright 2015-2019 Leonid Yuriev . +Copyright 2011-2015 Howard Chu, Symas Corp. +Copyright 2015,2016 Peter-Service R&D LLC. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted only as authorized by the OpenLDAP +Public License. + +A copy of this license is available in the file LICENSE in the +top-level directory of the distribution or, alternatively, at +. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Individual files and/or contributed packages may be copyright by +other parties and/or subject to additional restrictions. + +This work also contains materials derived from public sources. + +Additional information about OpenLDAP can be obtained at +. diff --git a/src/contrib/db/libmdbx/GNUmakefile b/src/contrib/db/libmdbx/GNUmakefile new file mode 100644 index 0000000..f6d0edf --- /dev/null +++ b/src/contrib/db/libmdbx/GNUmakefile @@ -0,0 +1,362 @@ +# This makefile is for GNU Make, and nowadays provided +# just for compatibility and preservation of traditions. +# Please use CMake in case of any difficulties or problems. +# +# Preprocessor macros (for MDBX_OPTIONS) of interest... +# Note that the defaults should already be correct for most platforms; +# you should not need to change any of these. Read their descriptions +# in README and source code if you do. There may be other macros of interest. +SHELL := /bin/bash + +# install sandbox +SANDBOX ?= + +# install prefixes (inside sandbox) +prefix ?= /usr/local +mandir ?= $(prefix)/man + +# lib/bin suffix for multiarch/biarch, e.g. '.x86_64' +suffix ?= + +CC ?= gcc +LD ?= ld +MDBX_OPTIONS ?= -DNDEBUG=1 +CFLAGS ?= -Os -g3 -Wall -Werror -Wextra -Wpedantic -ffunction-sections -fPIC -fvisibility=hidden -std=gnu11 -pthread -Wno-tautological-compare + +# LY: '--no-as-needed,-lrt' for ability to built with modern glibc, but then run with the old +LDFLAGS ?= $(shell $(LD) --help 2>/dev/null | grep -q -- --gc-sections && echo '-Wl,--gc-sections,-z,relro,-O1')$(shell $(LD) --help 2>/dev/null | grep -q -- -dead_strip && echo '-Wl,-dead_strip') +EXE_LDFLAGS ?= -pthread + +################################################################################ + +UNAME := $(shell uname -s 2>/dev/null || echo Unknown) +define uname2sosuffix + case "$(UNAME)" in + Darwin*|Mach*) echo dylib;; + CYGWIN*|MINGW*|MSYS*|Windows*) echo dll;; + *) echo so;; + esac +endef +SO_SUFFIX := $(shell $(uname2sosuffix)) + +HEADERS := mdbx.h +LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX) +TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk +MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 mdbx_chk.1 + +.PHONY: mdbx all install clean test dist check + +all: $(LIBRARIES) $(TOOLS) + +mdbx: libmdbx.a libmdbx.$(SO_SUFFIX) + +tools: $(TOOLS) + +strip: all + strip libmdbx.$(SO_SUFFIX) $(TOOLS) + +clean: + rm -rf $(TOOLS) mdbx_test @* *.[ao] *.[ls]o *~ tmp.db/* \ + *.gcov *.log *.err src/*.o test/*.o mdbx_example dist \ + config.h src/elements/config.h src/elements/version.c *.tar* + +libmdbx.a: mdbx-static.o + $(AR) rs $@ $? + +libmdbx.$(SO_SUFFIX): mdbx-dylib.o + $(CC) $(CFLAGS) $^ -pthread -shared $(LDFLAGS) -o $@ + +#> dist-cutoff-begin +ifeq ($(wildcard mdbx.c),mdbx.c) +#< dist-cutoff-end + +################################################################################ +# Amalgamated source code, i.e. distributed after `make dists` +MAN_SRCDIR := man1/ + +config.h: mdbx.c $(lastword $(MAKEFILE_LIST)) + (echo '#define MDBX_BUILD_TIMESTAMP "$(shell date +%Y-%m-%dT%H:%M:%S%z)"' \ + && echo '#define MDBX_BUILD_FLAGS "$(CFLAGS) $(LDFLAGS)"' \ + && echo '#define MDBX_BUILD_COMPILER "$(shell set -o pipefail; $(CC) --version | head -1 || echo 'Please use GCC or CLANG compatible compiler')"' \ + && echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || echo 'Please use GCC or CLANG compatible compiler')"' \ + ) > $@ + +mdbx-dylib.o: config.h mdbx.c $(lastword $(MAKEFILE_LIST)) + $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c mdbx.c -o $@ + +mdbx-static.o: config.h mdbx.c $(lastword $(MAKEFILE_LIST)) + $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c mdbx.c -o $@ + +mdbx_%: mdbx_%.c libmdbx.a + $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) -o $@ + +#> dist-cutoff-begin +else +################################################################################ +# Plain (non-amalgamated) sources with test + +define uname2osal + case "$(UNAME)" in + CYGWIN*|MINGW*|MSYS*|Windows*) echo windows;; + *) echo unix;; + esac +endef + +define uname2titer + case "$(UNAME)" in + Darwin*|Mach*) echo 2;; + *) echo 12;; + esac +endef + +DIST_EXTRA := LICENSE README.md CMakeLists.txt GNUmakefile $(addprefix man1/, $(MANPAGES)) +DIST_SRC := mdbx.h mdbx.c $(addsuffix .c, $(TOOLS)) + +TEST_DB ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.db +TEST_LOG ?= $(shell [ -d /dev/shm ] && echo /dev/shm || echo /tmp)/mdbx-test.log +TEST_OSAL := $(shell $(uname2osal)) +TEST_ITER := $(shell $(uname2titer)) +TEST_SRC := test/osal-$(TEST_OSAL).cc $(filter-out $(wildcard test/osal-*.cc), $(wildcard test/*.cc)) +TEST_INC := $(wildcard test/*.h) +TEST_OBJ := $(patsubst %.cc,%.o,$(TEST_SRC)) +CXX ?= g++ +CXXSTD ?= $(shell $(CXX) -std=c++27 -c test/test.cc -o /dev/null 2>/dev/null && echo -std=c++17 || echo -std=c++11) +CXXFLAGS := $(CXXSTD) $(filter-out -std=gnu11,$(CFLAGS)) + +MAN_SRCDIR := src/man1/ +ALLOY_DEPS := $(wildcard src/elements/*) +MDBX_VERSION_GIT = ${shell set -o pipefail; git describe --tags | sed -n 's|^v*\([0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}\)\(.*\)|\1|p' || echo 'Please fetch tags and/or install latest git version'} +MDBX_GIT_TIMESTAMP = $(shell git show --no-patch --format=%cI HEAD || echo 'Please install latest get version') +MDBX_GIT_DESCRIBE = $(shell git describe --tags --long --dirty=-dirty || echo 'Please fetch tags and/or install latest git version') +MDBX_VERSION_SUFFIX = $(shell set -o pipefail; echo -n '$(MDBX_GIT_DESCRIBE)' | tr -c -s '[a-zA-Z0-9]' _) +MDBX_BUILD_SOURCERY = $(shell set -o pipefail; $(MAKE) -s src/elements/version.c && (openssl dgst -r -sha256 src/elements/version.c || sha256sum src/elements/version.c || shasum -a 256 src/elements/version.c) 2>/dev/null | cut -d ' ' -f 1 || echo 'Please install openssl or sha256sum or shasum')_$(MDBX_VERSION_SUFFIX) + +test check: all mdbx_example mdbx_test + rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \ + (./mdbx_test --progress --console=no --repeat=$(TEST_ITER) --pathname=$(TEST_DB) --dont-cleanup-after basic && \ + ./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=1 --pathname=$(TEST_DB) --dont-cleanup-after basic) \ + | tee -a $(TEST_LOG) | tail -n 42) \ + && ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy + +mdbx_example: mdbx.h example/example-mdbx.c libmdbx.$(SO_SUFFIX) + $(CC) $(CFLAGS) -I. example/example-mdbx.c ./libmdbx.$(SO_SUFFIX) -o $@ + +check-singleprocess: all mdbx_test + rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \ + (./mdbx_test --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after --hill && \ + ./mdbx_test --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \ + ./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=42 --pathname=$(TEST_DB) --dont-cleanup-after --nested) \ + | tee -a $(TEST_LOG) | tail -n 42) \ + && ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy + +check-fault: all mdbx_test + rm -f $(TEST_DB) $(TEST_LOG) && (set -o pipefail; ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --inject-writefault=42 --dump-config --dont-cleanup-after basic | tee -a $(TEST_LOG) | tail -n 42) \ + ; ./mdbx_chk -vvnw $(TEST_DB) && ([ ! -e $(TEST_DB)-copy ] || ./mdbx_chk -vvn $(TEST_DB)-copy) + +VALGRIND=valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt +memcheck check-valgrind: all mdbx_test + @echo "$(MDBX_OPTIONS)" | grep -q MDBX_USE_VALGRIND || echo "WARNING: Please build libmdbx with -DMDBX_USE_VALGRIND to avoid false-positives from Valgrind !!!" >&2 + rm -f valgrind-*.log $(TEST_DB) $(TEST_LOG) && (set -o pipefail; \ + ($(VALGRIND) ./mdbx_test --mode=-writemap,-lifo --progress --console=no --repeat=4 --pathname=$(TEST_DB) --dont-cleanup-after basic && \ + $(VALGRIND) ./mdbx_test --progress --console=no --pathname=$(TEST_DB) --dont-cleanup-before --dont-cleanup-after --copy && \ + $(VALGRIND) ./mdbx_test --progress --console=no --repeat=2 --pathname=$(TEST_DB) --dont-cleanup-after basic) \ + | tee -a $(TEST_LOG) | tail -n 42) \ + && $(VALGRIND) ./mdbx_chk -vvn $(TEST_DB) && ./mdbx_chk -vvn $(TEST_DB)-copy + +define test-rule +$(patsubst %.cc,%.o,$(1)): $(1) $(TEST_INC) mdbx.h $(lastword $(MAKEFILE_LIST)) + $(CXX) $(CXXFLAGS) $(MDBX_OPTIONS) -c $(1) -o $$@ + +endef +$(foreach file,$(TEST_SRC),$(eval $(call test-rule,$(file)))) + +mdbx_%: src/tools/mdbx_%.c libmdbx.a + $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' $^ $(EXE_LDFLAGS) -o $@ + +mdbx_test: $(TEST_OBJ) libmdbx.$(SO_SUFFIX) + $(CXX) $(CXXFLAGS) $(TEST_OBJ) -Wl,-rpath . -L . -l mdbx $(EXE_LDFLAGS) -o $@ + +git_DIR := $(shell if [ -d .git ]; then echo .git; elif [ -s .git -a -f .git ]; then grep '^gitdir: ' .git | cut -d ':' -f 2; else echo "Please use libmdbx as a git-submodule or the amalgamated source code" >&2 && echo git_directory; fi) + +src/elements/version.c: src/elements/version.c.in $(lastword $(MAKEFILE_LIST)) $(git_DIR)/HEAD $(git_DIR)/index $(git_DIR)/refs/tags + sed \ + -e "s|@MDBX_GIT_TIMESTAMP@|$(MDBX_GIT_TIMESTAMP)|" \ + -e "s|@MDBX_GIT_TREE@|$(shell git show --no-patch --format=%T HEAD || echo 'Please install latest get version')|" \ + -e "s|@MDBX_GIT_COMMIT@|$(shell git show --no-patch --format=%H HEAD || echo 'Please install latest get version')|" \ + -e "s|@MDBX_GIT_DESCRIBE@|$(MDBX_GIT_DESCRIBE)|" \ + -e "s|\$${MDBX_VERSION_MAJOR}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 1)|" \ + -e "s|\$${MDBX_VERSION_MINOR}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 2)|" \ + -e "s|\$${MDBX_VERSION_RELEASE}|$(shell echo '$(MDBX_VERSION_GIT)' | cut -d . -f 3)|" \ + -e "s|\$${MDBX_VERSION_REVISION}|$(shell git rev-list --count --no-merges HEAD || echo 'Please fetch tags and/or install latest git version')|" \ + src/elements/version.c.in > $@ + +src/elements/config.h: src/elements/version.c $(lastword $(MAKEFILE_LIST)) + (echo '#define MDBX_BUILD_TIMESTAMP "$(shell date +%Y-%m-%dT%H:%M:%S%z)"' \ + && echo '#define MDBX_BUILD_FLAGS "$(CFLAGS) $(LDFLAGS)"' \ + && echo '#define MDBX_BUILD_COMPILER "$(shell set -o pipefail; $(CC) --version | head -1 || echo 'Please use GCC or CLANG compatible compiler')"' \ + && echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || echo 'Please use GCC or CLANG compatible compiler')"' \ + && echo '#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)' \ + ) > $@ + +mdbx-dylib.o: src/elements/config.h src/elements/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST)) + $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c src/alloy.c -o $@ + +mdbx-static.o: src/elements/config.h src/elements/version.c src/alloy.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST)) + $(CC) $(CFLAGS) $(MDBX_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c src/alloy.c -o $@ + +.PHONY: dist +dist: libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz $(lastword $(MAKEFILE_LIST)) + +libmdbx-sources-$(MDBX_VERSION_SUFFIX).tar.gz: $(addprefix dist/, $(DIST_SRC) $(DIST_EXTRA)) $(addprefix dist/man1/,$(MANPAGES)) + tar -c -a -f $@ --owner=0 --group=0 -C dist $(DIST_SRC) $(DIST_EXTRA) \ + && rm dist/@tmp-shared_internals.inc + +dist/mdbx.h: mdbx.h src/elements/version.c $(lastword $(MAKEFILE_LIST)) + mkdir -p dist && cp $< $@ + +dist/GNUmakefile: GNUmakefile + mkdir -p dist && sed -e '/^#> dist-cutoff-begin/,/^#< dist-cutoff-end/d' $< > $@ + +dist/@tmp-shared_internals.inc: src/elements/version.c $(ALLOY_DEPS) $(lastword $(MAKEFILE_LIST)) + mkdir -p dist && sed \ + -e 's|#pragma once|#define MDBX_ALLOY 1\n#define MDBX_BUILD_SOURCERY $(MDBX_BUILD_SOURCERY)|' \ + -e 's|#include "../../mdbx.h"|@INCLUDE "mdbx.h"|' \ + -e '/#include "defs.h"/r src/elements/defs.h' \ + -e '/#include "osal.h"/r src/elements/osal.h' \ + src/elements/internals.h > $@ + +dist/mdbx.c: dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST)) + mkdir -p dist && (cat dist/@tmp-shared_internals.inc \ + && cat src/elements/core.c src/elements/osal.c src/elements/version.c \ + && echo '#if defined(_WIN32) || defined(_WIN64)' \ + && cat src/elements/lck-windows.c && echo '#else /* LCK-implementation */' \ + && cat src/elements/lck-posix.c && echo '#endif /* LCK-implementation */' \ + ) | grep -v -e '#include "' -e '#pragma once' | sed 's|@INCLUDE|#include|' > $@ + +define dist-tool-rule +dist/$(1).c: src/tools/$(1).c src/tools/wingetopt.h src/tools/wingetopt.c \ + dist/@tmp-shared_internals.inc $(lastword $(MAKEFILE_LIST)) + mkdir -p dist && sed \ + -e '/#include "..\/elements\/internals.h"/r dist/@tmp-shared_internals.inc' \ + -e '/#include "wingetopt.h"/r src/tools/wingetopt.c' \ + src/tools/$(1).c \ + | grep -v -e '#include "' -e '#pragma once' -e '#define MDBX_ALLOY' \ + | sed 's|@INCLUDE|#include|' > $$@ + +endef +$(foreach file,$(TOOLS),$(eval $(call dist-tool-rule,$(file)))) + +dist/man1/mdbx_%.1: src/man1/mdbx_%.1 + mkdir -p dist/man1/ && cp $< $@ +dist/LICENSE: LICENSE + mkdir -p dist/man1/ && cp $< $@ +dist/README.md: README.md + mkdir -p dist/man1/ && cp $< $@ +dist/CMakeLists.txt: CMakeLists.dist-minimal + mkdir -p dist/man1/ && cp $< $@ +endif + +################################################################################ +# Cross-compilation simple test + +CROSS_LIST = mips-linux-gnu-gcc \ + powerpc64-linux-gnu-gcc powerpc-linux-gnu-gcc \ + arm-linux-gnueabihf-gcc aarch64-linux-gnu-gcc \ + sh4-linux-gnu-gcc mips64-linux-gnuabi64-gcc + +# hppa-linux-gnu-gcc - don't supported by current qemu release +# s390x-linux-gnu-gcc - qemu troubles (hang/abort) +# sparc64-linux-gnu-gcc - qemu troubles (fcntl for F_SETLK/F_GETLK) +# alpha-linux-gnu-gcc - qemu (or gcc) troubles (coredump) + +CROSS_LIST_NOQEMU = hppa-linux-gnu-gcc s390x-linux-gnu-gcc \ + sparc64-linux-gnu-gcc alpha-linux-gnu-gcc + +cross-gcc: + @echo "CORRESPONDING CROSS-COMPILERs ARE REQUIRED." + @echo "FOR INSTANCE: apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu" + @for CC in $(CROSS_LIST_NOQEMU) $(CROSS_LIST); do \ + echo "===================== $$CC"; \ + $(MAKE) clean && CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static $(MAKE) all || exit $$?; \ + done + +# Unfortunately qemu don't provide robust support for futexes. +# Therefore it is impossible to run full multi-process tests. +cross-qemu: + @echo "CORRESPONDING CROSS-COMPILERs AND QEMUs ARE REQUIRED." + @echo "FOR INSTANCE: " + @echo " 1) apt install g++-aarch64-linux-gnu g++-alpha-linux-gnu g++-arm-linux-gnueabihf g++-hppa-linux-gnu g++-mips-linux-gnu g++-mips64-linux-gnuabi64 g++-powerpc-linux-gnu g++-powerpc64-linux-gnu g++-s390x-linux-gnu g++-sh4-linux-gnu g++-sparc64-linux-gnu" + @echo " 2) apt install binfmt-support qemu-user-static qemu-user qemu-system-arm qemu-system-mips qemu-system-misc qemu-system-ppc qemu-system-sparc" + @for CC in $(CROSS_LIST); do \ + echo "===================== $$CC + qemu"; \ + $(MAKE) clean && \ + CC=$$CC CXX=$$(echo $$CC | sed 's/-gcc/-g++/') EXE_LDFLAGS=-static MDBX_OPTIONS="-DMDBX_SAFE4QEMU $(MDBX_OPTIONS)" \ + $(MAKE) check-singleprocess || exit $$?; \ + done + +#< dist-cutoff-end +install: $(LIBRARIES) $(TOOLS) $(HEADERS) + mkdir -p $(SANDBOX)$(prefix)/bin$(suffix) \ + && cp -t $(SANDBOX)$(prefix)/bin$(suffix) $(TOOLS) && \ + mkdir -p $(SANDBOX)$(prefix)/lib$(suffix) \ + && cp -t $(SANDBOX)$(prefix)/lib$(suffix) $(LIBRARIES) && \ + mkdir -p $(SANDBOX)$(prefix)/include \ + && cp -t $(SANDBOX)$(prefix)/include $(HEADERS) && \ + mkdir -p $(SANDBOX)$(mandir)/man1 \ + && cp -t $(SANDBOX)$(mandir)/man1 $(addprefix $(MAN_SRCDIR), $(MANPAGES)) + +################################################################################ +# Benchmarking by ioarena + +IOARENA ?= $(shell \ + (test -x ../ioarena/@BUILD/src/ioarena && echo ../ioarena/@BUILD/src/ioarena) || \ + (test -x ../../@BUILD/src/ioarena && echo ../../@BUILD/src/ioarena) || \ + (test -x ../../src/ioarena && echo ../../src/ioarena) || which ioarena) +NN ?= 25000000 + +ifneq ($(wildcard $(IOARENA)),) + +.PHONY: bench clean-bench re-bench + +clean-bench: + rm -rf bench-*.txt _ioarena/* + +re-bench: clean-bench bench + +define bench-rule +bench-$(1)_$(2).txt: $(3) $(IOARENA) $(lastword $(MAKEFILE_LIST)) + LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \ + $(IOARENA) -D $(1) -B crud -m nosync -n $(2) \ + | tee $$@ | grep throughput && \ + LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}" \ + $(IOARENA) -D $(1) -B get,iterate -m sync -r 4 -n $(2) \ + | tee -a $$@ | grep throughput \ + || mv -f $$@ $$@.error + +endef + +$(eval $(call bench-rule,mdbx,$(NN),libmdbx.$(SO_SUFFIX))) + +$(eval $(call bench-rule,sophia,$(NN))) +$(eval $(call bench-rule,leveldb,$(NN))) +$(eval $(call bench-rule,rocksdb,$(NN))) +$(eval $(call bench-rule,wiredtiger,$(NN))) +$(eval $(call bench-rule,forestdb,$(NN))) +$(eval $(call bench-rule,lmdb,$(NN))) +$(eval $(call bench-rule,nessdb,$(NN))) +$(eval $(call bench-rule,sqlite3,$(NN))) +$(eval $(call bench-rule,ejdb,$(NN))) +$(eval $(call bench-rule,vedisdb,$(NN))) +$(eval $(call bench-rule,dummy,$(NN))) + +$(eval $(call bench-rule,debug,10)) + +bench: bench-mdbx_$(NN).txt + +.PHONY: bench-debug + +bench-debug: bench-debug_10.txt + +bench-quartet: bench-mdbx_$(NN).txt bench-lmdb_$(NN).txt bench-rocksdb_$(NN).txt bench-wiredtiger_$(NN).txt + +endif diff --git a/src/contrib/db/libmdbx/LICENSE b/src/contrib/db/libmdbx/LICENSE new file mode 100644 index 0000000..05ad757 --- /dev/null +++ b/src/contrib/db/libmdbx/LICENSE @@ -0,0 +1,47 @@ +The OpenLDAP Public License + Version 2.8, 17 August 2003 + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided +that the following conditions are met: + +1. Redistributions in source form must retain copyright statements + and notices, + +2. Redistributions in binary form must reproduce applicable copyright + statements and notices, this list of conditions, and the following + disclaimer in the documentation and/or other materials provided + with the distribution, and + +3. Redistributions must contain a verbatim copy of this document. + +The OpenLDAP Foundation may revise this license from time to time. +Each revision is distinguished by a version number. You may use +this Software under terms of this license revision or under the +terms of any subsequent revision of the license. + +THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS +CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) +OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +The names of the authors and copyright holders must not be used in +advertising or otherwise to promote the sale, use or other dealing +in this Software without specific, written prior permission. Title +to copyright in this Software shall at all times remain with copyright +holders. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, +California, USA. All Rights Reserved. Permission to copy and +distribute verbatim copies of this document is granted. diff --git a/src/contrib/db/libmdbx/README.md b/src/contrib/db/libmdbx/README.md new file mode 100644 index 0000000..1e82f3f --- /dev/null +++ b/src/contrib/db/libmdbx/README.md @@ -0,0 +1,591 @@ +### The [repository now only mirrored on the Github](https://abf.io/erthink/libmdbx) due to illegal discriminatory restrictions for Russian Crimea and for sovereign crimeans. + +----- + +libmdbx +====================================== + +_libmdbx_ is an extremely fast, compact, powerful, embedded +transactional [key-value +store](https://en.wikipedia.org/wiki/Key-value_database) +database, with permissive [OpenLDAP Public License](LICENSE). +_libmdbx_ has a specific set of properties and capabilities, +focused on creating unique lightweight solutions with +extraordinary performance. + +The next version is under active non-public development and will be +released as **_MithrilDB_** and `libmithrildb` for libraries & packages. +Admittedly mythical [Mithril](https://en.wikipedia.org/wiki/Mithril) is +resembling silver but being stronger and lighter than steel. Therefore +_MithrilDB_ is rightly relevant name. +> _MithrilDB_ will be radically different from _libmdbx_ by the new +> database format and API based on C++17, as well as the [Apache 2.0 +> License](https://www.apache.org/licenses/LICENSE-2.0). The goal of this +> revolution is to provide a clearer and robust API, add more features and +> new valuable properties of database. + +*The Future will (be) [Positive](https://www.ptsecurity.com). Ð’ÑÑ‘ будет хорошо.* + +[![Build Status](https://travis-ci.org/leo-yuriev/libmdbx.svg?branch=master)](https://travis-ci.org/leo-yuriev/libmdbx) +[![Build status](https://ci.appveyor.com/api/projects/status/ue94mlopn50dqiqg/branch/master?svg=true)](https://ci.appveyor.com/project/leo-yuriev/libmdbx/branch/master) +[![Coverity Scan Status](https://scan.coverity.com/projects/12915/badge.svg)](https://scan.coverity.com/projects/reopen-libmdbx) + +## Table of Contents +- [Overview](#overview) + - [Comparison with other databases](#comparison-with-other-databases) + - [History & Acknowledgments](#history) +- [Description](#description) + - [Key features](#key-features) + - [Improvements over LMDB](#improvements-over-lmdb) + - [Gotchas](#gotchas) +- [Usage](#usage) + - [Building](#building) + - [Bindings](#bindings) +- [Performance comparison](#performance-comparison) + - [Integral performance](#integral-performance) + - [Read scalability](#read-scalability) + - [Sync-write mode](#sync-write-mode) + - [Lazy-write mode](#lazy-write-mode) + - [Async-write mode](#async-write-mode) + - [Cost comparison](#cost-comparison) + +----- + +## Overview + +_libmdbx_ is revised and extended descendant of amazing [Lightning +Memory-Mapped +Database](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database). +_libmdbx_ inherits all features and characteristics from +[LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database), +but resolves some issues and adds several features. + + - _libmdbx_ guarantee data integrity after crash unless this was explicitly +neglected in favour of write performance. + + - _libmdbx_ allows multiple processes to read and update several key-value +tables concurrently, while being +[ACID](https://en.wikipedia.org/wiki/ACID)-compliant, with minimal +overhead and Olog(N) operation cost. + + - _libmdbx_ enforce +[serializability](https://en.wikipedia.org/wiki/Serializability) for +writers by single +[mutex](https://en.wikipedia.org/wiki/Mutual_exclusion) and affords +[wait-free](https://en.wikipedia.org/wiki/Non-blocking_algorithm#Wait-freedom) +for parallel readers without atomic/interlocked operations, while +writing and reading transactions do not block each other. + + - _libmdbx_ uses [B+Trees](https://en.wikipedia.org/wiki/B%2B_tree) and +[Memory-Mapping](https://en.wikipedia.org/wiki/Memory-mapped_file), +doesn't use [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) +which might be a caveat for some workloads. + + - _libmdbx_ implements a simplified variant of the [Berkeley +DB](https://en.wikipedia.org/wiki/Berkeley_DB) and/or +[dbm](https://en.wikipedia.org/wiki/DBM_(computing)) API. + + - _libmdbx_ supports Linux, Windows, MacOS, FreeBSD and other systems +compliant with POSIX.1-2008. + +### Comparison with other databases +For now please refer to [chapter of "BoltDB comparison with other +databases"](https://github.com/coreos/bbolt#comparison-with-other-databases) +which is also (mostly) applicable to _libmdbx_. + +### History +At first the development was carried out within the +[ReOpenLDAP](https://github.com/leo-yuriev/ReOpenLDAP) project. About a +year later _libmdbx_ was separated into standalone project, which was +[presented at Highload++ 2015 +conference](http://www.highload.ru/2015/abstracts/1831.html). + +Since 2017 _libmdbx_ is used in [Fast Positive Tables](https://github.com/leo-yuriev/libfpta), +and development is funded by [Positive Technologies](https://www.ptsecurity.com). + +### Acknowledgments +Howard Chu is the author of LMDB, from which +originated the MDBX in 2015. + +Martin Hedenfalk is the author of `btree.c` code, which +was used for begin development of LMDB. + +----- + +Description +=========== + +## Key features + +1. Key-value pairs are stored in ordered map(s), keys are always sorted, +range lookups are supported. + +2. Data is [memory-mapped](https://en.wikipedia.org/wiki/Memory-mapped_file) +into each worker DB process, and could be accessed zero-copy from transactions. + +3. Transactions are +[ACID](https://en.wikipedia.org/wiki/ACID)-compliant, through to +[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) +and [CoW](https://en.wikipedia.org/wiki/Copy-on-write). Writes are +strongly serialized and aren't blocked by reads, transactions can't +conflict with each other. Reads are guaranteed to get only commited data +([relaxing serializability](https://en.wikipedia.org/wiki/Serializability#Relaxing_serializability)). + +4. Read transactions are +[non-blocking](https://en.wikipedia.org/wiki/Non-blocking_algorithm), +don't use [atomic operations](https://en.wikipedia.org/wiki/Linearizability#High-level_atomic_operations). +Readers don't block each other and aren't blocked by writers. Read +performance scales linearly with CPU core count. + > Nonetheless, "connect to DB" (starting the first read transaction in a thread) and + > "disconnect from DB" (closing DB or thread termination) requires a lock + > acquisition to register/unregister at the "readers table". + +5. Keys with multiple values are stored efficiently without key +duplication, sorted by value, including integers (valuable for +secondary indexes). + +6. Efficient operation on short fixed length keys, +including 32/64-bit integer types. + +7. [WAF](https://en.wikipedia.org/wiki/Write_amplification) (Write +Amplification Factor) и RAF (Read Amplification Factor) are Olog(N). + +8. No [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) and +transaction journal. In case of a crash no recovery needed. No need for +regular maintenance. Backups can be made on the fly on working DB +without freezing writers. + +9. No additional memory management, all done by basic OS services. + + +## Improvements over LMDB + +_libmdbx_ is superior to _legendary [LMDB](https://symas.com/lmdb/)_ in +terms of features and reliability, not inferior in performance. In +comparison to LMDB, _libmdbx_ make things "just work" perfectly and +out-of-the-box, not silently and catastrophically break down. The list +below is pruned down to the improvements most notable and obvious from +the user's point of view. + +1. Automatic on-the-fly database size control by preset parameters, both +reduction and increment. + > _libmdbx_ manage the database size according to parameters specified + > by `mdbx_env_set_geometry()` function, + > ones include the growth step and the truncation threshold. + +2. Automatic continuous zero-overhead database compactification. + > _libmdbx_ logically move as possible a freed pages + > at end of allocation area into unallocated space, + > and then release such space if a lot of. + +3. LIFO policy for recycling a Garbage Collection items. On systems with a disk +write-back cache, this can significantly increase write performance, up to +several times in a best case scenario. + > LIFO means that for reuse pages will be taken which became unused the lastest. + > Therefore the loop of database pages circulation becomes as short as possible. + > In other words, the number of pages, that are overwritten in memory + > and on disk during a series of write transactions, will be as small as possible. + > Thus creates ideal conditions for the efficient operation of the disk write-back cache. + +4. Fast estimation of range query result volume, i.e. how many items can +be found between a `KEY1` and a `KEY2`. This is prerequisite for build +and/or optimize query execution plans. + > _libmdbx_ performs a rough estimate based only on b-tree pages that + > are common for the both stacks of cursors that were set to corresponing + > keys. + +5. `mdbx_chk` tool for database integrity check. + +6. Guarantee of database integrity even in asynchronous unordered write-to-disk mode. + > _libmdbx_ propose additional trade-off by implementing append-like manner for updates + > in `NOSYNC` and `MAPASYNC` modes, that avoid database corruption after a system crash + > contrary to LMDB. Nevertheless, the `MDBX_UTTERLY_NOSYNC` mode available to match LMDB behaviour, + > and for a special use-cases. + +7. Automated steady flush to disk upon volume of changes and/or by +timeout via cheap polling. + +8. Sequence generation and three cheap persistent 64-bit markers with ACID. + +9. Support for keys and values of zero length, including multi-values +(aka sorted duplicates). + +10. The handler of lack-of-space condition with a callback, +that allow you to control and resolve such situations. + +11. Support for opening a database in the exclusive mode, including on a network share. + +12. Extended transaction info, including dirty and leftover space info +for a write transaction, reading lag and hold over space for read +transactions. + +13. Extended whole-database info (aka environment) and reader enumeration. + +14. Extended update or delete, _at once_ with getting previous value +and addressing the particular item from multi-value with the same key. + +15. Support for explicitly updating the existing record, not insertion a new one. + +16. All cursors are uniformly, can be reused and should be closed explicitly, +regardless ones were opened within write or read transaction. + +17. Correct update of current record with `MDBX_CURRENT` flag when size +of key or data was changed, including sorted duplicated. + +18. Opening database handles is spared from race conditions and +pre-opening is not needed. + +19. Ability to determine whether the particular data is on a dirty page +or not, that allows to avoid copy-out before updates. + +20. Ability to determine whether the cursor is pointed to a key-value +pair, to the first, to the last, or not set to anything. + +21. Returning `MDBX_EMULTIVAL` error in case of ambiguous update or delete. + +22. On **MacOS** the `fcntl(F_FULLFSYNC)` syscall is used _by +default_ to synchronize data with the disk, as this is [the only way to +guarantee data +durability](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html) +in case of power failure. Unfortunately, in scenarios with high write +intensity, the use of `F_FULLFSYNC` significant degrades performance +compared to LMDB, where the `fsync()` syscall is used. Therefore, +_libmdbx_ allows you to override this behavior by defining the +`MDBX_OSX_SPEED_INSTEADOF_DURABILITY=1` option while build the library. + +23. On **Windows** the `LockFileEx()` syscall is used for locking, since +it allows place the database on network drives, and provides protection +against incompetent user actions (aka +[poka-yoke](https://en.wikipedia.org/wiki/Poka-yoke)). Therefore +_libmdbx_ may be a little lag in performance tests from LMDB where a +named mutexes are used. + + +## Gotchas + +1. There cannot be more than one writer at a time. + > On the other hand, this allows serialize an updates and eliminate any + > possibility of conflicts, deadlocks or logical errors. + +2. No [WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) means +relatively big [WAF](https://en.wikipedia.org/wiki/Write_amplification) +(Write Amplification Factor). Because of this syncing data to disk might +be quite resource intensive and be main performance bottleneck during +intensive write workload. + > As compromise _libmdbx_ allows several modes of lazy and/or periodic + > syncing, including `MAPASYNC` mode, which modificate data in memory and + > asynchronously syncs data to disk, moment to sync is picked by OS. + > + > Although this should be used with care, synchronous transactions in a DB + > with transaction journal will require 2 IOPS minimum (probably 3-4 in + > practice) because of filesystem overhead, overhead depends on + > filesystem, not on record count or record size. In _libmdbx_ IOPS count + > will grow logarithmically depending on record count in DB (height of B+ + > tree) and will require at least 2 IOPS per transaction too. + +3. [CoW](https://en.wikipedia.org/wiki/Copy-on-write) for +[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) +is done on memory page level with +[B+trees](https://ru.wikipedia.org/wiki/B-%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE). +Therefore altering data requires to copy about Olog(N) memory pages, +which uses [memory bandwidth](https://en.wikipedia.org/wiki/Memory_bandwidth) and is main +performance bottleneck in `MDBX_MAPASYNC` mode. + > This is unavoidable, but isn't that bad. Syncing data to disk requires + > much more similar operations which will be done by OS, therefore this is + > noticeable only if data sync to persistent storage is fully disabled. + > _libmdbx_ allows to safely save data to persistent storage with minimal + > performance overhead. If there is no need to save data to persistent + > storage then it's much more preferable to use `std::map`. + +4. Massive altering of data during a parallel long read operation will +increase the process work set, may exhaust entire free database space and +result in subsequent write performance degradation. + > _libmdbx_ mostly solve this issue by lack-of-space callback and `MDBX_LIFORECLAIM` mode. + > See [`mdbx.h`](mdbx.h) with API description for details. + > The "next" version of libmdbx (MithrilDB) will completely solve this. + +5. There are no built-in checksums or digests to verify database integrity. + > The "next" version of _libmdbx_ (MithrilDB) will solve this issue employing [Merkle Tree](https://en.wikipedia.org/wiki/Merkle_tree). + +-------------------------------------------------------------------------------- + +Usage +===== + +## Source code embedding + +_libmdbx_ provides two official ways for integration in source code form: + +1. Using the amalgamated source code. + > The amalgamated source code includes all files requires to build and + > use _libmdbx_, but not for testing _libmdbx_ itself. + +2. Adding the complete original source code as a `git submodule`. + > This allows you to build as _libmdbx_ and testing tool. + > On the other hand, this way requires you to pull git tags, and use C++11 compiler for test tool. + +**_Please, avoid using any other techniques._** Otherwise, at least +don't ask for support and don't name such chimeras `libmdbx`. + +The amalgamated source code could be created from original clone of git +repository on Linux by executing `make dist`. As a result, the desired +set of files will be formed in the `dist` subdirectory. + +## Building + +Both amalgamated and original source code provides build through the use +[CMake](https://cmake.org/) or [GNU +Make](https://www.gnu.org/software/make/) with +[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)). All build ways +are completely traditional and have minimal prerequirements like +`build-essential`, i.e. the non-obsolete C/C++ compiler and a +[SDK](https://en.wikipedia.org/wiki/Software_development_kit) for the +target platform. Obviously you need building tools itself, i.e. `git`, +`cmake` or GNU `make` with `bash`. + +So just use CMake or GNU Make in your habitual manner and feel free to +fill an issue or make pull request in the case something will be +unexpected or broken down. + +#### DSO/DLL unloading and destructors of Thread-Local-Storage objects +When building _libmdbx_ as a shared library or use static _libmdbx_ as a +part of another dynamic library, it is advisable to make sure that your +system ensures the correctness of the call destructors of +Thread-Local-Storage objects when unloading dynamic libraries. + +If this is not the case, then unloading a dynamic-link library with +_libmdbx_ code inside, can result in either a resource leak or a crash +due to calling destructors from an already unloaded DSO/DLL object. The +problem can only manifest in a multithreaded application, which makes +the unloading of shared dynamic libraries with _libmdbx_ code inside, +after using _libmdbx_. It is known that TLS-destructors are properly +maintained in the following cases: + +- On all modern versions of Windows (Windows 7 and later). + +- On systems with the +[`__cxa_thread_atexit_impl()`](https://sourceware.org/glibc/wiki/Destructor%20support%20for%20thread_local%20variables) +function in the standard C library, including systems with GNU libc +version 2.18 and later. + +- On systems with libpthread/ntpl from GNU libc with bug fixes +[#21031](https://sourceware.org/bugzilla/show_bug.cgi?id=21031) and +[#21032](https://sourceware.org/bugzilla/show_bug.cgi?id=21032), or +where there are no similar bugs in the pthreads implementation. + +### Linux and other platforms with GNU Make +To build the library it is enough to execute `make all` in the directory +of source code, and `make check` for execute the basic tests. + +If the `make` installed on the system is not GNU Make, there will be a +lot of errors from make when trying to build. In this case, perhaps you +should use `gmake` instead of `make`, or even `gnu-make`, etc. + +### FreeBSD and related platforms +As a rule, in such systems, the default is to use Berkeley Make. And GNU +Make is called by the gmake command or may be missing. In addition, +[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) may be absent. + +You need to install the required components: GNU Make, bash, C and C++ +compilers compatible with GCC or CLANG. After that, to build the +library, it is enough execute `gmake all` (or `make all`) in the +directory with source code, and `gmake check` (or `make check`) to run +the basic tests. + +### Windows +For build _libmdbx_ on Windows the _original_ CMake and [Microsoft Visual +Studio](https://en.wikipedia.org/wiki/Microsoft_Visual_Studio) are +recommended. + +Building by MinGW, MSYS or Cygwin is potentially possible. However, +these scripts are not tested and will probably require you to modify the +CMakeLists.txt or Makefile respectively. + +It should be noted that in _libmdbx_ was efforts to resolve +runtime dependencies from CRT and other libraries Visual Studio. +For this is enough define the `MDBX_AVOID_CRT` during build. + +An example of running a basic test script can be found in the +[CI-script](appveyor.yml) for [AppVeyor](https://www.appveyor.com/). To +run the [long stochastic test scenario](test/long_stochastic.sh), +[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)) is required, and +the such testing is recommended with place the test data on the +[RAM-disk](https://en.wikipedia.org/wiki/RAM_drive). + +### MacOS +Current [native build tools](https://en.wikipedia.org/wiki/Xcode) for +MacOS include GNU Make, CLANG and an outdated version of bash. +Therefore, to build the library, it is enough to run `make all` in the +directory with source code, and run `make check` to execute the base +tests. If something goes wrong, it is recommended to install +[Homebrew](https://brew.sh/) and try again. + +To run the [long stochastic test scenario](test/long_stochastic.sh), you +will need to install the current (not outdated) version of +[bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell)). To do this, we +recommend that you install [Homebrew](https://brew.sh/) and then execute +`brew install bash`. + +## Bindings + + | Runtime | GitHub | Author | + | -------- | ------ | ------ | + | Java | [mdbxjni](https://github.com/castortech/mdbxjni) | [Castor Technologies](https://castortech.com/) | + | .NET | [mdbx.NET](https://github.com/wangjia184/mdbx.NET) | [Jerry Wang](https://github.com/wangjia184) | + + +-------------------------------------------------------------------------------- + +Performance comparison +====================== + +All benchmarks were done by [IOArena](https://github.com/pmwkaa/ioarena) +and multiple [scripts](https://github.com/pmwkaa/ioarena/tree/HL%2B%2B2015) +runs on Lenovo Carbon-2 laptop, i7-4600U 2.1 GHz, 8 Gb RAM, +SSD SAMSUNG MZNTD512HAGL-000L1 (DXT23L0Q) 512 Gb. + +## Integral performance + +Here showed sum of performance metrics in 3 benchmarks: + + - Read/Search on 4 CPU cores machine; + + - Transactions with [CRUD](https://en.wikipedia.org/wiki/CRUD) + operations in sync-write mode (fdatasync is called after each + transaction); + + - Transactions with [CRUD](https://en.wikipedia.org/wiki/CRUD) + operations in lazy-write mode (moment to sync data to persistent storage + is decided by OS). + +*Reasons why asynchronous mode isn't benchmarked here:* + + 1. It doesn't make sense as it has to be done with DB engines, oriented + for keeping data in memory e.g. [Tarantool](https://tarantool.io/), + [Redis](https://redis.io/)), etc. + + 2. Performance gap is too high to compare in any meaningful way. + +![Comparison #1: Integral Performance](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-1.png) + +-------------------------------------------------------------------------------- + +## Read Scalability + +Summary performance with concurrent read/search queries in 1-2-4-8 +threads on 4 CPU cores machine. + +![Comparison #2: Read Scalability](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-2.png) + +-------------------------------------------------------------------------------- + +## Sync-write mode + + - Linear scale on left and dark rectangles mean arithmetic mean + transactions per second; + + - Logarithmic scale on right is in seconds and yellow intervals mean + execution time of transactions. Each interval shows minimal and maximum + execution time, cross marks standard deviation. + +**10,000 transactions in sync-write mode**. In case of a crash all data +is consistent and state is right after last successful transaction. +[fdatasync](https://linux.die.net/man/2/fdatasync) syscall is used after +each write transaction in this mode. + +In the benchmark each transaction contains combined CRUD operations (2 +inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database +and after full run the database contains 10,000 small key-value records. + +![Comparison #3: Sync-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-3.png) + +-------------------------------------------------------------------------------- + +## Lazy-write mode + + - Linear scale on left and dark rectangles mean arithmetic mean of + thousands transactions per second; + + - Logarithmic scale on right in seconds and yellow intervals mean + execution time of transactions. Each interval shows minimal and maximum + execution time, cross marks standard deviation. + +**100,000 transactions in lazy-write mode**. In case of a crash all data +is consistent and state is right after one of last transactions, but +transactions after it will be lost. Other DB engines use +[WAL](https://en.wikipedia.org/wiki/Write-ahead_logging) or transaction +journal for that, which in turn depends on order of operations in +journaled filesystem. _libmdbx_ doesn't use WAL and hands I/O operations +to filesystem and OS kernel (mmap). + +In the benchmark each transaction contains combined CRUD operations (2 +inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database +and after full run the database contains 100,000 small key-value +records. + + +![Comparison #4: Lazy-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-4.png) + +-------------------------------------------------------------------------------- + +## Async-write mode + + - Linear scale on left and dark rectangles mean arithmetic mean of + thousands transactions per second; + + - Logarithmic scale on right in seconds and yellow intervals mean + execution time of transactions. Each interval shows minimal and maximum + execution time, cross marks standard deviation. + +**1,000,000 transactions in async-write mode**. In case of a crash all +data will be consistent and state will be right after one of last +transactions, but lost transaction count is much higher than in +lazy-write mode. All DB engines in this mode do as little writes as +possible on persistent storage. _libmdbx_ uses +[msync(MS_ASYNC)](https://linux.die.net/man/2/msync) in this mode. + +In the benchmark each transaction contains combined CRUD operations (2 +inserts, 1 read, 1 update, 1 delete). Benchmark starts on empty database +and after full run the database contains 10,000 small key-value records. + +![Comparison #5: Async-write mode](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-5.png) + +-------------------------------------------------------------------------------- + +## Cost comparison + +Summary of used resources during lazy-write mode benchmarks: + + - Read and write IOPS; + + - Sum of user CPU time and sys CPU time; + + - Used space on persistent storage after the test and closed DB, but not + waiting for the end of all internal housekeeping operations (LSM + compactification, etc). + +_ForestDB_ is excluded because benchmark showed it's resource +consumption for each resource (CPU, IOPS) much higher than other engines +which prevents to meaningfully compare it with them. + +All benchmark data is gathered by +[getrusage()](http://man7.org/linux/man-pages/man2/getrusage.2.html) +syscall and by scanning data directory. + +![Comparison #6: Cost comparison](https://raw.githubusercontent.com/wiki/leo-yuriev/libmdbx/img/perf-slide-6.png) + +-------------------------------------------------------------------------------- + +``` +$ objdump -f -h -j .text libmdbx.so + +libmdbx.so: file format elf64-x86-64 +architecture: i386:x86-64, flags 0x00000150: +HAS_SYMS, DYNAMIC, D_PAGED +start address 0x0000000000003710 + +Sections: +Idx Name Size VMA LMA File off Algn + 11 .text 00015eff 0000000000003710 0000000000003710 00003710 2**4 + CONTENTS, ALLOC, LOAD, READONLY, CODE +``` diff --git a/src/contrib/db/libmdbx/appveyor.yml b/src/contrib/db/libmdbx/appveyor.yml new file mode 100644 index 0000000..be7ee76 --- /dev/null +++ b/src/contrib/db/libmdbx/appveyor.yml @@ -0,0 +1,99 @@ +version: 0.3.2.{build} + +environment: + matrix: + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + CMAKE_GENERATOR: Visual Studio 16 2019 + TOOLSET: 142 + MDBX_BUILD_SHARED_LIBRARY: OFF + MDBX_AVOID_CRT: OFF + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + CMAKE_GENERATOR: Visual Studio 16 2019 + TOOLSET: 142 + MDBX_BUILD_SHARED_LIBRARY: ON + MDBX_AVOID_CRT: ON + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + CMAKE_GENERATOR: Visual Studio 16 2019 + TOOLSET: 142 + MDBX_BUILD_SHARED_LIBRARY: OFF + MDBX_AVOID_CRT: ON + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + CMAKE_GENERATOR: Visual Studio 16 2019 + TOOLSET: 142 + MDBX_BUILD_SHARED_LIBRARY: ON + MDBX_AVOID_CRT: OFF + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + CMAKE_GENERATOR: Visual Studio 15 2017 + TOOLSET: 141 + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + CMAKE_GENERATOR: Visual Studio 14 2015 + TOOLSET: 140 + +branches: + except: + - coverity_scan + +configuration: +- Debug +- Release + +platform: +- Win32 +- x64 + +before_build: +- git clean -x -f -d +- git submodule sync +- git fetch --tags --prune +- git submodule update --init --recursive +- git submodule foreach --recursive git fetch --tags --prune +- cmake --version + +build_script: +- ps: | + Write-Output "*******************************************************************************" + Write-Output "Configuration: $env:CONFIGURATION" + Write-Output "Platform: $env:PLATFORM" + Write-Output "Toolchain: $env:CMAKE_GENERATOR v$env:TOOLSET" + Write-Output "Options: MDBX_AVOID_CRT=$env:MDBX_AVOID_CRT MDBX_BUILD_SHARED_LIBRARY=$env:MDBX_BUILD_SHARED_LIBRARY" + Write-Output "*******************************************************************************" + + md _build -Force | Out-Null + cd _build + + $generator = $env:CMAKE_GENERATOR + if ($env:TOOLSET -lt 142) { + if ($env:PLATFORM -eq "x64") { + $generator = "$generator Win64" + } + & cmake -G "$generator" -D CMAKE_CONFIGURATION_TYPES="Debug;Release" -D MDBX_AVOID_CRT:BOOL=$env:MDBX_AVOID_CRT -D MDBX_BUILD_SHARED_LIBRARY:BOOL=$env:MDBX_BUILD_SHARED_LIBRARY .. + } else { + & cmake -G "$generator" -A $env:PLATFORM -D CMAKE_CONFIGURATION_TYPES="Debug;Release" -DMDBX_AVOID_CRT:BOOL=$env:MDBX_AVOID_CRT -D MDBX_BUILD_SHARED_LIBRARY:BOOL=$env:MDBX_BUILD_SHARED_LIBRARY .. + } + if ($LastExitCode -ne 0) { + throw "Exec: $ErrorMessage" + } + Write-Output "*******************************************************************************" + + & cmake --build . --config $env:CONFIGURATION + if ($LastExitCode -ne 0) { + throw "Exec: $ErrorMessage" + } + Write-Output "*******************************************************************************" + +test_script: +- ps: | + if (($env:PLATFORM -ne "ARM") -and ($env:PLATFORM -ne "ARM64")) { + & ./$env:CONFIGURATION/mdbx_test.exe --progress --console=no --pathname=test.db --dont-cleanup-after basic > test.log + Get-Content test.log | Select-Object -last 42 + if ($LastExitCode -ne 0) { + throw "Exec: $ErrorMessage" + } else { + & ./$env:CONFIGURATION/mdbx_chk.exe -nvv test.db | Tee-Object -file chk.log | Select-Object -last 42 + } + } + +on_failure: +- ps: Push-AppveyorArtifact \projects\libmdbx\_build\test.log +- ps: Push-AppveyorArtifact \projects\libmdbx\_build\test.db +- ps: Push-AppveyorArtifact \projects\libmdbx\_build\chk.log diff --git a/src/contrib/db/libmdbx/cmake/compiler.cmake b/src/contrib/db/libmdbx/cmake/compiler.cmake new file mode 100644 index 0000000..03b0805 --- /dev/null +++ b/src/contrib/db/libmdbx/cmake/compiler.cmake @@ -0,0 +1,666 @@ +## Copyright (c) 2012-2019 Leonid Yuriev . +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +cmake_minimum_required(VERSION 3.8.2) +cmake_policy(PUSH) +cmake_policy(VERSION 3.8.2) + +if (CMAKE_VERSION MATCHES ".*MSVC.*") + message(FATAL_ERROR "CMake from MSVC kit is unfit! " + "Please use the original CMake from https://cmake.org/download/") +endif() + +if (NOT (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED)) + message(FATAL_ERROR "This module required C or C++ to be enabled") +endif() + +include(CMakeDependentOption) + +if(CMAKE_CXX_COMPILER_LOADED) + include(CheckCXXSourceRuns) + include(CheckCXXSourceCompiles) + include(CheckCXXCompilerFlag) +endif() +if(CMAKE_C_COMPILER_LOADED) + include(CheckCSourceRuns) + include(CheckCSourceCompiles) + include(CheckCCompilerFlag) +endif() + +# Check if the same compile family is used for both C and CXX +if(CMAKE_C_COMPILER_LOADED AND CMAKE_CXX_COMPILER_LOADED AND + NOT (CMAKE_C_COMPILER_ID STREQUAL CMAKE_CXX_COMPILER_ID)) + message(WARNING "CMAKE_C_COMPILER_ID (${CMAKE_C_COMPILER_ID}) is different " + "from CMAKE_CXX_COMPILER_ID (${CMAKE_CXX_COMPILER_ID}). " + "The final binary may be unusable.") +endif() + +if(CMAKE_CXX_COMPILER_LOADED) + set(CMAKE_PRIMARY_LANG "CXX") +else() + set(CMAKE_PRIMARY_LANG "C") +endif() + +macro(check_compiler_flag flag variable) + if(CMAKE_CXX_COMPILER_LOADED) + check_cxx_compiler_flag(${flag} ${variable}) + else() + check_c_compiler_flag(${flag} ${variable}) + endif() +endmacro(check_compiler_flag) + +# We support building with Clang and gcc. First check +# what we're using for build. +if(CMAKE_C_COMPILER_LOADED AND CMAKE_C_COMPILER_ID STREQUAL "Clang") + set(CMAKE_COMPILER_IS_CLANG ON) + set(CMAKE_COMPILER_IS_GNUCC OFF) +endif() +if(CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXx_COMPILER_ID STREQUAL "Clang") + set(CMAKE_COMPILER_IS_CLANG ON) + set(CMAKE_COMPILER_IS_GNUCXX OFF) +endif() + +# Hard coding the compiler version is ugly from cmake POV, but +# at least gives user a friendly error message. The most critical +# demand for C++ compiler is support of C++11 lambdas, added +# only in version 4.5 https://gcc.gnu.org/projects/cxx0x.html +if(CMAKE_COMPILER_IS_GNUCC) + if(CMAKE_C_COMPILER_VERSION VERSION_LESS 4.5) + message(FATAL_ERROR " + Your GCC version is ${CMAKE_C_COMPILER_VERSION}, please update") + endif() +endif() +if(CMAKE_COMPILER_IS_GNUCXX) + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.5) + message(FATAL_ERROR " + Your G++ version is ${CMAKE_CXX_COMPILER_VERSION}, please update") + endif() +endif() + +if(CMAKE_C_COMPILER_LOADED) + # Check for Elbrus lcc + execute_process(COMMAND ${CMAKE_C_COMPILER} --version + OUTPUT_VARIABLE tmp_lcc_probe_version + RESULT_VARIABLE tmp_lcc_probe_result ERROR_QUIET) + if(tmp_lcc_probe_result EQUAL 0) + string(FIND "${tmp_lcc_probe_version}" "lcc:" tmp_lcc_marker) + string(FIND "${tmp_lcc_probe_version}" ":e2k-" tmp_e2k_marker) + if(tmp_lcc_marker GREATER -1 AND tmp_e2k_marker GREATER tmp_lcc_marker) + execute_process(COMMAND ${CMAKE_C_COMPILER} -print-version + OUTPUT_VARIABLE CMAKE_C_COMPILER_VERSION + RESULT_VARIABLE tmp_lcc_probe_result) + set(CMAKE_COMPILER_IS_ELBRUSC ON) + set(CMAKE_C_COMPILER_ID "Elbrus") + else() + set(CMAKE_COMPILER_IS_ELBRUSC OFF) + endif() + unset(tmp_lcc_marker) + unset(tmp_e2k_marker) + endif() + unset(tmp_lcc_probe_version) + unset(tmp_lcc_probe_result) +endif() + +if(CMAKE_CXX_COMPILER_LOADED) + # Check for Elbrus l++ + execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version + OUTPUT_VARIABLE tmp_lxx_probe_version + RESULT_VARIABLE tmp_lxx_probe_result ERROR_QUIET) + if(tmp_lxx_probe_result EQUAL 0) + string(FIND "${tmp_lxx_probe_version}" "lcc:" tmp_lcc_marker) + string(FIND "${tmp_lxx_probe_version}" ":e2k-" tmp_e2k_marker) + if(tmp_lcc_marker GREATER -1 AND tmp_e2k_marker GREATER tmp_lcc_marker) + execute_process(COMMAND ${CMAKE_CXX_COMPILER} -print-version + OUTPUT_VARIABLE CMAKE_CXX_COMPILER_VERSION + RESULT_VARIABLE tmp_lxx_probe_result) + set(CMAKE_COMPILER_IS_ELBRUSCXX ON) + set(CMAKE_CXX_COMPILER_ID "Elbrus") + else() + set(CMAKE_COMPILER_IS_ELBRUSCXX OFF) + endif() + unset(tmp_lcc_marker) + unset(tmp_e2k_marker) + endif() + unset(tmp_lxx_probe_version) + unset(tmp_lxx_probe_result) +endif() + +if(CMAKE_CL_64) + set(MSVC64 1) +endif() +if(WIN32 AND CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG}) + execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -dumpmachine + OUTPUT_VARIABLE __GCC_TARGET_MACHINE + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(__GCC_TARGET_MACHINE MATCHES "amd64|x86_64|AMD64") + set(MINGW64 1) + endif() + unset(__GCC_TARGET_MACHINE) +endif() + +if(CMAKE_COMPILER_IS_ELBRUSC OR CMAKE_SYSTEM_PROCESSOR MATCHES "e2k.*|E2K.*|elbrus.*|ELBRUS.*") + set(E2K TRUE) + set(CMAKE_SYSTEM_ARCH "Elbrus") +elseif((MSVC64 OR MINGW64) AND CMAKE_SIZEOF_VOID_P EQUAL 8) + set(X86_64 TRUE) + set(CMAKE_SYSTEM_ARCH "x86_64") +elseif(MINGW OR (MSVC AND NOT CMAKE_CROSSCOMPILING)) + set(X86_32 TRUE) + set(CMAKE_SYSTEM_ARCH "x86") +elseif(CMAKE_COMPILER_IS_ELBRUSC OR CMAKE_SYSTEM_PROCESSOR MATCHES "e2k.*|E2K.*|elbrus.*|ELBRUS.*") + set(E2K TRUE) + set(CMAKE_SYSTEM_ARCH "Elbrus") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8) + set(X86_64 TRUE) + set(CMAKE_SYSTEM_ARCH "x86_64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i686.*|i386.*|x86.*") + set(X86_32 TRUE) + set(CMAKE_SYSTEM_ARCH "x86") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|ARM64.*)" AND CMAKE_SIZEOF_VOID_P EQUAL 8) + set(AARCH64 TRUE) + set(CMAKE_SYSTEM_ARCH "ARM64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm.*|ARM.*)") + set(ARM32 TRUE) + set(CMAKE_SYSTEM_ARCH "ARM") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64le.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8) + set(PPC64LE TRUE) + set(CMAKE_SYSTEM_ARCH "PPC64LE") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8) + set(PPC64 TRUE) + set(CMAKE_SYSTEM_ARCH "PPC64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc).*") + set(PPC32 TRUE) + set(CMAKE_SYSTEM_ARCH "PPC") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(mips|MIPS)64.*" AND CMAKE_SIZEOF_VOID_P EQUAL 8) + set(MIPS64 TRUE) + set(CMAKE_SYSTEM_ARCH "MIPS64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(mips|MIPS).*") + set(MIPS32 TRUE) + set(CMAKE_SYSTEM_ARCH "MIPS") +endif() + +if(MSVC) + check_compiler_flag("/WX" CC_HAS_WERROR) +else() + # + # GCC started to warn for unused result starting from 4.2, and + # this is when it introduced -Wno-unused-result + # GCC can also be built on top of llvm runtime (on mac). + check_compiler_flag("-Wno-unknown-pragmas" CC_HAS_WNO_UNKNOWN_PRAGMAS) + check_compiler_flag("-Wextra" CC_HAS_WEXTRA) + check_compiler_flag("-Werror" CC_HAS_WERROR) + check_compiler_flag("-fexceptions" CC_HAS_FEXCEPTIONS) + check_cxx_compiler_flag("-fcxx-exceptions" CC_HAS_FCXX_EXCEPTIONS) + check_compiler_flag("-funwind-tables" CC_HAS_FUNWIND_TABLES) + check_compiler_flag("-fno-omit-frame-pointer" CC_HAS_FNO_OMIT_FRAME_POINTER) + check_compiler_flag("-fno-common" CC_HAS_FNO_COMMON) + check_compiler_flag("-ggdb" CC_HAS_GGDB) + check_compiler_flag("-fvisibility=hidden" CC_HAS_VISIBILITY) + check_compiler_flag("-march=native" CC_HAS_ARCH_NATIVE) + check_compiler_flag("-Og" CC_HAS_DEBUG_FRENDLY_OPTIMIZATION) + check_compiler_flag("-Wall" CC_HAS_WALL) + check_compiler_flag("-Ominimal" CC_HAS_OMINIMAL) + check_compiler_flag("-ffunction-sections -fdata-sections" CC_HAS_SECTIONS) + check_compiler_flag("-ffast-math" CC_HAS_FASTMATH) + + # Check for an omp support + set(CMAKE_REQUIRED_FLAGS "-fopenmp -Werror") + check_cxx_source_compiles("int main(void) { + #pragma omp parallel + return 0; + }" HAVE_OPENMP) + set(CMAKE_REQUIRED_FLAGS "") +endif() + +# Check for LTO support by GCC +if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG}) + unset(gcc_collect) + unset(gcc_lto_wrapper) + + if(NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 4.7) + execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -v + OUTPUT_VARIABLE gcc_info_v ERROR_VARIABLE gcc_info_v) + + string(REGEX MATCH "^(.+\nCOLLECT_GCC=)([^ \n]+)(\n.+)$" gcc_collect_valid ${gcc_info_v}) + if(gcc_collect_valid) + string(REGEX REPLACE "^(.+\nCOLLECT_GCC=)([^ \n]+)(\n.+)$" "\\2" gcc_collect ${gcc_info_v}) + endif() + + string(REGEX MATCH "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" gcc_lto_wrapper_valid ${gcc_info_v}) + if(gcc_lto_wrapper_valid) + string(REGEX REPLACE "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" "\\2" gcc_lto_wrapper ${gcc_info_v}) + endif() + + set(gcc_suffix "") + if(gcc_collect_valid AND gcc_collect) + string(REGEX MATCH "^(.*cc)(-.+)$" gcc_suffix_valid ${gcc_collect}) + if(gcc_suffix_valid) + string(REGEX MATCH "^(.*cc)(-.+)$" "\\2" gcc_suffix ${gcc_collect}) + endif() + endif() + + get_filename_component(gcc_dir ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} DIRECTORY) + if(NOT CMAKE_GCC_AR) + find_program(CMAKE_GCC_AR NAMES gcc${gcc_suffix}-ar gcc-ar${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH) + endif() + if(NOT CMAKE_GCC_NM) + find_program(CMAKE_GCC_NM NAMES gcc${gcc_suffix}-nm gcc-nm${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH) + endif() + if(NOT CMAKE_GCC_RANLIB) + find_program(CMAKE_GCC_RANLIB NAMES gcc${gcc_suffix}-ranlib gcc-ranlib${gcc_suffix} PATHS ${gcc_dir} NO_DEFAULT_PATH) + endif() + + unset(gcc_dir) + unset(gcc_suffix_valid) + unset(gcc_suffix) + unset(gcc_lto_wrapper_valid) + unset(gcc_collect_valid) + unset(gcc_collect) + unset(gcc_info_v) + endif() + + if(CMAKE_GCC_AR AND CMAKE_GCC_NM AND CMAKE_GCC_RANLIB AND gcc_lto_wrapper) + message(STATUS "Found GCC's LTO toolset: ${gcc_lto_wrapper}, ${CMAKE_GCC_AR}, ${CMAKE_GCC_RANLIB}") + set(GCC_LTO_CFLAGS "-flto -fno-fat-lto-objects -fuse-linker-plugin") + set(GCC_LTO_AVAILABLE TRUE) + message(STATUS "Link-Time Optimization by GCC is available") + else() + set(GCC_LTO_AVAILABLE FALSE) + message(STATUS "Link-Time Optimization by GCC is NOT available") + endif() + unset(gcc_lto_wrapper) +endif() + +# check for LTO by MSVC +if(MSVC) + if(NOT MSVC_VERSION LESS 1600) + set(MSVC_LTO_AVAILABLE TRUE) + message(STATUS "Link-Time Optimization by MSVC is available") + else() + set(MSVC_LTO_AVAILABLE FALSE) + message(STATUS "Link-Time Optimization by MSVC is NOT available") + endif() +endif() + +# Check for LTO support by CLANG +if(CMAKE_COMPILER_IS_CLANG) + if(NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 3.5) + execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -print-search-dirs + OUTPUT_VARIABLE clang_search_dirs) + + unset(clang_bindir) + unset(clang_libdir) + string(REGEX MATCH "^(.*programs: =)([^:]*:)*([^:]+/llvm[-.0-9]+/bin[^:]*)(:[^:]*)*(\n.+)$" clang_bindir_valid ${clang_search_dirs}) + if(clang_bindir_valid) + string(REGEX REPLACE "^(.*programs: =)([^:]*:)*([^:]+/llvm[-.0-9]+/bin[^:]*)(:[^:]*)*(\n.+)$" "\\3" clang_bindir ${clang_search_dirs}) + get_filename_component(clang_libdir "${clang_bindir}/../lib" REALPATH) + if(clang_libdir) + message(STATUS "Found CLANG/LLVM directories: ${clang_bindir}, ${clang_libdir}") + endif() + endif() + + if(NOT (clang_bindir AND clang_libdir)) + message(STATUS "Could NOT find CLANG/LLVM directories (bin and/or lib).") + endif() + + if(NOT CMAKE_CLANG_LD AND clang_bindir) + find_program(CMAKE_CLANG_LD NAMES llvm-link link llvm-ld ld PATHS ${clang_bindir} NO_DEFAULT_PATH) + endif() + if(NOT CMAKE_CLANG_AR AND clang_bindir) + find_program(CMAKE_CLANG_AR NAMES llvm-ar ar PATHS ${clang_bindir} NO_DEFAULT_PATH) + endif() + if(NOT CMAKE_CLANG_NM AND clang_bindir) + find_program(CMAKE_CLANG_NM NAMES llvm-nm nm PATHS ${clang_bindir} NO_DEFAULT_PATH) + endif() + if(NOT CMAKE_CLANG_RANLIB AND clang_bindir) + find_program(CMAKE_CLANG_RANLIB NAMES llvm-ranlib ranlib PATHS ${clang_bindir} NO_DEFAULT_PATH) + endif() + + set(clang_lto_plugin_name "LLVMgold${CMAKE_SHARED_LIBRARY_SUFFIX}") + if(NOT CMAKE_LD_GOLD AND clang_bindir) + find_program(CMAKE_LD_GOLD NAMES ld.gold PATHS) + endif() + if(NOT CLANG_LTO_PLUGIN AND clang_libdir) + find_file(CLANG_LTO_PLUGIN ${clang_lto_plugin_name} PATH ${clang_libdir} NO_DEFAULT_PATH) + endif() + if(CLANG_LTO_PLUGIN) + message(STATUS "Found CLANG/LLVM's plugin for LTO: ${CLANG_LTO_PLUGIN}") + else() + message(STATUS "Could NOT find CLANG/LLVM's plugin (${clang_lto_plugin_name}) for LTO.") + endif() + + if(CMAKE_CLANG_LD AND CMAKE_CLANG_AR AND CMAKE_CLANG_NM AND CMAKE_CLANG_RANLIB) + message(STATUS "Found CLANG/LLVM's binutils for LTO: ${CMAKE_CLANG_AR}, ${CMAKE_CLANG_RANLIB}") + else() + message(STATUS "Could NOT find CLANG/LLVM's binutils (ar, ranlib, nm) for LTO.") + endif() + + unset(clang_lto_plugin_name) + unset(clang_libdir) + unset(clang_bindir_valid) + unset(clang_bindir) + unset(clang_search_dirs) + endif() + + if((CLANG_LTO_PLUGIN AND CMAKE_LD_GOLD) AND + (CMAKE_CLANG_LD AND CMAKE_CLANG_AR AND CMAKE_CLANG_NM AND CMAKE_CLANG_RANLIB)) + set(CLANG_LTO_AVAILABLE TRUE) + message(STATUS "Link-Time Optimization by CLANG/LLVM is available") + elseif(CMAKE_TOOLCHAIN_FILE AND NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 7.0) + set(CLANG_LTO_AVAILABLE TRUE) + if (NOT CMAKE_CLANG_AR) + set(CMAKE_CLANG_AR ${CMAKE_AR}) + endif() + if (NOT CMAKE_CLANG_NM) + set(CMAKE_CLANG_NM ${CMAKE_NM}) + endif() + if (NOT CMAKE_CLANG_RANLIB) + set(CMAKE_CLANG_RANLIB ${CMAKE_RANLIB }) + endif() + message(STATUS "Assume Link-Time Optimization by CLANG/LLVM is available via ${CMAKE_TOOLCHAIN_FILE}") + else() + set(CLANG_LTO_AVAILABLE FALSE) + message(STATUS "Link-Time Optimization by CLANG/LLVM is NOT available") + endif() +endif() + +# Perform build type specific configuration. +option(ENABLE_BACKTRACE "Enable output of fiber backtrace information in 'show + fiber' administrative command. Only works on x86 architectures, if compiled + with gcc. If GNU binutils and binutils-dev libraries are installed, backtrace + is output with resolved function (symbol) names. Otherwise only frame + addresses are printed." OFF) + +set(HAVE_BFD False) +if(ENABLE_BACKTRACE) + if(NOT (X86_32 OR X86_64) OR NOT CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG}) + # We only know this option to work with gcc + message(FATAL_ERROR "ENABLE_BACKTRACE option is set but the system + is not x86 based (${CMAKE_SYSTEM_PROCESSOR}) or the compiler + is not GNU GCC (${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER}).") + endif() + # Use GNU bfd if present. + find_library(BFD_LIBRARY NAMES libbfd.a) + if(BFD_LIBRARY) + check_library_exists(${BFD_LIBRARY} bfd_init "" HAVE_BFD_LIB) + endif() + find_library(IBERTY_LIBRARY NAMES libiberty.a) + if(IBERTY_LIBRARY) + check_library_exists(${IBERTY_LIBRARY} cplus_demangle "" HAVE_IBERTY_LIB) + endif() + set(CMAKE_REQUIRED_DEFINITIONS -DPACKAGE=${PACKAGE} -DPACKAGE_VERSION=${PACKAGE_VERSION}) + check_include_files(bfd.h HAVE_BFD_H) + set(CMAKE_REQUIRED_DEFINITIONS) + find_package(ZLIB) + if(HAVE_BFD_LIB AND HAVE_BFD_H AND HAVE_IBERTY_LIB AND ZLIB_FOUND) + set(HAVE_BFD ON) + set(BFD_LIBRARIES ${BFD_LIBRARY} ${IBERTY_LIBRARY} ${ZLIB_LIBRARIES}) + find_package_message(BFD_LIBRARIES "Found libbfd and dependencies" + ${BFD_LIBRARIES}) + if(TARGET_OS_FREEBSD AND NOT TARGET_OS_DEBIAN_FREEBSD) + set(BFD_LIBRARIES ${BFD_LIBRARIES} iconv) + endif() + endif() +endif() + +macro(setup_compile_flags) + # LY: save initial C/CXX flags + if(NOT INITIAL_CMAKE_FLAGS_SAVED) + if(MSVC) + string(REGEX REPLACE "^(.*)(/EHsc)( *)(.*)$" "\\1/EHs\\3\\4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + endif() + set(INITIAL_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_C_FLAGS ${CMAKE_C_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_STATIC_LINKER_FLAGS ${CMAKE_STATIC_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_MODULE_LINKER_FLAGS ${CMAKE_MODULE_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_FLAGS_SAVED TRUE CACHE INTERNAL "State of initial CMake's flags" FORCE) + endif() + + # LY: reset C/CXX flags + set(CXX_FLAGS ${INITIAL_CMAKE_CXX_FLAGS}) + set(C_FLAGS ${INITIAL_CMAKE_C_FLAGS}) + set(EXE_LINKER_FLAGS ${INITIAL_CMAKE_EXE_LINKER_FLAGS}) + set(SHARED_LINKER_FLAGS ${INITIAL_CMAKE_SHARED_LINKER_FLAGS}) + set(STATIC_LINKER_FLAGS ${INITIAL_CMAKE_STATIC_LINKER_FLAGS}) + set(MODULE_LINKER_FLAGS ${INITIAL_CMAKE_MODULE_LINKER_FLAGS}) + + if(CC_HAS_FEXCEPTIONS) + add_compile_flags("C;CXX" "-fexceptions") + endif() + if(CC_HAS_FCXX_EXCEPTIONS) + add_compile_flags("CXX" "-fcxx-exceptions -frtti") + endif() + + # In C a global variable without a storage specifier (static/extern) and + # without an initialiser is called a ’tentative definition’. The + # language permits multiple tentative definitions in the single + # translation unit; i.e. int foo; int foo; is perfectly ok. GNU + # toolchain goes even further, allowing multiple tentative definitions + # in *different* translation units. Internally, variables introduced via + # tentative definitions are implemented as ‘common’ symbols. Linker + # permits multiple definitions if they are common symbols, and it picks + # one arbitrarily for inclusion in the binary being linked. + # + # -fno-common forces GNU toolchain to behave in a more + # standard-conformant way in respect to tentative definitions and it + # prevents common symbols generation. Since we are a cross-platform + # project it really makes sense. There are toolchains that don’t + # implement GNU style handling of the tentative definitions and there + # are platforms lacking proper support for common symbols (osx). + if(CC_HAS_FNO_COMMON) + add_compile_flags("C;CXX" "-fno-common") + endif() + + if(CC_HAS_GGDB) + add_compile_flags("C;CXX" "-ggdb") + endif() + + if(CC_HAS_WNO_UNKNOWN_PRAGMAS AND NOT HAVE_OPENMP) + add_compile_flags("C;CXX" -Wno-unknown-pragmas) + endif() + + if(CC_HAS_SECTIONS) + add_compile_flags("C;CXX" -ffunction-sections -fdata-sections) + elseif(MSVC) + add_compile_flags("C;CXX" /Gy) + endif() + + # We must set -fno-omit-frame-pointer here, since we rely + # on frame pointer when getting a backtrace, and it must + # be used consistently across all object files. + # The same reasoning applies to -fno-stack-protector switch. + if(ENABLE_BACKTRACE) + if(CC_HAS_FNO_OMIT_FRAME_POINTER) + add_compile_flags("C;CXX" "-fno-omit-frame-pointer") + endif() + endif() + + if(MSVC) + if (MSVC_VERSION LESS 1900) + message(FATAL_ERROR "At least \"Microsoft C/C++ Compiler\" version 19.0.24234.1 (Visual Studio 2015 Update 3) is required.") + endif() + add_compile_flags("CXX" "/Zc:__cplusplus") + add_compile_flags("C;CXX" "/W4") + add_compile_flags("C;CXX" "/utf-8") + else() + if(CC_HAS_WALL) + add_compile_flags("C;CXX" "-Wall") + endif() + if(CC_HAS_WEXTRA) + add_compile_flags("C;CXX" "-Wextra") + endif() + endif() + + if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} + AND CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 5) + # G++ bug. http://gcc.gnu.org/bugzilla/show_bug.cgi?id=31488 + add_compile_flags("CXX" "-Wno-invalid-offsetof") + endif() + + add_definitions("-D__STDC_FORMAT_MACROS=1") + add_definitions("-D__STDC_LIMIT_MACROS=1") + add_definitions("-D__STDC_CONSTANT_MACROS=1") + add_definitions("-D_HAS_EXCEPTIONS=1") + + # Only add -Werror if it's a debug build, done by developers. + # Release builds should not cause extra trouble. + if(CC_HAS_WERROR AND (CI OR CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE STREQUAL "Debug")) + if(MSVC) + add_compile_flags("C;CXX" "/WX") + elseif(CMAKE_COMPILER_IS_CLANG) + if (NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 6) + add_compile_flags("C;CXX" "-Werror") + endif() + elseif(CMAKE_COMPILER_IS_GNUCC) + if (NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 6) + add_compile_flags("C;CXX" "-Werror") + endif() + else() + add_compile_flags("C;CXX" "-Werror") + endif() + endif() + + if(HAVE_OPENMP) + add_compile_flags("C;CXX" "-fopenmp") + endif() + + if (ENABLE_ASAN) + add_compile_flags("C;CXX" -fsanitize=address) + endif() + + if(ENABLE_GCOV) + if(NOT HAVE_GCOV) + message(FATAL_ERROR + "ENABLE_GCOV option requested but gcov library is not found") + endif() + + add_compile_flags("C;CXX" "-fprofile-arcs" "-ftest-coverage") + set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") + set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") + set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") + # add_library(gcov SHARED IMPORTED) + endif() + + if(ENABLE_GPROF) + add_compile_flags("C;CXX" "-pg") + endif() + + if(CMAKE_COMPILER_IS_GNUCC AND LTO_ENABLED) + add_compile_flags("C;CXX" ${GCC_LTO_CFLAGS}) + set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm -fwhole-program") + set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm") + set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${GCC_LTO_CFLAGS} -fverbose-asm") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5) + # Pass the same optimization flags to the linker + set(compile_flags "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}}") + set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${compile_flags}") + set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${compile_flags}") + set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${compile_flags}") + unset(compile_flags) + else() + add_compile_flags("CXX" "-flto-odr-type-merging") + endif() + endif() + + if(MSVC AND LTO_ENABLED) + add_compile_flags("C;CXX" "/GL") + foreach(linkmode IN ITEMS EXE SHARED STATIC MODULE) + set(${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS} /LTCG") + string(REGEX REPLACE "^(.*)(/INCREMENTAL:NO *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}") + string(REGEX REPLACE "^(.*)(/INCREMENTAL:YES *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}") + string(REGEX REPLACE "^(.*)(/INCREMENTAL *)(.*)$" "\\1\\3" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}") + string(STRIP "${${linkmode}_LINKER_FLAGS}" ${linkmode}_LINKER_FLAGS) + foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES ITEMS Release MinSizeRel RelWithDebInfo Debug) + string(TOUPPER "${config}" config_uppercase) + if(DEFINED "CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}") + string(REGEX REPLACE "^(.*)(/INCREMENTAL:NO *)(.*)$" "\\1\\3" altered_flags "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}") + string(REGEX REPLACE "^(.*)(/INCREMENTAL:YES *)(.*)$" "\\1\\3" altered_flags "${altered_flags}") + string(REGEX REPLACE "^(.*)(/INCREMENTAL *)(.*)$" "\\1\\3" altered_flags "${altered_flags}") + string(STRIP "${altered_flags}" altered_flags) + if(NOT "${altered_flags}" STREQUAL "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}") + set(CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase} "${altered_flags}" CACHE STRING "Altered: '/INCREMENTAL' removed for LTO" FORCE) + endif() + endif() + endforeach(config) + endforeach(linkmode) + unset(linkmode) + + foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES ITEMS Release MinSizeRel RelWithDebInfo) + foreach(lang IN ITEMS C CXX) + string(TOUPPER "${config}" config_uppercase) + if(DEFINED "CMAKE_${lang}_FLAGS_${config_uppercase}") + string(REPLACE "/O2" "/Ox" altered_flags "${CMAKE_${lang}_FLAGS_${config_uppercase}}") + if(NOT "${altered_flags}" STREQUAL "${CMAKE_${lang}_FLAGS_${config_uppercase}}") + set(CMAKE_${lang}_FLAGS_${config_uppercase} "${altered_flags}" CACHE STRING "Altered: '/O2' replaced by '/Ox' for LTO" FORCE) + endif() + endif() + unset(config_uppercase) + endforeach(lang) + endforeach(config) + unset(altered_flags) + unset(lang) + unset(config) + endif() + + if(CMAKE_COMPILER_IS_CLANG AND OSX_ARCHITECTURES) + set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} -Wl,-keep_dwarf_unwind") + set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} -Wl,-keep_dwarf_unwind") + set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} -Wl,-keep_dwarf_unwind") + endif() + + if(CMAKE_COMPILER_IS_CLANG AND LTO_ENABLED) + if(CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 3.9) + set(CLANG_LTO_FLAG "-flto") + else() + set(CLANG_LTO_FLAG "-flto=thin") + endif() + add_compile_flags("C;CXX" ${CLANG_LTO_FLAG}) + set(EXE_LINKER_FLAGS "${EXE_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm -fwhole-program") + set(SHARED_LINKER_FLAGS "${SHARED_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm") + set(MODULE_LINKER_FLAGS "${MODULE_LINKER_FLAGS} ${CLANG_LTO_FLAG} -fverbose-asm") + endif() + + # LY: push C/CXX flags into the cache + set(CMAKE_CXX_FLAGS ${CXX_FLAGS} CACHE STRING "Flags used by the C++ compiler during all build types" FORCE) + set(CMAKE_C_FLAGS ${C_FLAGS} CACHE STRING "Flags used by the C compiler during all build types" FORCE) + set(CMAKE_EXE_LINKER_FLAGS ${EXE_LINKER_FLAGS} CACHE STRING "Flags used by the linker" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS ${SHARED_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of dll's" FORCE) + set(CMAKE_STATIC_LINKER_FLAGS ${STATIC_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of static libraries" FORCE) + set(CMAKE_MODULE_LINKER_FLAGS ${MODULE_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of modules" FORCE) + unset(CXX_FLAGS) + unset(C_FLAGS) + unset(EXE_LINKER_FLAGS) + unset(SHARED_LINKER_FLAGS) + unset(STATIC_LINKER_FLAGS) + unset(MODULE_LINKER_FLAGS) +endmacro(setup_compile_flags) + +# determine library for for std::filesystem +set(LIBCXX_FILESYSTEM "") +if(CMAKE_COMPILER_IS_GNUCXX) + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) + set(LIBCXX_FILESYSTEM "stdc++fs") + endif() +elseif (CMAKE_COMPILER_IS_CLANG) + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) + set(LIBCXX_FILESYSTEM "c++experimental") + else() + set(LIBCXX_FILESYSTEM "stdc++fs") + endif() +endif() + +cmake_policy(POP) diff --git a/src/contrib/db/libmdbx/cmake/profile.cmake b/src/contrib/db/libmdbx/cmake/profile.cmake new file mode 100644 index 0000000..6507e8d --- /dev/null +++ b/src/contrib/db/libmdbx/cmake/profile.cmake @@ -0,0 +1,45 @@ +## Copyright (c) 2012-2019 Leonid Yuriev . +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +cmake_minimum_required(VERSION 3.8.2) +cmake_policy(PUSH) +cmake_policy(VERSION 3.8.2) + +include(CheckLibraryExists) +check_library_exists(gcov __gcov_flush "" HAVE_GCOV) + +option(ENABLE_GCOV + "Enable integration with gcov, a code coverage program" OFF) + +option(ENABLE_GPROF + "Enable integration with gprof, a performance analyzing tool" OFF) + +if(CMAKE_CXX_COMPILER_LOADED) + include(CheckIncludeFileCXX) + check_include_file_cxx(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H) +else() + include(CheckIncludeFile) + check_include_file(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H) +endif() + +option(MDBX_USE_VALGRIND "Enable integration with valgrind, a memory analyzing tool" OFF) +if(MDBX_USE_VALGRIND AND NOT HAVE_VALGRIND_MEMCHECK_H) + message(FATAL_ERROR "MDBX_USE_VALGRIND option is set but valgrind/memcheck.h is not found") +endif() + +option(ENABLE_ASAN + "Enable AddressSanitizer, a fast memory error detector based on compiler instrumentation" OFF) + +cmake_policy(POP) diff --git a/src/contrib/db/libmdbx/cmake/utils.cmake b/src/contrib/db/libmdbx/cmake/utils.cmake new file mode 100644 index 0000000..c31f53c --- /dev/null +++ b/src/contrib/db/libmdbx/cmake/utils.cmake @@ -0,0 +1,183 @@ +## Copyright (c) 2012-2019 Leonid Yuriev . +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## + +cmake_minimum_required(VERSION 3.8.2) +cmake_policy(PUSH) +cmake_policy(VERSION 3.8.2) + +macro(add_compile_flags langs) + foreach(_lang ${langs}) + string(REPLACE ";" " " _flags "${ARGN}") + if(CMAKE_CXX_COMPILER_LOADED AND _lang STREQUAL "CXX") + set("${_lang}_FLAGS" "${${_lang}_FLAGS} ${_flags}") + endif() + if(CMAKE_C_COMPILER_LOADED AND _lang STREQUAL "C") + set("${_lang}_FLAGS" "${${_lang}_FLAGS} ${_flags}") + endif() + endforeach() + unset(_lang) + unset(_flags) +endmacro(add_compile_flags) + +macro(set_source_files_compile_flags) + foreach(file ${ARGN}) + get_filename_component(_file_ext ${file} EXT) + set(_lang "") + if("${_file_ext}" STREQUAL ".m") + set(_lang OBJC) + # CMake believes that Objective C is a flavor of C++, not C, + # and uses g++ compiler for .m files. + # LANGUAGE property forces CMake to use CC for ${file} + set_source_files_properties(${file} PROPERTIES LANGUAGE C) + elseif("${_file_ext}" STREQUAL ".mm") + set(_lang OBJCXX) + endif() + + if(_lang) + get_source_file_property(_flags ${file} COMPILE_FLAGS) + if("${_flags}" STREQUAL "NOTFOUND") + set(_flags "${CMAKE_${_lang}_FLAGS}") + else() + set(_flags "${_flags} ${CMAKE_${_lang}_FLAGS}") + endif() + # message(STATUS "Set (${file} ${_flags}") + set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS + "${_flags}") + endif() + endforeach() + unset(_file_ext) + unset(_lang) +endmacro(set_source_files_compile_flags) + +macro(fetch_version name version_file) + set(${name}_VERSION "") + set(${name}_GIT_DESCRIBE "") + set(${name}_GIT_TIMESTAMP "") + set(${name}_GIT_TREE "") + set(${name}_GIT_COMMIT "") + set(${name}_GIT_REVISION 0) + set(${name}_GIT_VERSION "") + if(GIT) + execute_process(COMMAND ${GIT} describe --tags --long --dirty=-dirty + OUTPUT_VARIABLE ${name}_GIT_DESCRIBE + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE rc) + if(rc OR "${name}_GIT_DESCRIBE" STREQUAL "") + message(FATAL_ERROR "Please fetch tags and/or install latest version of git ('describe --tags --long --dirty' failed)") + endif() + + execute_process(COMMAND ${GIT} show --no-patch --format=%cI HEAD + OUTPUT_VARIABLE ${name}_GIT_TIMESTAMP + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE rc) + if(rc OR "${name}_GIT_TIMESTAMP" STREQUAL "%cI") + execute_process(COMMAND ${GIT} show --no-patch --format=%ci HEAD + OUTPUT_VARIABLE ${name}_GIT_TIMESTAMP + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE rc) + if(rc OR "${name}_GIT_TIMESTAMP" STREQUAL "%ci") + message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%cI HEAD' failed)") + endif() + endif() + + execute_process(COMMAND ${GIT} show --no-patch --format=%T HEAD + OUTPUT_VARIABLE ${name}_GIT_TREE + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE rc) + if(rc OR "${name}_GIT_TREE" STREQUAL "") + message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%T HEAD' failed)") + endif() + + execute_process(COMMAND ${GIT} show --no-patch --format=%H HEAD + OUTPUT_VARIABLE ${name}_GIT_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE rc) + if(rc OR "${name}_GIT_COMMIT" STREQUAL "") + message(FATAL_ERROR "Please install latest version of git ('show --no-patch --format=%H HEAD' failed)") + endif() + + execute_process(COMMAND ${GIT} rev-list --count --no-merges HEAD + OUTPUT_VARIABLE ${name}_GIT_REVISION + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE rc) + if(rc OR "${name}_GIT_REVISION" STREQUAL "") + message(FATAL_ERROR "Please install latest version of git ('rev-list --count --no-merges HEAD' failed)") + endif() + + string(REGEX MATCH "^(v)?([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" git_version_valid "${${name}_GIT_DESCRIBE}") + if(git_version_valid) + string(REGEX REPLACE "^(v)?([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" "\\2;\\3;\\4" ${name}_GIT_VERSION ${${name}_GIT_DESCRIBE}) + else() + string(REGEX MATCH "^(v)?([0-9]+)\\.([0-9]+)(.*)?" git_version_valid "${${name}_GIT_DESCRIBE}") + if(git_version_valid) + string(REGEX REPLACE "^(v)?([0-9]+)\\.([0-9]+)(.*)?" "\\2;\\3;0" ${name}_GIT_VERSION ${${name}_GIT_DESCRIBE}) + else() + message(AUTHOR_WARNING "Bad ${name} version \"${${name}_GIT_DESCRIBE}\"; falling back to 0.0.0 (have you made an initial release?)") + set(${name}_GIT_VERSION "0;0;0") + endif() + endif() + endif() + + if(NOT ${name}_GIT_VERSION OR NOT ${name}_GIT_TIMESTAMP OR NOT ${name}_GIT_REVISION) + message(WARNING "Unable to retrive ${name} version from git.") + set(${name}_GIT_VERSION "0;0;0;0") + set(${name}_GIT_TIMESTAMP "") + set(${name}_GIT_REVISION 0) + + # Try to get version from VERSION file + if(EXISTS "${version_file}") + file(STRINGS "${version_file}" ${name}_VERSION) + endif() + + if(NOT ${name}_VERSION) + message(WARNING "Unable to retrive ${name} version from \"${version_file}\" file.") + set(${name}_VERSION_LIST ${${name}_GIT_VERSION}) + string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}") + else() + string(REPLACE "." ";" ${name}_VERSION_LIST ${${name}_VERSION}) + endif() + + else() + list(APPEND ${name}_GIT_VERSION ${${name}_GIT_REVISION}) + set(${name}_VERSION_LIST ${${name}_GIT_VERSION}) + string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}") + endif() + + list(GET ${name}_VERSION_LIST 0 "${name}_VERSION_MAJOR") + list(GET ${name}_VERSION_LIST 1 "${name}_VERSION_MINOR") + list(GET ${name}_VERSION_LIST 2 "${name}_VERSION_RELEASE") + list(GET ${name}_VERSION_LIST 3 "${name}_VERSION_REVISION") + + set(${name}_VERSION_MAJOR ${${name}_VERSION_MAJOR} PARENT_SCOPE) + set(${name}_VERSION_MINOR ${${name}_VERSION_MINOR} PARENT_SCOPE) + set(${name}_VERSION_RELEASE ${${name}_VERSION_RELEASE} PARENT_SCOPE) + set(${name}_VERSION_REVISION ${${name}_VERSION_REVISION} PARENT_SCOPE) + set(${name}_VERSION ${${name}_VERSION} PARENT_SCOPE) + + set(${name}_GIT_DESCRIBE ${${name}_GIT_DESCRIBE} PARENT_SCOPE) + set(${name}_GIT_TIMESTAMP ${${name}_GIT_TIMESTAMP} PARENT_SCOPE) + set(${name}_GIT_TREE ${${name}_GIT_TREE} PARENT_SCOPE) + set(${name}_GIT_COMMIT ${${name}_GIT_COMMIT} PARENT_SCOPE) + set(${name}_GIT_REVISION ${${name}_GIT_REVISION} PARENT_SCOPE) + set(${name}_GIT_VERSION ${${name}_GIT_VERSION} PARENT_SCOPE) +endmacro(fetch_version) + +cmake_policy(POP) diff --git a/src/contrib/db/libmdbx/example/CMakeLists.txt b/src/contrib/db/libmdbx/example/CMakeLists.txt new file mode 100644 index 0000000..d3e56e8 --- /dev/null +++ b/src/contrib/db/libmdbx/example/CMakeLists.txt @@ -0,0 +1,6 @@ +set(TARGET mdbx_example) +project(${TARGET}) + +add_executable(${TARGET} example-mdbx.c) + +target_link_libraries(${TARGET} mdbx) diff --git a/src/contrib/db/libmdbx/example/README.md b/src/contrib/db/libmdbx/example/README.md new file mode 100644 index 0000000..b819cf4 --- /dev/null +++ b/src/contrib/db/libmdbx/example/README.md @@ -0,0 +1 @@ +See [example-mdbx.c](example-mdbx.c) as an example of using _libmdbx_, and do a line-by-line comparison of it with the [sample-bdb.txt](sample-bdb.txt) file. diff --git a/src/contrib/db/libmdbx/example/example-mdbx.c b/src/contrib/db/libmdbx/example/example-mdbx.c new file mode 100644 index 0000000..1d25ef6 --- /dev/null +++ b/src/contrib/db/libmdbx/example/example-mdbx.c @@ -0,0 +1,112 @@ +/* MDBX usage examle + * + * Do a line-by-line comparison of this and sample-bdb.txt + */ + +/* + * Copyright 2015-2019 Leonid Yuriev . + * Copyright 2017 Ilya Shipitsin . + * Copyright 2012-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "mdbx.h" +#include +#include + +int main(int argc, char *argv[]) { + (void)argc; + (void)argv; + + int rc; + MDBX_env *env = NULL; + MDBX_dbi dbi = 0; + MDBX_val key, data; + MDBX_txn *txn = NULL; + MDBX_cursor *cursor = NULL; + char sval[32]; + + rc = mdbx_env_create(&env); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_env_create: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + rc = mdbx_env_open(env, "./example-db", + MDBX_NOSUBDIR | MDBX_COALESCE | MDBX_LIFORECLAIM, 0664); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_env_open: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + rc = mdbx_txn_begin(env, NULL, 0, &txn); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + rc = mdbx_dbi_open(txn, NULL, 0, &dbi); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_dbi_open: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + key.iov_len = sizeof(int); + key.iov_base = sval; + data.iov_len = sizeof(sval); + data.iov_base = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = mdbx_put(txn, dbi, &key, &data, 0); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_put: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + rc = mdbx_txn_commit(txn); + if (rc) { + fprintf(stderr, "mdbx_txn_commit: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + txn = NULL; + + rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_txn_begin: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + rc = mdbx_cursor_open(txn, dbi, &cursor); + if (rc != MDBX_SUCCESS) { + fprintf(stderr, "mdbx_cursor_open: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + int found = 0; + while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", key.iov_base, (int)key.iov_len, + (char *)key.iov_base, data.iov_base, (int)data.iov_len, + (char *)data.iov_base); + found += 1; + } + if (rc != MDBX_NOTFOUND || found == 0) { + fprintf(stderr, "mdbx_cursor_get: (%d) %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } else { + rc = MDBX_SUCCESS; + } +bailout: + if (cursor) + mdbx_cursor_close(cursor); + if (txn) + mdbx_txn_abort(txn); + if (dbi) + mdbx_dbi_close(env, dbi); + if (env) + mdbx_env_close(env); + return (rc != MDBX_SUCCESS) ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/contrib/db/libmdbx/example/sample-bdb.txt b/src/contrib/db/libmdbx/example/sample-bdb.txt new file mode 100644 index 0000000..5c89540 --- /dev/null +++ b/src/contrib/db/libmdbx/example/sample-bdb.txt @@ -0,0 +1,77 @@ +/* BerkeleyDB toy/sample + * + * Do a line-by-line comparison of this and example-mdbx.c + */ + +/* + * Copyright 2015-2019 Leonid Yuriev . + * Copyright 2012-2015 Howard Chu, Symas Corp. + * Copyright 2015,2016 Peter-Service R&D LLC. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include +#include +#include + +int main(int argc,char * argv[]) +{ + int rc; + DB_ENV *env; + DB *dbi; + DBT key, data; + DB_TXN *txn; + DBC *cursor; + char sval[32], kval[32]; + + /* Note: Most error checking omitted for simplicity */ + +#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD) + rc = db_env_create(&env, 0); + rc = env->open(env, "./testdb", FLAGS, 0664); + rc = db_create(&dbi, env, 0); + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664); + + memset(&key, 0, sizeof(DBT)); + memset(&data, 0, sizeof(DBT)); + key.size = sizeof(int); + key.data = sval; + data.size = sizeof(sval); + data.data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = dbi->put(dbi, txn, &key, &data, 0); + rc = txn->commit(txn, 0); + if (rc) { + fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc)); + goto leave; + } + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->cursor(dbi, txn, &cursor, 0); + key.flags = DB_DBT_USERMEM; + key.data = kval; + key.ulen = sizeof(kval); + data.flags = DB_DBT_USERMEM; + data.data = sval; + data.ulen = sizeof(sval); + while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.data, (int) key.size, (char *) key.data, + data.data, (int) data.size, (char *) data.data); + } + rc = cursor->c_close(cursor); + rc = txn->abort(txn); +leave: + rc = dbi->close(dbi, 0); + rc = env->close(env, 0); + return rc; +} diff --git a/src/contrib/db/libmdbx/mdbx.h b/src/contrib/db/libmdbx/mdbx.h new file mode 100644 index 0000000..a52335b --- /dev/null +++ b/src/contrib/db/libmdbx/mdbx.h @@ -0,0 +1,3497 @@ +/**** BRIEFLY ****************************************************************** + * + * libmdbx is superior to LMDB (https://bit.ly/26ts7tL) in terms of features + * and reliability, not inferior in performance. In comparison to LMDB, libmdbx + * makes many things just work perfectly, not silently and catastrophically + * break down. libmdbx supports Linux, Windows, MacOS, FreeBSD and other + * systems compliant with POSIX.1-2008. + * + * Look below for API description, for other information (build, embedding and + * amalgamation, improvements over LMDB, benchmarking, etc) please refer to + * README.md at https://abf.io/erthink/libmdbx. + * + * --- + * + * The next version is under active non-public development and will be released + * as MithrilDB and libmithrildb for libraries & packages. Admittedly mythical + * Mithril is resembling silver but being stronger and lighter than steel. + * Therefore MithrilDB is rightly relevant name. + * + * MithrilDB will be radically different from libmdbx by the new database format + * and API based on C++17, as well as the Apache 2.0 License. The goal of this + * revolution is to provide a clearer and robust API, add more features and new + * valuable properties of database. + * + * The Future will (be) Positive. Ð’ÑÑ‘ будет хорошо. + * + * + **** INTRODUCTION ************************************************************* + * + * // For the most part, this section is a copy of the corresponding text + * // from LMDB description, but with some edits reflecting the improvements + * // and enhancements were made in MDBX. + * + * MDBX is a Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. The entire database (aka "environment") + * is exposed in a memory map, and all data fetches return data directly from + * the mapped memory, so no malloc's or memcpy's occur during data fetches. + * As such, the library is extremely simple because it requires no page caching + * layer of its own, and it is extremely high performance and memory-efficient. + * It is also fully transactional with full ACID semantics, and when the memory + * map is read-only, the database integrity cannot be corrupted by stray pointer + * writes from application code. + * + * The library is fully thread-aware and supports concurrent read/write access + * from multiple processes and threads. Data pages use a copy-on-write strategy + * so no active data pages are ever overwritten, which also provides resistance + * to corruption and eliminates the need of any special recovery procedures + * after a system crash. Writes are fully serialized; only one write transaction + * may be active at a time, which guarantees that writers can never deadlock. + * The database structure is multi-versioned so readers run with no locks; + * writers cannot block readers, and readers don't block writers. + * + * Unlike other well-known database mechanisms which use either write-ahead + * transaction logs or append-only data writes, MDBX requires no maintenance + * during operation. Both write-ahead loggers and append-only databases require + * periodic checkpointing and/or compaction of their log or database files + * otherwise they grow without bound. MDBX tracks free pages within the database + * and re-uses them for new write operations, so the database size does not grow + * without bound in normal use. It is worth noting that the "next" version + * libmdbx (MithrilDB) will solve this problem. + * + * The memory map can be used as a read-only or read-write map. It is read-only + * by default as this provides total immunity to corruption. Using read-write + * mode offers much higher write performance, but adds the possibility for stray + * application writes thru pointers to silently corrupt the database. + * Of course if your application code is known to be bug-free (...) then this is + * not an issue. + * + * If this is your first time using a transactional embedded key-value store, + * you may find the "GETTING STARTED" section below to be helpful. + * + * + **** GETTING STARTED ********************************************************** + * + * // This section is based on Bert Hubert's intro "LMDB Semantics", with + * // edits reflecting the improvements and enhancements were made in MDBX. + * // See https://bit.ly/2maejGY for Bert Hubert's original. + * + * Everything starts with an environment, created by mdbx_env_create(). + * Once created, this environment must also be opened with mdbx_env_open(), + * and after use be closed by mdbx_env_close(). At that a non-zero value of the + * last argument "mode" supposes MDBX will create database and directory if ones + * does not exist. In this case the non-zero "mode" argument specifies the file + * mode bits be applied when a new files are created by open() function. + * + * Within that directory, a lock file (aka LCK-file) and a storage file (aka + * DXB-file) will be generated. If you don't want to use a directory, you can + * pass the MDBX_NOSUBDIR option, in which case the path you provided is used + * directly as the DXB-file, and another file with a "-lck" suffix added + * will be used for the LCK-file. + * + * Once the environment is open, a transaction can be created within it using + * mdbx_txn_begin(). Transactions may be read-write or read-only, and read-write + * transactions may be nested. A transaction must only be used by one thread at + * a time. Transactions are always required, even for read-only access. The + * transaction provides a consistent view of the data. + * + * Once a transaction has been created, a database (i.e. key-value space inside + * the environment) can be opened within it using mdbx_dbi_open(). If only one + * database will ever be used in the environment, a NULL can be passed as the + * database name. For named databases, the MDBX_CREATE flag must be used to + * create the database if it doesn't already exist. Also, mdbx_env_set_maxdbs() + * must be called after mdbx_env_create() and before mdbx_env_open() to set the + * maximum number of named databases you want to support. + * + * NOTE: a single transaction can open multiple databases. Generally databases + * should only be opened once, by the first transaction in the process. + * + * Within a transaction, mdbx_get() and mdbx_put() can store single key-value + * pairs if that is all you need to do (but see CURSORS below if you want to do + * more). + * + * A key-value pair is expressed as two MDBX_val structures. This struct that is + * exactly similar to POSIX's struct iovec and has two fields, iov_len and + * iov_base. The data is a void pointer to an array of iov_len bytes. + * (!) The notable difference between MDBX and LMDB is that MDBX support zero + * length keys. + * + * Because MDBX is very efficient (and usually zero-copy), the data returned in + * an MDBX_val structure may be memory-mapped straight from disk. In other words + * look but do not touch (or free() for that matter). Once a transaction is + * closed, the values can no longer be used, so make a copy if you need to keep + * them after that. + * + * + * CURSORS -- To do more powerful things, we must use a cursor. + * + * Within the transaction, a cursor can be created with mdbx_cursor_open(). + * With this cursor we can store/retrieve/delete (multiple) values using + * mdbx_cursor_get(), mdbx_cursor_put(), and mdbx_cursor_del(). + * + * mdbx_cursor_get() positions itself depending on the cursor operation + * requested, and for some operations, on the supplied key. For example, to list + * all key-value pairs in a database, use operation MDBX_FIRST for the first + * call to mdbx_cursor_get(), and MDBX_NEXT on subsequent calls, until the end + * is hit. + * + * To retrieve all keys starting from a specified key value, use MDBX_SET. For + * more cursor operations, see the API description below. + * + * When using mdbx_cursor_put(), either the function will position the cursor + * for you based on the key, or you can use operation MDBX_CURRENT to use the + * current position of the cursor. NOTE that key must then match the current + * position's key. + * + * + * SUMMARIZING THE OPENING + * + * So we have a cursor in a transaction which opened a database in an + * environment which is opened from a filesystem after it was separately + * created. + * + * Or, we create an environment, open it from a filesystem, create a transaction + * within it, open a database within that transaction, and create a cursor + * within all of the above. + * + * Got it? + * + * + * THREADS AND PROCESSES + * + * Do not have open an database twice in the same process at the same time, MDBX + * will track and prevent this. Instead, share the MDBX environment that has + * opened the file across all threads. The reason for this is: + * - When the "Open file description" locks (aka OFD-locks) are not available, + * MDBX uses POSIX locks on files, and these locks have issues if one process + * opens a file multiple times. + * - If a single process opens the same environment multiple times, closing it + * once will remove all the locks held on it, and the other instances will be + * vulnerable to corruption from other processes. + * + For compatibility with LMDB which allows multi-opening, MDBX can be + * configured at runtime by mdbx_setup_debug(MDBX_DBG_LEGACY_MULTIOPEN, ...) + * prior to calling other MDBX funcitons. In this way MDBX will track + * databases opening, detect multi-opening cases and then recover POSIX file + * locks as necessary. However, lock recovery can cause unexpected pauses, + * such as when another process opened the database in exclusive mode before + * the lock was restored - we have to wait until such a process releases the + * database, and so on. + * + * Do not use opened MDBX environment(s) after fork() in a child process(es), + * MDBX will check and prevent this at critical points. Instead, ensure there is + * no open MDBX-instance(s) during fork(), or atleast close it immediately after + * fork() in the child process and reopen if required - for instance by using + * pthread_atfork(). The reason for this is: + * - For competitive consistent reading, MDBX assigns a slot in the shared + * table for each process that interacts with the database. This slot is + * populated with process attributes, including the PID. + * - After fork(), in order to remain connected to a database, the child + * process must have its own such "slot", which can't be assigned in any + * simple and robust way another than the regular. + * - A write transaction from a parent process cannot continue in a child + * process for obvious reasons. + * - Moreover, in a multithreaded process at the fork() moment any number of + * threads could run in critical and/or intermediate sections of MDBX code + * with interaction and/or racing conditions with threads from other + * process(es). For instance: shrinking a database or copying it to a pipe, + * opening or closing environment, begining or finishing a transaction, + * and so on. + * = Therefore, any solution other than simply close database (and reopen if + * necessary) in a child process would be both extreme complicated and so + * fragile. + * + * Also note that a transaction is tied to one thread by default using Thread + * Local Storage. If you want to pass read-only transactions across threads, + * you can use the MDBX_NOTLS option on the environment. Nevertheless, a write + * transaction entirely should only be used in one thread from start to finish. + * MDBX checks this in a reasonable manner and return the MDBX_THREAD_MISMATCH + * error in rules violation. + * + * + * TRANSACTIONS, ROLLBACKS, etc. + * + * To actually get anything done, a transaction must be committed using + * mdbx_txn_commit(). Alternatively, all of a transaction's operations + * can be discarded using mdbx_txn_abort(). + * + * (!) An important difference between MDBX and LMDB is that MDBX required that + * any opened cursors can be reused and must be freed explicitly, regardless + * ones was opened in a read-only or write transaction. The REASON for this is + * eliminates ambiguity which helps to avoid errors such as: use-after-free, + * double-free, i.e. memory corruption and segfaults. + * + * For read-only transactions, obviously there is nothing to commit to storage. + * (!) An another notable difference between MDBX and LMDB is that MDBX make + * handles opened for existing databases immediately available for other + * transactions, regardless this transaction will be aborted or reset. The + * REASON for this is to avoiding the requirement for multiple opening a same + * handles in concurrent read transactions, and tracking of such open but hidden + * handles until the completion of read transactions which opened them. + * + * In addition, as long as a transaction is open, a consistent view of the + * database is kept alive, which requires storage. A read-only transaction that + * no longer requires this consistent view should be terminated (committed or + * aborted) when the view is no longer needed (but see below for an + * optimization). + * + * There can be multiple simultaneously active read-only transactions but only + * one that can write. Once a single read-write transaction is opened, all + * further attempts to begin one will block until the first one is committed or + * aborted. This has no effect on read-only transactions, however, and they may + * continue to be opened at any time. + * + * + * DUPLICATE KEYS + * + * mdbx_get() and mdbx_put() respectively have no and only some support or + * multiple key-value pairs with identical keys. If there are multiple values + * for a key, mdbx_get() will only return the first value. + * + * When multiple values for one key are required, pass the MDBX_DUPSORT flag to + * mdbx_dbi_open(). In an MDBX_DUPSORT database, by default mdbx_put() will not + * replace the value for a key if the key existed already. Instead it will add + * the new value to the key. In addition, mdbx_del() will pay attention to the + * value field too, allowing for specific values of a key to be deleted. + * + * Finally, additional cursor operations become available for traversing through + * and retrieving duplicate values. + * + * + * SOME OPTIMIZATION + * + * If you frequently begin and abort read-only transactions, as an optimization, + * it is possible to only reset and renew a transaction. + * + * mdbx_txn_reset() releases any old copies of data kept around for a read-only + * transaction. To reuse this reset transaction, call mdbx_txn_renew() on it. + * Any cursors in this transaction can also be renewed using mdbx_cursor_renew() + * or freed by mdbx_cursor_close(). + * + * To permanently free a transaction, reset or not, use mdbx_txn_abort(). + * + * + * CLEANING UP + * + * Any created cursors must be closed using mdbx_cursor_close(). It is advisable + * to repeat: + * (!) An important difference between MDBX and LMDB is that MDBX required that + * any opened cursors can be reused and must be freed explicitly, regardless + * ones was opened in a read-only or write transaction. The REASON for this is + * eliminates ambiguity which helps to avoid errors such as: use-after-free, + * double-free, i.e. memory corruption and segfaults. + * + * It is very rarely necessary to close a database handle, and in general they + * should just be left open. When you close a handle, it immediately becomes + * unavailable for all transactions in the environment. Therefore, you should + * avoid closing the handle while at least one transaction is using it. + * + * + * THE FULL API + * + * The full MDBX documentation lists further details below, + * like how to: + * + * - configure database size and automatic size management + * - drop and clean a database + * - detect and report errors + * - optimize (bulk) loading speed + * - (temporarily) reduce robustness to gain even more speed + * - gather statistics about the database + * - define custom sort orders + * - estimate size of range query result + * - double perfomance by LIFO reclaiming on storages with write-back + * - use sequences and canary markers + * - use lack-of-space callback (aka OOM-KICK) + * - use exclusive mode + * + * + **** RESTRICTIONS & CAVEATS *************************************************** + * in addition to those listed for some functions. + * + * - Troubleshooting the LCK-file. + * 1. A broken LCK-file can cause sync issues, including appearance of + * wrong/inconsistent data for readers. When database opened in the + * cooperative read-write mode the LCK-file requires to be mapped to + * memory in read-write access. In this case it is always possible for + * stray/malfunctioned application could writes thru pointers to + * silently corrupt the LCK-file. + * + * Unfortunately, there is no any portable way to prevent such + * corruption, since the LCK-file is updated concurrently by + * multiple processes in a lock-free manner and any locking is + * unwise due to a large overhead. + * + * The "next" version of libmdbx (MithrilDB) will solve this issue. + * + * Workaround: Just make all programs using the database close it; + * the LCK-file is always reset on first open. + * + * 2. Stale reader transactions left behind by an aborted program cause + * further writes to grow the database quickly, and stale locks can + * block further operation. + * MDBX checks for stale readers while opening environment and before + * growth the database. But in some cases, this may not be enough. + * + * Workaround: Check for stale readers periodically, using the + * mdbx_reader_check() function or the mdbx_stat tool. + * + * 3. Stale writers will be cleared automatically by MDBX on supprted + * platforms. But this is platform-specific, especially of + * implementation of shared POSIX-mutexes and support for robust + * mutexes. For instance there are no known issues on Linux, OSX, + * Windows and FreeBSD. + * + * Workaround: Otherwise just make all programs using the database + * close it; the LCK-file is always reset on first open + * of the environment. + * + * - Do not use MDBX databases on remote filesystems, even between processes + * on the same host. This breaks file locks on some platforms, possibly + * memory map sync, and certainly sync between programs on different hosts. + * + * On the other hand, MDBX support the exclusive database operation over + * a network, and cooperative read-only access to the database placed on + * a read-only network shares. + * + * - Do not use opened MDBX_env instance(s) in a child processes after fork(). + * It would be insane to call fork() and any MDBX-functions simultaneously + * from multiple threads. The best way is to prevent the presence of open + * MDBX-instances during fork(). + * + * The MDBX_TXN_CHECKPID build-time option, which is ON by default on + * non-Windows platforms (i.e. where fork() is available), enables PID + * checking at a few critical points. But this does not give any guarantees, + * but only allows you to detect such errors a little sooner. Depending on + * the platform, you should expect an application crash and/or database + * corruption in such cases. + * + * On the other hand, MDBX allow calling mdbx_close_env() in such cases to + * release resources, but no more and in general this is a wrong way. + * + * - There is no pure read-only mode in a normal explicitly way, since + * readers need write access to LCK-file to be ones visible for writer. + * MDBX always tries to open/create LCK-file for read-write, but switches + * to without-LCK mode on appropriate errors (EROFS, EACCESS, EPERM) + * if the read-only mode was requested by the MDBX_RDONLY flag which is + * described below. + * + * The "next" version of libmdbx (MithrilDB) will solve this issue. + * + * - A thread can only use one transaction at a time, plus any nested + * read-write transactions in the non-writemap mode. Each transaction + * belongs to one thread. The MDBX_NOTLS flag changes this for read-only + * transactions. See below. + * + * - Do not have open an MDBX database twice in the same process at the same + * time. By default MDBX prevent this in most cases by tracking databases + * opening and return MDBX_BUSY if anyone LCK-file is already open. + * + * The reason for this is that when the "Open file description" locks (aka + * OFD-locks) are not available, MDBX uses POSIX locks on files, and these + * locks have issues if one process opens a file multiple times. If a single + * process opens the same environment multiple times, closing it once will + * remove all the locks held on it, and the other instances will be + * vulnerable to corruption from other processes. + * + * For compatibility with LMDB which allows multi-opening, MDBX can be + * configured at runtime by mdbx_setup_debug(MDBX_DBG_LEGACY_MULTIOPEN, ...) + * prior to calling other MDBX funcitons. In this way MDBX will track + * databases opening, detect multi-opening cases and then recover POSIX file + * locks as necessary. However, lock recovery can cause unexpected pauses, + * such as when another process opened the database in exclusive mode before + * the lock was restored - we have to wait until such a process releases the + * database, and so on. + * + * - Avoid long-lived transactions, especially in the scenarios with a high + * rate of write transactions. Read transactions prevent reuse of pages + * freed by newer write transactions, thus the database can grow quickly. + * Write transactions prevent other write transactions, since writes are + * serialized. + * + * Understanding the problem of long-lived read transactions requires some + * explanation, but can be difficult for quick perception. So is is + * reasonable to simplify this as follows: + * 1. Garbage collection problem exists in all databases one way or + * another, e.g. VACUUM in PostgreSQL. But in _libmdbx_ it's even more + * discernible because of high transaction rate and intentional + * internals simplification in favor of performance. + * + * 2. MDBX employs Multiversion concurrency control on the Copy-on-Write + * basis, that allows multiple readers runs in parallel with a write + * transaction without blocking. An each write transaction needs free + * pages to put the changed data, that pages will be placed in the new + * b-tree snapshot at commit. MDBX efficiently recycling pages from + * previous created unused snapshots, BUT this is impossible if anyone + * a read transaction use such snapshot. + * + * 3. Thus massive altering of data during a parallel long read operation + * will increase the process's work set and may exhaust entire free + * database space. + * + * A good example of long readers is a hot backup to the slow destination + * or debugging of a client application while retaining an active read + * transaction. LMDB this results in MAP_FULL error and subsequent write + * performance degradation. + * + * MDBX mostly solve "long-lived" readers issue by the lack-of-space callback + * which allow to aborts long readers, and by the MDBX_LIFORECLAIM mode which + * addresses subsequent performance degradation. + * The "next" version of libmdbx (MithrilDB) will completely solve this. + * + * - Avoid suspending a process with active transactions. These would then be + * "long-lived" as above. + * + * The "next" version of libmdbx (MithrilDB) will solve this issue. + * + * - Avoid aborting a process with an active read-only transaction in scenaries + * with high rate of write transactions. The transaction becomes "long-lived" + * as above until a check for stale readers is performed or the LCK-file is + * reset, since the process may not remove it from the lockfile. This does + * not apply to write transactions if the system clears stale writers, see + * above. + * + * - An MDBX database configuration will often reserve considerable unused + * memory address space and maybe file size for future growth. This does + * not use actual memory or disk space, but users may need to understand + * the difference so they won't be scared off. + * + * - The Write Amplification Factor. + * TBD. + * + **** LICENSE AND COPYRUSTING ************************************************** + * + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * --- + * + * This code is derived from "LMDB engine" written by + * Howard Chu (Symas Corporation), which itself derived from btree.c + * written by Martin Hedenfalk. + * + * --- + * + * Portions Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * --- + * + * Portions Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + **** ACKNOWLEDGEMENTS ********************************************************* + * + * Howard Chu (Symas Corporation) - the author of LMDB, + * from which originated the MDBX in 2015. + * + * Martin Hedenfalk - the author of `btree.c` code, + * which was used for begin development of LMDB. + * + ******************************************************************************/ + +#pragma once +#ifndef LIBMDBX_H +#define LIBMDBX_H + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#include +#include +#include + +#if defined(_WIN32) || defined(_WIN64) + +#include +#include +#ifndef __mode_t_defined +typedef unsigned short mode_t; +#endif +typedef HANDLE mdbx_filehandle_t; +typedef DWORD mdbx_pid_t; +typedef DWORD mdbx_tid_t; +#define MDBX_ENODATA ERROR_HANDLE_EOF +#define MDBX_EINVAL ERROR_INVALID_PARAMETER +#define MDBX_EACCESS ERROR_ACCESS_DENIED +#define MDBX_ENOMEM ERROR_OUTOFMEMORY +#define MDBX_EROFS ERROR_FILE_READ_ONLY +#define MDBX_ENOSYS ERROR_NOT_SUPPORTED +#define MDBX_EIO ERROR_WRITE_FAULT +#define MDBX_EPERM ERROR_INVALID_FUNCTION +#define MDBX_EINTR ERROR_CANCELLED +#define MDBX_ENOFILE ERROR_FILE_NOT_FOUND + +#else + +#include /* for error codes */ +#include /* for pthread_t */ +#include /* for pid_t */ +#include /* for truct iovec */ +#define HAVE_STRUCT_IOVEC 1 +typedef int mdbx_filehandle_t; +typedef pid_t mdbx_pid_t; +typedef pthread_t mdbx_tid_t; +#ifdef ENODATA +#define MDBX_ENODATA ENODATA +#else +#define MDBX_ENODATA -1 +#endif +#define MDBX_EINVAL EINVAL +#define MDBX_EACCESS EACCES +#define MDBX_ENOMEM ENOMEM +#define MDBX_EROFS EROFS +#define MDBX_ENOSYS ENOSYS +#define MDBX_EIO EIO +#define MDBX_EPERM EPERM +#define MDBX_EINTR EINTR +#define MDBX_ENOFILE ENOENT + +#endif + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __dll_export +#if defined(_WIN32) || defined(__CYGWIN__) +#if defined(__GNUC__) || __has_attribute(dllexport) +#define __dll_export __attribute__((dllexport)) +#elif defined(_MSC_VER) +#define __dll_export __declspec(dllexport) +#else +#define __dll_export +#endif +#elif defined(__GNUC__) || __has_attribute(__visibility__) +#define __dll_export __attribute__((__visibility__("default"))) +#else +#define __dll_export +#endif +#endif /* __dll_export */ + +#ifndef __dll_import +#if defined(_WIN32) || defined(__CYGWIN__) +#if defined(__GNUC__) || __has_attribute(dllimport) +#define __dll_import __attribute__((dllimport)) +#elif defined(_MSC_VER) +#define __dll_import __declspec(dllimport) +#else +#define __dll_import +#endif +#else +#define __dll_import +#endif +#endif /* __dll_import */ + +/*----------------------------------------------------------------------------*/ + +#define MDBX_VERSION_MAJOR 0 +#define MDBX_VERSION_MINOR 3 + +#ifndef LIBMDBX_API +#if defined(LIBMDBX_EXPORTS) +#define LIBMDBX_API __dll_export +#elif defined(LIBMDBX_IMPORTS) +#define LIBMDBX_API __dll_import +#else +#define LIBMDBX_API +#endif +#endif /* LIBMDBX_API */ + +#ifdef __cplusplus +extern "C" { +#endif + +/**** MDBX version information ************************************************/ + +#if defined(LIBMDBX_IMPORTS) +#define LIBMDBX_VERINFO_API __dll_import +#else +#define LIBMDBX_VERINFO_API __dll_export +#endif /* LIBMDBX_VERINFO_API */ + +typedef struct mdbx_version_info { + uint8_t major; + uint8_t minor; + uint16_t release; + uint32_t revision; + struct /* source info from git */ { + const char *datetime /* committer date, strict ISO-8601 format */; + const char *tree /* commit hash (hexadecimal digits) */; + const char *commit /* tree hash, i.e. digest of the source code */; + const char *describe /* git-describe string */; + } git; + const char *sourcery /* sourcery anchor for pinning */; +} mdbx_version_info; +extern LIBMDBX_VERINFO_API const mdbx_version_info mdbx_version; + +/* MDBX build information. + * WARNING: Some strings could be NULL in case no corresponding information was + * provided at build time (i.e. flags). */ +typedef struct mdbx_build_info { + const char *datetime /* build timestamp (ISO-8601 or __DATE__ __TIME__) */; + const char *target /* cpu/arch-system-config triplet */; + const char *options /* mdbx-related options */; + const char *compiler /* compiler */; + const char *flags /* CFLAGS */; +} mdbx_build_info; +extern LIBMDBX_VERINFO_API const mdbx_build_info mdbx_build; + +#if defined(_WIN32) || defined(_WIN64) +#if !MDBX_BUILD_SHARED_LIBRARY + +/* MDBX internally uses global and thread local storage destructors to + * automatically (de)initialization, releasing reader lock table slots + * and so on. + * + * If MDBX builded as a DLL this is done out-of-the-box by DllEntry() function, + * which called automatically by Windows core with passing corresponding reason + * argument. + * + * Otherwise, if MDBX was builded not as a DLL, some black magic + * may be required depending of Windows version: + * - Modern Windows versions, including Windows Vista and later, provides + * support for "TLS Directory" (e.g .CRT$XL[A-Z] sections in executable + * or dll file). In this case, MDBX capable of doing all automatically, + * and you do not need to call mdbx_dll_handler(). + * - Obsolete versions of Windows, prior to Windows Vista, REQUIRES calling + * mdbx_dll_handler() manually from corresponding DllMain() or WinMain() + * of your DLL or application. + * - This behavior is under control of the MODX_CONFIG_MANUAL_TLS_CALLBACK + * option, which is determined by default according to the target version + * of Windows at build time. + * But you may override MODX_CONFIG_MANUAL_TLS_CALLBACK in special cases. + * + * Therefore, building MDBX as a DLL is recommended for all version of Windows. + * So, if you doubt, just build MDBX as the separate DLL and don't worry. */ + +#ifndef MDBX_CONFIG_MANUAL_TLS_CALLBACK +#if defined(_WIN32_WINNT_VISTA) && WINVER >= _WIN32_WINNT_VISTA +/* As described above mdbx_dll_handler() is NOT needed forWindows Vista + * and later. */ +#define MDBX_CONFIG_MANUAL_TLS_CALLBACK 0 +#else +/* As described above mdbx_dll_handler() IS REQUIRED for Windows versions + * prior to Windows Vista. */ +#define MDBX_CONFIG_MANUAL_TLS_CALLBACK 1 +#endif +#endif /* MDBX_CONFIG_MANUAL_TLS_CALLBACK */ + +#if MDBX_CONFIG_MANUAL_TLS_CALLBACK +void LIBMDBX_API NTAPI mdbx_dll_handler(PVOID module, DWORD reason, + PVOID reserved); +#endif /* MDBX_CONFIG_MANUAL_TLS_CALLBACK */ +#endif /* !MDBX_BUILD_SHARED_LIBRARY */ +#endif /* Windows */ + +/**** OPACITY STRUCTURES ******************************************************/ + +/* Opaque structure for a database environment. + * + * An environment supports multiple key-value databases (aka key-value spaces + * or tables), all residing in the same shared-memory map. */ +typedef struct MDBX_env MDBX_env; + +/* Opaque structure for a transaction handle. + * + * All database operations require a transaction handle. Transactions may be + * read-only or read-write. */ +typedef struct MDBX_txn MDBX_txn; + +/* A handle for an individual database (key-value spaces) in the environment. + * Zero handle is used internally (hidden Garbage Collection DB). + * So, any valid DBI-handle great than 0 and less than or equal MDBX_MAX_DBI. */ +typedef uint32_t MDBX_dbi; +#define MDBX_MAX_DBI UINT32_C(32765) + +/* Opaque structure for navigating through a database */ +typedef struct MDBX_cursor MDBX_cursor; + +/* Generic structure used for passing keys and data in and out of the database. + * + * Values returned from the database are valid only until a subsequent + * update operation, or the end of the transaction. Do not modify or + * free them, they commonly point into the database itself. + * + * Key sizes must be between 0 and mdbx_env_get_maxkeysize() inclusive. + * The same applies to data sizes in databases with the MDBX_DUPSORT flag. + * Other data items can in theory be from 0 to 0x7fffffff bytes long. + * + * (!) The notable difference between MDBX and LMDB is that MDBX support zero + * length keys. */ +#ifndef HAVE_STRUCT_IOVEC +struct iovec { + void *iov_base /* pointer to some data */; + size_t iov_len /* the length of data in bytes */; +}; +#define HAVE_STRUCT_IOVEC +#endif /* HAVE_STRUCT_IOVEC */ + +typedef struct iovec MDBX_val; + +/* The maximum size of a data item. + * MDBX only store a 32 bit value for node sizes. */ +#define MDBX_MAXDATASIZE INT32_MAX + +/**** DEBUG & LOGGING ********************************************************** + * Logging and runtime debug flags. + * + * NOTE: Most of debug feature enabled only when libmdbx builded with + * MDBX_DEBUG options. + */ + +/* Log level (requires build libmdbx with MDBX_DEBUG) */ +#define MDBX_LOG_FATAL 0 /* critical conditions, i.e. assertion failures */ +#define MDBX_LOG_ERROR 1 /* error conditions */ +#define MDBX_LOG_WARN 2 /* warning conditions */ +#define MDBX_LOG_NOTICE 3 /* normal but significant condition */ +#define MDBX_LOG_VERBOSE 4 /* verbose informational */ +#define MDBX_LOG_DEBUG 5 /* debug-level messages */ +#define MDBX_LOG_TRACE 6 /* trace debug-level messages */ +#define MDBX_LOG_EXTRA 7 /* extra debug-level messages (dump pgno lists) */ + +/* Runtime debug flags. + * + * MDBX_DBG_DUMP and MDBX_DBG_LEGACY_MULTIOPEN always have an effect, + * but MDBX_DBG_ASSERT, MDBX_DBG_AUDIT and MDBX_DBG_JITTER only if libmdbx + * builded with MDBX_DEBUG. */ + +#define MDBX_DBG_ASSERT 1 /* Enable assertion checks */ +#define MDBX_DBG_AUDIT 2 /* Enable pages usage audit at commit transactions */ +#define MDBX_DBG_JITTER 4 /* Enable small random delays in critical points */ +#define MDBX_DBG_DUMP 8 /* Include or not database(s) in coredump files */ +#define MDBX_DBG_LEGACY_MULTIOPEN 16 /* Enable multi-opening environment(s) */ + +/* A debug-logger callback function, + * called before printing the message and aborting. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] msg The assertion message, not including newline. */ +typedef void MDBX_debug_func(int loglevel, const char *function, int line, + const char *msg, va_list args); + +/* FIXME: Complete description */ +LIBMDBX_API int mdbx_setup_debug(int loglevel, int flags, + MDBX_debug_func *logger); + +/* A callback function for most MDBX assert() failures, + * called before printing the message and aborting. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] msg The assertion message, not including newline. */ +typedef void MDBX_assert_func(const MDBX_env *env, const char *msg, + const char *function, unsigned line); + +/* Set or reset the assert() callback of the environment. + * + * Does nothing if libmdbx was built with MDBX_DEBUG=0 or with NDEBUG, + * and will return MDBX_ENOSYS in such case. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] func An MDBX_assert_func function, or 0. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func); + +/* FIXME: Complete description */ +LIBMDBX_API const char *mdbx_dump_val(const MDBX_val *key, char *const buf, + const size_t bufsize); + +/**** THE FILES **************************************************************** + * At the file system level, the environment corresponds to a pair of files. */ + +/* The name of the lock file in the environment */ +#define MDBX_LOCKNAME "/mdbx.lck" +/* The name of the data file in the environment */ +#define MDBX_DATANAME "/mdbx.dat" + +/* The suffix of the lock file when MDBX_NOSUBDIR is used */ +#define MDBX_LOCK_SUFFIX "-lck" + +/**** ENVIRONMENT FLAGS *******************************************************/ + +/* MDBX_NOSUBDIR = no environment directory. + * + * By default, MDBX creates its environment in a directory whose pathname is + * given in path, and creates its data and lock files under that directory. + * With this option, path is used as-is for the database main data file. + * The database lock file is the path with "-lck" appended. + * + * - with MDBX_NOSUBDIR = in a filesystem we have the pair of MDBX-files which + * names derived from given pathname by appending predefined suffixes. + * + * - without MDBX_NOSUBDIR = in a filesystem we have the MDBX-directory with + * given pathname, within that a pair of MDBX-files with predefined names. + * + * This flag affects only at environment opening and can't be changed after. */ +#define MDBX_NOSUBDIR 0x4000u + +/* MDBX_RDONLY = read only mode. + * + * Open the environment in read-only mode. No write operations will be allowed. + * MDBX will still modify the lock file - except on read-only filesystems, + * where MDBX does not use locks. + * + * - with MDBX_RDONLY = open environment in read-only mode. + * MDBX supports pure read-only mode (i.e. without opening LCK-file) only + * when environment directory and/or both files are not writable (and the + * LCK-file may be missing). In such case allowing file(s) to be placed + * on a network read-only share. + * + * - without MDBX_RDONLY = open environment in read-write mode. + * + * This flag affects only at environment opening but can't be changed after. */ +#define MDBX_RDONLY 0x20000u + +/* MDBX_EXCLUSIVE = open environment in exclusive/monopolistic mode. + * + * MDBX_EXCLUSIVE flag can be used as a replacement for MDB_NOLOCK, which don't + * supported by MDBX. In this way, you can get the minimal overhead, but with + * the correct multi-process and mutli-thread locking. + * + * - with MDBX_EXCLUSIVE = open environment in exclusive/monopolistic mode + * or return MDBX_BUSY if environment already used by other process. + * The main feature of the exclusive mode is the ability to open the + * environment placed on a network share. + * + * - without MDBX_EXCLUSIVE = open environment in cooperative mode, + * i.e. for multi-process access/interaction/cooperation. + * The main requirements of the cooperative mode are: + * 1. data files MUST be placed in the LOCAL file system, + * but NOT on a network share. + * 2. environment MUST be opened only by LOCAL processes, + * but NOT over a network. + * 3. OS kernel (i.e. file system and memory mapping implementation) and + * all processes that open the given environment MUST be running + * in the physically single RAM with cache-coherency. The only + * exception for cache-consistency requirement is Linux on MIPS + * architecture, but this case has not been tested for a long time). + + * This flag affects only at environment opening but can't be changed after. */ +#define MDBX_EXCLUSIVE 0x400000u + +/* MDBX_WRITEMAP = map data into memory with write permission. + * + * Use a writeable memory map unless MDBX_RDONLY is set. This uses fewer mallocs + * and requires much less work for tracking database pages, but loses protection + * from application bugs like wild pointer writes and other bad updates into the + * database. This may be slightly faster for DBs that fit entirely in RAM, but + * is slower for DBs larger than RAM. Also adds the possibility for stray + * application writes thru pointers to silently corrupt the database. + * Incompatible with nested transactions. + * + * NOTE: The MDBX_WRITEMAP mode is incompatible with nested transactions, since + * this is unreasonable. I.e. nested transactions requires mallocation of + * database pages and more work for tracking ones, which neuters a + * performance boost caused by the MDBX_WRITEMAP mode. + * + * NOTE: MDBX don't allow to mix processes with and without MDBX_WRITEMAP on + * the same environment. In such case MDBX_INCOMPATIBLE will be generated. + * + * - with MDBX_WRITEMAP = all data will be mapped into memory in the read-write + * mode. This offers a significant performance benefit, since the data will + * be modified directly in mapped memory and then flushed to disk by + * single system call, without any memory management nor copying. + * (!) On the other hand, MDBX_WRITEMAP adds the possibility for stray + * application writes thru pointers to silently corrupt the database. + * Moreover, MDBX_WRITEMAP disallows nested write transactions. + * + * - without MDBX_WRITEMAP = data will be mapped into memory in the read-only + * mode. This requires stocking all modified database pages in memory and + * then writing them to disk through file operations. + * + * This flag affects only at environment opening but can't be changed after. */ +#define MDBX_WRITEMAP 0x80000u + +/* MDBX_NOTLS = tie reader locktable slots to read-only transactions instead + * of to threads. + * + * Don't use Thread-Local Storage, instead tie reader locktable slots to + * MDBX_txn objects instead of to threads. So, mdbx_txn_reset() keeps the slot + * reserved for the MDBX_txn object. A thread may use parallel read-only + * transactions. And a read-only transaction may span threads if you + * synchronizes its use. + * + * Applications that multiplex many user threads over individual OS threads need + * this option. Such an application must also serialize the write transactions + * in an OS thread, since MDBX's write locking is unaware of the user threads. + * + * NOTE: Regardless to MDBX_NOTLS flag a write transaction entirely should + * always be used in one thread from start to finish. MDBX checks this in a + * reasonable manner and return the MDBX_THREAD_MISMATCH error in rules + * violation. + * + * This flag affects only at environment opening but can't be changed after. */ +#define MDBX_NOTLS 0x200000u + +/* MDBX_NORDAHEAD = don't do readahead. + * + * Turn off readahead. Most operating systems perform readahead on read requests + * by default. This option turns it off if the OS supports it. Turning it off + * may help random read performance when the DB is larger than RAM and system + * RAM is full. + * + * NOTE: The mdbx_is_readahead_reasonable() function allows to quickly find out + * whether to use readahead or not based on the size of the data and the + * amount of available memory. + * + * This flag affects only at environment opening and can't be changed after. */ +#define MDBX_NORDAHEAD 0x800000u + +/* MDBX_NOMEMINIT = don't initialize malloc'd memory before writing to datafile. + * + * Don't initialize malloc'd memory before writing to unused spaces in the data + * file. By default, memory for pages written to the data file is obtained using + * malloc. While these pages may be reused in subsequent transactions, freshly + * malloc'd pages will be initialized to zeroes before use. This avoids + * persisting leftover data from other code (that used the heap and subsequently + * freed the memory) into the data file. + * + * Note that many other system libraries may allocate and free memory from the + * heap for arbitrary uses. E.g., stdio may use the heap for file I/O buffers. + * This initialization step has a modest performance cost so some applications + * may want to disable it using this flag. This option can be a problem for + * applications which handle sensitive data like passwords, and it makes memory + * checkers like Valgrind noisy. This flag is not needed with MDBX_WRITEMAP, + * which writes directly to the mmap instead of using malloc for pages. The + * initialization is also skipped if MDBX_RESERVE is used; the caller is + * expected to overwrite all of the memory that was reserved in that case. + * + * This flag may be changed at any time using mdbx_env_set_flags(). */ +#define MDBX_NOMEMINIT 0x1000000u + +/* MDBX_COALESCE = aims to coalesce a Garbage Collection items. + * + * With MDBX_COALESCE flag MDBX will aims to coalesce items while recycling + * a Garbage Collection. Technically, when possible short lists of pages will + * be combined into longer ones, but to fit on one database page. As a result, + * there will be fewer items in Garbage Collection and a page lists are longer, + * which slightly increases the likelihood of returning pages to Unallocated + * space and reducing the database file. + * + * This flag may be changed at any time using mdbx_env_set_flags(). */ +#define MDBX_COALESCE 0x2000000u + +/* MDBX_LIFORECLAIM = LIFO policy for recycling a Garbage Collection items. + * + * MDBX_LIFORECLAIM flag turns on LIFO policy for recycling a Garbage + * Collection items, instead of FIFO by default. On systems with a disk + * write-back cache, this can significantly increase write performance, up to + * several times in a best case scenario. + * + * LIFO recycling policy means that for reuse pages will be taken which became + * unused the lastest (i.e. just now or most recently). Therefore the loop of + * database pages circulation becomes as short as possible. In other words, the + * number of pages, that are overwritten in memory and on disk during a series + * of write transactions, will be as small as possible. Thus creates ideal + * conditions for the efficient operation of the disk write-back cache. + * + * MDBX_LIFORECLAIM is compatible with all no-sync flags (i.e. MDBX_NOMETASYNC, + * MDBX_NOSYNC, MDBX_UTTERLY_NOSYNC, MDBX_MAPASYNC), but gives no noticeable + * impact in combination with MDB_NOSYNC and MDX_MAPASYNC. Because MDBX will + * not reused paged from the last "steady" MVCC-snapshot and later, i.e. the + * loop length of database pages circulation will be mostly defined by frequency + * of calling mdbx_env_sync() rather than LIFO and FIFO difference. + * + * This flag may be changed at any time using mdbx_env_set_flags(). */ +#define MDBX_LIFORECLAIM 0x4000000u + +/* Debuging option, fill/perturb released pages. */ +#define MDBX_PAGEPERTURB 0x8000000u + +/**** SYNC MODES *************************************************************** + * (!!!) Using any combination of MDBX_NOSYNC, MDBX_NOMETASYNC, MDBX_MAPASYNC + * and especially MDBX_UTTERLY_NOSYNC is always a deal to reduce durability + * for gain write performance. You must know exactly what you are doing and + * what risks you are taking! + * + * NOTE for LMDB users: MDBX_NOSYNC is NOT similar to LMDB_NOSYNC, but + * MDBX_UTTERLY_NOSYNC is exactly match LMDB_NOSYNC. + * See details below. + * + * THE SCENE: + * - The DAT-file contains several MVCC-snapshots of B-tree at same time, + * each of those B-tree has its own root page. + * - Each of meta pages at the beginning of the DAT file contains a pointer + * to the root page of B-tree which is the result of the particular + * transaction, and a number of this transaction. + * - For data durability, MDBX must first write all MVCC-snapshot data pages + * and ensure that are written to the disk, then update a meta page with + * the new transaction number and a pointer to the corresponding new root + * page, and flush any buffers yet again. + * - Thus during commit a I/O buffers should be flushed to the disk twice; + * i.e. fdatasync(), FlushFileBuffers() or similar syscall should be called + * twice for each commit. This is very expensive for performance, but + * guaranteed durability even on unexpected system failure or power outage. + * Of course, provided that the operating system and the underlying hardware + * (e.g. disk) work correctly. + * + * TRADE-OFF: By skipping some stages described above, you can significantly + * benefit in speed, while partially or completely losing in the guarantee of + * data durability and/or consistency in the event of system or power failure. + * Moreover, if for any reason disk write order is not preserved, then at moment + * of a system crash, a meta-page with a pointer to the new B-tree may be + * written to disk, while the itself B-tree not yet. In that case, the database + * will be corrupted! + * + * + * MDBX_NOMETASYNC = don't sync the meta-page after commit. + * + * Flush system buffers to disk only once per transaction, omit the + * metadata flush. Defer that until the system flushes files to disk, + * or next non-MDBX_RDONLY commit or mdbx_env_sync(). Depending on the + * platform and hardware, with MDBX_NOMETASYNC you may get a doubling of + * write performance. + * + * This trade-off maintains database integrity, but a system crash may + * undo the last committed transaction. I.e. it preserves the ACI + * (atomicity, consistency, isolation) but not D (durability) database + * property. + * + * MDBX_NOMETASYNC flag may be changed at any time using + * mdbx_env_set_flags() or by passing to mdbx_txn_begin() for particular + * write transaction. + * + * + * MDBX_UTTERLY_NOSYNC = don't sync anything and wipe previous steady commits. + * + * Don't flush system buffers to disk when committing a transaction. This + * optimization means a system crash can corrupt the database, if buffers + * are not yet flushed to disk. Depending on the platform and hardware, + * with MDBX_UTTERLY_NOSYNC you may get a multiple increase of write + * performance, even 100 times or more. + * + * If the filesystem preserves write order (which is rare and never + * provided unless explicitly noted) and the MDBX_WRITEMAP and + * MDBX_LIFORECLAIM flags are not used, then a system crash can't corrupt + * the database, but you can lose the last transactions, if at least one + * buffer is not yet flushed to disk. The risk is governed by how often the + * system flushes dirty buffers to disk and how often mdbx_env_sync() is + * called. So, transactions exhibit ACI (atomicity, consistency, isolation) + * properties and only lose D (durability). I.e. database integrity is + * maintained, but a system crash may undo the final transactions. + * + * Otherwise, if the filesystem not preserves write order (which is + * typically) or MDBX_WRITEMAP or MDBX_LIFORECLAIM flags are used, you + * should expect the corrupted database after a system crash. + * + * So, most important thing about MDBX_UTTERLY_NOSYNC: + * - a system crash immediately after commit the write transaction + * high likely lead to database corruption. + * - successful completion of mdbx_env_sync(force = true) after one or + * more commited transactions guarantees consystency and durability. + * - BUT by committing two or more transactions you back database into a + * weak state, in which a system crash may lead to database corruption! + * In case single transaction after mdbx_env_sync, you may lose + * transaction itself, but not a whole database. + * + * Nevertheless, MDBX_UTTERLY_NOSYNC provides ACID in case of a application + * crash, and therefore may be very useful in scenarios where data + * durability is not required over a system failure (e.g for short-lived + * data), or if you can ignore such risk. + * + * MDBX_UTTERLY_NOSYNC flag may be changed at any time using + * mdbx_env_set_flags(), but don't has effect if passed to mdbx_txn_begin() + * for particular write transaction. + * + * + * MDBX_NOSYNC = don't sync anything but keep previous steady commits. + * + * Like MDBX_UTTERLY_NOSYNC the MDBX_NOSYNC flag similarly disable flush + * system buffers to disk when committing a transaction. But there is a + * huge difference in how are recycled the MVCC snapshots corresponding + * to previous "steady" transactions (see below). + * + * Depending on the platform and hardware, with MDBX_NOSYNC you may get + * a multiple increase of write performance, even 10 times or more. + * NOTE that (MDBX_NOSYNC | MDBX_WRITEMAP) leaves the system with no hint + * for when to write transactions to disk. Therefore the (MDBX_MAPASYNC | + * MDBX_WRITEMAP) may be preferable, but without MDBX_NOSYNC because + * the (MDBX_MAPASYNC | MDBX_NOSYNC) actually gives MDBX_UTTERLY_NOSYNC. + * + * In contrast to MDBX_UTTERLY_NOSYNC mode, with MDBX_NOSYNC flag MDBX will + * keeps untouched pages within B-tree of the last transaction "steady" + * which was synced to disk completely. This has big implications for both + * data durability and (unfortunately) performance: + * - a system crash can't corrupt the database, but you will lose the + * last transactions; because MDBX will rollback to last steady commit + * since it kept explicitly. + * - the last steady transaction makes an effect similar to "long-lived" + * read transaction (see above in the "RESTRICTIONS & CAVEATS" section) + * since prevents reuse of pages freed by newer write transactions, + * thus the any data changes will be placed in newly allocated pages. + * - to avoid rapid database growth, the system will sync data and issue + * a steady commit-point to resume reuse pages, each time there is + * insufficient space and before increasing the size of the file on + * disk. + * + * In other words, with MDBX_NOSYNC flag MDBX insures you from the whole + * database corruption, at the cost increasing database size and/or number + * of disk IOPS. So, MDBX_NOSYNC flag could be used with mdbx_env_synv() + * as alternatively for batch committing or nested transaction (in some + * cases). As well, auto-sync feature exposed by mdbx_env_set_syncbytes() + * and mdbx_env_set_syncperiod() functions could be very usefull with + * MDBX_NOSYNC flag. + * + * The number and volume of of disk IOPS with MDBX_NOSYNC flag will + * exactly the as without any no-sync flags. However, you should expect + * a larger process's work set (https://bit.ly/2kA2tFX) and significantly + * worse a locality of reference (https://bit.ly/2mbYq2J), due to the + * more intensive allocation of previously unused pages and increase the + * size of the database. + * + * MDBX_NOSYNC flag may be changed at any time using + * mdbx_env_set_flags() or by passing to mdbx_txn_begin() for particular + * write transaction. + * + * + * MDBX_MAPASYNC = use asynchronous msync when MDBX_WRITEMAP is used. + * + * MDBX_MAPASYNC meaningful and give effect only in conjunction + * with MDBX_WRITEMAP or MDBX_NOSYNC: + * - with MDBX_NOSYNC actually gives MDBX_UTTERLY_NOSYNC, which + * wipe previous steady commits for reuse pages as described above. + * - with MDBX_WRITEMAP but without MDBX_NOSYNC instructs MDBX to use + * asynchronous mmap-flushes to disk as described below. + * - with both MDBX_WRITEMAP and MDBX_NOSYNC you get the both effects. + * + * Asynchronous mmap-flushes means that actually all writes will scheduled + * and performed by operation system on it own manner, i.e. unordered. + * MDBX itself just notify operating system that it would be nice to write + * data to disk, but no more. + * + * With MDBX_MAPASYNC flag, but without MDBX_UTTERLY_NOSYNC (i.e. without + * OR'ing with MDBX_NOSYNC) MDBX will keeps untouched pages within B-tree + * of the last transaction "steady" which was synced to disk completely. + * So, this makes exactly the same "long-lived" impact and the same + * consequences as described above for MDBX_NOSYNC flag. + * + * Depending on the platform and hardware, with combination of + * MDBX_WRITEMAP and MDBX_MAPASYNC you may get a multiple increase of write + * performance, even 25 times or more. MDBX_MAPASYNC flag may be changed at + * any time using mdbx_env_set_flags() or by passing to mdbx_txn_begin() + * for particular write transaction. + */ + +/* Don't sync meta-page after commit, + * see description in the "SYNC MODES" section above. */ +#define MDBX_NOMETASYNC 0x40000u + +/* Don't sync anything but keep previous steady commits, + * see description in the "SYNC MODES" section above. + * + * (!) don't combine this flag with MDBX_MAPASYNC + * since you will got MDBX_UTTERLY_NOSYNC in that way (see below) */ +#define MDBX_NOSYNC 0x10000u + +/* Use asynchronous msync when MDBX_WRITEMAP is used, + * see description in the "SYNC MODES" section above. + * + * (!) don't combine this flag with MDBX_NOSYNC + * since you will got MDBX_UTTERLY_NOSYNC in that way (see below) */ +#define MDBX_MAPASYNC 0x100000u + +/* Don't sync anything and wipe previous steady commits, + * see description in the "SYNC MODES" section above. */ +#define MDBX_UTTERLY_NOSYNC (MDBX_NOSYNC | MDBX_MAPASYNC) + +/**** DATABASE FLAGS **********************************************************/ +/* Use reverse string keys */ +#define MDBX_REVERSEKEY 0x02u +/* Use sorted duplicates */ +#define MDBX_DUPSORT 0x04u +/* Numeric keys in native byte order, either uint32_t or uint64_t. + * The keys must all be of the same size. */ +#define MDBX_INTEGERKEY 0x08u +/* With MDBX_DUPSORT, sorted dup items have fixed size */ +#define MDBX_DUPFIXED 0x10u +/* With MDBX_DUPSORT, dups are MDBX_INTEGERKEY-style integers */ +#define MDBX_INTEGERDUP 0x20u +/* With MDBX_DUPSORT, use reverse string dups */ +#define MDBX_REVERSEDUP 0x40u +/* Create DB if not already existing */ +#define MDBX_CREATE 0x40000u + +/**** DATA UPDATE FLAGS *******************************************************/ +/* For put: Don't write if the key already exists. */ +#define MDBX_NOOVERWRITE 0x10u +/* Only for MDBX_DUPSORT + * For put: don't write if the key and data pair already exist. + * For mdbx_cursor_del: remove all duplicate data items. */ +#define MDBX_NODUPDATA 0x20u +/* For mdbx_cursor_put: overwrite the current key/data pair + * MDBX allows this flag for mdbx_put() for explicit overwrite/update without + * insertion. */ +#define MDBX_CURRENT 0x40u +/* For put: Just reserve space for data, don't copy it. Return a + * pointer to the reserved space. */ +#define MDBX_RESERVE 0x10000u +/* Data is being appended, don't split full pages. */ +#define MDBX_APPEND 0x20000u +/* Duplicate data is being appended, don't split full pages. */ +#define MDBX_APPENDDUP 0x40000u +/* Store multiple data items in one call. Only for MDBX_DUPFIXED. */ +#define MDBX_MULTIPLE 0x80000u + +/**** TRANSACTION FLAGS *******************************************************/ +/* Do not block when starting a write transaction */ +#define MDBX_TRYTXN 0x10000000u + +/**** ENVIRONMENT COPY FLAGS **************************************************/ +/* Compacting: Omit free space from copy, and renumber all pages sequentially */ +#define MDBX_CP_COMPACT 1u + +/**** CURSOR OPERATIONS ******************************************************** + * + * This is the set of all operations for retrieving data + * using a cursor. */ +typedef enum MDBX_cursor_op { + MDBX_FIRST, /* Position at first key/data item */ + MDBX_FIRST_DUP, /* MDBX_DUPSORT-only: Position at first data item + * of current key. */ + MDBX_GET_BOTH, /* MDBX_DUPSORT-only: Position at key/data pair. */ + MDBX_GET_BOTH_RANGE, /* MDBX_DUPSORT-only: position at key, nearest data. */ + MDBX_GET_CURRENT, /* Return key/data at current cursor position */ + MDBX_GET_MULTIPLE, /* MDBX_DUPFIXED-only: Return up to a page of duplicate + * data items from current cursor position. + * Move cursor to prepare for MDBX_NEXT_MULTIPLE. */ + MDBX_LAST, /* Position at last key/data item */ + MDBX_LAST_DUP, /* MDBX_DUPSORT-only: Position at last data item + * of current key. */ + MDBX_NEXT, /* Position at next data item */ + MDBX_NEXT_DUP, /* MDBX_DUPSORT-only: Position at next data item + * of current key. */ + MDBX_NEXT_MULTIPLE, /* MDBX_DUPFIXED-only: Return up to a page of + * duplicate data items from next cursor position. + * Move cursor to prepare for MDBX_NEXT_MULTIPLE. */ + MDBX_NEXT_NODUP, /* Position at first data item of next key */ + MDBX_PREV, /* Position at previous data item */ + MDBX_PREV_DUP, /* MDBX_DUPSORT-only: Position at previous data item + * of current key. */ + MDBX_PREV_NODUP, /* Position at last data item of previous key */ + MDBX_SET, /* Position at specified key */ + MDBX_SET_KEY, /* Position at specified key, return both key and data */ + MDBX_SET_RANGE, /* Position at first key greater than or equal to + * specified key. */ + MDBX_PREV_MULTIPLE /* MDBX_DUPFIXED-only: Position at previous page and + * return up to a page of duplicate data items. */ +} MDBX_cursor_op; + +/**** ERRORS & RETURN CODES **************************************************** + * BerkeleyDB uses -30800 to -30999, we'll go under them */ + +/* Successful result */ +#define MDBX_SUCCESS 0 +#define MDBX_RESULT_FALSE MDBX_SUCCESS +/* Successful result with special meaning or a flag */ +#define MDBX_RESULT_TRUE (-1) + +/* key/data pair already exists */ +#define MDBX_KEYEXIST (-30799) +/* key/data pair not found (EOF) */ +#define MDBX_NOTFOUND (-30798) +/* Requested page not found - this usually indicates corruption */ +#define MDBX_PAGE_NOTFOUND (-30797) +/* Database is corrupted (page was wrong type and so on) */ +#define MDBX_CORRUPTED (-30796) +/* Environment had fatal error (i.e. update of meta page failed and so on) */ +#define MDBX_PANIC (-30795) +/* DB file version mismatch with libmdbx */ +#define MDBX_VERSION_MISMATCH (-30794) +/* File is not a valid MDBX file */ +#define MDBX_INVALID (-30793) +/* Environment mapsize reached */ +#define MDBX_MAP_FULL (-30792) +/* Environment maxdbs reached */ +#define MDBX_DBS_FULL (-30791) +/* Environment maxreaders reached */ +#define MDBX_READERS_FULL (-30790) +/* Txn has too many dirty pages */ +#define MDBX_TXN_FULL (-30788) +/* Cursor stack too deep - internal error */ +#define MDBX_CURSOR_FULL (-30787) +/* Page has not enough space - internal error */ +#define MDBX_PAGE_FULL (-30786) +/* Database contents grew beyond environment mapsize */ +#define MDBX_MAP_RESIZED (-30785) +/* Operation and DB incompatible, or DB type changed. This can mean: + * - The operation expects an MDBX_DUPSORT / MDBX_DUPFIXED database. + * - Opening a named DB when the unnamed DB has MDBX_DUPSORT/MDBX_INTEGERKEY. + * - Accessing a data record as a database, or vice versa. + * - The database was dropped and recreated with different flags. */ +#define MDBX_INCOMPATIBLE (-30784) +/* Invalid reuse of reader locktable slot */ +#define MDBX_BAD_RSLOT (-30783) +/* Transaction must abort, has a child, or is invalid */ +#define MDBX_BAD_TXN (-30782) +/* Unsupported size of key/DB name/data, or wrong DUPFIXED size */ +#define MDBX_BAD_VALSIZE (-30781) +/* The specified DBI was changed unexpectedly */ +#define MDBX_BAD_DBI (-30780) +/* Unexpected problem - txn should abort */ +#define MDBX_PROBLEM (-30779) +/* Another write transaction is running or environment is already used while + * opening with MDBX_EXCLUSIVE flag */ +#define MDBX_BUSY (-30778) +/* The last defined error code */ +#define MDBX_LAST_ERRCODE MDBX_BUSY + +/* The mdbx_put() or mdbx_replace() was called for key, + * that has more that one associated value. */ +#define MDBX_EMULTIVAL (-30421) + +/* Bad signature of a runtime object(s), this can mean: + * - memory corruption or double-free; + * - ABI version mismatch (rare case); */ +#define MDBX_EBADSIGN (-30420) + +/* Database should be recovered, but this could NOT be done automatically + * right now (e.g. in readonly mode and so forth). */ +#define MDBX_WANNA_RECOVERY (-30419) + +/* The given key value is mismatched to the current cursor position, + * when mdbx_cursor_put() called with MDBX_CURRENT option. */ +#define MDBX_EKEYMISMATCH (-30418) + +/* Database is too large for current system, + * e.g. could NOT be mapped into RAM. */ +#define MDBX_TOO_LARGE (-30417) + +/* A thread has attempted to use a not owned object, + * e.g. a transaction that started by another thread. */ +#define MDBX_THREAD_MISMATCH (-30416) + +/**** FUNCTIONS & RELATED STRUCTURES ******************************************/ + +/* Return a string describing a given error code. + * + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) + * function. If the error code is greater than or equal to 0, then the string + * returned by the system function strerror(3) is returned. If the error code + * is less than 0, an error string corresponding to the MDBX library error is + * returned. See errors for a list of MDBX-specific error codes. + * + * mdbx_strerror() - is NOT thread-safe because may share common internal + * buffer for system maessages. The returned string must + * NOT be modified by the application, but MAY be modified + * by a subsequent call to mdbx_strerror(), strerror() and + * other related functions. + * + * mdbx_strerror_r() - is thread-safe since uses user-supplied buffer where + * appropriate. The returned string must NOT be modified + * by the application, since it may be pointer to internal + * constatn string. However, there is no restriction if the + * returned string points to the supplied buffer. + * + * [in] err The error code. + * + * Returns "error message" The description of the error. */ +LIBMDBX_API const char *mdbx_strerror(int errnum); +LIBMDBX_API const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen); + +#if defined(_WIN32) || defined(_WIN64) +/* Bit of Windows' madness. The similar functions but returns Windows + * error-messages in the OEM-encoding for console utilities. */ +LIBMDBX_API const char *mdbx_strerror_ANSI2OEM(int errnum); +LIBMDBX_API const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, + size_t buflen); +#endif /* Bit of Windows' madness */ + +/* Create an MDBX environment instance. + * + * This function allocates memory for a MDBX_env structure. To release + * the allocated memory and discard the handle, call mdbx_env_close(). + * Before the handle may be used, it must be opened using mdbx_env_open(). + * + * Various other options may also need to be set before opening the handle, + * e.g. mdbx_env_set_geometry(), mdbx_env_set_maxreaders(), + * mdbx_env_set_maxdbs(), depending on usage requirements. + * + * [out] env The address where the new handle will be stored. + * + * Returns a non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_create(MDBX_env **penv); + +/* Open an environment instance. + * + * Indifferently this function will fails or not, the mdbx_env_close() must be + * called later to discard the MDBX_env handle and release associated resources. + * + * [in] env An environment handle returned by mdbx_env_create() + * [in] pathname The directory in which the database files reside. + * This directory must already exist and be writable. + * [in] flags Special options for this environment. This parameter + * must be set to 0 or by bitwise OR'ing together one + * or more of the values described above in the + * "ENVIRONMENT FLAGS" and "SYNC MODES" sections. + * + * Flags set by mdbx_env_set_flags() are also used: + * - MDBX_NOSUBDIR, MDBX_RDONLY, MDBX_EXCLUSIVE, MDBX_WRITEMAP, MDBX_NOTLS, + * MDBX_NORDAHEAD, MDBX_NOMEMINIT, MDBX_COALESCE, MDBX_LIFORECLAIM. + * See "ENVIRONMENT FLAGS" section above. + * + * - MDBX_NOMETASYNC, MDBX_NOSYNC, MDBX_UTTERLY_NOSYNC, MDBX_MAPASYNC. + * See "SYNC MODES" section above. + * + * NOTE: MDB_NOLOCK flag don't supported by MDBX, + * try use MDBX_EXCLUSIVE as a replacement. + * + * NOTE: MDBX don't allow to mix processes with different MDBX_WRITEMAP, + * MDBX_NOSYNC, MDBX_NOMETASYNC, MDBX_MAPASYNC flags onthe same + * environment. In such case MDBX_INCOMPATIBLE will be returned. + * + * If the database is already exist and parameters specified early by + * mdbx_env_set_geometry() are incompatible (i.e. for instance, different page + * size) then mdbx_env_open() will return MDBX_INCOMPATIBLE error. + * + * [in] mode The UNIX permissions to set on created files. Zero value means + * to open existing, but do not create. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_VERSION_MISMATCH = the version of the MDBX library doesn't match the + * version that created the database environment. + * - MDBX_INVALID = the environment file headers are corrupted. + * - MDBX_ENOENT = the directory specified by the path parameter + * doesn't exist. + * - MDBX_EACCES = the user didn't have permission to access + * the environment files. + * - MDBX_EAGAIN = the environment was locked by another process. + * - MDBX_BUSY = MDBX_EXCLUSIVE flag was specified and the + * environment is in use by another process, + * or the current process tries to open environment + * more than once. + * - MDBX_INCOMPATIBLE = Environment is already opened by another process, + * but with different set of MDBX_WRITEMAP, + * MDBX_NOSYNC, MDBX_NOMETASYNC, MDBX_MAPASYNC + * flags. + * Or if the database is already exist and + * parameters specified early by + * mdbx_env_set_geometry() are incompatible (i.e. + * for instance, different page size). + * - MDBX_WANNA_RECOVERY = MDBX_RDONLY flag was specified but read-write + * access is required to rollback inconsistent state + * after a system crash. + * - MDBX_TOO_LARGE = Database is too large for this process, i.e. + * 32-bit process tries to open >4Gb database. */ +LIBMDBX_API int mdbx_env_open(MDBX_env *env, const char *pathname, + unsigned flags, mode_t mode); + +/* Copy an MDBX environment to the specified path, with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * NOTE: This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under "Caveats" section. + * + * [in] env An environment handle returned by mdbx_env_create(). It must + * have already been opened successfully. + * [in] dest The directory in which the copy will reside. This directory + * must already exist and be writable but must otherwise be empty. + * [in] flags Special options for this operation. This parameter must be set + * to 0 or by bitwise OR'ing together one or more of the values + * described here: + * + * - MDBX_CP_COMPACT + * Perform compaction while copying: omit free pages and sequentially + * renumber all pages in output. This option consumes little bit more + * CPU for processing, but may running quickly than the default, on + * account skipping free pages. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_copy(MDBX_env *env, const char *dest, unsigned flags); + +/* Copy an MDBX environment to the specified file descriptor, + * with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. See + * mdbx_env_copy() for further details. + * + * NOTE: This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under "Caveats" section. + * + * NOTE: Fails if the environment has suffered a page leak and the destination + * file descriptor is associated with a pipe, socket, or FIFO. + * + * [in] env An environment handle returned by mdbx_env_create(). It must + * have already been opened successfully. + * [in] fd The filedescriptor to write the copy to. It must have already + * been opened for Write access. + * [in] flags Special options for this operation. See mdbx_env_copy() for + * options. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, + unsigned flags); + +/* Statistics for a database in the environment */ +typedef struct MDBX_stat { + uint32_t ms_psize; /* Size of a database page. + * This is the same for all databases. */ + uint32_t ms_depth; /* Depth (height) of the B-tree */ + uint64_t ms_branch_pages; /* Number of internal (non-leaf) pages */ + uint64_t ms_leaf_pages; /* Number of leaf pages */ + uint64_t ms_overflow_pages; /* Number of overflow pages */ + uint64_t ms_entries; /* Number of data items */ + uint64_t ms_mod_txnid; /* Transaction ID of commited last modification */ +} MDBX_stat; + +/* Return statistics about the MDBX environment. + * + * At least one of env or txn argument must be non-null. If txn is passed + * non-null then stat will be filled accordingly to the given transaction. + * Otherwise, if txn is null, then stat will be populated by a snapshot from the + * last committed write transaction, and at next time, other information can be + * returned. + * + * Legacy mdbx_env_stat() correspond to calling mdbx_env_stat_ex() with the null + * txn argument. + * + * [in] env An environment handle returned by mdbx_env_create() + * [in] txn A transaction handle returned by mdbx_txn_begin() + * [out] stat The address of an MDBX_stat structure where the statistics + * will be copied + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, + MDBX_stat *stat, size_t bytes); +LIBMDBX_API int mdbx_env_stat(MDBX_env *env, MDBX_stat *stat, size_t bytes); + +/* Information about the environment */ +typedef struct MDBX_envinfo { + struct { + uint64_t lower; /* lower limit for datafile size */ + uint64_t upper; /* upper limit for datafile size */ + uint64_t current; /* current datafile size */ + uint64_t shrink; /* shrink threshold for datafile */ + uint64_t grow; /* growth step for datafile */ + } mi_geo; + uint64_t mi_mapsize; /* Size of the data memory map */ + uint64_t mi_last_pgno; /* ID of the last used page */ + uint64_t mi_recent_txnid; /* ID of the last committed transaction */ + uint64_t mi_latter_reader_txnid; /* ID of the last reader transaction */ + uint64_t mi_self_latter_reader_txnid; /* ID of the last reader transaction of + caller process */ + uint64_t mi_meta0_txnid, mi_meta0_sign; + uint64_t mi_meta1_txnid, mi_meta1_sign; + uint64_t mi_meta2_txnid, mi_meta2_sign; + uint32_t mi_maxreaders; /* max reader slots in the environment */ + uint32_t mi_numreaders; /* max reader slots used in the environment */ + uint32_t mi_dxb_pagesize; /* database pagesize */ + uint32_t mi_sys_pagesize; /* system pagesize */ + + uint64_t + mi_bootid[2]; /* A mostly unique ID that is regenerated on each boot. + As such it can be used to identify the local + machine's current boot. MDBX uses such when open + the database to determine whether rollback required + to the last steady sync point or not. I.e. if current + bootid is differ from the value within a database then + the system was rebooted and all changes since last steady + sync must be reverted for data integrity. Zeros mean that + no relevant information is available from the system. */ + uint64_t mi_unsync_volume; /* bytes not explicitly synchronized to disk */ + uint64_t mi_autosync_threshold; /* current auto-sync threshold, see + mdbx_env_set_syncbytes(). */ + uint32_t mi_since_sync_seconds16dot16; /* time since the last steady sync in + 1/65536 of second */ + uint32_t mi_autosync_period_seconds16dot16 /* current auto-sync period in + 1/65536 of second, see + mdbx_env_set_syncperiod(). */ + ; + uint32_t mi_since_reader_check_seconds16dot16; /* time since the last readers + check in 1/65536 of second, + see mdbx_reader_check(). */ + uint32_t mi_mode; /* current environment mode, the same as + mdbx_env_get_flags() returns. */ +} MDBX_envinfo; + +/* Return information about the MDBX environment. + * + * At least one of env or txn argument must be non-null. If txn is passed + * non-null then stat will be filled accordingly to the given transaction. + * Otherwise, if txn is null, then stat will be populated by a snapshot from the + * last committed write transaction, and at next time, other information can be + * returned. + * + * Legacy mdbx_env_info() correspond to calling mdbx_env_info_ex() with the null + * txn argument. + + * [in] env An environment handle returned by mdbx_env_create() + * [in] txn A transaction handle returned by mdbx_txn_begin() + * [out] stat The address of an MDBX_envinfo structure + * where the information will be copied + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, + MDBX_envinfo *info, size_t bytes); +LIBMDBX_API int mdbx_env_info(MDBX_env *env, MDBX_envinfo *info, size_t bytes); + +/* Flush the environment data buffers to disk. + * + * Unless the environment was opened with no-sync flags (MDBX_NOMETASYNC, + * MDBX_NOSYNC, MDBX_UTTERLY_NOSYNC and MDBX_MAPASYNC), then data is always + * written an flushed to disk when mdbx_txn_commit() is called. Otherwise + * mdbx_env_sync() may be called to manually write and flush unsynced data to + * disk. + * + * Besides, mdbx_env_sync_ex() with argument force=false may be used to + * provide polling mode for lazy/asynchronous sync in conjunction with + * mdbx_env_set_syncbytes() and/or mdbx_env_set_syncperiod(). + * + * The mdbx_env_sync() is shortcut to calling mdbx_env_sync_ex() with + * try force=true and nonblock=false arguments. + * + * The mdbx_env_sync_poll() is shortcut to calling mdbx_env_sync_ex() with + * the force=false and nonblock=true arguments. + * + * NOTE: This call is not valid if the environment was opened with MDBX_RDONLY. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] force If non-zero, force a flush. Otherwise, if force is zero, then + * will run in polling mode, i.e. it will check the thresholds + * that were set mdbx_env_set_syncbytes() and/or + * mdbx_env_set_syncperiod() and perform flush If at least one + * of the thresholds is reached. + * [in] nonblock Don't wait if write transaction is running by other thread. + * + * Returns A non-zero error value on failure and MDBX_RESULT_TRUE or 0 on + * success. The MDBX_RESULT_TRUE means no data pending for flush to disk, + * and 0 otherwise. Some possible errors are: + * - MDBX_EACCES = the environment is read-only. + * - MDBX_BUSY = the environment is used by other thread and nonblock=true. + * - MDBX_EINVAL = an invalid parameter was specified. + * - MDBX_EIO = an error occurred during synchronization. */ +LIBMDBX_API int mdbx_env_sync_ex(MDBX_env *env, int force, int nonblock); +LIBMDBX_API int mdbx_env_sync(MDBX_env *env); +LIBMDBX_API int mdbx_env_sync_poll(MDBX_env *env); + +/* Sets threshold to force flush the data buffers to disk, even of MDBX_NOSYNC, + * MDBX_NOMETASYNC and MDBX_MAPASYNC flags in the environment. The threshold + * value affects all processes which operates with given environment until the + * last process close environment or a new value will be settled. + * + * Data is always written to disk when mdbx_txn_commit() is called, but the + * operating system may keep it buffered. MDBX always flushes the OS buffers + * upon commit as well, unless the environment was opened with MDBX_NOSYNC, + * MDBX_MAPASYNC or in part MDBX_NOMETASYNC. + * + * The default is 0, than mean no any threshold checked, and no additional + * flush will be made. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] threshold The size in bytes of summary changes when a synchronous + * flush would be made. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold); + +/* Sets relative period since the last unsteay commit to force flush the data + * buffers to disk, even of MDBX_NOSYNC, MDBX_NOMETASYNC and MDBX_MAPASYNC flags + * in the environment. The relative period value affects all processes which + * operates with given environment until the last process close environment or a + * new value will be settled. + * + * Data is always written to disk when mdbx_txn_commit() is called, but the + * operating system may keep it buffered. MDBX always flushes the OS buffers + * upon commit as well, unless the environment was opened with MDBX_NOSYNC, + * MDBX_MAPASYNC or in part MDBX_NOMETASYNC. + * + * Settled period don't checked asynchronously, but only by the + * mdbx_txn_commit() and mdbx_env_sync() functions. Therefore, in cases where + * transactions are committed infrequently and/or irregularly, polling by + * mdbx_env_sync() may be a reasonable solution to timeout enforcement. + * + * The default is 0, than mean no any timeout checked, and no additional + * flush will be made. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] seconds_16dot16 The period in 1/65536 of second when a synchronous + * flush would be made since the last unsteay commit. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_set_syncperiod(MDBX_env *env, + unsigned seconds_16dot16); + +/* Close the environment and release the memory map. + * + * Only a single thread may call this function. All transactions, databases, + * and cursors must already be closed before calling this function. Attempts + * to use any such handles after calling this function will cause a SIGSEGV. + * The environment handle will be freed and must not be used again after this + * call. + * + * Legacy mdbx_env_close() correspond to calling mdbx_env_close_ex() with the + * argument dont_sync=false. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] dont_sync A dont'sync flag, if non-zero the last checkpoint (meta-page + * update) will be kept "as is" and may be still "weak" in the + * NOSYNC/MAPASYNC modes. Such "weak" checkpoint will be + * ignored on opening next time, and transactions since the + * last non-weak checkpoint (meta-page update) will rolledback + * for consistency guarantee. + * + * Returns A non-zero error value on failure and 0 on success. + * Some possible errors are: + * - MDBX_BUSY = The write transaction is running by other thread, in such + * case MDBX_env instance has NOT be destroyed not released! + * NOTE: if any OTHER error code was returned then given + * MDBX_env instance has been destroyed and released. + * - MDBX_PANIC = If mdbx_env_close_ex() was called in the child process + * after fork(). In this case MDBX_PANIC is a expecte, + * i.e. MDBX_env instance was freed in proper manner. + * - MDBX_EIO = an error occurred during synchronization. */ +LIBMDBX_API int mdbx_env_close_ex(MDBX_env *env, int dont_sync); +LIBMDBX_API int mdbx_env_close(MDBX_env *env); + +/* Set environment flags. + * + * This may be used to set some flags in addition to those from + * mdbx_env_open(), or to unset these flags. + * + * NOTE: In contrast to LMDB, the MDBX serialize threads via mutex while + * changing the flags. Therefore this function will be blocked while a write + * transaction running by other thread, or MDBX_BUSY will be returned if + * function called within a write transaction. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] flags The flags to change, bitwise OR'ed together. + * [in] onoff A non-zero value sets the flags, zero clears them. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_env_set_flags(MDBX_env *env, unsigned flags, int onoff); + +/* Get environment flags. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [out] flags The address of an integer to store the flags. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_env_get_flags(MDBX_env *env, unsigned *flags); + +/* Return the path that was used in mdbx_env_open(). + * + * [in] env An environment handle returned by mdbx_env_create() + * [out] dest Address of a string pointer to contain the path. + * This is the actual string in the environment, not a copy. + * It should not be altered in any way. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_env_get_path(MDBX_env *env, const char **dest); + +/* Return the file descriptor for the given environment. + * + * NOTE: All MDBX file descriptors have FD_CLOEXEC and + * could't be used after exec() and or fork(). + * + * [in] env An environment handle returned by mdbx_env_create(). + * [out] fd Address of a int to contain the descriptor. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_env_get_fd(MDBX_env *env, mdbx_filehandle_t *fd); + +/* Set all size-related parameters of environment, including page size and the + * min/max size of the memory map. + * + * In contrast to LMDB, the MDBX provide automatic size management of an + * database according the given parameters, including shrinking and resizing + * on the fly. From user point of view all of these just working. Nevertheless, + * it is reasonable to know some details in order to make optimal decisions when + * choosing parameters. + * + * Both mdbx_env_info_ex() and legacy mdbx_env_info() are inapplicable to + * read-only opened environment. + * + * Both mdbx_env_info_ex() and legacy mdbx_env_info() could be called either + * before or after mdbx_env_open(), either within the write transaction running + * by current thread or not: + * + * - In case mdbx_env_info_ex() or legacy mdbx_env_info() was called BEFORE + * mdbx_env_open(), i.e. for closed environment, then the specified + * parameters will be used for new database creation, or will be appliend + * during openeing if database exists and no other process using it. + * + * If the database is already exist, opened with MDBX_EXCLUSIVE or not used + * by any other process, and parameters specified by mdbx_env_set_geometry() + * are incompatible (i.e. for instance, different page size) then + * mdbx_env_open() will return MDBX_INCOMPATIBLE error. + * + * In another way, if database will opened read-only or will used by other + * process during calling mdbx_env_open() that specified parameters will + * silently discarded (open the database with MDBX_EXCLUSIVE flag to avoid + * this). + * + * - In case mdbx_env_info_ex() or legacy mdbx_env_info() was called after + * mdbx_env_open() WITHIN the write transaction running by current thread, + * then specified parameters will be appliad as a part of write transaction, + * i.e. will not be visible to any others processes until the current write + * transaction has been committed by the current process. However, if + * transaction will be aborted, then the database file will be reverted to + * the previous size not immediately, but when a next transaction will be + * committed or when the database will be opened next time. + * + * - In case mdbx_env_info_ex() or legacy mdbx_env_info() was called after + * mdbx_env_open() but OUTSIDE a write transaction, then MDBX will execute + * internal pseudo-transaction to apply new parameters (but only if anything + * has been changed), and changes be visible to any others processes + * immediatelly after succesfull competeion of function. + * + * Essentially a concept of "automatic size management" is simple and useful: + * - There are the lower and upper bound of the database file size; + * - There is the growth step by which the database file will be increased, + * in case of lack of space. + * - There is the threshold for unused space, beyond which the database file + * will be shrunk. + * - The size of the memory map is also the maximum size of the database. + * - MDBX will automatically manage both the size of the database and the size + * of memory map, according to the given parameters. + * + * So, there some considerations about choosing these parameters: + * - The lower bound allows you to prevent database shrinking below some + * rational size to avoid unnecessary resizing costs. + * - The upper bound allows you to prevent database growth above some rational + * size. Besides, the upper bound defines the linear address space + * reservation in each process that opens the database. Therefore changing + * the upper bound is costly and may be required reopening environment in + * case of MDBX_MAP_RESIZED errors, and so on. Therefore, this value should + * be chosen reasonable as large as possible, to accommodate future growth + * of the database. + * - The growth step must be greater than zero to allow the database to grow, + * but also reasonable not too small, since increasing the size by little + * steps will result a large overhead. + * - The shrink threshold must be greater than zero to allow the database + * to shrink but also reasonable not too small (to avoid extra overhead) and + * not less than growth step to avoid up-and-down flouncing. + * - The current size (i.e. size_now argument) is an auxiliary parameter for + * simulation legacy mdbx_env_set_mapsize() and as workaround Windows issues + * (see below). + * + * Unfortunately, Windows has is a several issues + * with resizing of memory-mapped file: + * - Windows unable shrinking a memory-mapped file (i.e memory-mapped section) + * in any way except unmapping file entirely and then map again. Moreover, + * it is impossible in any way if a memory-mapped file is used more than + * one process. + * - Windows does not provide the usual API to augment a memory-mapped file + * (that is, a memory-mapped partition), but only by using "Native API" + * in an undocumented way. + * MDBX bypasses all Windows issues, but at a cost: + * - Ability to resize database on the fly requires an additional lock + * and release SlimReadWriteLock during each read-only transaction. + * - During resize all in-process threads should be paused and then resumed. + * - Shrinking of database file is performed only when it used by single + * process, i.e. when a database closes by the last process or opened + * by the first. + * = Therefore, the size_now argument may be useful to set database size + * by the first process which open a database, and thus avoid expensive + * remapping further. + * + * For create a new database with particular parameters, including the page + * size, mdbx_env_set_geometry() should be called after mdbx_env_create() and + * before mdbx_env_open(). Once the database is created, the page size cannot be + * changed. If you do not specify all or some of the parameters, the + * corresponding default values will be used. For instance, the default for + * database size is 10485760 bytes. + * + * If the mapsize is increased by another process, MDBX silently and + * transparently adopt these changes at next transaction start. However, + * mdbx_txn_begin() will return MDBX_MAP_RESIZED if new mapping size could not + * be applied for current process (for instance if address space is busy). + * Therefore, in the case of MDBX_MAP_RESIZED error you need close and reopen + * the environment to resolve error. + * + * NOTE: Actual values may be different than your have specified because of + * rounding to specified database page size, the system page size and/or the + * size of the system virtual memory management unit. You can get actual values + * by mdbx_env_sync_ex() or see by using the tool "mdbx_chk" with the "-v" + * option. + * + * Legacy mdbx_env_set_mapsize() correspond to calling mdbx_env_set_geometry() + * with the arguments size_lower, size_now, size_upper equal to the size + * and -1 (i.e. default) for all other parameters. + * + * [in] env An environment handle returned by mdbx_env_create() + * + * [in] size_lower The lower bound of database sive in bytes. + * Zero value means "minimal acceptable", + * and negative means "keep current or use default". + * + * [in] size_now The size in bytes to setup the database size for now. + * Zero value means "minimal acceptable", + * and negative means "keep current or use default". + * So, it is recommended always pass -1 in this argument + * except some special cases. + * + * [in] size_upper The upper bound of database sive in bytes. + * Zero value means "minimal acceptable", + * and negative means "keep current or use default". + * It is recommended to avoid change upper bound while + * database is used by other processes or threaded + * (i.e. just pass -1 in this argument except absolutely + * necessity). Otherwise you must be ready for + * MDBX_MAP_RESIZED error(s), unexpected pauses during + * remapping and/or system errors like "addtress busy", + * and so on. In other words, there is no way to handle + * a growth of the upper bound robustly because there may + * be a lack of appropriate system resources (which are + * extremely volatile in a multi-process multi-threaded + * environment). + * + * [in] growth_step The growth step in bytes, must be greater than zero + * to allow the database to grow. + * Negative value means "keep current or use default". + * + * [in] shrink_threshold The shrink threshold in bytes, must be greater than + * zero to allow the database to shrink. + * Negative value means "keep current or use default". + * + * [in] pagesize The database page size for new database creation + * or -1 otherwise. Must be power of 2 in the range + * between MDBX_MIN_PAGESIZE and MDBX_MAX_PAGESIZE. + * Zero value means "minimal acceptable", + * and negative means "keep current or use default". + * + * Returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * - MDBX_EINVAL = An invalid parameter was specified, + * or the environment has an active write transaction. + * - MDBX_EPERM = specific for Windows: Shrinking was disabled before and + * now it wanna be enabled, but there are reading threads + * that don't use the additional SRWL (that is required to + * avoid Windows issues). + * - MDBX_EACCESS = The environment opened in read-only. + * - MDBX_MAP_FULL = Specified size smaller than the space already + * consumed by the environment. + * - MDBX_TOO_LARGE = Specified size is too large, i.e. too many pages for + * given size, or a 32-bit process requests too much bytes + * for the 32-bit address space. */ +LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, + intptr_t size_now, intptr_t size_upper, + intptr_t growth_step, + intptr_t shrink_threshold, + intptr_t pagesize); +LIBMDBX_API int mdbx_env_set_mapsize(MDBX_env *env, size_t size); + +/* Find out whether to use readahead or not, based on the given database size + * and the amount of available memory. + * + * [in] volume The expected database size in bytes. + * [in] redundancy Additional reserve or overload in case of negative value. + * + * Returns: + * - MDBX_RESULT_TRUE = readahead is reasonable. + * - MDBX_RESULT_FALSE = readahead is NOT reasonable, i.e. MDBX_NORDAHEAD + * is useful to open environment by mdbx_env_open(). + * - Otherwise the error code. */ +LIBMDBX_API int mdbx_is_readahead_reasonable(size_t volume, + intptr_t redundancy); + +/* The minimal database page size in bytes. */ +#define MDBX_MIN_PAGESIZE 256 +__inline intptr_t mdbx_limits_pgsize_min(void) { return MDBX_MIN_PAGESIZE; } + +/* The maximal database page size in bytes. */ +#define MDBX_MAX_PAGESIZE 65536 +__inline intptr_t mdbx_limits_pgsize_max(void) { return MDBX_MAX_PAGESIZE; } + +/* Returns minimal database size in bytes for given page size, + * or the negative error code. */ +LIBMDBX_API intptr_t mdbx_limits_dbsize_min(intptr_t pagesize); + +/* Returns maximal database size in bytes for given page size, + * or the negative error code. */ +LIBMDBX_API intptr_t mdbx_limits_dbsize_max(intptr_t pagesize); + +/* Returns maximal key size in bytes for given page size, + * or the negative error code. */ +LIBMDBX_API intptr_t mdbx_limits_keysize_max(intptr_t pagesize); + +/* Returns maximal write transaction size (i.e. limit for summary volume of + * dirty pages) in bytes for given page size, or the negative error code. */ +LIBMDBX_API intptr_t mdbx_limits_txnsize_max(intptr_t pagesize); + +/* Set the maximum number of threads/reader slots for the environment. + * + * This defines the number of slots in the lock table that is used to track + * readers in the the environment. The default is 119 for 4K system page size. + * Starting a read-only transaction normally ties a lock table slot to the + * current thread until the environment closes or the thread exits. If + * MDBX_NOTLS is in use, mdbx_txn_begin() instead ties the slot to the + * MDBX_txn object until it or the MDBX_env object is destroyed. + * This function may only be called after mdbx_env_create() and before + * mdbx_env_open(). + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] readers The maximum number of reader lock table slots. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. + * - MDBX_EPERM = the environment is already open. */ +LIBMDBX_API int mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers); + +/* Get the maximum number of threads/reader slots for the environment. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [out] readers Address of an integer to store the number of readers. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_env_get_maxreaders(MDBX_env *env, unsigned *readers); + +/* Set the maximum number of named databases for the environment. + * + * This function is only needed if multiple databases will be used in the + * environment. Simpler applications that use the environment as a single + * unnamed database can ignore this option. + * This function may only be called after mdbx_env_create() and before + * mdbx_env_open(). + * + * Currently a moderate number of slots are cheap but a huge number gets + * expensive: 7-120 words per transaction, and every mdbx_dbi_open() + * does a linear search of the opened slots. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] dbs The maximum number of databases. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. + * - MDBX_EPERM = the environment is already open. */ +LIBMDBX_API int mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs); + +/* Get the maximum size of keys and MDBX_DUPSORT data we can write. + * + * [in] env An environment handle returned by mdbx_env_create(). + * + * Returns The maximum size of a key we can write. */ +LIBMDBX_API int mdbx_env_get_maxkeysize(MDBX_env *env); + +/* Set application information associated with the MDBX_env. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] ctx An arbitrary pointer for whatever the application needs. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_set_userctx(MDBX_env *env, void *ctx); + +/* Get the application information associated with the MDBX_env. + * + * [in] env An environment handle returned by mdbx_env_create() + * Returns The pointer set by mdbx_env_set_userctx(). */ +LIBMDBX_API void *mdbx_env_get_userctx(MDBX_env *env); + +/* Create a transaction for use with the environment. + * + * The transaction handle may be discarded using mdbx_txn_abort() + * or mdbx_txn_commit(). + * + * NOTE: A transaction and its cursors must only be used by a single thread, + * and a thread may only have a single transaction at a time. If MDBX_NOTLS is + * in use, this does not apply to read-only transactions. + * + * NOTE: Cursors may not span transactions. + * + * [in] env An environment handle returned by mdbx_env_create() + * [in] parent If this parameter is non-NULL, the new transaction will be + * a nested transaction, with the transaction indicated by parent + * as its parent. Transactions may be nested to any level. + * A parent transaction and its cursors may not issue any other + * operations than mdbx_txn_commit and mdbx_txn_abort while it + * has active child transactions. + * [in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one or more + * of the values described here. + * + * - MDBX_RDONLY + * This transaction will not perform any write operations. + * + * - MDBX_TRYTXN + * Do not block when starting a write transaction. + * + * - MDBX_NOSYNC, MDBX_NOMETASYNC or MDBX_MAPASYNC + * Do not sync data to disk corresponding to MDBX_NOMETASYNC + * or MDBX_NOSYNC description (see abobe). + * + * [out] txn Address where the new MDBX_txn handle will be stored + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_PANIC = a fatal error occurred earlier and the environment + * must be shut down. + * - MDBX_MAP_RESIZED = another process wrote data beyond this MDBX_env's + * mapsize and this environment's map must be resized + * as well. See mdbx_env_set_mapsize(). + * - MDBX_READERS_FULL = a read-only transaction was requested and the reader + * lock table is full. See mdbx_env_set_maxreaders(). + * - MDBX_ENOMEM = out of memory. + * - MDBX_BUSY = the write transaction is already started by the + * current thread. */ +LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, unsigned flags, + MDBX_txn **txn); + +/* Information about the transaction */ +typedef struct MDBX_txn_info { + uint64_t txn_id; /* The ID of the transaction. For a READ-ONLY transaction, + this corresponds to the snapshot being read. */ + + uint64_t + txn_reader_lag; /* For READ-ONLY transaction: the lag from a recent + MVCC-snapshot, i.e. the number of committed + transaction since read transaction started. + For WRITE transaction (provided if scan_rlt=true): the + lag of the oldest reader from current transaction (i.e. + atleast 1 if any reader running). */ + + uint64_t txn_space_used; /* Used space by this transaction, i.e. corresponding + to the last used database page. */ + + uint64_t txn_space_limit_soft; /* Current size of database file. */ + + uint64_t + txn_space_limit_hard; /* Upper bound for size the database file, + i.e. the value "size_upper" argument of the + approriate call of mdbx_env_set_geometry(). */ + + uint64_t txn_space_retired; /* For READ-ONLY transaction: The total size of + the database pages that were retired by + committed write transactions after the reader's + MVCC-snapshot, i.e. the space which would be + freed after the Reader releases the + MVCC-snapshot for reuse by completion read + transaction. + For WRITE transaction: The summarized size of + the database pages that were retired for now + due Copy-On-Write during this transaction. */ + + uint64_t + txn_space_leftover; /* For READ-ONLY transaction: the space available for + writer(s) and that must be exhausted for reason to + call the OOM-killer for this read transaction. + For WRITE transaction: the space inside transaction + that left to MDBX_TXN_FULL error. */ + + uint64_t txn_space_dirty; /* For READ-ONLY transaction (provided if + scan_rlt=true): The space that actually become + available for reuse when only this transaction + will be finished. + For WRITE transaction: The summarized size of the + dirty database pages that generated during this + transaction. */ +} MDBX_txn_info; + +/* Return information about the MDBX transaction. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [out] stat The address of an MDBX_txn_info structure + * where the information will be copied. + * [in[ scan_rlt The boolean flag controls the scan of the read lock table to + * provide complete information. Such scan is relatively + * expensive and you can avoid it if corresponding fields are + * not needed (see description of MDBX_txn_info above). + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_info(MDBX_txn *txn, MDBX_txn_info *info, int scan_rlt); + +/* Returns the transaction's MDBX_env. + * + * [in] txn A transaction handle returned by mdbx_txn_begin() */ +LIBMDBX_API MDBX_env *mdbx_txn_env(MDBX_txn *txn); + +/* Return the transaction's flags. + * + * This returns the flags associated with this transaction. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * + * Returns A transaction flags, valid if input is an valid transaction, + * otherwise -1. */ +LIBMDBX_API int mdbx_txn_flags(MDBX_txn *txn); + +/* Return the transaction's ID. + * + * This returns the identifier associated with this transaction. For a read-only + * transaction, this corresponds to the snapshot being read; concurrent readers + * will frequently have the same transaction ID. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * + * Returns A transaction ID, valid if input is an active transaction, + * otherwise 0. */ +LIBMDBX_API uint64_t mdbx_txn_id(MDBX_txn *txn); + +/* Commit all the operations of a transaction into the database. + * + * The transaction handle is freed. It and its cursors must not be used again + * after this call, except with mdbx_cursor_renew() and mdbx_cursor_close(). + * + * A cursor must be closed explicitly always, before or after its transaction + * ends. It can be reused with mdbx_cursor_renew() before finally closing it. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. + * - MDBX_ENOSPC = no more disk space. + * - MDBX_EIO = a low-level I/O error occurred while writing. + * - MDBX_ENOMEM = out of memory. */ +LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn); + +/* Abandon all the operations of the transaction instead of saving them. + * + * The transaction handle is freed. It and its cursors must not be used again + * after this call, except with mdbx_cursor_renew() and mdbx_cursor_close(). + * + * A cursor must be closed explicitly always, before or after its transaction + * ends. It can be reused with mdbx_cursor_renew() before finally closing it. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_abort(MDBX_txn *txn); + +/* Reset a read-only transaction. + * + * Abort the read-only transaction like mdbx_txn_abort(), but keep the + * transaction handle. Therefore mdbx_txn_renew() may reuse the handle. This + * saves allocation overhead if the process will start a new read-only + * transaction soon, and also locking overhead if MDBX_NOTLS is in use. The + * reader table lock is released, but the table slot stays tied to its thread or + * MDBX_txn. Use mdbx_txn_abort() to discard a reset handle, and to free its + * lock table slot if MDBX_NOTLS is in use. + * + * Cursors opened within the transaction must not be used again after this call, + * except with mdbx_cursor_renew() and mdbx_cursor_close(). + * + * Reader locks generally don't interfere with writers, but they keep old + * versions of database pages allocated. Thus they prevent the old pages from + * being reused when writers commit new data, and so under heavy load the + * database size may grow much more rapidly than otherwise. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_reset(MDBX_txn *txn); + +/* Renew a read-only transaction. + * + * This acquires a new reader lock for a transaction handle that had been + * released by mdbx_txn_reset(). It must be called before a reset transaction + * may be used again. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_PANIC = a fatal error occurred earlier and the environment + * must be shut down. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_txn_renew(MDBX_txn *txn); + +/* The fours integers markers (aka "canary") associated with the environment. + * + * The `x`, `y` and `z` values could be set by mdbx_canary_put(), while the 'v' + * will be always set to the transaction number. Updated values becomes visible + * outside the current transaction only after it was committed. Current values + * could be retrieved by mdbx_canary_get(). */ +typedef struct mdbx_canary { + uint64_t x, y, z, v; +} mdbx_canary; + +/* Set integers markers (aka "canary") associated with the environment. + * + * [in] txn A transaction handle returned by mdbx_txn_begin() + * [in] canary A optional pointer to mdbx_canary structure for `x`, `y` + * and `z` values from. + * - If canary is NOT NULL then the `x`, `y` and `z` values will be + * updated from given canary argument, but the 'v' be always set + * to the current transaction number if at least one `x`, `y` or + * `z` values have changed (i.e. if `x`, `y` and `z` have the same + * values as currently present then nothing will be changes or + * updated). + * - if canary is NULL then the `v` value will be explicitly update + * to the current transaction number without changes `x`, `y` nor + * `z`. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_canary_put(MDBX_txn *txn, const mdbx_canary *canary); + +/* Returns fours integers markers (aka "canary") associated with the + * environment. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] canary The address of an mdbx_canary structure where the information + * will be copied. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_canary_get(MDBX_txn *txn, mdbx_canary *canary); + +/* A callback function used to compare two keys in a database */ +typedef int(MDBX_cmp_func)(const MDBX_val *a, const MDBX_val *b); + +/* Open a database in the environment. + * + * A database handle denotes the name and parameters of a database, + * independently of whether such a database exists. The database handle may be + * discarded by calling mdbx_dbi_close(). The old database handle is returned if + * the database was already open. The handle may only be closed once. + * + * (!) A notable difference between MDBX and LMDB is that MDBX make handles + * opened for existing databases immediately available for other transactions, + * regardless this transaction will be aborted or reset. The REASON for this is + * to avoiding the requirement for multiple opening a same handles in concurrent + * read transactions, and tracking of such open but hidden handles until the + * completion of read transactions which opened them. + * + * Nevertheless, the handle for the NEWLY CREATED database will be invisible for + * other transactions until the this write transaction is successfully + * committed. If the write transaction is aborted the handle will be closed + * automatically. After a successful commit the such handle will reside in the + * shared environment, and may be used by other transactions. + * + * In contrast to LMDB, the MDBX allow this function to be called from multiple + * concurrent transactions or threads in the same process. + * + * Legacy mdbx_dbi_open() correspond to calling mdbx_dbi_open_ex() with the null + * keycmp and datacmp arguments. + * + * To use named database (with name != NULL), mdbx_env_set_maxdbs() + * must be called before opening the environment. Table names are + * keys in the internal unnamed database, and may be read but not written. + * + * [in] txn transaction handle returned by mdbx_txn_begin(). + * [in] name The name of the database to open. If only a single + * database is needed in the environment, this value may be NULL. + * [in] flags Special options for this database. This parameter must be set + * to 0 or by bitwise OR'ing together one or more of the values + * described here: + * - MDBX_REVERSEKEY + * Keys are strings to be compared in reverse order, from the end + * of the strings to the beginning. By default, Keys are treated as + * strings and compared from beginning to end. + * - MDBX_DUPSORT + * Duplicate keys may be used in the database. Or, from another point of + * view, keys may have multiple data items, stored in sorted order. By + * default keys must be unique and may have only a single data item. + * - MDBX_INTEGERKEY + * Keys are binary integers in native byte order, either uin32_t or + * uint64_t, and will be sorted as such. The keys must all be of the + * same size. + * - MDBX_DUPFIXED + * This flag may only be used in combination with MDBX_DUPSORT. This + * option tells the library that the data items for this database are + * all the same size, which allows further optimizations in storage and + * retrieval. When all data items are the same size, the MDBX_GET_MULTIPLE, + * MDBX_NEXT_MULTIPLE and MDBX_PREV_MULTIPLE cursor operations may be used + * to retrieve multiple items at once. + * - MDBX_INTEGERDUP + * This option specifies that duplicate data items are binary integers, + * similar to MDBX_INTEGERKEY keys. + * - MDBX_REVERSEDUP + * This option specifies that duplicate data items should be compared as + * strings in reverse order (the comparison is performed in the direction + * from the last byte to the first). + * - MDBX_CREATE + * Create the named database if it doesn't exist. This option is not + * allowed in a read-only transaction or a read-only environment. + * + * [in] keycmp Optional custom key comparison function for a database. + * [in] datacmp Optional custom data comparison function for a database, takes + * effect only if database was opened with the MDB_DUPSORT flag. + * [out] dbi Address where the new MDBX_dbi handle will be stored. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_NOTFOUND = the specified database doesn't exist in the + * environment and MDBX_CREATE was not specified. + * - MDBX_DBS_FULL = too many databases have been opened. + * See mdbx_env_set_maxdbs(). + * - MDBX_INCOMPATIBLE = Database is incompatible with given flags, + * i.e. the passed flags is different with which the + * database was created, or the database was already + * opened with a different comparison function(s). */ +LIBMDBX_API int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, + unsigned flags, MDBX_dbi *dbi, + MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); +LIBMDBX_API int mdbx_dbi_open(MDBX_txn *txn, const char *name, unsigned flags, + MDBX_dbi *dbi); + +/* Retrieve statistics for a database. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [out] stat The address of an MDBX_stat structure where the statistics + * will be copied. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_dbi_stat(MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *stat, + size_t bytes); + +/* Retrieve the DB flags and status for a database handle. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [out] flags Address where the flags will be returned. + * [out] state Address where the state will be returned. + * + * Legacy mdbx_dbi_flags() correspond to calling mdbx_dbi_flags_ex() with + * discarding result from the last argument. + * + * Returns A non-zero error value on failure and 0 on success. */ +#define MDBX_TBL_DIRTY 0x01 /* DB was written in this txn */ +#define MDBX_TBL_STALE 0x02 /* Named-DB record is older than txnID */ +#define MDBX_TBL_FRESH 0x04 /* Named-DB handle opened in this txn */ +#define MDBX_TBL_CREAT 0x08 /* Named-DB handle created in this txn */ +LIBMDBX_API int mdbx_dbi_flags_ex(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, + unsigned *state); +LIBMDBX_API int mdbx_dbi_flags(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags); + +/* Close a database handle. Normally unnecessary. + * + * NOTE: Use with care. + * This call is synchronized via mutex with mdbx_dbi_close(), but NOT with + * other transactions running by other threads. The "next" version of libmdbx + * (MithrilDB) will solve this issue. + * + * Handles should only be closed if no other threads are going to reference + * the database handle or one of its cursors any further. Do not close a handle + * if an existing transaction has modified its database. Doing so can cause + * misbehavior from database corruption to errors like MDBX_BAD_VALSIZE (since + * the DB name is gone). + * + * Closing a database handle is not necessary, but lets mdbx_dbi_open() reuse + * the handle value. Usually it's better to set a bigger mdbx_env_set_maxdbs(), + * unless that value would be large. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi); + +/* Empty or delete and close a database. + * + * See mdbx_dbi_close() for restrictions about closing the DB handle. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] del 0 to empty the DB, 1 to delete it from the environment + * and close the DB handle. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, int del); + +/* Get items from a database. + * + * This function retrieves key/data pairs from the database. The address + * and length of the data associated with the specified key are returned + * in the structure to which data refers. + * If the database supports duplicate keys (MDBX_DUPSORT) then the + * first data item for the key will be returned. Retrieval of other + * items requires the use of mdbx_cursor_get(). + * + * NOTE: The memory pointed to by the returned values is owned by the + * database. The caller need not dispose of the memory, and may not + * modify it in any way. For values returned in a read-only transaction + * any modification attempts will cause a SIGSEGV. + * + * NOTE: Values returned from the database are valid only until a + * subsequent update operation, or the end of the transaction. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] key The key to search for in the database. + * [in,out] data The data corresponding to the key. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_NOTFOUND = the key was not in the database. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_get(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data); + +/* Get items from a database and optionaly number of data items for a given key. + * + * Briefly this function does the same as mdbx_get() with a few differences: + * 1. If values_count is NOT NULL, then returns the count + * of multi-values/duplicates for a given key. + * 2. Updates BOTH the key and the data for pointing to the actual key-value + * pair inside the database. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in,out] key The key to search for in the database. + * [in,out] data The data corresponding to the key. + * [out] values_count The optional address to return number of values + * associated with given key, i.e. + * = 0 - in case MDBX_NOTFOUND error; + * = 1 - exactly for databases WITHOUT MDBX_DUPSORT; + * >= 1 for databases WITH MDBX_DUPSORT. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_NOTFOUND = the key was not in the database. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_get_ex(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data, size_t *values_count); + +/* Get nearest items from a database. + * + * Briefly this function does the same as mdbx_get() with a few differences: + * 1. Return nearest (i.e. equal or great due comparison function) key-value + * pair, but not only exactly matching with the key. + * 2. On success return MDBX_SUCCESS if key found exactly, + * and MDBX_RESULT_TRUE otherwise. Moreover, for databases with MDBX_DUPSORT + * flag the data argument also will be used to match over + * multi-value/duplicates, and MDBX_SUCCESS will be returned only when BOTH + * the key and the data match exactly. + * 3. Updates BOTH the key and the data for pointing to the actual key-value + * pair inside the database. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in,out] key The key to search for in the database. + * [in,out] data The data corresponding to the key. + * + * Returns A non-zero error value on failure and MDBX_RESULT_TRUE (0) or + * MDBX_RESULT_TRUE on success (as described above). + * Some possible errors are: + * - MDBX_NOTFOUND = the key was not in the database. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_get_nearest(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data); + +/* Store items into a database. + * + * This function stores key/data pairs in the database. The default behavior + * is to enter the new key/data pair, replacing any previously existing key + * if duplicates are disallowed, or adding a duplicate data item if + * duplicates are allowed (MDBX_DUPSORT). + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] key The key to store in the database. + * [in,out] data The data to store. + * [in] flags Special options for this operation. This parameter must be + * set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * + * - MDBX_NODUPDATA + * Enter the new key/data pair only if it does not already appear + * in the database. This flag may only be specified if the database + * was opened with MDBX_DUPSORT. The function will return MDBX_KEYEXIST + * if the key/data pair already appears in the database. + * + * - MDBX_NOOVERWRITE + * Enter the new key/data pair only if the key does not already appear + * in the database. The function will return MDBX_KEYEXIST if the key + * already appears in the database, even if the database supports + * duplicates (MDBX_DUPSORT). The data parameter will be set to point + * to the existing item. + * + * - MDBX_CURRENT + * Update an single existing entry, but not add new ones. The function + * will return MDBX_NOTFOUND if the given key not exist in the database. + * Or the MDBX_EMULTIVAL in case duplicates for the given key. + * + * - MDBX_RESERVE + * Reserve space for data of the given size, but don't copy the given + * data. Instead, return a pointer to the reserved space, which the + * caller can fill in later - before the next update operation or the + * transaction ends. This saves an extra memcpy if the data is being + * generated later. MDBX does nothing else with this memory, the caller + * is expected to modify all of the space requested. This flag must not + * be specified if the database was opened with MDBX_DUPSORT. + * + * - MDBX_APPEND + * Append the given key/data pair to the end of the database. This option + * allows fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * a MDBX_EKEYMISMATCH error. + * + * - MDBX_APPENDDUP + * As above, but for sorted dup data. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_KEYEXIST + * - MDBX_MAP_FULL = the database is full, see mdbx_env_set_mapsize(). + * - MDBX_TXN_FULL = the transaction has too many dirty pages. + * - MDBX_EACCES = an attempt was made to write in a read-only transaction. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data, unsigned flags); + +/* Replace items in a database. + * + * This function allows to update or delete an existing value at the same time + * as the previous value is retrieved. If the argument new_data equal is NULL + * zero, the removal is performed, otherwise the update/insert. + * + * The current value may be in an already changed (aka dirty) page. In this + * case, the page will be overwritten during the update, and the old value will + * be lost. Therefore, an additional buffer must be passed via old_data argument + * initially to copy the old value. If the buffer passed in is too small, the + * function will return MDBX_RESULT_TRUE (-1) by setting iov_len field pointed + * by old_data argument to the appropriate value, without performing any + * changes. + * + * For databases with non-unique keys (i.e. with MDBX_DUPSORT flag), another use + * case is also possible, when by old_data argument selects a specific item from + * multi-value/duplicates with the same key for deletion or update. To select + * this scenario in flags should simultaneously specify MDBX_CURRENT and + * MDBX_NOOVERWRITE. This combination is chosen because it makes no sense, and + * thus allows you to identify the request of such a scenario. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] key The key to store in the database. + * [in,out] new_data The data to store, if NULL then deletion will be + * performed. + * [in,out] old_data The buffer for retrieve previous value as describe + * above. + * [in] flags Special options for this operation. This parameter must + * be set to 0 or by bitwise OR'ing together one or more of + * the values described in mdbx_put() description above, + * and additionally (MDBX_CURRENT | MDBX_NOOVERWRITE) + * combination for selection particular item from + * multi-value/duplicates. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *new_data, MDBX_val *old_data, + unsigned flags); + +/* Delete items from a database. + * + * This function removes key/data pairs from the database. + * + * NOTE: The data parameter is NOT ignored regardless the database does + * support sorted duplicate data items or not. If the data parameter + * is non-NULL only the matching data item will be deleted. + * + * This function will return MDBX_NOTFOUND if the specified key/data + * pair is not in the database. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] key The key to delete from the database. + * [in] data The data to delete. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EACCES = an attempt was made to write in a read-only transaction. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data); + +/* Create a cursor handle. + * + * A cursor is associated with a specific transaction and database. A cursor + * cannot be used when its database handle is closed. Nor when its transaction + * has ended, except with mdbx_cursor_renew(). Also it can be discarded with + * mdbx_cursor_close(). + * + * A cursor must be closed explicitly always, before or after its transaction + * ends. It can be reused with mdbx_cursor_renew() before finally closing it. + * + * NOTE: In contrast to LMDB, the MDBX required that any opened cursors can be + * reused and must be freed explicitly, regardless ones was opened in a + * read-only or write transaction. The REASON for this is eliminates ambiguity + * which helps to avoid errors such as: use-after-free, double-free, i.e. memory + * corruption and segfaults. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [out] cursor Address where the new MDBX_cursor handle will be stored. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, + MDBX_cursor **cursor); + +/* Close a cursor handle. + * + * The cursor handle will be freed and must not be used again after this call, + * but its transaction may still be live. + * + * NOTE: In contrast to LMDB, the MDBX required that any opened cursors can be + * reused and must be freed explicitly, regardless ones was opened in a + * read-only or write transaction. The REASON for this is eliminates ambiguity + * which helps to avoid errors such as: use-after-free, double-free, i.e. memory + * corruption and segfaults. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). */ +LIBMDBX_API void mdbx_cursor_close(MDBX_cursor *cursor); + +/* Renew a cursor handle. + * + * A cursor is associated with a specific transaction and database. The cursor + * may be associated with a new transaction, and referencing the same database + * handle as it was created with. This may be done whether the previous + * transaction is live or dead. + * + * NOTE: In contrast to LMDB, the MDBX allow any cursor to be re-used by using + * mdbx_cursor_renew(), to avoid unnecessary malloc/free overhead until it freed + * by mdbx_cursor_close(). + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *cursor); + +/* Return the cursor's transaction handle. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). */ +LIBMDBX_API MDBX_txn *mdbx_cursor_txn(MDBX_cursor *cursor); + +/* Return the cursor's database handle. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). */ +LIBMDBX_API MDBX_dbi mdbx_cursor_dbi(MDBX_cursor *cursor); + +/* Retrieve by cursor. + * + * This function retrieves key/data pairs from the database. The address and + * length of the key are returned in the object to which key refers (except + * for the case of the MDBX_SET option, in which the key object is unchanged), + * and the address and length of the data are returned in the object to which + * data refers. See mdbx_get() for restrictions on using the output values. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * [in,out] key The key for a retrieved item. + * [in,out] data The data of a retrieved item. + * [in] op A cursor operation MDBX_cursor_op. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_NOTFOUND = no matching key found. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_get(MDBX_cursor *cursor, MDBX_val *key, + MDBX_val *data, MDBX_cursor_op op); + +/* Store by cursor. + * + * This function stores key/data pairs into the database. The cursor is + * positioned at the new item, or on failure usually near it. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * [in] key The key operated on. + * [in] data The data operated on. + * [in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here: + * + * - MDBX_CURRENT + * Replace the item at the current cursor position. The key parameter + * must still be provided, and must match it, otherwise the function + * return MDBX_EKEYMISMATCH. + * + * NOTE: MDBX unlike LMDB allows you to change the size of the data and + * automatically handles reordering for sorted duplicates (MDBX_DUPSORT). + * + * - MDBX_NODUPDATA + * Enter the new key/data pair only if it does not already appear in the + * database. This flag may only be specified if the database was opened + * with MDBX_DUPSORT. The function will return MDBX_KEYEXIST if the + * key/data pair already appears in the database. + * + * - MDBX_NOOVERWRITE + * Enter the new key/data pair only if the key does not already appear + * in the database. The function will return MDBX_KEYEXIST if the key + * already appears in the database, even if the database supports + * duplicates (MDBX_DUPSORT). + * + * - MDBX_RESERVE + * Reserve space for data of the given size, but don't copy the given + * data. Instead, return a pointer to the reserved space, which the + * caller can fill in later - before the next update operation or the + * transaction ends. This saves an extra memcpy if the data is being + * generated later. This flag must not be specified if the database + * was opened with MDBX_DUPSORT. + * + * - MDBX_APPEND + * Append the given key/data pair to the end of the database. No key + * comparisons are performed. This option allows fast bulk loading when + * keys are already known to be in the correct order. Loading unsorted + * keys with this flag will cause a MDBX_KEYEXIST error. + * + * - MDBX_APPENDDUP + * As above, but for sorted dup data. + * + * - MDBX_MULTIPLE + * Store multiple contiguous data elements in a single request. This flag + * may only be specified if the database was opened with MDBX_DUPFIXED. + * The data argument must be an array of two MDBX_vals. The iov_len of the + * first MDBX_val must be the size of a single data element. The iov_base + * of the first MDBX_val must point to the beginning of the array of + * contiguous data elements. The iov_len of the second MDBX_val must be + * the count of the number of data elements to store. On return this + * field will be set to the count of the number of elements actually + * written. The iov_base of the second MDBX_val is unused. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EKEYMISMATCH + * - MDBX_MAP_FULL = the database is full, see mdbx_env_set_mapsize(). + * - MDBX_TXN_FULL = the transaction has too many dirty pages. + * - MDBX_EACCES = an attempt was made to write in a read-only transaction. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_put(MDBX_cursor *cursor, MDBX_val *key, + MDBX_val *data, unsigned flags); + +/* Delete current key/data pair. + * + * This function deletes the key/data pair to which the cursor refers. This does + * not invalidate the cursor, so operations such as MDBX_NEXT can still be used + * on it. Both MDBX_NEXT and MDBX_GET_CURRENT will return the same record after + * this operation. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * [in] flags Options for this operation. This parameter must be set to 0 + * or one of the values described here. + * + * - MDBX_NODUPDATA + * Delete all of the data items for the current key. This flag may only + * be specified if the database was opened with MDBX_DUPSORT. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EACCES = an attempt was made to write in a read-only transaction. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_del(MDBX_cursor *cursor, unsigned flags); + +/* Return count of duplicates for current key. + * + * This call is valid for all databases, but reasonable only for that support + * sorted duplicate data items MDBX_DUPSORT. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * [out] countp Address where the count will be stored. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EINVAL = cursor is not initialized, or an invalid parameter + * was specified. */ +LIBMDBX_API int mdbx_cursor_count(MDBX_cursor *cursor, size_t *countp); + +/* Determines whether the cursor is pointed to a key-value pair or not, + * i.e. was not positioned or points to the end of data. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * + * Returns: + * - MDBX_RESULT_TRUE = no more data available or cursor not positioned; + * - MDBX_RESULT_FALSE = data available; + * - Otherwise the error code. */ +LIBMDBX_API int mdbx_cursor_eof(MDBX_cursor *mc); + +/* Determines whether the cursor is pointed to the first key-value pair or not. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * + * Returns: + * - MDBX_RESULT_TRUE = cursor positioned to the first key-value pair. + * - MDBX_RESULT_FALSE = cursor NOT positioned to the first key-value pair. + * - Otherwise the error code. */ +LIBMDBX_API int mdbx_cursor_on_first(MDBX_cursor *mc); + +/* Determines whether the cursor is pointed to the last key-value pair or not. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * + * Returns: + * - MDBX_RESULT_TRUE = cursor positioned to the last key-value pair. + * - MDBX_RESULT_FALSE = cursor NOT positioned to the last key-value pair. + * - Otherwise the error code. */ +LIBMDBX_API int mdbx_cursor_on_last(MDBX_cursor *mc); + +/* Estimates the distance between cursors as a number of elements. The results + * of such estimation can be used to build and/or optimize query execution + * plans. + * + * This function performs a rough estimate based only on b-tree pages that are + * common for the both cursor's stacks. + * + * NOTE: The result varies greatly depending on the filling of specific pages + * and the overall balance of the b-tree: + * + * 1. The number of items is estimated by analyzing the height and fullness of + * the b-tree. The accuracy of the result directly depends on the balance of the + * b-tree, which in turn is determined by the history of previous insert/delete + * operations and the nature of the data (i.e. variability of keys length and so + * on). Therefore, the accuracy of the estimation can vary greatly in a + * particular situation. + * + * 2. To understand the potential spread of results, you should consider a + * possible situations basing on the general criteria for splitting and merging + * b-tree pages: + * - the page is split into two when there is no space for added data; + * - two pages merge if the result fits in half a page; + * - thus, the b-tree can consist of an arbitrary combination of pages filled + * both completely and only 1/4. Therefore, in the worst case, the result + * can diverge 4 times for each level of the b-tree excepting the first and + * the last. + * + * 3. In practice, the probability of extreme cases of the above situation is + * close to zero and in most cases the error does not exceed a few percent. On + * the other hand, it's just a chance you shouldn't overestimate. + * + * Both cursors must be initialized for the same database and the same + * transaction. + * + * [in] first The first cursor for estimation. + * [in] last The second cursor for estimation. + * [out] distance_items A pointer to store estimated distance value, + * i.e. *distance_items = distance(first, last). + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_estimate_distance(const MDBX_cursor *first, + const MDBX_cursor *last, + ptrdiff_t *distance_items); + +/* Estimates the move distance, i.e. between the current cursor position and + * next position after the specified move-operation with given key and data. + * The results of such estimation can be used to build and/or optimize query + * execution plans. Current cursor position and state are preserved. + * + * Please see notes on accuracy of the result in mdbx_estimate_distance() + * description above. + * + * [in] cursor Cursor for estimation. + * [in,out] key The key for a retrieved item. + * [in,out] data The data of a retrieved item. + * [in] op A cursor operation MDBX_cursor_op. + * [out] distance_items A pointer to store estimated move distance + * as the number of elements. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, + MDBX_val *data, MDBX_cursor_op move_op, + ptrdiff_t *distance_items); + +/* Estimates the size of a range as a number of elements. The results + * of such estimation can be used to build and/or optimize query execution + * plans. + * + * Please see notes on accuracy of the result in mdbx_estimate_distance() + * description above. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] begin_key The key of range beginning or NULL for explicit FIRST. + * [in] begin_data Optional additional data to seeking among sorted + * duplicates. Only for MDBX_DUPSORT, NULL otherwise. + * [in] end_key The key of range ending or NULL for explicit LAST. + * [in] end_data Optional additional data to seeking among sorted + * duplicates. Only for MDBX_DUPSORT, NULL otherwise. + * [out] distance_items A pointer to store range estimation result. + * + * Returns A non-zero error value on failure and 0 on success. */ +#define MDBX_EPSILON ((MDBX_val *)((ptrdiff_t)-1)) +LIBMDBX_API int mdbx_estimate_range(MDBX_txn *txn, MDBX_dbi dbi, + MDBX_val *begin_key, MDBX_val *begin_data, + MDBX_val *end_key, MDBX_val *end_data, + ptrdiff_t *size_items); + +/* Determines whether the given address is on a dirty database page of the + * transaction or not. Ultimately, this allows to avoid copy data from non-dirty + * pages. + * + * "Dirty" pages are those that have already been changed during a write + * transaction. Accordingly, any further changes may result in such pages being + * overwritten. Therefore, all functions libmdbx performing changes inside the + * database as arguments should NOT get pointers to data in those pages. In + * turn, "not dirty" pages before modification will be copied. + * + * In other words, data from dirty pages must either be copied before being + * passed as arguments for further processing or rejected at the argument + * validation stage. Thus, mdbx_is_dirty() allows you to get rid of unnecessary + * copying, and perform a more complete check of the arguments. + * + * NOTE: The address passed must point to the beginning of the data. This is the + * only way to ensure that the actual page header is physically located in the + * same memory page, including for multi-pages with long data. + * + * NOTE: In rare cases the function may return a false positive answer + * (DBX_RESULT_TRUE when data is NOT on a dirty page), but never a false + * negative if the arguments are correct. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] ptr The address of data to check. + * + * Returns: + * - MDBX_RESULT_TRUE = given address is on the dirty page. + * - MDBX_RESULT_FALSE = given address is NOT on the dirty page. + * - Otherwise the error code. */ +LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr); + +/* Sequence generation for a database. + * + * The function allows to create a linear sequence of unique positive integers + * for each database. The function can be called for a read transaction to + * retrieve the current sequence value, and the increment must be zero. + * Sequence changes become visible outside the current write transaction after + * it is committed, and discarded on abort. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [out] result The optional address where the value of sequence before the + * change will be stored. + * [in] increment Value to increase the sequence, + * must be 0 for read-only transactions. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_RESULT_TRUE = Increasing the sequence has resulted in an overflow + * and therefore cannot be executed. */ +LIBMDBX_API int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, + uint64_t increment); + +/* Compare two data items according to a particular database. + * + * This returns a comparison as if the two data items were keys in the + * specified database. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] a The first item to compare. + * [in] b The second item to compare. + * + * Returns < 0 if a < b, 0 if a == b, > 0 if a > b */ +LIBMDBX_API int mdbx_cmp(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, + const MDBX_val *b); + +/* Compare two data items according to a particular database. + * + * This returns a comparison as if the two items were data items of the + * specified database. The database must have the MDBX_DUPSORT flag. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] a The first item to compare. + * [in] b The second item to compare. + * + * Returns < 0 if a < b, 0 if a == b, > 0 if a > b */ +LIBMDBX_API int mdbx_dcmp(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, + const MDBX_val *b); + +/* A callback function used to enumerate the reader lock table. + * + * [in] ctx An arbitrary context pointer for the callback. + * [in] num The serial number during enumeration, starting from 1. + * [in] slot The reader lock table slot number. + * [in] txnid The ID of the transaction being read, + * i.e. the MVCC-snaphot number. + * [in] lag The lag from a recent MVCC-snapshot, i.e. the number of + * committed transaction since read transaction started. + * [in] pid The reader process ID. + * [in] thread The reader thread ID. + * [in] bytes_used The number of last used page in the MVCC-snapshot which + * being read, i.e. database file can't shrinked beyond this. + * [in] bytes_retired The total size of the database pages that were retired by + * committed write transactions after the reader's + * MVCC-snapshot, i.e. the space which would be freed after + * the Reader releases the MVCC-snapshot for reuse by + * completion read transaction. + * + * Returns < 0 on failure, >= 0 on success. */ +typedef int(MDBX_reader_list_func)(void *ctx, int num, int slot, mdbx_pid_t pid, + mdbx_tid_t thread, uint64_t txnid, + uint64_t lag, size_t bytes_used, + size_t bytes_retained); + +/* Enumarete the entries in the reader lock table. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] func A MDBX_reader_list_func function. + * [in] ctx An arbitrary context pointer for the enumeration function. + * + * Returns A non-zero error value on failure and 0 on success, + * or MDBX_RESULT_TRUE (-1) if the reader lock table is empty. */ +LIBMDBX_API int mdbx_reader_list(MDBX_env *env, MDBX_reader_list_func *func, + void *ctx); + +/* Check for stale entries in the reader lock table. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [out] dead Number of stale slots that were cleared. + * + * Returns A non-zero error value on failure and 0 on success, + * or MDBX_RESULT_TRUE (-1) if a dead reader(s) found or mutex was recovered. */ +LIBMDBX_API int mdbx_reader_check(MDBX_env *env, int *dead); + +/* Returns a lag of the reading for the given transaction. + * + * Returns an information for estimate how much given read-only + * transaction is lagging relative the to actual head. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [out] percent Percentage of page allocation in the database. + * + * Returns Number of transactions committed after the given was started for + * read, or negative value on failure. */ +LIBMDBX_API int mdbx_txn_straggler(MDBX_txn *txn, int *percent); + +/* A lack-of-space callback function to resolve issues with a laggard readers. + * + * Read transactions prevent reuse of pages freed by newer write transactions, + * thus the database can grow quickly. This callback will be called when there + * is not enough space in the database (ie. before increasing the database size + * or before MDBX_MAP_FULL error) and thus can be used to resolve issues with + * a "long-lived" read transactions. + * + * Depending on the arguments and needs, your implementation may wait, terminate + * a process or thread that is performing a long read, or perform some other + * action. In doing so it is important that the returned code always corresponds + * to the performed action. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] pid A pid of the reader process. + * [in] tid A thread_id of the reader thread. + * [in] txn A transaction number on which stalled. + * [in] gap A lag from the last commited txn. + * [in] space A space that actually become available for reuse after this + * reader finished. The callback function can take this value into + * account to evaluate the impact that a long-running transaction + * has. + * [in] retry A retry number starting from 0. if callback has returned 0 + * at least once, then at end of current OOM-handler loop callback + * will be called additionally with negative value to notify about + * the end of loop. The callback function can use this value to + * implement timeout logic while waiting for readers. + * + * The RETURN CODE determines the further actions libmdbx and must match the + * action which was executed by the callback: + * + * -2 or less = An error condition and the reader was not killed. + * + * -1 = The callback was unable to solve the problem and agreed + * on MDBX_MAP_FULL error, libmdbx should increase the + * database size or return MDBX_MAP_FULL error. + * + * 0 (zero) = The callback solved the problem or just waited for + * a while, libmdbx should rescan the reader lock table and + * retry. This also includes a situation when corresponding + * transaction terminated in normal way by mdbx_txn_abort() + * or mdbx_txn_reset(), and my be restarted. I.e. reader + * slot don't needed to be cleaned from transaction. + * + * 1 = Transaction aborted asynchronous and reader slot should + * be cleared immediately, i.e. read transaction will not + * continue but mdbx_txn_abort() or mdbx_txn_reset() will + * be called later. + * + * 2 or great = The reader process was terminated or killed, and libmdbx + * should entirely reset reader registration. */ +typedef int(MDBX_oom_func)(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, + uint64_t txn, unsigned gap, size_t space, int retry); + +/* Set the OOM callback. + * + * The callback will only be triggered on lack of space to resolve issues with + * lagging reader(s) (i.e. to kill it) for resume reuse pages from the garbage + * collector. + * + * [in] env An environment handle returned by mdbx_env_create(). + * [in] oom_func A MDBX_oom_func function or NULL to disable. + * + * Returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_env_set_oomfunc(MDBX_env *env, MDBX_oom_func *oom_func); + +/* Get the current oom_func callback. + * + * [in] env An environment handle returned by mdbx_env_create(). + * + * Returns A MDBX_oom_func function or NULL if disabled. */ +LIBMDBX_API MDBX_oom_func *mdbx_env_get_oomfunc(MDBX_env *env); + +/**** B-tree Traversal ********************************************************* + * This is internal API for mdbx_chk tool. You should avoid to use it, except + * some extremal special cases. */ + +/* Page types for traverse the b-tree. */ +typedef enum { + MDBX_page_void, + MDBX_page_meta, + MDBX_page_large, + MDBX_page_branch, + MDBX_page_leaf, + MDBX_page_dupfixed_leaf, + MDBX_subpage_leaf, + MDBX_subpage_dupfixed_leaf +} MDBX_page_type_t; + +#define MDBX_PGWALK_MAIN ((const char *)((ptrdiff_t)0)) +#define MDBX_PGWALK_GC ((const char *)((ptrdiff_t)-1)) +#define MDBX_PGWALK_META ((const char *)((ptrdiff_t)-2)) + +/* Callback function for traverse the b-tree. */ +typedef int +MDBX_pgvisitor_func(const uint64_t pgno, const unsigned number, void *const ctx, + const int deep, const char *const dbi, + const size_t page_size, const MDBX_page_type_t type, + const size_t nentries, const size_t payload_bytes, + const size_t header_bytes, const size_t unused_bytes); + +/* B-tree traversal function. */ +LIBMDBX_API int mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor, + void *ctx); + +/**** Attribute support functions for Nexenta *********************************/ +#ifdef MDBX_NEXENTA_ATTRS +typedef uint_fast64_t mdbx_attr_t; + +/* Store by cursor with attribute. + * + * This function stores key/data pairs into the database. The cursor is + * positioned at the new item, or on failure usually near it. + * + * NOTE: Internally based on MDBX_RESERVE feature, + * therefore doesn't support MDBX_DUPSORT. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open() + * [in] key The key operated on. + * [in] data The data operated on. + * [in] attr The attribute. + * [in] flags Options for this operation. This parameter must be set to 0 + * or one of the values described here: + * + * - MDBX_CURRENT + * Replace the item at the current cursor position. The key parameter + * must still be provided, and must match it, otherwise the function + * return MDBX_EKEYMISMATCH. + * + * - MDBX_APPEND + * Append the given key/data pair to the end of the database. No key + * comparisons are performed. This option allows fast bulk loading when + * keys are already known to be in the correct order. Loading unsorted + * keys with this flag will cause a MDBX_KEYEXIST error. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_EKEYMISMATCH + * - MDBX_MAP_FULL = the database is full, see mdbx_env_set_mapsize(). + * - MDBX_TXN_FULL = the transaction has too many dirty pages. + * - MDBX_EACCES = an attempt was made to write in a read-only transaction. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_put_attr(MDBX_cursor *cursor, MDBX_val *key, + MDBX_val *data, mdbx_attr_t attr, + unsigned flags); + +/* Store items and attributes into a database. + * + * This function stores key/data pairs in the database. The default behavior + * is to enter the new key/data pair, replacing any previously existing key + * if duplicates are disallowed. + * + * NOTE: Internally based on MDBX_RESERVE feature, + * therefore doesn't support MDBX_DUPSORT. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] key The key to store in the database. + * [in] attr The attribute to store in the database. + * [in,out] data The data to store. + * [in] flags Special options for this operation. This parameter must be + * set to 0 or by bitwise OR'ing together one or more of the + * values described here: + * + * - MDBX_NOOVERWRITE + * Enter the new key/data pair only if the key does not already appear + * in the database. The function will return MDBX_KEYEXIST if the key + * already appears in the database. The data parameter will be set to + * point to the existing item. + * + * - MDBX_CURRENT + * Update an single existing entry, but not add new ones. The function + * will return MDBX_NOTFOUND if the given key not exist in the database. + * Or the MDBX_EMULTIVAL in case duplicates for the given key. + * + * - MDBX_APPEND + * Append the given key/data pair to the end of the database. This option + * allows fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * a MDBX_EKEYMISMATCH error. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_KEYEXIST + * - MDBX_MAP_FULL = the database is full, see mdbx_env_set_mapsize(). + * - MDBX_TXN_FULL = the transaction has too many dirty pages. + * - MDBX_EACCES = an attempt was made to write in a read-only transaction. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_put_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data, mdbx_attr_t attr, unsigned flags); + +/* Set items attribute from a database. + * + * This function stores key/data pairs attribute to the database. + * + * NOTE: Internally based on MDBX_RESERVE feature, + * therefore doesn't support MDBX_DUPSORT. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] key The key to search for in the database. + * [in] data The data to be stored or NULL to save previous value. + * [in] attr The attribute to be stored. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_NOTFOUND = the key-value pair was not in the database. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_set_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data, mdbx_attr_t attr); + +/* Get items attribute from a database cursor. + * + * This function retrieves key/data pairs from the database. The address and + * length of the key are returned in the object to which key refers (except + * for the case of the MDBX_SET option, in which the key object is unchanged), + * and the address and length of the data are returned in the object to which + * data refers. See mdbx_get() for restrictions on using the output values. + * + * [in] cursor A cursor handle returned by mdbx_cursor_open(). + * [in,out] key The key for a retrieved item. + * [in,out] data The data of a retrieved item. + * [in] op A cursor operation MDBX_cursor_op. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_NOTFOUND = no matching key found. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_get_attr(MDBX_cursor *mc, MDBX_val *key, + MDBX_val *data, mdbx_attr_t *attrptr, + MDBX_cursor_op op); + +/* Get items attribute from a database. + * + * This function retrieves key/data pairs from the database. The address + * and length of the data associated with the specified key are returned + * in the structure to which data refers. + * If the database supports duplicate keys (MDBX_DUPSORT) then the + * first data item for the key will be returned. Retrieval of other + * items requires the use of mdbx_cursor_get(). + * + * NOTE: The memory pointed to by the returned values is owned by the + * database. The caller need not dispose of the memory, and may not + * modify it in any way. For values returned in a read-only transaction + * any modification attempts will cause a SIGSEGV. + * + * NOTE: Values returned from the database are valid only until a + * subsequent update operation, or the end of the transaction. + * + * [in] txn A transaction handle returned by mdbx_txn_begin(). + * [in] dbi A database handle returned by mdbx_dbi_open(). + * [in] key The key to search for in the database. + * [in,out] data The data corresponding to the key. + * + * Returns A non-zero error value on failure and 0 on success, some + * possible errors are: + * - MDBX_NOTFOUND = the key was not in the database. + * - MDBX_EINVAL = an invalid parameter was specified. */ +LIBMDBX_API int mdbx_get_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data, mdbx_attr_t *attrptr); +#endif /* MDBX_NEXENTA_ATTRS */ + +/******************************************************************************* + * LY: temporary workaround for Elbrus's memcmp() bug. */ +#ifndef __GLIBC_PREREQ +#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +#define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#else +#define __GLIBC_PREREQ(maj, min) (0) +#endif +#endif /* __GLIBC_PREREQ */ +#if defined(__e2k__) && !__GLIBC_PREREQ(2, 24) +LIBMDBX_API int mdbx_e2k_memcmp_bug_workaround(const void *s1, const void *s2, + size_t n); +LIBMDBX_API int mdbx_e2k_strcmp_bug_workaround(const char *s1, const char *s2); +LIBMDBX_API int mdbx_e2k_strncmp_bug_workaround(const char *s1, const char *s2, + size_t n); +LIBMDBX_API size_t mdbx_e2k_strlen_bug_workaround(const char *s); +LIBMDBX_API size_t mdbx_e2k_strnlen_bug_workaround(const char *s, + size_t maxlen); +#include +#include +#undef memcmp +#define memcmp mdbx_e2k_memcmp_bug_workaround +#undef bcmp +#define bcmp mdbx_e2k_memcmp_bug_workaround +#undef strcmp +#define strcmp mdbx_e2k_strcmp_bug_workaround +#undef strncmp +#define strncmp mdbx_e2k_strncmp_bug_workaround +#undef strlen +#define strlen mdbx_e2k_strlen_bug_workaround +#undef strnlen +#define strnlen mdbx_e2k_strnlen_bug_workaround + +#endif /* Elbrus's memcmp() bug. */ + +#ifdef __cplusplus +} +#endif + +#endif /* LIBMDBX_H */ diff --git a/src/contrib/db/libmdbx/packages/rpm/CMakeLists.txt b/src/contrib/db/libmdbx/packages/rpm/CMakeLists.txt new file mode 100644 index 0000000..5949e9f --- /dev/null +++ b/src/contrib/db/libmdbx/packages/rpm/CMakeLists.txt @@ -0,0 +1,184 @@ +cmake_minimum_required(VERSION 2.8.7) +set(TARGET mdbx) +project(${TARGET}) + +set(MDBX_VERSION_MAJOR 0) +set(MDBX_VERSION_MINOR 3) +set(MDBX_VERSION_RELEASE 1) +set(MDBX_VERSION_REVISION 0) + +set(MDBX_VERSION_STRING ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}.${MDBX_VERSION_RELEASE}) + +enable_language(C) +enable_language(CXX) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED on) + +add_definitions(-DNDEBUG=1 -DMDBX_DEBUG=0 -DLIBMDBX_EXPORTS=1 -D_GNU_SOURCE=1) + +find_package(Threads REQUIRED) + +get_directory_property(hasParent PARENT_DIRECTORY) +if(hasParent) + set(STANDALONE_BUILD 0) +else() + set(STANDALONE_BUILD 1) + enable_testing() + + if (CMAKE_C_COMPILER_ID MATCHES GNU) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g3") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wextra") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffunction-sections") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu11") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread") + endif() + + if (CMAKE_CXX_COMPILER_ID MATCHES GNU) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpointer-arith") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wformat-security") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Woverloaded-virtual") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wwrite-strings") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmax-errors=20") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wunused-function -Wunused-variable -Wunused-value -Wmissing-declarations") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wcast-qual") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finline-functions-called-once") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-packed-bitfield-compat") + + set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3") + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g3") + endif() + + if (COVERAGE) + if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + message(FATAL_ERROR "Coverage requires -DCMAKE_BUILD_TYPE=Debug Current value=${CMAKE_BUILD_TYPE}") + endif() + + message(STATUS "Setting coverage compiler flags") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -ggdb3 -O0 --coverage -fprofile-arcs -ftest-coverage") + add_definitions(-DCOVERAGE_TEST) + endif() + + if (NOT TRAVIS) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -fsanitize=leak -fstack-protector-strong -static-libasan") + endif() +endif() + +set(${TARGET}_SRC + mdbx.h + src/bits.h + src/defs.h + src/lck-linux.c + src/mdbx.c + src/osal.c + src/osal.h + src/version.c + ) + +add_library(${TARGET}_STATIC STATIC + ${${TARGET}_SRC} + ) + +add_library(${TARGET} ALIAS ${TARGET}_STATIC) + +add_library(${TARGET}_SHARED SHARED + ${${TARGET}_SRC} + ) + +set_target_properties(${TARGET}_SHARED PROPERTIES + VERSION ${MDBX_VERSION_STRING} + SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR} + OUTPUT_NAME ${TARGET} + CLEAN_DIRECT_OUTPUT 1 + ) + +set_target_properties(${TARGET}_STATIC PROPERTIES + VERSION ${MDBX_VERSION_STRING} + SOVERSION ${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR} + OUTPUT_NAME ${TARGET} + CLEAN_DIRECT_OUTPUT 1 + ) + +target_include_directories(${TARGET}_STATIC PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(${TARGET}_SHARED PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries(${TARGET}_STATIC ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET}_SHARED ${CMAKE_THREAD_LIBS_INIT}) +if(UNIX AND NOT APPLE) + target_link_libraries(${TARGET}_STATIC rt) + target_link_libraries(${TARGET}_SHARED rt) +endif() + +install(TARGETS ${TARGET}_STATIC DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx) +install(TARGETS ${TARGET}_SHARED DESTINATION ${CMAKE_INSTALL_PREFIX}/lib64 COMPONENT mdbx) +install(FILES mdbx.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT mdbx-devel) + +add_subdirectory(src/tools) +add_subdirectory(test) +add_subdirectory(test/pcrf) +add_subdirectory(tutorial) + +############################################################################## + +set(CPACK_GENERATOR "RPM") +set(CPACK_RPM_COMPONENT_INSTALL ON) + +# Version +if (NOT "$ENV{BUILD_NUMBER}" STREQUAL "") + set(CPACK_PACKAGE_RELEASE $ENV{BUILD_NUMBER}) +else() + if (NOT "$ENV{CI_PIPELINE_ID}" STREQUAL "") + set(CPACK_PACKAGE_RELEASE $ENV{CI_PIPELINE_ID}) + else() + set(CPACK_PACKAGE_RELEASE 1) + endif() +endif() +set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE}) + +set(CPACK_PACKAGE_VERSION ${MDBX_VERSION_STRING}) +set(CPACK_PACKAGE_VERSION_FULL ${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}) + +set(CPACK_RPM_mdbx-devel_PACKAGE_REQUIRES "mdbx = ${CPACK_PACKAGE_VERSION}") + +set(CPACK_RPM_SPEC_INSTALL_POST "/bin/true") +set(CPACK_RPM_mdbx_PACKAGE_NAME mdbx) +set(CPACK_RPM_mdbx-devel_PACKAGE_NAME mdbx-devel) +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "The revised and extended descendant of Symas LMDB") + +set(CPACK_PACKAGE_VENDOR "???") +set(CPACK_PACKAGE_CONTACT "Vladimir Romanov") +set(CPACK_PACKAGE_RELOCATABLE false) +set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64") +set(CPACK_RPM_PACKAGE_REQUIRES "") +set(CPACK_RPM_PACKAGE_GROUP "Applications/Database") + +set(CPACK_RPM_mdbx_FILE_NAME "${CPACK_RPM_mdbx_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm") +set(CPACK_RPM_mdbx-devel_FILE_NAME "${CPACK_RPM_mdbx-devel_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_FULL}.${CPACK_RPM_PACKAGE_ARCHITECTURE}.rpm") + +set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION + /usr/local + /usr/local/bin + /usr/local/lib64 + /usr/local/include + /usr/local/man + /usr/local/man/man1 + ) + +include(CPack) diff --git a/src/contrib/db/libmdbx/packages/rpm/build.sh b/src/contrib/db/libmdbx/packages/rpm/build.sh new file mode 100644 index 0000000..5170882 --- /dev/null +++ b/src/contrib/db/libmdbx/packages/rpm/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e +CONFIG=$1 + +if [[ -z "${CONFIG}" ]]; then + CONFIG=Debug +fi +if [[ -r /opt/rh/devtoolset-6/enable ]]; then + source /opt/rh/devtoolset-6/enable +fi +#rm -f -r build || true +mkdir -p cmake-build-${CONFIG} +pushd cmake-build-${CONFIG} &> /dev/null +if [[ ! -r Makefile ]]; then + cmake .. -DCMAKE_BUILD_TYPE=${CONFIG} +fi +make -j8 || exit 1 +popd &> /dev/null diff --git a/src/contrib/db/libmdbx/packages/rpm/package.sh b/src/contrib/db/libmdbx/packages/rpm/package.sh new file mode 100644 index 0000000..d7f9ab2 --- /dev/null +++ b/src/contrib/db/libmdbx/packages/rpm/package.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +CONFIG=$1 + +if [[ -z "${CONFIG}" ]]; then + CONFIG=Debug +fi + +DIRNAME=`dirname ${BASH_SOURCE[0]}` +DIRNAME=`readlink --canonicalize ${DIRNAME}` + +if [[ -r /opt/rh/devtoolset-6/enable ]]; then + source /opt/rh/devtoolset-6/enable +fi + +mkdir -p cmake-build-${CONFIG} +pushd cmake-build-${CONFIG} &> /dev/null +if [[ ! -r Makefile ]]; then + cmake .. -DCMAKE_BUILD_TYPE=${CONFIG} +fi +rm -f *.rpm +make -j8 package || exit 1 +rm -f *-Unspecified.rpm +popd &> /dev/null diff --git a/src/contrib/db/libmdbx/src/CMakeLists.txt b/src/contrib/db/libmdbx/src/CMakeLists.txt new file mode 100644 index 0000000..04aead5 --- /dev/null +++ b/src/contrib/db/libmdbx/src/CMakeLists.txt @@ -0,0 +1,225 @@ +## +## Copyright 2019 Leonid Yuriev +## and other libmdbx authors: please see AUTHORS file. +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted only as authorized by the OpenLDAP +## Public License. +## +## A copy of this license is available in the file LICENSE in the +## top-level directory of the distribution or, alternatively, at +## . +## + +# Get version +fetch_version(MDBX "${CMAKE_CURRENT_SOURCE_DIR}/../VERSION") +message(STATUS "libmdbx version is ${MDBX_VERSION}") + +if(MDBX_ALLOY_MODE) + set(LIBMDBX_SOURCES alloy.c) +else() + if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set(LIBMDBX_OSAL windows) + else() + set(LIBMDBX_OSAL posix) + endif() + set(LIBMDBX_SOURCES + elements/defs.h elements/internals.h elements/osal.h + elements/core.c elements/osal.c elements/lck-${LIBMDBX_OSAL}.c) +endif() +list(APPEND LIBMDBX_SOURCES ../mdbx.h + "${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c" + "${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h") + +if(MDBX_BUILD_SHARED_LIBRARY) + add_library(mdbx SHARED ${LIBMDBX_SOURCES}) + target_compile_definitions(mdbx PRIVATE LIBMDBX_EXPORTS INTERFACE LIBMDBX_IMPORTS) + set(MDBX_LIBDEP_MODE PRIVATE) +else() + add_library(mdbx STATIC ${LIBMDBX_SOURCES}) + set(MDBX_LIBDEP_MODE PUBLIC) +endif() + +if(CC_HAS_VISIBILITY AND (LTO_ENABLED OR INTERPROCEDURAL_OPTIMIZATION)) + set_target_properties(mdbx PROPERTIES LINK_FLAGS "-fvisibility=hidden") +endif() + +if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + if(MSVC) + if(NOT MSVC_LIB_EXE) + # Find lib.exe + get_filename_component(CL_NAME ${CMAKE_C_COMPILER} NAME) + string(REPLACE cl.exe lib.exe MSVC_LIB_EXE ${CL_NAME}) + find_program(MSVC_LIB_EXE ${MSVC_LIB_EXE}) + endif() + if(MSVC_LIB_EXE) + message(STATUS "Found MSVC's lib tool: ${MSVC_LIB_EXE}") + set(MDBX_NTDLL_EXTRA_IMPLIB ${CMAKE_CURRENT_BINARY_DIR}/mdbx_ntdll_extra.lib) + add_custom_command(OUTPUT ${MDBX_NTDLL_EXTRA_IMPLIB} + COMMENT "Create extra-import-library for ntdll.dll" + MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def" + COMMAND ${MSVC_LIB_EXE} /def:"${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def" /out:"${MDBX_NTDLL_EXTRA_IMPLIB}" ${INITIAL_CMAKE_STATIC_LINKER_FLAGS}) + else() + message(SEND_ERROR "MSVC's lib tool not found") + endif() + elseif(MINGW OR MINGW64) + if(NOT DLLTOOL) + # Find dlltool + get_filename_component(GCC_NAME ${CMAKE_C_COMPILER} NAME) + string(REPLACE gcc dlltool DLLTOOL_NAME ${GCC_NAME}) + find_program(DLLTOOL NAMES ${DLLTOOL_NAME}) + endif() + if(DLLTOOL) + message(STATUS "Found dlltool: ${DLLTOOL}") + set(MDBX_NTDLL_EXTRA_IMPLIB "${CMAKE_CURRENT_BINARY_DIR}/mdbx_ntdll_extra.a") + add_custom_command(OUTPUT ${MDBX_NTDLL_EXTRA_IMPLIB} + COMMENT "Create extra-import-library for ntdll.dll" + MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def" + COMMAND ${DLLTOOL} -d "${CMAKE_CURRENT_SOURCE_DIR}/elements/ntdll.def" -l "${MDBX_NTDLL_EXTRA_IMPLIB}") + else() + message(SEND_ERROR "dlltool not found") + endif() + endif() +endif() + +target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ${CMAKE_THREAD_LIBS_INIT}) +if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ntdll.lib) + if(MDBX_NTDLL_EXTRA_IMPLIB) + # LY: Sometimes Cmake requires a nightmarish magic for simple things. + # 1) create a target out of the library compilation result + add_custom_target(ntdll_extra_target DEPENDS ${MDBX_NTDLL_EXTRA_IMPLIB}) + # 2) create an library target out of the library compilation result + add_library(ntdll_extra STATIC IMPORTED GLOBAL) + add_dependencies(ntdll_extra ntdll_extra_target) + # 3) specify where the library is (and where to find the headers) + set_target_properties(ntdll_extra + PROPERTIES + IMPORTED_LOCATION ${MDBX_NTDLL_EXTRA_IMPLIB}) + target_link_libraries(mdbx ${MDBX_LIBDEP_MODE} ntdll_extra) + endif() +endif() + +set_target_properties(mdbx PROPERTIES + INTERPROCEDURAL_OPTIMIZATION $ + C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON + PUBLIC_HEADER "../mdbx.h") + +if(CC_HAS_FASTMATH) + target_compile_options(mdbx PRIVATE "-ffast-math") +endif() +if(BUILD_FOR_NATIVE_CPU AND CC_HAS_ARCH_NATIVE) + target_compile_options(mdbx PUBLIC "-march=native") +endif() +if(CC_HAS_VISIBILITY) + target_compile_options(mdbx PRIVATE "-fvisibility=hidden") +endif() + +install(TARGETS mdbx + LIBRARY DESTINATION lib COMPONENT runtime + RUNTIME DESTINATION bin COMPONENT runtime + ARCHIVE DESTINATION lib/static COMPONENT devel + PUBLIC_HEADER DESTINATION include + INCLUDES DESTINATION include COMPONENT devel) + +################################################################################ +# +# library build info (used in library version output) +# +set(MDBX_BUILD_FLAGS "") + +# append cmake's build-type flags and defines +if(NOT CMAKE_CONFIGURATION_TYPES) + list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPERCASE}}) + list(APPEND MDBX_BUILD_FLAGS ${CMAKE_C_DEFINES_${CMAKE_BUILD_TYPE_UPPERCASE}}) +endif() + +# append linker dll's options +if(LIBMDBX_TYPE STREQUAL "SHARED") + list(APPEND MDBX_BUILD_FLAGS ${CMAKE_SHARED_LINKER_FLAGS}) +endif() + +# get definitions +get_target_property(defs_list mdbx COMPILE_DEFINITIONS) +if(defs_list) + list(APPEND MDBX_BUILD_FLAGS ${defs_list}) +endif() + +# get target compile options +get_target_property(options_list mdbx COMPILE_OPTIONS) +if(options_list) + list(APPEND MDBX_BUILD_FLAGS ${options_list}) +endif() + +list(REMOVE_DUPLICATES MDBX_BUILD_FLAGS) +string(REPLACE ";" " " MDBX_BUILD_FLAGS "${MDBX_BUILD_FLAGS}") +if(CMAKE_CONFIGURATION_TYPES) + # add dynamic part via per-configuration define + message(STATUS "MDBX Compile Flags: ${MDBX_BUILD_FLAGS} ") + add_definitions(-DMDBX_BUILD_FLAGS_CONFIG="$<$:${CMAKE_C_FLAGS_DEBUG} ${CMAKE_C_DEFINES_DEBUG}>$<$:${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_DEFINES_RELEASE}>$<$:${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_DEFINES_RELWITHDEBINFO}>$<$:${CMAKE_C_FLAGS_MINSIZEREL} ${CMAKE_C_DEFINES_MINSIZEREL}>") +else() + message(STATUS "MDBX Compile Flags: ${MDBX_BUILD_FLAGS}") +endif() + +# get compiler info +execute_process(COMMAND sh -c "${CMAKE_C_COMPILER} --version | head -1" + OUTPUT_VARIABLE MDBX_BUILD_COMPILER + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + RESULT_VARIABLE rc) +if(rc OR NOT MDBX_BUILD_COMPILER) + string(STRIP "${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION}" MDBX_BUILD_COMPILER) +endif() + +# make a build-target triplet +if(CMAKE_C_COMPILER_TARGET) + set(MDBX_BUILD_TARGET "${CMAKE_C_COMPILER_TARGET}") +elseif(CMAKE_C_PLATFORM_ID AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME) + string(STRIP "${CMAKE_C_PLATFORM_ID}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +elseif(CMAKE_LIBRARY_ARCHITECTURE) + string(STRIP "${CMAKE_LIBRARY_ARCHITECTURE}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +elseif(CMAKE_GENERATOR_PLATFORM AND NOT CMAKE_C_PLATFORM_ID STREQUAL CMAKE_SYSTEM_NAME) + string(STRIP "${CMAKE_GENERATOR_PLATFORM}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +elseif(CMAKE_SYSTEM_ARCH) + string(STRIP "${CMAKE_SYSTEM_ARCH}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +else() + string(STRIP "${CMAKE_SYSTEM_PROCESSOR}-${CMAKE_SYSTEM_NAME}" MDBX_BUILD_TARGET) +endif() +if(CMAKE_CONFIGURATION_TYPES) + add_definitions(-DMDBX_BUILD_CONFIG="$") +else() + set(MDBX_BUILD_CONFIG ${CMAKE_BUILD_TYPE}) +endif() + +# options +string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC) +set(options VERSION C_COMPILER CXX_COMPILER) +foreach(item IN LISTS options) + if(DEFINED ${item}) + set(value "${${item}}") + elseif(DEFINED MDBX_${item}) + set(item MDBX_${item}) + set(value "${${item}}") + elseif(DEFINED CMAKE_${item}) + set(item CMAKE_${item}) + set(value "${${item}}") + else() + set(value "undefined") + endif() + message(STATUS "${item}: ${value}") +endforeach(item) + +# generate version and config files +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c.in" + "${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c" ESCAPE_QUOTES) + +file(SHA256 "${CMAKE_CURRENT_SOURCE_DIR}/elements/version.c" MDBX_SOURCERY_DIGEST) +string(MAKE_C_IDENTIFIER "${MDBX_GIT_DESCRIBE}" MDBX_SOURCERY_SUFFIX) +set(MDBX_BUILD_SOURCERY "${MDBX_SOURCERY_DIGEST}_${MDBX_SOURCERY_SUFFIX}") + +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h.in" + "${CMAKE_CURRENT_SOURCE_DIR}/elements/config.h" ESCAPE_QUOTES) +add_definitions(-DMDBX_CONFIG_H="config.h") + +add_subdirectory(tools) diff --git a/src/contrib/db/libmdbx/src/alloy.c b/src/contrib/db/libmdbx/src/alloy.c new file mode 100644 index 0000000..98f3aac --- /dev/null +++ b/src/contrib/db/libmdbx/src/alloy.c @@ -0,0 +1,26 @@ +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . */ + +/* Amalgamated build */ +#define MDBX_ALLOY 1 +#include "elements/internals.h" /* must be included fisrt */ + +#include "elements/core.c" +#include "elements/osal.c" +#include "elements/version.c" + +#if defined(_WIN32) || defined(_WIN64) +#include "elements/lck-windows.c" +#else +#include "elements/lck-posix.c" +#endif diff --git a/src/contrib/db/libmdbx/src/elements/config.h.in b/src/contrib/db/libmdbx/src/elements/config.h.in new file mode 100644 index 0000000..44ae150 --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/config.h.in @@ -0,0 +1,56 @@ +/* This is CMake-template for libmdbx's config.h + ******************************************************************************/ + +/* *INDENT-OFF* */ +/* clang-format off */ + +#cmakedefine HAVE_VALGRIND_MEMCHECK_H +#cmakedefine HAS_RELAXED_CONSTEXPR + +#cmakedefine LTO_ENABLED +#cmakedefine MDBX_USE_VALGRIND +#cmakedefine ENABLE_GPROF +#cmakedefine ENABLE_GCOV +#cmakedefine ENABLE_ASAN + +/* Common */ +#cmakedefine01 MDBX_TXN_CHECKPID +#cmakedefine01 MDBX_TXN_CHECKOWNER +#cmakedefine01 MDBX_BUILD_SHARED_LIBRARY + +/* Windows */ +#cmakedefine01 MDBX_CONFIG_MANUAL_TLS_CALLBACK +#cmakedefine01 MDBX_AVOID_CRT + +/* MacOS */ +#cmakedefine01 MDBX_OSX_SPEED_INSTEADOF_DURABILITY + +/* POSIX */ +#cmakedefine01 MDBX_USE_ROBUST +#cmakedefine01 MDBX_USE_OFDLOCKS +#cmakedefine01 MDBX_DISABLE_GNU_SOURCE + +/* Simulate "AUTO" values of tristate options */ +#cmakedefine MDBX_TXN_CHECKPID_AUTO +#ifdef MDBX_TXN_CHECKPID_AUTO +#undef MDBX_TXN_CHECKPID +#endif +#cmakedefine MDBX_USE_ROBUST_AUTO +#ifdef MDBX_USE_ROBUST_AUTO +#undef MDBX_USE_ROBUST +#endif +#cmakedefine MDBX_USE_OFDLOCKS_AUTO +#ifdef MDBX_USE_OFDLOCKS_AUTO +#undef MDBX_USE_OFDLOCKS +#endif + +/* Build Info */ +#cmakedefine MDBX_BUILD_TIMESTAMP "@MDBX_BUILD_TIMESTAMP@" +#cmakedefine MDBX_BUILD_TARGET "@MDBX_BUILD_TARGET@" +#cmakedefine MDBX_BUILD_CONFIG "@MDBX_BUILD_CONFIG@" +#cmakedefine MDBX_BUILD_COMPILER "@MDBX_BUILD_COMPILER@" +#cmakedefine MDBX_BUILD_FLAGS "@MDBX_BUILD_FLAGS@" +#cmakedefine MDBX_BUILD_SOURCERY @MDBX_BUILD_SOURCERY@ + +/* *INDENT-ON* */ +/* clang-format on */ diff --git a/src/contrib/db/libmdbx/src/elements/core.c b/src/contrib/db/libmdbx/src/elements/core.c new file mode 100644 index 0000000..c64259c --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/core.c @@ -0,0 +1,16709 @@ +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * This code is derived from "LMDB engine" written by + * Howard Chu (Symas Corporation), which itself derived from btree.c + * written by Martin Hedenfalk. + * + * --- + * + * Portions Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * --- + * + * Portions Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + +#include "internals.h" + +/*------------------------------------------------------------------------------ + * Internal inlines */ + +static __pure_function __inline bool is_powerof2(size_t x) { + return (x & (x - 1)) == 0; +} + +static __pure_function __inline size_t roundup_powerof2(size_t value, + size_t granularity) { + assert(is_powerof2(granularity)); + return (value + granularity - 1) & ~(granularity - 1); +} + +static __pure_function unsigned log2n(size_t value) { + assert(value > 0 && value < INT32_MAX && is_powerof2(value)); + assert((value & -(int32_t)value) == value); +#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl) + return __builtin_ctzl(value); +#elif defined(_MSC_VER) + unsigned long index; + _BitScanForward(&index, (unsigned long)value); + return index; +#else + static const uint8_t debruijn_ctz32[32] = { + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; + return debruijn_ctz32[(uint32_t)(value * 0x077CB531u) >> 27]; +#endif +} + +/*------------------------------------------------------------------------------ + * Unaligned access */ + +static __pure_function __maybe_unused __inline unsigned +field_alignment(unsigned alignment_baseline, size_t field_offset) { + unsigned merge = alignment_baseline | (unsigned)field_offset; + return merge & -(int)merge; +} + +/* read-thunk for UB-sanitizer */ +static __pure_function __inline uint8_t peek_u8(const uint8_t *ptr) { + return *ptr; +} + +/* write-thunk for UB-sanitizer */ +static __inline void poke_u8(uint8_t *ptr, const uint8_t v) { *ptr = v; } + +static __pure_function __inline uint16_t +unaligned_peek_u16(const unsigned expected_alignment, const void *ptr) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(uint16_t)) == 0) + return *(const uint16_t *)ptr; + else { + uint16_t v; + memcpy(&v, ptr, sizeof(v)); + return v; + } +} + +static __inline void unaligned_poke_u16(const unsigned expected_alignment, + void *ptr, const uint16_t v) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(v)) == 0) + *(uint16_t *)ptr = v; + else + memcpy(ptr, &v, sizeof(v)); +} + +static __pure_function __inline uint32_t +unaligned_peek_u32(const unsigned expected_alignment, const void *ptr) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(uint32_t)) == 0) + return *(const uint32_t *)ptr; + else if ((expected_alignment % sizeof(uint16_t)) == 0) { + const uint16_t lo = + ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; + const uint16_t hi = + ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; + return lo | (uint32_t)hi << 16; + } else { + uint32_t v; + memcpy(&v, ptr, sizeof(v)); + return v; + } +} + +static __inline void unaligned_poke_u32(const unsigned expected_alignment, + void *ptr, const uint32_t v) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(v)) == 0) + *(uint32_t *)ptr = v; + else if ((expected_alignment % sizeof(uint16_t)) == 0) { + ((uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint16_t)v; + ((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = + (uint16_t)(v >> 16); + } else + memcpy(ptr, &v, sizeof(v)); +} + +static __pure_function __inline uint64_t +unaligned_peek_u64(const unsigned expected_alignment, const void *ptr) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(uint64_t)) == 0) + return *(const uint64_t *)ptr; + else if ((expected_alignment % sizeof(uint32_t)) == 0) { + const uint32_t lo = + ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; + const uint32_t hi = + ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; + return lo | (uint64_t)hi << 32; + } else { + uint64_t v; + memcpy(&v, ptr, sizeof(v)); + return v; + } +} + +static __inline void unaligned_poke_u64(const unsigned expected_alignment, + void *ptr, const uint64_t v) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK || (expected_alignment % sizeof(v)) == 0) + *(uint64_t *)ptr = v; + else if ((expected_alignment % sizeof(uint32_t)) == 0) { + ((uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint32_t)v; + ((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = + (uint32_t)(v >> 32); + } else + memcpy(ptr, &v, sizeof(v)); +} + +#define UNALIGNED_PEEK_8(ptr, struct, field) \ + peek_u8((const uint8_t *)(ptr) + offsetof(struct, field)) +#define UNALIGNED_POKE_8(ptr, struct, field, value) \ + poke_u8((uint8_t *)(ptr) + offsetof(struct, field), value) + +#define UNALIGNED_PEEK_16(ptr, struct, field) \ + unaligned_peek_u16(1, (const char *)(ptr) + offsetof(struct, field)) +#define UNALIGNED_POKE_16(ptr, struct, field, value) \ + unaligned_poke_u16(1, (char *)(ptr) + offsetof(struct, field), value) + +#define UNALIGNED_PEEK_32(ptr, struct, field) \ + unaligned_peek_u32(1, (const char *)(ptr) + offsetof(struct, field)) +#define UNALIGNED_POKE_32(ptr, struct, field, value) \ + unaligned_poke_u32(1, (char *)(ptr) + offsetof(struct, field), value) + +#define UNALIGNED_PEEK_64(ptr, struct, field) \ + unaligned_peek_u64(1, (const char *)(ptr) + offsetof(struct, field)) +#define UNALIGNED_POKE_64(ptr, struct, field, value) \ + unaligned_poke_u64(1, (char *)(ptr) + offsetof(struct, field), value) + +/* Get the page number pointed to by a branch node */ +static __pure_function __inline pgno_t node_pgno(const MDBX_node *node) { + pgno_t pgno = UNALIGNED_PEEK_32(node, MDBX_node, mn_pgno32); + if (sizeof(pgno) > 4) + pgno |= ((uint64_t)UNALIGNED_PEEK_8(node, MDBX_node, mn_extra)) << 32; + return pgno; +} + +/* Set the page number in a branch node */ +static __inline void node_set_pgno(MDBX_node *node, pgno_t pgno) { + assert(pgno >= MIN_PAGENO && pgno <= MAX_PAGENO); + + UNALIGNED_POKE_32(node, MDBX_node, mn_pgno32, (uint32_t)pgno); + if (sizeof(pgno) > 4) + UNALIGNED_POKE_8(node, MDBX_node, mn_extra, + (uint8_t)((uint64_t)pgno >> 32)); +} + +/* Get the size of the data in a leaf node */ +static __pure_function __inline size_t node_ds(const MDBX_node *node) { + return UNALIGNED_PEEK_32(node, MDBX_node, mn_dsize); +} + +/* Set the size of the data for a leaf node */ +static __inline void node_set_ds(MDBX_node *node, size_t size) { + assert(size < INT_MAX); + UNALIGNED_POKE_32(node, MDBX_node, mn_dsize, (uint32_t)size); +} + +/* The size of a key in a node */ +static __pure_function __inline size_t node_ks(const MDBX_node *node) { + return UNALIGNED_PEEK_16(node, MDBX_node, mn_ksize); +} + +/* Set the size of the key for a leaf node */ +static __inline void node_set_ks(MDBX_node *node, size_t size) { + assert(size < INT16_MAX); + UNALIGNED_POKE_16(node, MDBX_node, mn_ksize, (uint16_t)size); +} + +static __pure_function __inline uint8_t node_flags(const MDBX_node *node) { + return UNALIGNED_PEEK_8(node, MDBX_node, mn_flags); +} + +static __inline void node_set_flags(MDBX_node *node, uint8_t flags) { + UNALIGNED_POKE_8(node, MDBX_node, mn_flags, flags); +} + +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE offsetof(MDBX_node, mn_data) + +/* Address of the key for the node */ +static __pure_function __inline void *node_key(const MDBX_node *node) { + return (char *)node + NODESIZE; +} + +/* Address of the data for a node */ +static __pure_function __inline void *node_data(const MDBX_node *node) { + return (char *)node_key(node) + node_ks(node); +} + +/* Size of a node in a leaf page with a given key and data. + * This is node header plus key plus data size. */ +static __pure_function __inline size_t node_size(const MDBX_val *key, + const MDBX_val *value) { + return NODESIZE + + EVEN((key ? key->iov_len : 0) + (value ? value->iov_len : 0)); +} + +static __pure_function __inline pgno_t peek_pgno(const void *ptr) { + if (sizeof(pgno_t) == sizeof(uint32_t)) + return (pgno_t)unaligned_peek_u32(1, ptr); + else if (sizeof(pgno_t) == sizeof(uint64_t)) + return (pgno_t)unaligned_peek_u64(1, ptr); + else { + pgno_t pgno; + memcpy(&pgno, ptr, sizeof(pgno)); + return pgno; + } +} + +static __inline void poke_pgno(void *ptr, const pgno_t pgno) { + if (sizeof(pgno) == sizeof(uint32_t)) + unaligned_poke_u32(1, ptr, pgno); + else if (sizeof(pgno) == sizeof(uint64_t)) + unaligned_poke_u64(1, ptr, pgno); + else + memcpy(ptr, &pgno, sizeof(pgno)); +} + +static __pure_function __inline pgno_t +node_largedata_pgno(const MDBX_node *node) { + assert(node_flags(node) & F_BIGDATA); + return peek_pgno(node_data(node)); +} + +/* Calculate the size of a leaf node. + * + * The size depends on the environment's page size; if a data item + * is too large it will be put onto an overflow page and the node + * size will only include the key and not the data. Sizes are always + * rounded up to an even number of bytes, to guarantee 2-byte alignment + * of the MDBX_node headers. + * + * [in] env The environment handle. + * [in] key The key for the node. + * [in] data The data for the node. + * + * Returns The number of bytes needed to store the node. */ +static __pure_function __inline size_t +leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) { + size_t node_bytes = node_size(key, data); + if (node_bytes > env->me_nodemax) { + /* put on overflow page */ + node_bytes = node_size(key, nullptr) + sizeof(pgno_t); + } + + return node_bytes + sizeof(indx_t); +} + +/* Calculate the size of a branch node. + * + * The size should depend on the environment's page size but since + * we currently don't support spilling large keys onto overflow + * pages, it's simply the size of the MDBX_node header plus the + * size of the key. Sizes are always rounded up to an even number + * of bytes, to guarantee 2-byte alignment of the MDBX_node headers. + * + * [in] env The environment handle. + * [in] key The key for the node. + * + * Returns The number of bytes needed to store the node. */ +static __pure_function __inline size_t branch_size(const MDBX_env *env, + const MDBX_val *key) { + /* Size of a node in a branch page with a given key. + * This is just the node header plus the key, there is no data. */ + size_t node_bytes = node_size(key, nullptr); + if (unlikely(node_bytes > env->me_nodemax)) { + /* put on overflow page */ + /* not implemented */ + mdbx_assert_fail(env, "INDXSIZE(key) <= env->me_nodemax", __func__, + __LINE__); + node_bytes = node_size(key, nullptr) + sizeof(pgno_t); + } + + return node_bytes + sizeof(indx_t); +} + +/*----------------------------------------------------------------------------*/ + +static __pure_function __inline size_t pgno2bytes(const MDBX_env *env, + pgno_t pgno) { + mdbx_assert(env, (1u << env->me_psize2log) == env->me_psize); + return ((size_t)pgno) << env->me_psize2log; +} + +static __pure_function __inline MDBX_page *pgno2page(const MDBX_env *env, + pgno_t pgno) { + return (MDBX_page *)(env->me_map + pgno2bytes(env, pgno)); +} + +static __pure_function __inline pgno_t bytes2pgno(const MDBX_env *env, + size_t bytes) { + mdbx_assert(env, (env->me_psize >> env->me_psize2log) == 1); + return (pgno_t)(bytes >> env->me_psize2log); +} + +static __pure_function __inline size_t pgno_align2os_bytes(const MDBX_env *env, + pgno_t pgno) { + return roundup_powerof2(pgno2bytes(env, pgno), env->me_os_psize); +} + +static __pure_function __inline pgno_t pgno_align2os_pgno(const MDBX_env *env, + pgno_t pgno) { + return bytes2pgno(env, pgno_align2os_bytes(env, pgno)); +} + +static __pure_function __inline size_t bytes_align2os_bytes(const MDBX_env *env, + size_t bytes) { + return roundup_powerof2(roundup_powerof2(bytes, env->me_psize), + env->me_os_psize); +} + +/* Address of first usable data byte in a page, after the header */ +static __pure_function __inline void *page_data(const MDBX_page *mp) { + return (char *)mp + PAGEHDRSZ; +} + +static __pure_function __inline const MDBX_page *data_page(const void *data) { + return container_of(data, MDBX_page, mp_ptrs); +} + +static __pure_function __inline MDBX_meta *page_meta(MDBX_page *mp) { + return (MDBX_meta *)page_data(mp); +} + +/* Number of nodes on a page */ +static __pure_function __inline unsigned page_numkeys(const MDBX_page *mp) { + return mp->mp_lower >> 1; +} + +/* The amount of space remaining in the page */ +static __pure_function __inline unsigned page_room(const MDBX_page *mp) { + return mp->mp_upper - mp->mp_lower; +} + +static __pure_function __inline unsigned page_space(const MDBX_env *env) { + STATIC_ASSERT(PAGEHDRSZ % 2 == 0); + return env->me_psize - PAGEHDRSZ; +} + +static __pure_function __inline unsigned page_used(const MDBX_env *env, + const MDBX_page *mp) { + return page_space(env) - page_room(mp); +} + +/* The percentage of space used in the page, in a percents. */ +static __pure_function __maybe_unused __inline double +page_fill(const MDBX_env *env, const MDBX_page *mp) { + return page_used(env, mp) * 100.0 / page_space(env); +} + +static __pure_function __inline bool +page_fill_enough(const MDBX_page *mp, unsigned spaceleft_threshold, + unsigned minkeys_threshold) { + return page_room(mp) < spaceleft_threshold && + page_numkeys(mp) >= minkeys_threshold; +} + +/* The number of overflow pages needed to store the given size. */ +static __pure_function __inline pgno_t number_of_ovpages(const MDBX_env *env, + size_t bytes) { + return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1; +} + +/* Address of node i in page p */ +static __pure_function __inline MDBX_node *page_node(const MDBX_page *mp, + unsigned i) { + assert((mp->mp_flags & (P_LEAF2 | P_OVERFLOW | P_META)) == 0); + assert(page_numkeys(mp) > (unsigned)(i)); + assert(mp->mp_ptrs[i] % 2 == 0); + return (MDBX_node *)((char *)mp + mp->mp_ptrs[i] + PAGEHDRSZ); +} + +/* The address of a key in a LEAF2 page. + * LEAF2 pages are used for MDBX_DUPFIXED sorted-duplicate sub-DBs. + * There are no node headers, keys are stored contiguously. */ +static __pure_function __inline void * +page_leaf2key(const MDBX_page *mp, unsigned i, size_t keysize) { + assert(mp->mp_leaf2_ksize == keysize); + (void)keysize; + return (char *)mp + PAGEHDRSZ + (i * mp->mp_leaf2_ksize); +} + +/* Set the node's key into keyptr. */ +static __inline void get_key(const MDBX_node *node, MDBX_val *keyptr) { + keyptr->iov_len = node_ks(node); + keyptr->iov_base = node_key(node); +} + +/* Set the node's key into keyptr, if requested. */ +static __inline void get_key_optional(const MDBX_node *node, + MDBX_val *keyptr /* __may_null */) { + if (keyptr) + get_key(node, keyptr); +} + +/*------------------------------------------------------------------------------ + * LY: temporary workaround for Elbrus's memcmp() bug. */ + +#if defined(__e2k__) && !__GLIBC_PREREQ(2, 24) +int __hot mdbx_e2k_memcmp_bug_workaround(const void *s1, const void *s2, + size_t n) { + if (unlikely(n > 42 + /* LY: align followed access if reasonable possible */ + && (((uintptr_t)s1) & 7) != 0 && + (((uintptr_t)s1) & 7) == (((uintptr_t)s2) & 7))) { + if (((uintptr_t)s1) & 1) { + const int diff = *(uint8_t *)s1 - *(uint8_t *)s2; + if (diff) + return diff; + s1 = (char *)s1 + 1; + s2 = (char *)s2 + 1; + n -= 1; + } + + if (((uintptr_t)s1) & 2) { + const uint16_t a = *(uint16_t *)s1; + const uint16_t b = *(uint16_t *)s2; + if (likely(a != b)) + return (__builtin_bswap16(a) > __builtin_bswap16(b)) ? 1 : -1; + s1 = (char *)s1 + 2; + s2 = (char *)s2 + 2; + n -= 2; + } + + if (((uintptr_t)s1) & 4) { + const uint32_t a = *(uint32_t *)s1; + const uint32_t b = *(uint32_t *)s2; + if (likely(a != b)) + return (__builtin_bswap32(a) > __builtin_bswap32(b)) ? 1 : -1; + s1 = (char *)s1 + 4; + s2 = (char *)s2 + 4; + n -= 4; + } + } + + while (n >= 8) { + const uint64_t a = *(uint64_t *)s1; + const uint64_t b = *(uint64_t *)s2; + if (likely(a != b)) + return (__builtin_bswap64(a) > __builtin_bswap64(b)) ? 1 : -1; + s1 = (char *)s1 + 8; + s2 = (char *)s2 + 8; + n -= 8; + } + + if (n & 4) { + const uint32_t a = *(uint32_t *)s1; + const uint32_t b = *(uint32_t *)s2; + if (likely(a != b)) + return (__builtin_bswap32(a) > __builtin_bswap32(b)) ? 1 : -1; + s1 = (char *)s1 + 4; + s2 = (char *)s2 + 4; + } + + if (n & 2) { + const uint16_t a = *(uint16_t *)s1; + const uint16_t b = *(uint16_t *)s2; + if (likely(a != b)) + return (__builtin_bswap16(a) > __builtin_bswap16(b)) ? 1 : -1; + s1 = (char *)s1 + 2; + s2 = (char *)s2 + 2; + } + + return (n & 1) ? *(uint8_t *)s1 - *(uint8_t *)s2 : 0; +} + +int __hot mdbx_e2k_strcmp_bug_workaround(const char *s1, const char *s2) { + while (true) { + int diff = *(uint8_t *)s1 - *(uint8_t *)s2; + if (likely(diff != 0) || *s1 == '\0') + return diff; + s1 += 1; + s2 += 1; + } +} + +int __hot mdbx_e2k_strncmp_bug_workaround(const char *s1, const char *s2, + size_t n) { + while (n > 0) { + int diff = *(uint8_t *)s1 - *(uint8_t *)s2; + if (likely(diff != 0) || *s1 == '\0') + return diff; + s1 += 1; + s2 += 1; + n -= 1; + } + return 0; +} + +size_t __hot mdbx_e2k_strlen_bug_workaround(const char *s) { + size_t n = 0; + while (*s) { + s += 1; + n += 1; + } + return n; +} + +size_t __hot mdbx_e2k_strnlen_bug_workaround(const char *s, size_t maxlen) { + size_t n = 0; + while (maxlen > n && *s) { + s += 1; + n += 1; + } + return n; +} +#endif /* Elbrus's memcmp() bug. */ + +/*------------------------------------------------------------------------------ + * safe read/write volatile 64-bit fields on 32-bit architectures. */ + +static __inline void atomic_yield(void) { +#if defined(_WIN32) || defined(_WIN64) + YieldProcessor(); +#elif defined(__x86_64__) || defined(__i386__) || defined(__e2k__) + __builtin_ia32_pause(); +#elif defined(__ia64__) +#if defined(__HP_cc__) || defined(__HP_aCC__) + _Asm_hint(_HINT_PAUSE); +#else + __asm__ __volatile__("hint @pause"); +#endif +#elif defined(__arm__) || defined(__aarch64__) +#ifdef __CC_ARM + __yield(); +#else + __asm__ __volatile__("yield"); +#endif +#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && \ + __mips_isa_rev >= 2 + __asm__ __volatile__("pause"); +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) + __asm__ __volatile__(".word 0x00000140"); +#else + pthread_yield(); +#endif +} + +#if MDBX_64BIT_CAS +static __inline bool atomic_cas64(volatile uint64_t *p, uint64_t c, + uint64_t v) { +#if defined(ATOMIC_VAR_INIT) || defined(ATOMIC_LLONG_LOCK_FREE) + STATIC_ASSERT(sizeof(long long int) == 8); + STATIC_ASSERT(atomic_is_lock_free(p)); + return atomic_compare_exchange_strong((_Atomic uint64_t *)p, &c, v); +#elif defined(__GNUC__) || defined(__clang__) + return __sync_bool_compare_and_swap(p, c, v); +#elif defined(_MSC_VER) + return c == + (uint64_t)_InterlockedCompareExchange64((volatile int64_t *)p, v, c); +#elif defined(__APPLE__) + return OSAtomicCompareAndSwap64Barrier(c, v, (volatile uint64_t *)p); +#else +#error FIXME: Unsupported compiler +#endif +} +#endif /* MDBX_64BIT_CAS */ + +static __inline bool atomic_cas32(volatile uint32_t *p, uint32_t c, + uint32_t v) { +#if defined(ATOMIC_VAR_INIT) || defined(ATOMIC_LONG_LOCK_FREE) + STATIC_ASSERT(sizeof(long int) >= 4); + STATIC_ASSERT(atomic_is_lock_free(p)); + return atomic_compare_exchange_strong((_Atomic uint32_t *)p, &c, v); +#elif defined(__GNUC__) || defined(__clang__) + return __sync_bool_compare_and_swap(p, c, v); +#elif defined(_MSC_VER) + STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t)); + return c == (uint32_t)_InterlockedCompareExchange((volatile long *)p, v, c); +#elif defined(__APPLE__) + return OSAtomicCompareAndSwap32Barrier(c, v, (volatile int32_t *)p); +#else +#error FIXME: Unsupported compiler +#endif +} + +static __inline uint32_t atomic_add32(volatile uint32_t *p, uint32_t v) { +#if defined(ATOMIC_VAR_INIT) || defined(ATOMIC_LONG_LOCK_FREE) + STATIC_ASSERT(sizeof(long int) >= 4); + STATIC_ASSERT(atomic_is_lock_free(p)); + return atomic_fetch_add((_Atomic uint32_t *)p, v); +#elif defined(__GNUC__) || defined(__clang__) + return __sync_fetch_and_add(p, v); +#elif defined(_MSC_VER) + STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t)); + return _InterlockedExchangeAdd((volatile long *)p, v); +#elif defined(__APPLE__) + return OSAtomicAdd32Barrier(v, (volatile int32_t *)p); +#else +#error FIXME: Unsupported compiler +#endif +} + +#define atomic_sub32(p, v) atomic_add32(p, 0 - (v)) + +static __maybe_unused __inline bool safe64_is_valid(uint64_t v) { +#if MDBX_WORDBITS >= 64 + return v < SAFE64_INVALID_THRESHOLD; +#else + return (v >> 32) != UINT32_MAX; +#endif /* MDBX_WORDBITS */ +} + +static __maybe_unused __inline bool +safe64_is_valid_ptr(const mdbx_safe64_t *ptr) { + mdbx_compiler_barrier(); +#if MDBX_64BIT_ATOMIC + return ptr->atomic < SAFE64_INVALID_THRESHOLD; +#else + return ptr->high != UINT32_MAX; +#endif /* MDBX_64BIT_ATOMIC */ +} + +static __inline uint64_t safe64_txnid_next(uint64_t txnid) { + txnid += MDBX_TXNID_STEP; +#if !MDBX_64BIT_CAS + /* avoid overflow of low-part in safe64_reset() */ + txnid += (UINT32_MAX == (uint32_t)txnid); +#endif + return txnid; +} + +static __inline void safe64_reset(mdbx_safe64_t *ptr, bool single_writer) { + mdbx_compiler_barrier(); +#if !MDBX_64BIT_CAS + if (!single_writer) { + STATIC_ASSERT(MDBX_TXNID_STEP > 1); + /* it is safe to increment low-part to avoid ABA, since MDBX_TXNID_STEP > 1 + * and overflow was preserved in safe64_txnid_next() */ + atomic_add32(&ptr->low, 1) /* avoid ABA in safe64_reset_compare() */; + ptr->high = UINT32_MAX /* atomically make >= SAFE64_INVALID_THRESHOLD */; + atomic_add32(&ptr->low, 1) /* avoid ABA in safe64_reset_compare() */; + } else +#else + (void)single_writer; +#endif /* !MDBX_64BIT_CAS */ +#if MDBX_64BIT_ATOMIC + ptr->atomic = UINT64_MAX; +#else + /* atomically make value >= SAFE64_INVALID_THRESHOLD */ + ptr->high = UINT32_MAX; +#endif /* MDBX_64BIT_ATOMIC */ + assert(ptr->inconsistent >= SAFE64_INVALID_THRESHOLD); + mdbx_flush_noncoherent_cpu_writeback(); + mdbx_jitter4testing(true); +} + +static __inline bool safe64_reset_compare(mdbx_safe64_t *ptr, txnid_t compare) { + mdbx_compiler_barrier(); + /* LY: This function is used to reset `mr_txnid` from OOM-kick in case + * the asynchronously cancellation of read transaction. Therefore, + * there may be a collision between the cleanup performed here and + * asynchronous termination and restarting of the read transaction + * in another proces/thread. In general we MUST NOT reset the `mr_txnid` + * if a new transaction was started (i.e. if `mr_txnid` was changed). */ +#if MDBX_64BIT_CAS + bool rc = atomic_cas64(&ptr->inconsistent, compare, UINT64_MAX); + mdbx_flush_noncoherent_cpu_writeback(); +#else + /* LY: There is no gold ratio here since shared mutex is too costly, + * in such way we must acquire/release it for every update of mr_txnid, + * i.e. twice for each read transaction). */ + bool rc = false; + if (likely(ptr->low == (uint32_t)compare && + atomic_cas32(&ptr->high, (uint32_t)(compare >> 32), UINT32_MAX))) { + if (unlikely(ptr->low != (uint32_t)compare)) + atomic_cas32(&ptr->high, UINT32_MAX, (uint32_t)(compare >> 32)); + else + rc = true; + } +#endif /* MDBX_64BIT_CAS */ + mdbx_jitter4testing(true); + return rc; +} + +static __inline void safe64_write(mdbx_safe64_t *ptr, const uint64_t v) { + mdbx_compiler_barrier(); + assert(ptr->inconsistent >= SAFE64_INVALID_THRESHOLD); +#if MDBX_64BIT_ATOMIC + ptr->atomic = v; +#else /* MDBX_64BIT_ATOMIC */ + /* update low-part but still value >= SAFE64_INVALID_THRESHOLD */ + ptr->low = (uint32_t)v; + assert(ptr->inconsistent >= SAFE64_INVALID_THRESHOLD); + mdbx_flush_noncoherent_cpu_writeback(); + mdbx_jitter4testing(true); + /* update high-part from SAFE64_INVALID_THRESHOLD to actual value */ + ptr->high = (uint32_t)(v >> 32); +#endif /* MDBX_64BIT_ATOMIC */ + assert(ptr->inconsistent == v); + mdbx_flush_noncoherent_cpu_writeback(); + mdbx_jitter4testing(true); +} + +static __always_inline uint64_t safe64_read(const mdbx_safe64_t *ptr) { + mdbx_compiler_barrier(); + mdbx_jitter4testing(true); + uint64_t v; +#if MDBX_64BIT_ATOMIC + v = ptr->atomic; +#else /* MDBX_64BIT_ATOMIC */ + uint32_t hi, lo; + do { + hi = ptr->high; + mdbx_compiler_barrier(); + mdbx_jitter4testing(true); + lo = ptr->low; + mdbx_compiler_barrier(); + mdbx_jitter4testing(true); + } while (unlikely(hi != ptr->high)); + v = lo | (uint64_t)hi << 32; +#endif /* MDBX_64BIT_ATOMIC */ + mdbx_jitter4testing(true); + return v; +} + +static __inline void safe64_update(mdbx_safe64_t *ptr, const uint64_t v) { + safe64_reset(ptr, true); + safe64_write(ptr, v); +} + +/*----------------------------------------------------------------------------*/ +/* rthc (tls keys and destructors) */ + +typedef struct rthc_entry_t { + MDBX_reader *begin; + MDBX_reader *end; + mdbx_thread_key_t thr_tls_key; + bool key_valid; +} rthc_entry_t; + +#if MDBX_DEBUG +#define RTHC_INITIAL_LIMIT 1 +#else +#define RTHC_INITIAL_LIMIT 16 +#endif + +static bin128_t bootid; +#if defined(_WIN32) || defined(_WIN64) +static CRITICAL_SECTION rthc_critical_section; +static CRITICAL_SECTION lcklist_critical_section; +#else +int __cxa_thread_atexit_impl(void (*dtor)(void *), void *obj, void *dso_symbol) + __attribute__((__weak__)); +#ifdef __APPLE__ /* FIXME: Thread-Local Storage destructors & DSO-unloading */ +int __cxa_thread_atexit_impl(void (*dtor)(void *), void *obj, + void *dso_symbol) { + (void)dtor; + (void)obj; + (void)dso_symbol; + return -1; +} +#endif /* __APPLE__ */ + +static pthread_mutex_t lcklist_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t rthc_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t rthc_cond = PTHREAD_COND_INITIALIZER; +static mdbx_thread_key_t rthc_key; +static volatile uint32_t rthc_pending; + +static void __cold workaround_glibc_bug21031(void) { + /* Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031 + * + * Due race between pthread_key_delete() and __nptl_deallocate_tsd() + * The destructor(s) of thread-local-storate object(s) may be running + * in another thread(s) and be blocked or not finished yet. + * In such case we get a SEGFAULT after unload this library DSO. + * + * So just by yielding a few timeslices we give a chance + * to such destructor(s) for completion and avoids segfault. */ + sched_yield(); + sched_yield(); + sched_yield(); +} +#endif + +static unsigned rthc_count, rthc_limit; +static rthc_entry_t *rthc_table; +static rthc_entry_t rthc_table_static[RTHC_INITIAL_LIMIT]; + +static __inline void rthc_lock(void) { +#if defined(_WIN32) || defined(_WIN64) + EnterCriticalSection(&rthc_critical_section); +#else + mdbx_ensure(nullptr, pthread_mutex_lock(&rthc_mutex) == 0); +#endif +} + +static __inline void rthc_unlock(void) { +#if defined(_WIN32) || defined(_WIN64) + LeaveCriticalSection(&rthc_critical_section); +#else + mdbx_ensure(nullptr, pthread_mutex_unlock(&rthc_mutex) == 0); +#endif +} + +static __inline int thread_key_create(mdbx_thread_key_t *key) { + int rc; +#if defined(_WIN32) || defined(_WIN64) + *key = TlsAlloc(); + rc = (*key != TLS_OUT_OF_INDEXES) ? MDBX_SUCCESS : GetLastError(); +#else + rc = pthread_key_create(key, nullptr); +#endif + mdbx_trace("&key = %p, value 0x%x, rc %d", __Wpedantic_format_voidptr(key), + (unsigned)*key, rc); + return rc; +} + +static __inline void thread_key_delete(mdbx_thread_key_t key) { + mdbx_trace("key = 0x%x", (unsigned)key); +#if defined(_WIN32) || defined(_WIN64) + mdbx_ensure(nullptr, TlsFree(key)); +#else + mdbx_ensure(nullptr, pthread_key_delete(key) == 0); + workaround_glibc_bug21031(); +#endif +} + +static __inline void *thread_rthc_get(mdbx_thread_key_t key) { +#if defined(_WIN32) || defined(_WIN64) + return TlsGetValue(key); +#else + return pthread_getspecific(key); +#endif +} + +static void thread_rthc_set(mdbx_thread_key_t key, const void *value) { +#if defined(_WIN32) || defined(_WIN64) + mdbx_ensure(nullptr, TlsSetValue(key, (void *)value)); +#else +#define MDBX_THREAD_RTHC_ZERO 0 +#define MDBX_THREAD_RTHC_REGISTERD 1 +#define MDBX_THREAD_RTHC_COUNTED 2 + static __thread uint32_t thread_registration_state; + if (value && unlikely(thread_registration_state == MDBX_THREAD_RTHC_ZERO)) { + thread_registration_state = MDBX_THREAD_RTHC_REGISTERD; + mdbx_trace("thread registered 0x%" PRIxPTR, (uintptr_t)mdbx_thread_self()); + if (&__cxa_thread_atexit_impl == nullptr || + __cxa_thread_atexit_impl(mdbx_rthc_thread_dtor, + &thread_registration_state, + (void *)&mdbx_version /* dso_anchor */)) { + mdbx_ensure(nullptr, pthread_setspecific( + rthc_key, &thread_registration_state) == 0); + thread_registration_state = MDBX_THREAD_RTHC_COUNTED; + const unsigned count_before = atomic_add32(&rthc_pending, 1); + mdbx_ensure(nullptr, count_before < INT_MAX); + mdbx_trace("fallback to pthreads' tsd, key 0x%x, count %u", + (unsigned)rthc_key, count_before); + (void)count_before; + } + } + mdbx_ensure(nullptr, pthread_setspecific(key, value) == 0); +#endif +} + +__cold void mdbx_rthc_global_init(void) { + rthc_limit = RTHC_INITIAL_LIMIT; + rthc_table = rthc_table_static; +#if defined(_WIN32) || defined(_WIN64) + InitializeCriticalSection(&rthc_critical_section); + InitializeCriticalSection(&lcklist_critical_section); +#else + mdbx_ensure(nullptr, + pthread_key_create(&rthc_key, mdbx_rthc_thread_dtor) == 0); + mdbx_trace("pid %d, &mdbx_rthc_key = %p, value 0x%x", mdbx_getpid(), + __Wpedantic_format_voidptr(&rthc_key), (unsigned)rthc_key); +#endif + /* checking time conversion, this also avoids racing on 32-bit architectures + * during writing calculated 64-bit ratio(s) into memory. */ + uint32_t proba = UINT32_MAX; + while (true) { + unsigned time_conversion_checkup = + mdbx_osal_monotime_to_16dot16(mdbx_osal_16dot16_to_monotime(proba)); + unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba; + unsigned one_less = (proba > 0) ? proba - 1 : proba; + mdbx_ensure(nullptr, time_conversion_checkup >= one_less && + time_conversion_checkup <= one_more); + if (proba == 0) + break; + proba >>= 1; + } + + bootid = mdbx_osal_bootid(); +} + +/* dtor called for thread, i.e. for all mdbx's environment objects */ +__cold void mdbx_rthc_thread_dtor(void *ptr) { + rthc_lock(); + mdbx_trace(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", mdbx_getpid(), + (uintptr_t)mdbx_thread_self(), ptr); + + const uint32_t self_pid = mdbx_getpid(); + for (unsigned i = 0; i < rthc_count; ++i) { + if (!rthc_table[i].key_valid) + continue; + const mdbx_thread_key_t key = rthc_table[i].thr_tls_key; + MDBX_reader *const rthc = thread_rthc_get(key); + if (rthc < rthc_table[i].begin || rthc >= rthc_table[i].end) + continue; +#if !defined(_WIN32) && !defined(_WIN64) + if (pthread_setspecific(key, nullptr) != 0) { + mdbx_trace("== thread 0x%" PRIxPTR + ", rthc %p: ignore race with tsd-key deletion", + (uintptr_t)mdbx_thread_self(), ptr); + continue /* ignore race with tsd-key deletion by mdbx_env_close() */; + } +#endif + + mdbx_trace("== thread 0x%" PRIxPTR + ", rthc %p, [%i], %p ... %p (%+i), rtch-pid %i, " + "current-pid %i", + (uintptr_t)mdbx_thread_self(), __Wpedantic_format_voidptr(rthc), + i, __Wpedantic_format_voidptr(rthc_table[i].begin), + __Wpedantic_format_voidptr(rthc_table[i].end), + (int)(rthc - rthc_table[i].begin), rthc->mr_pid, self_pid); + if (rthc->mr_pid == self_pid) { + mdbx_trace("==== thread 0x%" PRIxPTR ", rthc %p, cleanup", + (uintptr_t)mdbx_thread_self(), + __Wpedantic_format_voidptr(rthc)); + rthc->mr_pid = 0; + } + } + +#if defined(_WIN32) || defined(_WIN64) + mdbx_trace("<< thread 0x%" PRIxPTR ", rthc %p", (uintptr_t)mdbx_thread_self(), + ptr); + rthc_unlock(); +#else + const char self_registration = *(char *)ptr; + *(char *)ptr = MDBX_THREAD_RTHC_ZERO; + mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %d", + (uintptr_t)mdbx_thread_self(), ptr, mdbx_getpid(), + self_registration); + if (self_registration == MDBX_THREAD_RTHC_COUNTED) + mdbx_ensure(nullptr, atomic_sub32(&rthc_pending, 1) > 0); + + if (rthc_pending == 0) { + mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake", + (uintptr_t)mdbx_thread_self(), ptr, mdbx_getpid()); + mdbx_ensure(nullptr, pthread_cond_broadcast(&rthc_cond) == 0); + } + + mdbx_trace("<< thread 0x%" PRIxPTR ", rthc %p", (uintptr_t)mdbx_thread_self(), + ptr); + /* Allow tail call optimization, i.e. gcc should generate the jmp instruction + * instead of a call for pthread_mutex_unlock() and therefore CPU could not + * return to current DSO's code section, which may be unloaded immediately + * after the mutex got released. */ + pthread_mutex_unlock(&rthc_mutex); +#endif +} + +__cold void mdbx_rthc_global_dtor(void) { + mdbx_trace(">> pid %d", mdbx_getpid()); + + rthc_lock(); +#if !defined(_WIN32) && !defined(_WIN64) + char *rthc = (char *)pthread_getspecific(rthc_key); + mdbx_trace("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %d", + (uintptr_t)mdbx_thread_self(), __Wpedantic_format_voidptr(rthc), + mdbx_getpid(), rthc ? *rthc : -1); + if (rthc) { + const char self_registration = *(char *)rthc; + *rthc = MDBX_THREAD_RTHC_ZERO; + if (self_registration == MDBX_THREAD_RTHC_COUNTED) + mdbx_ensure(nullptr, atomic_sub32(&rthc_pending, 1) > 0); + } + + struct timespec abstime; + mdbx_ensure(nullptr, clock_gettime(CLOCK_REALTIME, &abstime) == 0); + abstime.tv_nsec += 1000000000l / 10; + if (abstime.tv_nsec >= 1000000000l) { + abstime.tv_nsec -= 1000000000l; + abstime.tv_sec += 1; + } +#if MDBX_DEBUG > 0 + abstime.tv_sec += 600; +#endif + + for (unsigned left; (left = rthc_pending) > 0;) { + mdbx_trace("pid %d, pending %u, wait for...", mdbx_getpid(), left); + const int rc = pthread_cond_timedwait(&rthc_cond, &rthc_mutex, &abstime); + if (rc && rc != EINTR) + break; + } + thread_key_delete(rthc_key); +#endif + + const uint32_t self_pid = mdbx_getpid(); + for (unsigned i = 0; i < rthc_count; ++i) { + if (!rthc_table[i].key_valid) + continue; + const mdbx_thread_key_t key = rthc_table[i].thr_tls_key; + thread_key_delete(key); + for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end; + ++rthc) { + mdbx_trace("== [%i] = key %zu, %p ... %p, rthc %p (%+i), " + "rthc-pid %i, current-pid %i", + i, (size_t)key, + __Wpedantic_format_voidptr(rthc_table[i].begin), + __Wpedantic_format_voidptr(rthc_table[i].end), + __Wpedantic_format_voidptr(rthc), + (int)(rthc - rthc_table[i].begin), rthc->mr_pid, self_pid); + if (rthc->mr_pid == self_pid) { + rthc->mr_pid = 0; + mdbx_trace("== cleanup %p", __Wpedantic_format_voidptr(rthc)); + } + } + } + + rthc_limit = rthc_count = 0; + if (rthc_table != rthc_table_static) + mdbx_free(rthc_table); + rthc_table = nullptr; + rthc_unlock(); + +#if defined(_WIN32) || defined(_WIN64) + DeleteCriticalSection(&lcklist_critical_section); + DeleteCriticalSection(&rthc_critical_section); +#else + /* LY: yielding a few timeslices to give a more chance + * to racing destructor(s) for completion. */ + workaround_glibc_bug21031(); +#endif + + mdbx_trace("<< pid %d\n", mdbx_getpid()); +} + +__cold int mdbx_rthc_alloc(mdbx_thread_key_t *key, MDBX_reader *begin, + MDBX_reader *end) { + int rc; + if (key) { +#ifndef NDEBUG + *key = (mdbx_thread_key_t)0xBADBADBAD; +#endif /* NDEBUG */ + rc = thread_key_create(key); + if (rc != MDBX_SUCCESS) + return rc; + } + + rthc_lock(); + const mdbx_thread_key_t new_key = key ? *key : 0; + mdbx_trace(">> key %zu, rthc_count %u, rthc_limit %u", (size_t)new_key, + rthc_count, rthc_limit); + if (rthc_count == rthc_limit) { + rthc_entry_t *new_table = + mdbx_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table, + sizeof(rthc_entry_t) * rthc_limit * 2); + if (new_table == nullptr) { + rc = MDBX_ENOMEM; + goto bailout; + } + if (rthc_table == rthc_table_static) + memcpy(new_table, rthc_table_static, sizeof(rthc_table_static)); + rthc_table = new_table; + rthc_limit *= 2; + } + mdbx_trace("== [%i] = key %zu, %p ... %p", rthc_count, (size_t)new_key, + __Wpedantic_format_voidptr(begin), + __Wpedantic_format_voidptr(end)); + rthc_table[rthc_count].key_valid = key ? true : false; + rthc_table[rthc_count].thr_tls_key = key ? new_key : 0; + rthc_table[rthc_count].begin = begin; + rthc_table[rthc_count].end = end; + ++rthc_count; + mdbx_trace("<< key %zu, rthc_count %u, rthc_limit %u", (size_t)new_key, + rthc_count, rthc_limit); + rthc_unlock(); + return MDBX_SUCCESS; + +bailout: + if (key) + thread_key_delete(*key); + rthc_unlock(); + return rc; +} + +__cold void mdbx_rthc_remove(const mdbx_thread_key_t key) { + thread_key_delete(key); + rthc_lock(); + mdbx_trace(">> key %zu, rthc_count %u, rthc_limit %u", (size_t)key, + rthc_count, rthc_limit); + + for (unsigned i = 0; i < rthc_count; ++i) { + if (rthc_table[i].key_valid && key == rthc_table[i].thr_tls_key) { + const uint32_t self_pid = mdbx_getpid(); + mdbx_trace("== [%i], %p ...%p, current-pid %d", i, + __Wpedantic_format_voidptr(rthc_table[i].begin), + __Wpedantic_format_voidptr(rthc_table[i].end), self_pid); + + for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end; + ++rthc) { + if (rthc->mr_pid == self_pid) { + rthc->mr_pid = 0; + mdbx_trace("== cleanup %p", __Wpedantic_format_voidptr(rthc)); + } + } + if (--rthc_count > 0) + rthc_table[i] = rthc_table[rthc_count]; + else if (rthc_table != rthc_table_static) { + mdbx_free(rthc_table); + rthc_table = rthc_table_static; + rthc_limit = RTHC_INITIAL_LIMIT; + } + break; + } + } + + mdbx_trace("<< key %zu, rthc_count %u, rthc_limit %u", (size_t)key, + rthc_count, rthc_limit); + rthc_unlock(); +} + +//------------------------------------------------------------------------------ + +#define RTHC_ENVLIST_END ((MDBX_env *)((size_t)50459)) +static MDBX_env *inprocess_lcklist_head = RTHC_ENVLIST_END; + +static __inline void lcklist_lock(void) { +#if defined(_WIN32) || defined(_WIN64) + EnterCriticalSection(&lcklist_critical_section); +#else + mdbx_ensure(nullptr, pthread_mutex_lock(&lcklist_mutex) == 0); +#endif +} + +static __inline void lcklist_unlock(void) { +#if defined(_WIN32) || defined(_WIN64) + LeaveCriticalSection(&lcklist_critical_section); +#else + mdbx_ensure(nullptr, pthread_mutex_unlock(&lcklist_mutex) == 0); +#endif +} + +static uint64_t rrxmrrxmsx_0(uint64_t v) { + /* Pelle Evensen's mixer, https://bit.ly/2HOfynt */ + v ^= (v << 39 | v >> 25) ^ (v << 14 | v >> 50); + v *= UINT64_C(0xA24BAED4963EE407); + v ^= (v << 40 | v >> 24) ^ (v << 15 | v >> 49); + v *= UINT64_C(0x9FB21C651E98DF25); + return v ^ v >> 28; +} + +static int uniq_peek(const mdbx_mmap_t *pending, mdbx_mmap_t *scan) { + int rc; + uint64_t bait; + if (pending->address) { + bait = pending->lck->mti_bait_uniqueness; + rc = MDBX_SUCCESS; + } else { + bait = 0 /* hush MSVC warning */; + rc = mdbx_msync(scan, 0, sizeof(MDBX_lockinfo), true); + if (rc == MDBX_SUCCESS) + rc = + mdbx_pread(pending->fd, &bait, sizeof(scan->lck->mti_bait_uniqueness), + offsetof(MDBX_lockinfo, mti_bait_uniqueness)); + } + if (likely(rc == MDBX_SUCCESS) && bait == scan->lck->mti_bait_uniqueness) + rc = MDBX_RESULT_TRUE; + + mdbx_trace("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d", + pending->lck ? "mem" : "file", bait, + (rc == MDBX_RESULT_TRUE) ? " found," : (rc ? " FAILED," : ""), rc); + return rc; +} + +static int uniq_poke(const mdbx_mmap_t *pending, mdbx_mmap_t *scan, + uint64_t *abra) { + if (*abra == 0) { + const size_t tid = mdbx_thread_self(); + size_t uit = 0; + memcpy(&uit, &tid, (sizeof(tid) < sizeof(uit)) ? sizeof(tid) : sizeof(uit)); + *abra = + rrxmrrxmsx_0(mdbx_osal_monotime() + UINT64_C(5873865991930747) * uit); + } + const uint64_t cadabra = + rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)mdbx_getpid()) + << 24 | + *abra >> 40; + scan->lck->mti_bait_uniqueness = cadabra; + mdbx_flush_noncoherent_cpu_writeback(); + *abra = *abra * UINT64_C(6364136223846793005) + 1; + return uniq_peek(pending, scan); +} + +__cold static int uniq_check(const mdbx_mmap_t *pending, MDBX_env **found) { + *found = nullptr; + uint64_t salt = 0; + for (MDBX_env *scan = inprocess_lcklist_head; scan != RTHC_ENVLIST_END; + scan = scan->me_lcklist_next) { + int err = scan->me_lck_mmap.lck->mti_bait_uniqueness + ? uniq_peek(pending, &scan->me_lck_mmap) + : uniq_poke(pending, &scan->me_lck_mmap, &salt); + if (err == MDBX_ENODATA) { + uint64_t length; + if (likely(mdbx_filesize(pending->fd, &length) == MDBX_SUCCESS && + length == 0)) { + /* LY: skip checking since LCK-file is empty, i.e. just created. */ + mdbx_debug("uniq-probe: %s", "unique (new/empty lck)"); + return MDBX_RESULT_TRUE; + } + } + if (err == MDBX_RESULT_TRUE) + err = uniq_poke(pending, &scan->me_lck_mmap, &salt); + if (err == MDBX_RESULT_TRUE) { + (void)mdbx_msync(&scan->me_lck_mmap, 0, sizeof(MDBX_lockinfo), false); + err = uniq_poke(pending, &scan->me_lck_mmap, &salt); + } + if (err == MDBX_RESULT_TRUE) { + err = uniq_poke(pending, &scan->me_lck_mmap, &salt); + *found = scan; + mdbx_debug("uniq-probe: found %p", __Wpedantic_format_voidptr(*found)); + return MDBX_RESULT_FALSE; + } + if (unlikely(err != MDBX_SUCCESS)) { + mdbx_debug("uniq-probe: failed rc %d", err); + return err; + } + } + + mdbx_debug("uniq-probe: %s", "unique"); + return MDBX_RESULT_TRUE; +} + +static int lcklist_detach_locked(MDBX_env *env) { + MDBX_env *inprocess_neighbor = nullptr; + int rc = MDBX_SUCCESS; + if (env->me_lcklist_next != nullptr) { + mdbx_ensure(env, env->me_lcklist_next != nullptr); + mdbx_ensure(env, inprocess_lcklist_head != RTHC_ENVLIST_END); + for (MDBX_env **ptr = &inprocess_lcklist_head; *ptr != RTHC_ENVLIST_END; + ptr = &(*ptr)->me_lcklist_next) { + if (*ptr == env) { + *ptr = env->me_lcklist_next; + env->me_lcklist_next = nullptr; + break; + } + } + mdbx_ensure(env, env->me_lcklist_next == nullptr); + } + + rc = likely(mdbx_getpid() == env->me_pid) + ? uniq_check(&env->me_lck_mmap, &inprocess_neighbor) + : MDBX_PANIC; + if (!inprocess_neighbor && env->me_live_reader) + (void)mdbx_rpid_clear(env); + if (!MDBX_IS_ERROR(rc)) + rc = mdbx_lck_destroy(env, inprocess_neighbor); + return rc; +} + +/*------------------------------------------------------------------------------ + * LY: State of the art quicksort-based sorting, with internal stack and + * shell-insertion-sort for small chunks (less than half of SORT_THRESHOLD). + */ + +/* LY: Large threshold give some boost due less overhead in the inner qsort + * loops, but also a penalty in cases reverse-sorted data. + * So, 42 is magically but reasonable: + * - 0-3% faster than std::sort (from GNU C++ STL 2018) in most cases. + * - slower by a few ticks in a few cases for sequences shorter than 21. */ +#define SORT_THRESHOLD 42 + +#define SORT_SWAP(TYPE, a, b) \ + do { \ + const TYPE swap_tmp = (a); \ + (a) = (b); \ + (b) = swap_tmp; \ + } while (0) + +#define SORT_SHELLPASS(TYPE, CMP, begin, end, gap) \ + for (TYPE *i = begin + gap; i < end; ++i) { \ + for (TYPE *j = i - (gap); j >= begin && CMP(*i, *j); j -= gap) { \ + const TYPE tmp = *i; \ + do { \ + j[gap] = *j; \ + j -= gap; \ + } while (j >= begin && CMP(tmp, *j)); \ + j[gap] = tmp; \ + break; \ + } \ + } + +#define SORT_PUSH(low, high) \ + do { \ + top->lo = (low); \ + top->hi = (high); \ + ++top; \ + } while (0) + +#define SORT_POP(low, high) \ + do { \ + --top; \ + low = top->lo; \ + high = top->hi; \ + } while (0) + +#define SORT_IMPL(NAME, TYPE, CMP) \ + \ + typedef struct { \ + TYPE *lo, *hi; \ + } NAME##_stack; \ + \ + static __hot void NAME(TYPE *const begin, TYPE *const end) { \ + const ptrdiff_t length = end - begin; \ + if (length < 2) \ + return; \ + \ + if (length > SORT_THRESHOLD / 2) { \ + NAME##_stack stack[sizeof(unsigned) * CHAR_BIT], *top = stack; \ + \ + TYPE *hi = end - 1; \ + TYPE *lo = begin; \ + while (true) { \ + TYPE *mid = lo + ((hi - lo) >> 1); \ + if (CMP(*mid, *lo)) \ + SORT_SWAP(TYPE, *mid, *lo); \ + if (CMP(*hi, *mid)) { \ + SORT_SWAP(TYPE, *hi, *mid); \ + if (CMP(*mid, *lo)) \ + SORT_SWAP(TYPE, *mid, *lo); \ + } \ + \ + TYPE *right = hi - 1; \ + TYPE *left = lo + 1; \ + do { \ + while (CMP(*mid, *right)) \ + --right; \ + while (CMP(*left, *mid)) \ + ++left; \ + if (left < right) { \ + SORT_SWAP(TYPE, *left, *right); \ + if (mid == left) \ + mid = right; \ + else if (mid == right) \ + mid = left; \ + ++left; \ + --right; \ + } else if (left == right) { \ + ++left; \ + --right; \ + break; \ + } \ + } while (left <= right); \ + \ + if (lo + SORT_THRESHOLD > right) { \ + if (left + SORT_THRESHOLD > hi) { \ + if (top == stack) \ + break; \ + else \ + SORT_POP(lo, hi); \ + } else \ + lo = left; \ + } else if (left + SORT_THRESHOLD > hi) \ + hi = right; \ + else if (right - lo > hi - left) { \ + SORT_PUSH(lo, right); \ + lo = left; \ + } else { \ + SORT_PUSH(left, hi); \ + hi = right; \ + } \ + } \ + } \ + \ + SORT_SHELLPASS(TYPE, CMP, begin, end, 8); \ + SORT_SHELLPASS(TYPE, CMP, begin, end, 1); \ + for (TYPE *scan = begin + 1; scan < end; ++scan) \ + assert(CMP(scan[-1], scan[0])); \ + } + +/*------------------------------------------------------------------------------ + * LY: Binary search */ + +#define SEARCH_IMPL(NAME, TYPE_LIST, TYPE_ARG, CMP) \ + static __always_inline TYPE_LIST *NAME(TYPE_LIST *first, unsigned length, \ + const TYPE_ARG item) { \ + TYPE_LIST *const begin = first, *const end = begin + length; \ + \ + while (length > 3) { \ + const unsigned half = length >> 1; \ + TYPE_LIST *const middle = first + half; \ + if (CMP(*middle, item)) { \ + first = middle + 1; \ + length -= half + 1; \ + } else \ + length = half; \ + } \ + \ + switch (length) { \ + case 3: \ + if (!CMP(*first, item)) \ + break; \ + ++first; \ + /* fall through */ \ + __fallthrough; \ + case 2: \ + if (!CMP(*first, item)) \ + break; \ + ++first; \ + /* fall through */ \ + __fallthrough; \ + case 1: \ + if (CMP(*first, item)) \ + ++first; \ + } \ + \ + for (TYPE_LIST *scan = begin; scan < first; ++scan) \ + assert(CMP(*scan, item)); \ + for (TYPE_LIST *scan = first; scan < end; ++scan) \ + assert(!CMP(*scan, item)); \ + (void)begin, (void)end; \ + \ + return first; \ + } + +/*----------------------------------------------------------------------------*/ + +static __inline size_t pnl2bytes(const size_t size) { + assert(size > 0 && size <= MDBX_PNL_MAX * 2); + size_t bytes = roundup_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + + sizeof(pgno_t) * (size + 2), + MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; +} + +static __inline pgno_t bytes2pnl(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 2 && size <= MDBX_PNL_MAX * 2); + return (pgno_t)size - 2; +} + +static MDBX_PNL mdbx_pnl_alloc(size_t size) { + size_t bytes = pnl2bytes(size); + MDBX_PNL pl = mdbx_malloc(bytes); + if (likely(pl)) { +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(malloc_usable_size) + bytes = malloc_usable_size(pl); +#endif /* malloc_usable_size */ + pl[0] = bytes2pnl(bytes); + assert(pl[0] >= size); + pl[1] = 0; + pl += 1; + } + return pl; +} + +static void mdbx_pnl_free(MDBX_PNL pl) { + if (likely(pl)) + mdbx_free(pl - 1); +} + +/* Shrink the PNL to the default size if it has grown larger */ +static void mdbx_pnl_shrink(MDBX_PNL *ppl) { + assert(bytes2pnl(pnl2bytes(MDBX_PNL_INITIAL)) == MDBX_PNL_INITIAL); + assert(MDBX_PNL_SIZE(*ppl) <= MDBX_PNL_MAX && + MDBX_PNL_ALLOCLEN(*ppl) >= MDBX_PNL_SIZE(*ppl)); + MDBX_PNL_SIZE(*ppl) = 0; + if (unlikely(MDBX_PNL_ALLOCLEN(*ppl) > + MDBX_PNL_INITIAL * 2 - MDBX_CACHELINE_SIZE / sizeof(pgno_t))) { + size_t bytes = pnl2bytes(MDBX_PNL_INITIAL); + MDBX_PNL pl = mdbx_realloc(*ppl - 1, bytes); + if (likely(pl)) { +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(malloc_usable_size) + bytes = malloc_usable_size(pl); +#endif /* malloc_usable_size */ + *pl = bytes2pnl(bytes); + *ppl = pl + 1; + } + } +} + +/* Grow the PNL to the size growed to at least given size */ +static int mdbx_pnl_reserve(MDBX_PNL *ppl, const size_t wanna) { + const size_t allocated = MDBX_PNL_ALLOCLEN(*ppl); + assert(MDBX_PNL_SIZE(*ppl) <= MDBX_PNL_MAX && + MDBX_PNL_ALLOCLEN(*ppl) >= MDBX_PNL_SIZE(*ppl)); + if (likely(allocated >= wanna)) + return MDBX_SUCCESS; + + if (unlikely(wanna > /* paranoia */ MDBX_PNL_MAX)) + return MDBX_TXN_FULL; + + const size_t size = (wanna + wanna - allocated < MDBX_PNL_MAX) + ? wanna + wanna - allocated + : MDBX_PNL_MAX; + size_t bytes = pnl2bytes(size); + MDBX_PNL pl = mdbx_realloc(*ppl - 1, bytes); + if (likely(pl)) { +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(malloc_usable_size) + bytes = malloc_usable_size(pl); +#endif /* malloc_usable_size */ + *pl = bytes2pnl(bytes); + assert(*pl >= wanna); + *ppl = pl + 1; + return MDBX_SUCCESS; + } + return MDBX_ENOMEM; +} + +/* Make room for num additional elements in an PNL */ +static __inline int __must_check_result mdbx_pnl_need(MDBX_PNL *ppl, + size_t num) { + assert(MDBX_PNL_SIZE(*ppl) <= MDBX_PNL_MAX && + MDBX_PNL_ALLOCLEN(*ppl) >= MDBX_PNL_SIZE(*ppl)); + assert(num <= MDBX_PNL_MAX); + const size_t wanna = MDBX_PNL_SIZE(*ppl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppl) >= wanna) + ? MDBX_SUCCESS + : mdbx_pnl_reserve(ppl, wanna); +} + +static __inline void mdbx_pnl_xappend(MDBX_PNL pl, pgno_t pgno) { + assert(MDBX_PNL_SIZE(pl) < MDBX_PNL_ALLOCLEN(pl)); + if (mdbx_assert_enabled()) { + for (unsigned i = MDBX_PNL_SIZE(pl); i > 0; --i) + assert(pgno != pl[i]); + } + MDBX_PNL_SIZE(pl) += 1; + MDBX_PNL_LAST(pl) = pgno; +} + +/* Append an pgno onto an unsorted PNL */ +static __hot int __must_check_result mdbx_pnl_append(MDBX_PNL *ppl, + pgno_t pgno) { + /* Too big? */ + if (unlikely(MDBX_PNL_SIZE(*ppl) == MDBX_PNL_ALLOCLEN(*ppl))) { + int rc = mdbx_pnl_need(ppl, MDBX_PNL_GRANULATE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + mdbx_pnl_xappend(*ppl, pgno); + return MDBX_SUCCESS; +} + +/* Append an PNL onto an unsorted PNL */ +static int __must_check_result mdbx_pnl_append_list(MDBX_PNL *ppl, + MDBX_PNL append) { + const unsigned len = MDBX_PNL_SIZE(append); + if (likely(len)) { + int rc = mdbx_pnl_need(ppl, MDBX_PNL_SIZE(append)); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + const MDBX_PNL pnl = *ppl; + unsigned w = MDBX_PNL_SIZE(pnl), r = 1; + do + pnl[++w] = append[r]; + while (++r <= len); + MDBX_PNL_SIZE(pnl) = w; + } + return MDBX_SUCCESS; +} + +/* Append an pgno range onto an unsorted PNL */ +static __hot int __must_check_result mdbx_pnl_append_range(MDBX_PNL *ppl, + pgno_t pgno, + unsigned n) { + assert(n > 0); + int rc = mdbx_pnl_need(ppl, n); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + const MDBX_PNL pnl = *ppl; +#if MDBX_PNL_ASCENDING + unsigned w = MDBX_PNL_SIZE(pnl); + do + pnl[++w] = pgno++; + while (--n); + MDBX_PNL_SIZE(pnl) = w; +#else + unsigned w = MDBX_PNL_SIZE(pnl) + n; + MDBX_PNL_SIZE(pnl) = w; + do + pnl[w--] = --n + pgno; + while (n); +#endif + + return MDBX_SUCCESS; +} + +/* Append an pgno range into the sorted PNL */ +static __hot int __must_check_result mdbx_pnl_insert_range(MDBX_PNL *ppl, + pgno_t pgno, + unsigned n) { + assert(n > 0); + int rc = mdbx_pnl_need(ppl, n); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + const MDBX_PNL pnl = *ppl; + unsigned r = MDBX_PNL_SIZE(pnl), w = r + n; + MDBX_PNL_SIZE(pnl) = w; + while (r && MDBX_PNL_DISORDERED(pnl[r], pgno)) + pnl[w--] = pnl[r--]; + + for (pgno_t fill = MDBX_PNL_ASCENDING ? pgno + n : pgno; w > r; --w) + pnl[w] = MDBX_PNL_ASCENDING ? --fill : fill++; + + return MDBX_SUCCESS; +} + +static bool __hot mdbx_pnl_check(const MDBX_PNL pl, const pgno_t limit) { + assert(limit >= MIN_PAGENO && limit <= MAX_PAGENO + 1); + if (likely(MDBX_PNL_SIZE(pl))) { + assert(MDBX_PNL_LEAST(pl) >= MIN_PAGENO); + assert(MDBX_PNL_MOST(pl) < limit); + assert(MDBX_PNL_SIZE(pl) <= MDBX_PNL_MAX); + if (unlikely(MDBX_PNL_SIZE(pl) > MDBX_PNL_MAX * 3 / 2)) + return false; + if (unlikely(MDBX_PNL_LEAST(pl) < MIN_PAGENO)) + return false; + if (unlikely(MDBX_PNL_MOST(pl) >= limit)) + return false; + for (const pgno_t *scan = &MDBX_PNL_LAST(pl); --scan > pl;) { + assert(MDBX_PNL_ORDERED(scan[0], scan[1])); + if (unlikely(!MDBX_PNL_ORDERED(scan[0], scan[1]))) + return false; + } + } + return true; +} + +static __inline bool mdbx_pnl_check4assert(const MDBX_PNL pl, + const pgno_t limit) { + if (unlikely(pl == nullptr)) + return true; + assert(MDBX_PNL_ALLOCLEN(pl) >= MDBX_PNL_SIZE(pl)); + if (unlikely(MDBX_PNL_ALLOCLEN(pl) < MDBX_PNL_SIZE(pl))) + return false; + return mdbx_pnl_check(pl, limit); +} + +/* Merge an PNL onto an PNL. The destination PNL must be big enough */ +static void __hot mdbx_pnl_xmerge(MDBX_PNL dst, const MDBX_PNL src) { + assert(mdbx_pnl_check4assert(dst, MAX_PAGENO + 1)); + assert(mdbx_pnl_check(src, MAX_PAGENO + 1)); + const size_t total = MDBX_PNL_SIZE(dst) + MDBX_PNL_SIZE(src); + assert(MDBX_PNL_ALLOCLEN(dst) >= total); + pgno_t *w = dst + total; + pgno_t *d = dst + MDBX_PNL_SIZE(dst); + const pgno_t *s = src + MDBX_PNL_SIZE(src); + dst[0] = /* detent for scan below */ (MDBX_PNL_ASCENDING ? 0 : ~(pgno_t)0); + while (s > src) { + while (MDBX_PNL_ORDERED(*s, *d)) + *w-- = *d--; + *w-- = *s--; + } + MDBX_PNL_SIZE(dst) = (pgno_t)total; + assert(mdbx_pnl_check4assert(dst, MAX_PAGENO + 1)); +} + +SORT_IMPL(pgno_sort, pgno_t, MDBX_PNL_ORDERED) +static __hot void mdbx_pnl_sort(MDBX_PNL pnl) { + pgno_sort(MDBX_PNL_BEGIN(pnl), MDBX_PNL_END(pnl)); + assert(mdbx_pnl_check(pnl, MAX_PAGENO + 1)); +} + +/* Search for an pgno in an PNL. + * Returns The index of the first item greater than or equal to pgno. */ +SEARCH_IMPL(pgno_bsearch, pgno_t, pgno_t, MDBX_PNL_ORDERED) + +static __hot unsigned mdbx_pnl_search(MDBX_PNL pnl, pgno_t id) { + assert(mdbx_pnl_check4assert(pnl, MAX_PAGENO + 1)); + pgno_t *begin = MDBX_PNL_BEGIN(pnl); + pgno_t *it = pgno_bsearch(begin, MDBX_PNL_SIZE(pnl), id); + pgno_t *end = begin + MDBX_PNL_SIZE(pnl); + assert(it >= begin && it <= end); + if (it != begin) + assert(MDBX_PNL_ORDERED(it[-1], id)); + if (it != end) + assert(!MDBX_PNL_ORDERED(it[0], id)); + return (unsigned)(it - begin + 1); +} + +static __hot unsigned mdbx_pnl_exist(MDBX_PNL pnl, pgno_t id) { + unsigned n = mdbx_pnl_search(pnl, id); + return (n <= MDBX_PNL_SIZE(pnl) && pnl[n] == id) ? n : 0; +} + +/*----------------------------------------------------------------------------*/ + +static __inline size_t txl2bytes(const size_t size) { + assert(size > 0 && size <= MDBX_TXL_MAX * 2); + size_t bytes = roundup_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + + sizeof(txnid_t) * (size + 2), + MDBX_TXL_GRANULATE * sizeof(txnid_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; +} + +static __inline size_t bytes2txl(const size_t bytes) { + size_t size = bytes / sizeof(txnid_t); + assert(size > 2 && size <= MDBX_TXL_MAX * 2); + return size - 2; +} + +static MDBX_TXL mdbx_txl_alloc(void) { + size_t bytes = txl2bytes(MDBX_TXL_INITIAL); + MDBX_TXL tl = mdbx_malloc(bytes); + if (likely(tl)) { +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(malloc_usable_size) + bytes = malloc_usable_size(tl); +#endif /* malloc_usable_size */ + tl[0] = bytes2txl(bytes); + assert(tl[0] >= MDBX_TXL_INITIAL); + tl[1] = 0; + tl += 1; + } + return tl; +} + +static void mdbx_txl_free(MDBX_TXL tl) { + if (likely(tl)) + mdbx_free(tl - 1); +} + +static int mdbx_txl_reserve(MDBX_TXL *ptl, const size_t wanna) { + const size_t allocated = (size_t)MDBX_PNL_ALLOCLEN(*ptl); + assert(MDBX_PNL_SIZE(*ptl) <= MDBX_TXL_MAX && + MDBX_PNL_ALLOCLEN(*ptl) >= MDBX_PNL_SIZE(*ptl)); + if (likely(allocated >= wanna)) + return MDBX_SUCCESS; + + if (unlikely(wanna > /* paranoia */ MDBX_TXL_MAX)) + return MDBX_TXN_FULL; + + const size_t size = (wanna + wanna - allocated < MDBX_TXL_MAX) + ? wanna + wanna - allocated + : MDBX_TXL_MAX; + size_t bytes = txl2bytes(size); + MDBX_TXL tl = mdbx_realloc(*ptl - 1, bytes); + if (likely(tl)) { +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(malloc_usable_size) + bytes = malloc_usable_size(tl); +#endif /* malloc_usable_size */ + *tl = bytes2txl(bytes); + assert(*tl >= wanna); + *ptl = tl + 1; + return MDBX_SUCCESS; + } + return MDBX_ENOMEM; +} + +static __inline int __must_check_result mdbx_txl_need(MDBX_TXL *ptl, + size_t num) { + assert(MDBX_PNL_SIZE(*ptl) <= MDBX_TXL_MAX && + MDBX_PNL_ALLOCLEN(*ptl) >= MDBX_PNL_SIZE(*ptl)); + assert(num <= MDBX_PNL_MAX); + const size_t wanna = (size_t)MDBX_PNL_SIZE(*ptl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ptl) >= wanna) + ? MDBX_SUCCESS + : mdbx_txl_reserve(ptl, wanna); +} + +static __inline void mdbx_txl_xappend(MDBX_TXL tl, txnid_t id) { + assert(MDBX_PNL_SIZE(tl) < MDBX_PNL_ALLOCLEN(tl)); + MDBX_PNL_SIZE(tl) += 1; + MDBX_PNL_LAST(tl) = id; +} + +#define TXNID_SORT_CMP(first, last) ((first) > (last)) +SORT_IMPL(txnid_sort, txnid_t, TXNID_SORT_CMP) +static void mdbx_txl_sort(MDBX_TXL tl) { + txnid_sort(MDBX_PNL_BEGIN(tl), MDBX_PNL_END(tl)); +} + +static int __must_check_result mdbx_txl_append(MDBX_TXL *ptl, txnid_t id) { + if (unlikely(MDBX_PNL_SIZE(*ptl) == MDBX_PNL_ALLOCLEN(*ptl))) { + int rc = mdbx_txl_need(ptl, MDBX_TXL_GRANULATE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + mdbx_txl_xappend(*ptl, id); + return MDBX_SUCCESS; +} + +/*----------------------------------------------------------------------------*/ + +#define DP_SORT_CMP(first, last) ((first).pgno < (last).pgno) +SORT_IMPL(dp_sort, MDBX_DP, DP_SORT_CMP) +static __inline MDBX_DPL mdbx_dpl_sort(MDBX_DPL dl) { + assert(dl->length <= MDBX_DPL_TXNFULL); + assert(dl->sorted <= dl->length); + if (dl->sorted != dl->length) { + dl->sorted = dl->length; + dp_sort(dl + 1, dl + dl->length + 1); + } + return dl; +} + +/* Returns the index of the first dirty-page whose pgno + * member is greater than or equal to id. */ +#define DP_SEARCH_CMP(dp, id) ((dp).pgno < (id)) +SEARCH_IMPL(dp_bsearch, MDBX_DP, pgno_t, DP_SEARCH_CMP) + +static unsigned __hot mdbx_dpl_search(MDBX_DPL dl, pgno_t pgno) { + if (dl->sorted < dl->length) { + /* unsorted tail case */ +#if MDBX_DEBUG + for (const MDBX_DP *ptr = dl + dl->sorted; --ptr > dl;) { + assert(ptr[0].pgno < ptr[1].pgno); + assert(ptr[0].pgno >= NUM_METAS); + } +#endif + + /* try linear search until the threshold */ + if (dl->length - dl->sorted < SORT_THRESHOLD / 2) { + unsigned i = dl->length; + while (i - dl->sorted > 7) { + if (dl[i].pgno == pgno) + return i; + if (dl[i - 1].pgno == pgno) + return i - 1; + if (dl[i - 2].pgno == pgno) + return i - 2; + if (dl[i - 3].pgno == pgno) + return i - 3; + if (dl[i - 4].pgno == pgno) + return i - 4; + if (dl[i - 5].pgno == pgno) + return i - 5; + if (dl[i - 6].pgno == pgno) + return i - 6; + if (dl[i - 7].pgno == pgno) + return i - 7; + i -= 8; + } + while (i > dl->sorted) { + if (dl[i].pgno == pgno) + return i; + --i; + } + + MDBX_DPL it = dp_bsearch(dl + 1, i, pgno); + return (unsigned)(it - dl); + } + + /* sort a whole */ + dl->sorted = dl->length; + dp_sort(dl + 1, dl + dl->length + 1); + } + +#if MDBX_DEBUG + for (const MDBX_DP *ptr = dl + dl->length; --ptr > dl;) { + assert(ptr[0].pgno < ptr[1].pgno); + assert(ptr[0].pgno >= NUM_METAS); + } +#endif + + MDBX_DPL it = dp_bsearch(dl + 1, dl->length, pgno); + return (unsigned)(it - dl); +} + +static __inline MDBX_page *mdbx_dpl_find(MDBX_DPL dl, pgno_t pgno) { + const unsigned i = mdbx_dpl_search(dl, pgno); + assert((int)i > 0); + return (i <= dl->length && dl[i].pgno == pgno) ? dl[i].ptr : nullptr; +} + +static __hot MDBX_page *mdbx_dpl_remove(MDBX_DPL dl, pgno_t prno) { + unsigned i = mdbx_dpl_search(dl, prno); + assert((int)i > 0); + MDBX_page *mp = nullptr; + if (i <= dl->length && dl[i].pgno == prno) { + dl->sorted -= dl->sorted >= i; + mp = dl[i].ptr; + while (i < dl->length) { + dl[i] = dl[i + 1]; + ++i; + } + dl->length -= 1; + } + return mp; +} + +static __inline int __must_check_result mdbx_dpl_append(MDBX_DPL dl, + pgno_t pgno, + MDBX_page *page) { + assert(dl->length <= MDBX_DPL_TXNFULL); +#if MDBX_DEBUG + for (unsigned i = dl->length; i > 0; --i) { + assert(dl[i].pgno != pgno); + if (unlikely(dl[i].pgno == pgno)) + return MDBX_PROBLEM; + } +#endif + + if (unlikely(dl->length == MDBX_DPL_TXNFULL)) + return MDBX_TXN_FULL; + + /* append page */ + const unsigned n = dl->length + 1; + if (n == 1 || (dl->sorted >= dl->length && dl[n - 1].pgno < pgno)) + dl->sorted = n; + dl->length = n; + dl[n].pgno = pgno; + dl[n].ptr = page; + return MDBX_SUCCESS; +} + +static __inline void mdbx_dpl_clear(MDBX_DPL dl) { + dl->sorted = dl->length = 0; +} + +/*----------------------------------------------------------------------------*/ + +#ifndef MDBX_ALLOY +uint8_t mdbx_runtime_flags = MDBX_RUNTIME_FLAGS_INIT; +uint8_t mdbx_loglevel = MDBX_DEBUG; +MDBX_debug_func *mdbx_debug_logger; +#endif /* MDBX_ALLOY */ + +static bool mdbx_refund(MDBX_txn *txn); +static __must_check_result int mdbx_page_retire(MDBX_cursor *mc, MDBX_page *mp); +static __must_check_result int mdbx_page_loose(MDBX_txn *txn, MDBX_page *mp); +static int mdbx_page_alloc(MDBX_cursor *mc, unsigned num, MDBX_page **mp, + int flags); +static int mdbx_page_new(MDBX_cursor *mc, uint32_t flags, unsigned num, + MDBX_page **mp); +static int mdbx_page_touch(MDBX_cursor *mc); +static int mdbx_cursor_touch(MDBX_cursor *mc); + +#define MDBX_END_NAMES \ + { \ + "committed", "empty-commit", "abort", "reset", "reset-tmp", "fail-begin", \ + "fail-beginchild" \ + } +enum { + /* mdbx_txn_end operation number, for logging */ + MDBX_END_COMMITTED, + MDBX_END_EMPTY_COMMIT, + MDBX_END_ABORT, + MDBX_END_RESET, + MDBX_END_RESET_TMP, + MDBX_END_FAIL_BEGIN, + MDBX_END_FAIL_BEGINCHILD +}; +#define MDBX_END_OPMASK 0x0F /* mask for mdbx_txn_end() operation number */ +#define MDBX_END_UPDATE 0x10 /* update env state (DBIs) */ +#define MDBX_END_FREE 0x20 /* free txn unless it is MDBX_env.me_txn0 */ +#define MDBX_END_EOTDONE 0x40 /* txn's cursors already closed */ +#define MDBX_END_SLOT 0x80 /* release any reader slot if MDBX_NOTLS */ +static int mdbx_txn_end(MDBX_txn *txn, unsigned mode); + +static int __must_check_result mdbx_page_get(MDBX_cursor *mc, pgno_t pgno, + MDBX_page **mp, int *lvl); +static int __must_check_result mdbx_page_search_root(MDBX_cursor *mc, + MDBX_val *key, int modify); + +#define MDBX_PS_MODIFY 1 +#define MDBX_PS_ROOTONLY 2 +#define MDBX_PS_FIRST 4 +#define MDBX_PS_LAST 8 +static int __must_check_result mdbx_page_search(MDBX_cursor *mc, MDBX_val *key, + int flags); +static int __must_check_result mdbx_page_merge(MDBX_cursor *csrc, + MDBX_cursor *cdst); +static int __must_check_result mdbx_page_flush(MDBX_txn *txn, + const unsigned keep); + +#define MDBX_SPLIT_REPLACE MDBX_APPENDDUP /* newkey is not new */ +static int __must_check_result mdbx_page_split(MDBX_cursor *mc, + const MDBX_val *newkey, + MDBX_val *newdata, + pgno_t newpgno, unsigned nflags); + +static int __must_check_result mdbx_read_header(MDBX_env *env, MDBX_meta *meta, + uint64_t *filesize); +static int __must_check_result mdbx_sync_locked(MDBX_env *env, unsigned flags, + MDBX_meta *const pending); +static int mdbx_env_close0(MDBX_env *env); + +static MDBX_node *mdbx_node_search(MDBX_cursor *mc, MDBX_val *key, int *exactp); + +static int __must_check_result mdbx_node_add_branch(MDBX_cursor *mc, + unsigned indx, + const MDBX_val *key, + pgno_t pgno); +static int __must_check_result mdbx_node_add_leaf(MDBX_cursor *mc, + unsigned indx, + const MDBX_val *key, + MDBX_val *data, + unsigned flags); +static int __must_check_result mdbx_node_add_leaf2(MDBX_cursor *mc, + unsigned indx, + const MDBX_val *key); + +static void mdbx_node_del(MDBX_cursor *mc, size_t ksize); +static void mdbx_node_shrink(MDBX_page *mp, unsigned indx); +static int __must_check_result mdbx_node_move(MDBX_cursor *csrc, + MDBX_cursor *cdst, int fromleft); +static int __must_check_result mdbx_node_read(MDBX_cursor *mc, MDBX_node *leaf, + MDBX_val *data); +static int __must_check_result mdbx_rebalance(MDBX_cursor *mc); +static int __must_check_result mdbx_update_key(MDBX_cursor *mc, + const MDBX_val *key); + +static void mdbx_cursor_pop(MDBX_cursor *mc); +static int __must_check_result mdbx_cursor_push(MDBX_cursor *mc, MDBX_page *mp); + +static int __must_check_result mdbx_audit_ex(MDBX_txn *txn, + unsigned retired_stored, + bool dont_filter_gc); +static __maybe_unused __inline int __must_check_result +mdbx_audit(MDBX_txn *txn) { + return mdbx_audit_ex(txn, 0, (txn->mt_flags & MDBX_RDONLY) != 0); +} + +static int __must_check_result mdbx_page_check(MDBX_env *env, + const MDBX_page *const mp, + bool maybe_unfinished); +static int __must_check_result mdbx_cursor_check(MDBX_cursor *mc, bool pending); +static int __must_check_result mdbx_cursor_del0(MDBX_cursor *mc); +static int __must_check_result mdbx_del0(MDBX_txn *txn, MDBX_dbi dbi, + MDBX_val *key, MDBX_val *data, + unsigned flags); +static int __must_check_result mdbx_cursor_sibling(MDBX_cursor *mc, + int move_right); +static int __must_check_result mdbx_cursor_next(MDBX_cursor *mc, MDBX_val *key, + MDBX_val *data, + MDBX_cursor_op op); +static int __must_check_result mdbx_cursor_prev(MDBX_cursor *mc, MDBX_val *key, + MDBX_val *data, + MDBX_cursor_op op); +static int __must_check_result mdbx_cursor_set(MDBX_cursor *mc, MDBX_val *key, + MDBX_val *data, + MDBX_cursor_op op, int *exactp); +static int __must_check_result mdbx_cursor_first(MDBX_cursor *mc, MDBX_val *key, + MDBX_val *data); +static int __must_check_result mdbx_cursor_last(MDBX_cursor *mc, MDBX_val *key, + MDBX_val *data); + +static int __must_check_result mdbx_cursor_init(MDBX_cursor *mc, MDBX_txn *txn, + MDBX_dbi dbi); +static int __must_check_result mdbx_xcursor_init0(MDBX_cursor *mc); +static int __must_check_result mdbx_xcursor_init1(MDBX_cursor *mc, + MDBX_node *node); +static int __must_check_result mdbx_xcursor_init2(MDBX_cursor *mc, + MDBX_xcursor *src_mx, + int force); +static void mdbx_cursor_copy(const MDBX_cursor *csrc, MDBX_cursor *cdst); + +static int __must_check_result mdbx_drop0(MDBX_cursor *mc, int subs); +static int __must_check_result mdbx_fetch_sdb(MDBX_txn *txn, MDBX_dbi dbi); + +static MDBX_cmp_func mdbx_cmp_memn, mdbx_cmp_memnr, mdbx_cmp_int_align4, + mdbx_cmp_int_align2, mdbx_cmp_int_unaligned; + +static const char *__mdbx_strerr(int errnum) { + /* Table of descriptions for MDBX errors */ + static const char *const tbl[] = { + "MDBX_KEYEXIST: Key/data pair already exists", + "MDBX_NOTFOUND: No matching key/data pair found", + "MDBX_PAGE_NOTFOUND: Requested page not found", + "MDBX_CORRUPTED: Database is corrupted", + "MDBX_PANIC: Environment had fatal error", + "MDBX_VERSION_MISMATCH: DB version mismatch libmdbx", + "MDBX_INVALID: File is not an MDBX file", + "MDBX_MAP_FULL: Environment mapsize limit reached", + "MDBX_DBS_FULL: Too may DBI (maxdbs reached)", + "MDBX_READERS_FULL: Too many readers (maxreaders reached)", + NULL /* MDBX_TLS_FULL (-30789): unused in MDBX */, + "MDBX_TXN_FULL: Transaction has too many dirty pages, " + "i.e transaction too big", + "MDBX_CURSOR_FULL: Internal error - cursor stack limit reached", + "MDBX_PAGE_FULL: Internal error - page has no more space", + "MDBX_MAP_RESIZED: Database contents grew beyond environment mapsize", + "MDBX_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", + "MDBX_BAD_RSLOT: Invalid reuse of reader locktable slot", + "MDBX_BAD_TXN: Transaction must abort, has a child, or is invalid", + "MDBX_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong " + "DUPFIXED size", + "MDBX_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", + "MDBX_PROBLEM: Unexpected problem - txn should abort", + "MDBX_BUSY: Another write transaction is running or " + "environment is already used while opening with MDBX_EXCLUSIVE flag", + }; + + if (errnum >= MDBX_KEYEXIST && errnum <= MDBX_LAST_ERRCODE) { + int i = errnum - MDBX_KEYEXIST; + return tbl[i]; + } + + switch (errnum) { + case MDBX_SUCCESS: + return "MDBX_SUCCESS: Successful"; + case MDBX_EMULTIVAL: + return "MDBX_EMULTIVAL: Unable to update multi-value for the given key"; + case MDBX_EBADSIGN: + return "MDBX_EBADSIGN: Wrong signature of a runtime object(s)"; + case MDBX_WANNA_RECOVERY: + return "MDBX_WANNA_RECOVERY: Database should be recovered, but this could " + "NOT be done in a read-only mode"; + case MDBX_EKEYMISMATCH: + return "MDBX_EKEYMISMATCH: The given key value is mismatched to the " + "current cursor position"; + case MDBX_TOO_LARGE: + return "MDBX_TOO_LARGE: Database is too large for current system, " + "e.g. could NOT be mapped into RAM"; + case MDBX_THREAD_MISMATCH: + return "MDBX_THREAD_MISMATCH: A thread has attempted to use a not " + "owned object, e.g. a transaction that started by another thread"; + default: + return NULL; + } +} + +const char *__cold mdbx_strerror_r(int errnum, char *buf, size_t buflen) { + const char *msg = __mdbx_strerr(errnum); + if (!msg && buflen > 0 && buflen < INT_MAX) { +#if defined(_WIN32) || defined(_WIN64) + const DWORD size = FormatMessageA( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, + errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, + NULL); + return size ? buf : NULL; +#elif defined(_GNU_SOURCE) && defined(__GLIBC__) + /* GNU-specific */ + if (errnum > 0) + msg = strerror_r(errnum, buf, buflen); +#elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) + /* XSI-compliant */ + if (errnum > 0 && strerror_r(errnum, buf, buflen) == 0) + msg = buf; +#else + if (errnum > 0) { + msg = strerror(errnum); + if (msg) { + strncpy(buf, msg, buflen); + msg = buf; + } + } +#endif + if (!msg) { + (void)snprintf(buf, buflen, "error %d", errnum); + msg = buf; + } + buf[buflen - 1] = '\0'; + } + return msg; +} + +const char *__cold mdbx_strerror(int errnum) { +#if defined(_WIN32) || defined(_WIN64) + static char buf[1024]; + return mdbx_strerror_r(errnum, buf, sizeof(buf)); +#else + const char *msg = __mdbx_strerr(errnum); + if (!msg) { + if (errnum > 0) + msg = strerror(errnum); + if (!msg) { + static char buf[32]; + (void)snprintf(buf, sizeof(buf) - 1, "error %d", errnum); + msg = buf; + } + } + return msg; +#endif +} + +#if defined(_WIN32) || defined(_WIN64) /* Bit of madness for Windows */ +const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen) { + const char *msg = __mdbx_strerr(errnum); + if (!msg && buflen > 0 && buflen < INT_MAX) { + const DWORD size = FormatMessageA( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, + errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, + NULL); + if (size && CharToOemBuffA(buf, buf, size)) + msg = buf; + } + return msg; +} + +const char *mdbx_strerror_ANSI2OEM(int errnum) { + static char buf[1024]; + return mdbx_strerror_r_ANSI2OEM(errnum, buf, sizeof(buf)); +} +#endif /* Bit of madness for Windows */ + +static txnid_t mdbx_oomkick(MDBX_env *env, const txnid_t laggard); + +void __cold mdbx_debug_log(int level, const char *function, int line, + const char *fmt, ...) { + va_list args; + + va_start(args, fmt); + if (mdbx_debug_logger) + mdbx_debug_logger(level, function, line, fmt, args); + else { +#if defined(_WIN32) || defined(_WIN64) + if (IsDebuggerPresent()) { + int prefix_len = 0; + char *prefix = nullptr; + if (function && line > 0) + prefix_len = mdbx_asprintf(&prefix, "%s:%d ", function, line); + else if (function) + prefix_len = mdbx_asprintf(&prefix, "%s: ", function); + else if (line > 0) + prefix_len = mdbx_asprintf(&prefix, "%d: ", line); + if (prefix_len > 0 && prefix) { + OutputDebugStringA(prefix); + mdbx_free(prefix); + } + char *msg = nullptr; + int msg_len = mdbx_vasprintf(&msg, fmt, args); + if (msg_len > 0 && msg) { + OutputDebugStringA(msg); + mdbx_free(msg); + } + } +#else + if (function && line > 0) + fprintf(stderr, "%s:%d ", function, line); + else if (function) + fprintf(stderr, "%s: ", function); + else if (line > 0) + fprintf(stderr, "%d: ", line); + vfprintf(stderr, fmt, args); + fflush(stderr); +#endif + } + va_end(args); +} + +/* Dump a key in ascii or hexadecimal. */ +const char *mdbx_dump_val(const MDBX_val *key, char *const buf, + const size_t bufsize) { + if (!key) + return ""; + if (!buf || bufsize < 4) + return nullptr; + if (!key->iov_len) + return ""; + + const uint8_t *const data = key->iov_base; + bool is_ascii = true; + unsigned i; + for (i = 0; is_ascii && i < key->iov_len; i++) + if (data[i] < ' ' || data[i] > 127) + is_ascii = false; + + if (is_ascii) { + int len = + snprintf(buf, bufsize, "%.*s", + (key->iov_len > INT_MAX) ? INT_MAX : (int)key->iov_len, data); + assert(len > 0 && (unsigned)len < bufsize); + (void)len; + } else { + char *const detent = buf + bufsize - 2; + char *ptr = buf; + *ptr++ = '<'; + for (i = 0; i < key->iov_len; i++) { + const ptrdiff_t left = detent - ptr; + assert(left > 0); + int len = snprintf(ptr, left, "%02x", data[i]); + if (len < 0 || len >= left) + break; + ptr += len; + } + if (ptr < detent) { + ptr[0] = '>'; + ptr[1] = '\0'; + } + } + return buf; +} + +/*------------------------------------------------------------------------------ + LY: debug stuff */ + +static const char *mdbx_leafnode_type(MDBX_node *n) { + static const char *const tp[2][2] = {{"", ": DB"}, + {": sub-page", ": sub-DB"}}; + return F_ISSET(node_flags(n), F_BIGDATA) + ? ": overflow page" + : tp[F_ISSET(node_flags(n), F_DUPDATA)] + [F_ISSET(node_flags(n), F_SUBDATA)]; +} + +/* Display all the keys in the page. */ +static __maybe_unused void mdbx_page_list(MDBX_page *mp) { + pgno_t pgno = mp->mp_pgno; + const char *type, *state = IS_DIRTY(mp) ? ", dirty" : ""; + MDBX_node *node; + unsigned i, nkeys, nsize, total = 0; + MDBX_val key; + DKBUF; + + switch (mp->mp_flags & + (P_BRANCH | P_LEAF | P_LEAF2 | P_META | P_OVERFLOW | P_SUBP)) { + case P_BRANCH: + type = "Branch page"; + break; + case P_LEAF: + type = "Leaf page"; + break; + case P_LEAF | P_SUBP: + type = "Leaf sub-page"; + break; + case P_LEAF | P_LEAF2: + type = "Leaf2 page"; + break; + case P_LEAF | P_LEAF2 | P_SUBP: + type = "Leaf2 sub-page"; + break; + case P_OVERFLOW: + mdbx_verbose("Overflow page %" PRIaPGNO " pages %u%s\n", pgno, mp->mp_pages, + state); + return; + case P_META: + mdbx_verbose("Meta-page %" PRIaPGNO " txnid %" PRIu64 "\n", pgno, + page_meta(mp)->mm_txnid_a.inconsistent); + return; + default: + mdbx_verbose("Bad page %" PRIaPGNO " flags 0x%X\n", pgno, mp->mp_flags); + return; + } + + nkeys = page_numkeys(mp); + mdbx_verbose("%s %" PRIaPGNO " numkeys %u%s\n", type, pgno, nkeys, state); + + for (i = 0; i < nkeys; i++) { + if (IS_LEAF2(mp)) { /* LEAF2 pages have no mp_ptrs[] or node headers */ + key.iov_len = nsize = mp->mp_leaf2_ksize; + key.iov_base = page_leaf2key(mp, i, nsize); + total += nsize; + mdbx_verbose("key %u: nsize %u, %s\n", i, nsize, DKEY(&key)); + continue; + } + node = page_node(mp, i); + key.iov_len = node_ks(node); + key.iov_base = node->mn_data; + nsize = (unsigned)(NODESIZE + key.iov_len); + if (IS_BRANCH(mp)) { + mdbx_verbose("key %u: page %" PRIaPGNO ", %s\n", i, node_pgno(node), + DKEY(&key)); + total += nsize; + } else { + if (F_ISSET(node_flags(node), F_BIGDATA)) + nsize += sizeof(pgno_t); + else + nsize += (unsigned)node_ds(node); + total += nsize; + nsize += sizeof(indx_t); + mdbx_verbose("key %u: nsize %u, %s%s\n", i, nsize, DKEY(&key), + mdbx_leafnode_type(node)); + } + total = EVEN(total); + } + mdbx_verbose("Total: header %u + contents %u + unused %u\n", + IS_LEAF2(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->mp_lower, total, + page_room(mp)); +} + +/*----------------------------------------------------------------------------*/ + +/* Check if there is an inited xcursor, so XCURSOR_REFRESH() is proper */ +#define XCURSOR_INITED(mc) \ + ((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + +/* Update sub-page pointer, if any, in mc->mc_xcursor. + * Needed when the node which contains the sub-page may have moved. + * Called with mp = mc->mc_pg[mc->mc_top], ki = mc->mc_ki[mc->mc_top]. */ +#define XCURSOR_REFRESH(mc, mp, ki) \ + do { \ + MDBX_page *xr_pg = (mp); \ + MDBX_node *xr_node = page_node(xr_pg, ki); \ + if ((node_flags(xr_node) & (F_DUPDATA | F_SUBDATA)) == F_DUPDATA) \ + (mc)->mc_xcursor->mx_cursor.mc_pg[0] = node_data(xr_node); \ + } while (0) + +/* Perform act while tracking temporary cursor mn */ +#define WITH_CURSOR_TRACKING(mn, act) \ + do { \ + mdbx_cassert(&(mn), \ + mn.mc_txn->mt_cursors != NULL /* must be not rdonly txt */); \ + MDBX_cursor mc_dummy, **tp = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \ + MDBX_cursor *tracked = &(mn); \ + if ((mn).mc_flags & C_SUB) { \ + mc_dummy.mc_flags = C_INITIALIZED; \ + mc_dummy.mc_xcursor = (MDBX_xcursor *)&(mn); \ + tracked = &mc_dummy; \ + } \ + tracked->mc_next = *tp; \ + *tp = tracked; \ + { act; } \ + *tp = tracked->mc_next; \ + } while (0) + +int mdbx_cmp(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, + const MDBX_val *b) { + mdbx_assert(NULL, txn->mt_signature == MDBX_MT_SIGNATURE); + return txn->mt_dbxs[dbi].md_cmp(a, b); +} + +int mdbx_dcmp(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, + const MDBX_val *b) { + mdbx_assert(NULL, txn->mt_signature == MDBX_MT_SIGNATURE); + return txn->mt_dbxs[dbi].md_dcmp(a, b); +} + +/* Allocate memory for a page. + * Re-use old malloc'd pages first for singletons, otherwise just malloc. + * Set MDBX_TXN_ERROR on failure. */ +static MDBX_page *mdbx_page_malloc(MDBX_txn *txn, unsigned num) { + MDBX_env *env = txn->mt_env; + MDBX_page *np = env->me_dpages; + size_t size = env->me_psize; + if (likely(num == 1 && np)) { + ASAN_UNPOISON_MEMORY_REGION(np, size); + VALGRIND_MEMPOOL_ALLOC(env, np, size); + VALGRIND_MAKE_MEM_DEFINED(&np->mp_next, sizeof(np->mp_next)); + env->me_dpages = np->mp_next; + } else { + size = pgno2bytes(env, num); + np = mdbx_malloc(size); + if (unlikely(!np)) { + txn->mt_flags |= MDBX_TXN_ERROR; + return np; + } + VALGRIND_MEMPOOL_ALLOC(env, np, size); + } + + if ((env->me_flags & MDBX_NOMEMINIT) == 0) { + /* For a single page alloc, we init everything after the page header. + * For multi-page, we init the final page; if the caller needed that + * many pages they will be filling in at least up to the last page. */ + size_t skip = PAGEHDRSZ; + if (num > 1) + skip += pgno2bytes(env, num - 1); + memset((char *)np + skip, 0, size - skip); + } +#if MDBX_DEBUG + np->mp_pgno = 0; +#endif + VALGRIND_MAKE_MEM_UNDEFINED(np, size); + np->mp_flags = 0; + np->mp_pages = num; + return np; +} + +/* Free a dirty page */ +static void mdbx_dpage_free(MDBX_env *env, MDBX_page *dp, unsigned pages) { +#if MDBX_DEBUG + dp->mp_pgno = MAX_PAGENO + 1; +#endif + if (pages == 1) { + dp->mp_next = env->me_dpages; + VALGRIND_MEMPOOL_FREE(env, dp); + env->me_dpages = dp; + } else { + /* large pages just get freed directly */ + VALGRIND_MEMPOOL_FREE(env, dp); + mdbx_free(dp); + } +} + +/* Return all dirty pages to dpage list */ +static void mdbx_dlist_free(MDBX_txn *txn) { + MDBX_env *env = txn->mt_env; + const MDBX_DPL dl = txn->tw.dirtylist; + const size_t n = dl->length; + + for (size_t i = 1; i <= n; i++) { + MDBX_page *dp = dl[i].ptr; + mdbx_dpage_free(env, dp, IS_OVERFLOW(dp) ? dp->mp_pages : 1); + } + + mdbx_dpl_clear(dl); +} + +static __inline MDBX_db *mdbx_outer_db(MDBX_cursor *mc) { + mdbx_cassert(mc, (mc->mc_flags & C_SUB) != 0); + MDBX_xcursor *mx = container_of(mc->mc_db, MDBX_xcursor, mx_db); + MDBX_cursor_couple *couple = container_of(mx, MDBX_cursor_couple, inner); + mdbx_cassert(mc, mc->mc_db == &couple->outer.mc_xcursor->mx_db); + mdbx_cassert(mc, mc->mc_dbx == &couple->outer.mc_xcursor->mx_dbx); + return couple->outer.mc_db; +} + +static __cold __maybe_unused bool mdbx_dirtylist_check(MDBX_txn *txn) { + unsigned loose = 0; + for (unsigned i = txn->tw.dirtylist->length; i > 0; --i) { + const MDBX_page *const dp = txn->tw.dirtylist[i].ptr; + if (!dp) + continue; + mdbx_tassert(txn, dp->mp_pgno == txn->tw.dirtylist[i].pgno); + if (unlikely(dp->mp_pgno != txn->tw.dirtylist[i].pgno)) + return false; + + mdbx_tassert(txn, dp->mp_flags & P_DIRTY); + if (unlikely((dp->mp_flags & P_DIRTY) == 0)) + return false; + if (dp->mp_flags & P_LOOSE) { + mdbx_tassert(txn, dp->mp_flags == (P_LOOSE | P_DIRTY)); + if (unlikely(dp->mp_flags != (P_LOOSE | P_DIRTY))) + return false; + loose += 1; + } + + const unsigned num = IS_OVERFLOW(dp) ? dp->mp_pages : 1; + mdbx_tassert(txn, txn->mt_next_pgno >= dp->mp_pgno + num); + if (unlikely(txn->mt_next_pgno < dp->mp_pgno + num)) + return false; + + if (i < txn->tw.dirtylist->sorted) { + mdbx_tassert(txn, txn->tw.dirtylist[i + 1].pgno >= dp->mp_pgno + num); + if (unlikely(txn->tw.dirtylist[i + 1].pgno < dp->mp_pgno + num)) + return false; + } + + const unsigned rpa = mdbx_pnl_search(txn->tw.reclaimed_pglist, dp->mp_pgno); + mdbx_tassert(txn, rpa > MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) || + txn->tw.reclaimed_pglist[rpa] != dp->mp_pgno); + if (rpa <= MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) && + unlikely(txn->tw.reclaimed_pglist[rpa] == dp->mp_pgno)) + return false; + if (num > 1) { + const unsigned rpb = + mdbx_pnl_search(txn->tw.reclaimed_pglist, dp->mp_pgno + num - 1); + mdbx_tassert(txn, rpa == rpb); + if (unlikely(rpa != rpb)) + return false; + } + } + + mdbx_tassert(txn, loose == txn->tw.loose_count); + if (unlikely(loose != txn->tw.loose_count)) + return false; + + if (txn->tw.dirtylist->length - txn->tw.dirtylist->sorted < + SORT_THRESHOLD / 2) { + for (unsigned i = 1; i <= MDBX_PNL_SIZE(txn->tw.retired_pages); ++i) { + const MDBX_page *const dp = + mdbx_dpl_find(txn->tw.dirtylist, txn->tw.retired_pages[i]); + mdbx_tassert(txn, !dp); + if (unlikely(dp)) + return false; + } + } + + return true; +} + +static void mdbx_refund_reclaimed(MDBX_txn *txn) { + /* Scanning in descend order */ + pgno_t next_pgno = txn->mt_next_pgno; + const MDBX_PNL pnl = txn->tw.reclaimed_pglist; + mdbx_tassert(txn, MDBX_PNL_SIZE(pnl) && MDBX_PNL_MOST(pnl) == next_pgno - 1); +#if MDBX_PNL_ASCENDING + unsigned i = MDBX_PNL_SIZE(pnl); + mdbx_tassert(txn, pnl[i] == next_pgno - 1); + while (--next_pgno, --i > 0 && pnl[i] == next_pgno - 1) + ; + MDBX_PNL_SIZE(pnl) = i; +#else + unsigned i = 1; + mdbx_tassert(txn, pnl[i] == next_pgno - 1); + unsigned len = MDBX_PNL_SIZE(pnl); + while (--next_pgno, ++i <= len && pnl[i] == next_pgno - 1) + ; + MDBX_PNL_SIZE(pnl) = len -= i - 1; + for (unsigned move = 0; move < len; ++move) + pnl[1 + move] = pnl[i + move]; +#endif + mdbx_verbose("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, + txn->mt_next_pgno - next_pgno, txn->mt_next_pgno, next_pgno); + txn->mt_next_pgno = next_pgno; + mdbx_tassert( + txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, txn->mt_next_pgno)); +} + +static void mdbx_refund_loose(MDBX_txn *txn) { + mdbx_tassert(txn, mdbx_dirtylist_check(txn)); + mdbx_tassert(txn, txn->tw.loose_pages != nullptr); + mdbx_tassert(txn, txn->tw.loose_count > 0); + + const MDBX_DPL dl = txn->tw.dirtylist; + mdbx_tassert(txn, dl->length >= txn->tw.loose_count); + mdbx_tassert(txn, txn->tw.spill_pages == nullptr || + dl->length >= MDBX_PNL_SIZE(txn->tw.spill_pages)); + + pgno_t onstack[MDBX_CACHELINE_SIZE * 8 / sizeof(pgno_t)]; + MDBX_PNL suitable = onstack; + + if (dl->length - dl->sorted > txn->tw.loose_count) { + /* Dirty list is useless since unsorted. */ + if (bytes2pnl(sizeof(onstack)) < txn->tw.loose_count) { + suitable = mdbx_pnl_alloc(txn->tw.loose_count); + if (unlikely(!suitable)) + return /* this is not a reason for transaction fail */; + } + + /* Collect loose-pages which may be refunded. */ + mdbx_tassert(txn, txn->mt_next_pgno >= MIN_PAGENO + txn->tw.loose_count); + pgno_t most = MIN_PAGENO; + unsigned w = 0; + for (const MDBX_page *dp = txn->tw.loose_pages; dp; dp = dp->mp_next) { + mdbx_tassert(txn, dp->mp_flags == (P_LOOSE | P_DIRTY)); + mdbx_tassert(txn, txn->mt_next_pgno > dp->mp_pgno); + if (likely(txn->mt_next_pgno - txn->tw.loose_count <= dp->mp_pgno)) { + mdbx_tassert(txn, + w < ((suitable == onstack) ? bytes2pnl(sizeof(onstack)) + : MDBX_PNL_ALLOCLEN(suitable))); + suitable[++w] = dp->mp_pgno; + most = (dp->mp_pgno > most) ? dp->mp_pgno : most; + } + } + + if (most + 1 == txn->mt_next_pgno) { + /* Sort suitable list and refund pages at the tail. */ + MDBX_PNL_SIZE(suitable) = w; + mdbx_pnl_sort(suitable); + + /* Scanning in descend order */ + const int step = MDBX_PNL_ASCENDING ? -1 : 1; + const int begin = MDBX_PNL_ASCENDING ? MDBX_PNL_SIZE(suitable) : 1; + const int end = MDBX_PNL_ASCENDING ? 0 : MDBX_PNL_SIZE(suitable) + 1; + mdbx_tassert(txn, suitable[begin] >= suitable[end - step]); + mdbx_tassert(txn, most == suitable[begin]); + + for (int i = begin + step; i != end; i += step) { + if (suitable[i] != most - 1) + break; + most -= 1; + } + const unsigned refunded = txn->mt_next_pgno - most; + mdbx_verbose("refund-sorted %u pages %" PRIaPGNO " -> %" PRIaPGNO, + refunded, most, txn->mt_next_pgno); + txn->tw.loose_count -= refunded; + txn->tw.dirtyroom += refunded; + txn->mt_next_pgno = most; + + /* Filter-out dirty list */ + unsigned r = 0; + w = 0; + if (dl->sorted) { + do { + if (dl[++r].pgno < most) { + if (++w != r) + dl[w] = dl[r]; + } + } while (r < dl->sorted); + dl->sorted = w; + } + while (r < dl->length) { + if (dl[++r].pgno < most) { + if (++w != r) + dl[w] = dl[r]; + } + } + dl->length = w; + mdbx_tassert(txn, txn->mt_parent || + txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + goto unlink_loose; + } + } else { + /* Dirtylist is mostly sorted, just refund loose pages at the end. */ + mdbx_dpl_sort(dl); + mdbx_tassert(txn, dl->length < 2 || dl[1].pgno < dl[dl->length].pgno); + mdbx_tassert(txn, dl->sorted == dl->length); + + /* Scan dirtylist tail-forward and cutoff suitable pages. */ + while (dl->length && dl[dl->length].pgno == txn->mt_next_pgno - 1 && + dl[dl->length].ptr->mp_flags == (P_LOOSE | P_DIRTY)) { + MDBX_page *dp = dl[dl->length].ptr; + mdbx_verbose("refund-unsorted page %" PRIaPGNO, dp->mp_pgno); + mdbx_tassert(txn, dp->mp_pgno == dl[dl->length].pgno); + dl->length -= 1; + } + + if (dl->sorted != dl->length) { + const unsigned refunded = dl->sorted - dl->length; + dl->sorted = dl->length; + txn->tw.loose_count -= refunded; + txn->tw.dirtyroom += refunded; + txn->mt_next_pgno -= refunded; + mdbx_tassert(txn, txn->mt_parent || + txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + + /* Filter-out loose chain & dispose refunded pages. */ + unlink_loose: + for (MDBX_page **link = &txn->tw.loose_pages; *link;) { + MDBX_page *dp = *link; + mdbx_tassert(txn, dp->mp_flags == (P_LOOSE | P_DIRTY)); + if (txn->mt_next_pgno > dp->mp_pgno) { + link = &dp->mp_next; + } else { + *link = dp->mp_next; + if ((txn->mt_flags & MDBX_WRITEMAP) == 0) + mdbx_dpage_free(txn->mt_env, dp, 1); + } + } + } + } + + mdbx_tassert(txn, mdbx_dirtylist_check(txn)); + mdbx_tassert(txn, txn->mt_parent || + txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + if (suitable != onstack) + mdbx_pnl_free(suitable); + txn->tw.loose_refund_wl = txn->mt_next_pgno; +} + +static bool mdbx_refund(MDBX_txn *txn) { + const pgno_t before = txn->mt_next_pgno; + + if (txn->tw.loose_pages && txn->tw.loose_refund_wl > txn->mt_next_pgno) + mdbx_refund_loose(txn); + + while (true) { + if (MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) == 0 || + MDBX_PNL_MOST(txn->tw.reclaimed_pglist) != txn->mt_next_pgno - 1) + break; + + mdbx_refund_reclaimed(txn); + if (!txn->tw.loose_pages || txn->tw.loose_refund_wl <= txn->mt_next_pgno) + break; + + const pgno_t memo = txn->mt_next_pgno; + mdbx_refund_loose(txn); + if (memo == txn->mt_next_pgno) + break; + } + + return before != txn->mt_next_pgno; +} + +static __cold void mdbx_kill_page(MDBX_env *env, MDBX_page *mp, pgno_t pgno, + unsigned npages) { + mdbx_assert(env, pgno >= NUM_METAS && npages); + if (IS_DIRTY(mp) || (env->me_flags & MDBX_WRITEMAP)) { + const size_t bytes = pgno2bytes(env, npages); + memset(mp, 0, bytes); + mp->mp_pgno = pgno; + if ((env->me_flags & MDBX_WRITEMAP) == 0) + mdbx_pwrite(env->me_fd, mp, bytes, pgno2bytes(env, pgno)); + } else { + struct iovec iov[MDBX_COMMIT_PAGES]; + iov[0].iov_len = env->me_psize; + iov[0].iov_base = (char *)env->me_pbuf + env->me_psize; + size_t iov_off = pgno2bytes(env, pgno); + unsigned n = 1; + while (--npages) { + iov[n] = iov[0]; + if (++n == MDBX_COMMIT_PAGES) { + mdbx_pwritev(env->me_fd, iov, MDBX_COMMIT_PAGES, iov_off, + pgno2bytes(env, MDBX_COMMIT_PAGES)); + iov_off += pgno2bytes(env, MDBX_COMMIT_PAGES); + n = 0; + } + } + mdbx_pwritev(env->me_fd, iov, n, iov_off, pgno2bytes(env, n)); + } +} + +/* Retire, loosen or free a single page. + * + * Saves single pages to a list for future reuse + * in this same txn. It has been pulled from the GC + * and already resides on the dirty list, but has been + * deleted. Use these pages first before pulling again + * from the GC. + * + * If the page wasn't dirtied in this txn, just add it + * to this txn's free list. */ + +static __hot int mdbx_page_loose(MDBX_txn *txn, MDBX_page *mp) { + const unsigned npages = IS_OVERFLOW(mp) ? mp->mp_pages : 1; + const pgno_t pgno = mp->mp_pgno; + + if (txn->mt_parent) { + mdbx_tassert(txn, (txn->mt_env->me_flags & MDBX_WRITEMAP) == 0); + mdbx_tassert(txn, mp != pgno2page(txn->mt_env, pgno)); + /* If txn has a parent, make sure the page is in our dirty list. */ + MDBX_page *dp = mdbx_dpl_find(txn->tw.dirtylist, pgno); + /* TODO: use extended flag-mask to track parent's dirty-pages */ + if (dp == nullptr) { + mp->mp_next = txn->tw.retired2parent_pages; + txn->tw.retired2parent_pages = mp; + txn->tw.retired2parent_count += npages; + return MDBX_SUCCESS; + } + if (unlikely(mp != dp)) { /* bad cursor? */ + mdbx_error( + "wrong page 0x%p #%" PRIaPGNO " in the dirtylist, expecting %p", + __Wpedantic_format_voidptr(dp), pgno, __Wpedantic_format_voidptr(mp)); + txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_PROBLEM; + } + /* ok, it's ours */ + } + + mdbx_debug("loosen page %" PRIaPGNO, pgno); + const bool is_dirty = IS_DIRTY(mp); + if (MDBX_DEBUG || unlikely((txn->mt_env->me_flags & MDBX_PAGEPERTURB) != 0)) { + mdbx_kill_page(txn->mt_env, mp, pgno, npages); + VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ); + } + VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), + pgno2bytes(txn->mt_env, npages) - PAGEHDRSZ); + ASAN_POISON_MEMORY_REGION(page_data(mp), + pgno2bytes(txn->mt_env, npages) - PAGEHDRSZ); + + if (unlikely(npages > + 1 /* overflow pages doesn't comes to the loose-list */)) { + if (is_dirty) { + /* Remove from dirty list */ + MDBX_page *dp = mdbx_dpl_remove(txn->tw.dirtylist, pgno); + if (unlikely(dp != mp)) { + mdbx_error("not found page 0x%p #%" PRIaPGNO " in the dirtylist", + __Wpedantic_format_voidptr(mp), pgno); + txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_PROBLEM; + } + txn->tw.dirtyroom++; + mdbx_tassert(txn, txn->mt_parent || + txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + if ((txn->mt_flags & MDBX_WRITEMAP) == 0) + mdbx_dpage_free(txn->mt_env, mp, npages); + } + + if (unlikely(pgno + npages == txn->mt_next_pgno)) { + txn->mt_next_pgno = pgno; + mdbx_refund(txn); + return MDBX_SUCCESS; + } + + int rc = mdbx_pnl_insert_range(&txn->tw.reclaimed_pglist, pgno, npages); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + return MDBX_SUCCESS; + } + + mp->mp_flags = P_LOOSE | P_DIRTY; + mp->mp_next = txn->tw.loose_pages; + txn->tw.loose_pages = mp; + txn->tw.loose_count++; + if (unlikely(txn->mt_next_pgno == mp->mp_pgno + 1)) + mdbx_refund(txn); + + return MDBX_SUCCESS; +} + +static __hot int mdbx_page_retire(MDBX_cursor *mc, MDBX_page *mp) { + const unsigned npages = IS_OVERFLOW(mp) ? mp->mp_pages : 1; + const pgno_t pgno = mp->mp_pgno; + MDBX_txn *const txn = mc->mc_txn; + + if (unlikely(mc->mc_flags & C_SUB)) { + MDBX_db *outer = mdbx_outer_db(mc); + mdbx_cassert(mc, !IS_BRANCH(mp) || outer->md_branch_pages > 0); + outer->md_branch_pages -= IS_BRANCH(mp); + mdbx_cassert(mc, !IS_LEAF(mp) || outer->md_leaf_pages > 0); + outer->md_leaf_pages -= IS_LEAF(mp); + mdbx_cassert(mc, !IS_OVERFLOW(mp)); + } + mdbx_cassert(mc, !IS_BRANCH(mp) || mc->mc_db->md_branch_pages > 0); + mc->mc_db->md_branch_pages -= IS_BRANCH(mp); + mdbx_cassert(mc, !IS_LEAF(mp) || mc->mc_db->md_leaf_pages > 0); + mc->mc_db->md_leaf_pages -= IS_LEAF(mp); + mdbx_cassert(mc, !IS_OVERFLOW(mp) || mc->mc_db->md_overflow_pages >= npages); + mc->mc_db->md_overflow_pages -= IS_OVERFLOW(mp) ? npages : 0; + + if (IS_DIRTY(mp)) { + int rc = mdbx_page_loose(txn, mp); + if (unlikely(rc != MDBX_SUCCESS)) + mc->mc_flags &= ~(C_INITIALIZED | C_EOF); + return rc; + } + + if (txn->tw.spill_pages) { + const unsigned i = mdbx_pnl_exist(txn->tw.spill_pages, pgno << 1); + if (i) { + /* This page is no longer spilled */ + mdbx_tassert(txn, i == MDBX_PNL_SIZE(txn->tw.spill_pages) || + txn->tw.spill_pages[i + 1] >= (pgno + npages) << 1); + txn->tw.spill_pages[i] |= 1; + if (i == MDBX_PNL_SIZE(txn->tw.spill_pages)) + MDBX_PNL_SIZE(txn->tw.spill_pages) -= 1; + int rc = mdbx_page_loose(txn, mp); + if (unlikely(rc != MDBX_SUCCESS)) + mc->mc_flags &= ~(C_INITIALIZED | C_EOF); + return rc; + } + } + + mdbx_tassert(txn, mp == pgno2page(txn->mt_env, pgno)); + int rc = mdbx_pnl_append_range(&txn->tw.retired_pages, pgno, npages); + mdbx_tassert(txn, mdbx_dpl_find(txn->tw.dirtylist, pgno) == nullptr); + return rc; +} + +static __must_check_result __inline int mdbx_retire_pgno(MDBX_cursor *mc, + const pgno_t pgno) { + MDBX_page *mp; + int rc = mdbx_page_get(mc, pgno, &mp, NULL); + if (likely(rc == MDBX_SUCCESS)) + rc = mdbx_page_retire(mc, mp); + return rc; +} + +/* Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. + * + * [in] mc A cursor handle for the current operation. + * [in] pflags Flags of the pages to update: + * - P_DIRTY to set P_KEEP, + * - P_DIRTY|P_KEEP to clear it. + * [in] all No shortcuts. Needed except after a full mdbx_page_flush(). + * + * Returns 0 on success, non-zero on failure. */ +static int mdbx_pages_xkeep(MDBX_cursor *mc, unsigned pflags, bool all) { + const unsigned Mask = P_SUBP | P_DIRTY | P_LOOSE | P_KEEP; + MDBX_txn *txn = mc->mc_txn; + MDBX_cursor *m3, *m0 = mc; + MDBX_xcursor *mx; + MDBX_page *dp, *mp; + unsigned i, j; + int rc = MDBX_SUCCESS; + + /* Mark pages seen by cursors: First m0, then tracked cursors */ + for (i = txn->mt_numdbs;;) { + if (mc->mc_flags & C_INITIALIZED) { + for (m3 = mc;; m3 = &mx->mx_cursor) { + mp = NULL; + for (j = 0; j < m3->mc_snum; j++) { + mp = m3->mc_pg[j]; + if ((mp->mp_flags & Mask) == pflags) + mp->mp_flags ^= P_KEEP; + } + mx = m3->mc_xcursor; + /* Proceed to mx if it is at a sub-database */ + if (!(mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) + break; + if (!(mp && IS_LEAF(mp))) + break; + if (!(node_flags(page_node(mp, m3->mc_ki[j - 1])) & F_SUBDATA)) + break; + } + } + mc = mc->mc_next; + for (; !mc || mc == m0; mc = txn->mt_cursors[--i]) + if (i == 0) + goto mark_done; + } + +mark_done: + if (all) { + /* Mark dirty root pages */ + for (i = 0; i < txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + pgno_t pgno = txn->mt_dbs[i].md_root; + if (pgno == P_INVALID) + continue; + int level; + if (unlikely((rc = mdbx_page_get(m0, pgno, &dp, &level)) != + MDBX_SUCCESS)) + break; + if ((dp->mp_flags & Mask) == pflags && level <= 1) + dp->mp_flags ^= P_KEEP; + } + } + } + + return rc; +} + +/* Spill pages from the dirty list back to disk. + * This is intended to prevent running into MDBX_TXN_FULL situations, + * but note that they may still occur in a few cases: + * + * 1) our estimate of the txn size could be too small. Currently this + * seems unlikely, except with a large number of MDBX_MULTIPLE items. + * + * 2) child txns may run out of space if their parents dirtied a + * lot of pages and never spilled them. TODO: we probably should do + * a preemptive spill during mdbx_txn_begin() of a child txn, if + * the parent's dirtyroom is below a given threshold. + * + * Otherwise, if not using nested txns, it is expected that apps will + * not run into MDBX_TXN_FULL any more. The pages are flushed to disk + * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared. + * If the txn never references them again, they can be left alone. + * If the txn only reads them, they can be used without any fuss. + * If the txn writes them again, they can be dirtied immediately without + * going thru all of the work of mdbx_page_touch(). Such references are + * handled by mdbx_page_unspill(). + * + * Also note, we never spill DB root pages, nor pages of active cursors, + * because we'll need these back again soon anyway. And in nested txns, + * we can't spill a page in a child txn if it was already spilled in a + * parent txn. That would alter the parent txns' data even though + * the child hasn't committed yet, and we'd have no way to undo it if + * the child aborted. + * + * [in] mc cursor A cursor handle identifying the transaction and + * database for which we are checking space. + * [in] key For a put operation, the key being stored. + * [in] data For a put operation, the data being stored. + * + * Returns 0 on success, non-zero on failure. */ +static int mdbx_page_spill(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { + if (mc->mc_flags & C_SUB) + return MDBX_SUCCESS; + + MDBX_txn *txn = mc->mc_txn; + MDBX_DPL dl = txn->tw.dirtylist; + + /* Estimate how much space this op will take */ + pgno_t i = mc->mc_db->md_depth; + /* Named DBs also dirty the main DB */ + if (mc->mc_dbi >= CORE_DBS) + i += txn->mt_dbs[MAIN_DBI].md_depth; + /* For puts, roughly factor in the key+data size */ + if (key) + i += bytes2pgno(txn->mt_env, node_size(key, data) + txn->mt_env->me_psize); + i += i; /* double it for good measure */ + pgno_t need = i; + + if (txn->tw.dirtyroom > i) + return MDBX_SUCCESS; + + if (!txn->tw.spill_pages) { + txn->tw.spill_pages = mdbx_pnl_alloc(MDBX_DPL_TXNFULL / 8); + if (unlikely(!txn->tw.spill_pages)) + return MDBX_ENOMEM; + } else { + /* purge deleted slots */ + MDBX_PNL sl = txn->tw.spill_pages; + pgno_t num = MDBX_PNL_SIZE(sl), j = 0; + for (i = 1; i <= num; i++) { + if ((sl[i] & 1) == 0) + sl[++j] = sl[i]; + } + MDBX_PNL_SIZE(sl) = j; + } + + /* Preserve pages which may soon be dirtied again */ + int rc = mdbx_pages_xkeep(mc, P_DIRTY, true); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + /* Less aggressive spill - we originally spilled the entire dirty list, + * with a few exceptions for cursor pages and DB root pages. But this + * turns out to be a lot of wasted effort because in a large txn many + * of those pages will need to be used again. So now we spill only 1/8th + * of the dirty pages. Testing revealed this to be a good tradeoff, + * better than 1/2, 1/4, or 1/10. */ + if (need < MDBX_DPL_TXNFULL / 8) + need = MDBX_DPL_TXNFULL / 8; + + /* Save the page IDs of all the pages we're flushing */ + /* flush from the tail forward, this saves a lot of shifting later on. */ + for (i = dl->length; i && need; i--) { + pgno_t pn = dl[i].pgno << 1; + MDBX_page *dp = dl[i].ptr; + if (dp->mp_flags & (P_LOOSE | P_KEEP)) + continue; + /* Can't spill twice, + * make sure it's not already in a parent's spill list. */ + if (txn->mt_parent) { + MDBX_txn *parent; + for (parent = txn->mt_parent; parent; parent = parent->mt_parent) { + if (parent->tw.spill_pages && + mdbx_pnl_exist(parent->tw.spill_pages, pn)) { + dp->mp_flags |= P_KEEP; + break; + } + } + if (parent) + continue; + } + rc = mdbx_pnl_append(&txn->tw.spill_pages, pn); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + need--; + } + mdbx_pnl_sort(txn->tw.spill_pages); + + /* Flush the spilled part of dirty list */ + rc = mdbx_page_flush(txn, i); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + /* Reset any dirty pages we kept that page_flush didn't see */ + rc = mdbx_pages_xkeep(mc, P_DIRTY | P_KEEP, i != 0); + +bailout: + txn->mt_flags |= rc ? MDBX_TXN_ERROR : MDBX_TXN_SPILLS; + return rc; +} + +/*----------------------------------------------------------------------------*/ + +#define METAPAGE(env, n) page_meta(pgno2page(env, n)) +#define METAPAGE_END(env) METAPAGE(env, NUM_METAS) + +static __inline txnid_t meta_txnid(const MDBX_env *env, const MDBX_meta *meta, + bool allow_volatile) { + mdbx_assert(env, meta >= METAPAGE(env, 0) || meta < METAPAGE_END(env)); + txnid_t a = safe64_read(&meta->mm_txnid_a); + txnid_t b = safe64_read(&meta->mm_txnid_b); + if (allow_volatile) + return (a == b) ? a : 0; + mdbx_assert(env, a == b); + return a; +} + +static __inline txnid_t mdbx_meta_txnid_stable(const MDBX_env *env, + const MDBX_meta *meta) { + return meta_txnid(env, meta, false); +} + +static __inline txnid_t mdbx_meta_txnid_fluid(const MDBX_env *env, + const MDBX_meta *meta) { + return meta_txnid(env, meta, true); +} + +static __inline void mdbx_meta_update_begin(const MDBX_env *env, + MDBX_meta *meta, txnid_t txnid) { + mdbx_assert(env, meta >= METAPAGE(env, 0) || meta < METAPAGE_END(env)); + mdbx_assert(env, meta->mm_txnid_a.inconsistent < txnid && + meta->mm_txnid_b.inconsistent < txnid); + (void)env; + safe64_update(&meta->mm_txnid_a, txnid); +} + +static __inline void mdbx_meta_update_end(const MDBX_env *env, MDBX_meta *meta, + txnid_t txnid) { + mdbx_assert(env, meta >= METAPAGE(env, 0) || meta < METAPAGE_END(env)); + mdbx_assert(env, meta->mm_txnid_a.inconsistent == txnid); + mdbx_assert(env, meta->mm_txnid_b.inconsistent < txnid); + (void)env; + mdbx_jitter4testing(true); + safe64_update(&meta->mm_txnid_b, txnid); +} + +static __inline void mdbx_meta_set_txnid(const MDBX_env *env, MDBX_meta *meta, + txnid_t txnid) { + mdbx_assert(env, meta < METAPAGE(env, 0) || meta > METAPAGE_END(env)); + (void)env; + /* update inconsistent since this function used ONLY for filling meta-image + * for writing, but not the actual meta-page */ + meta->mm_txnid_a.inconsistent = txnid; + meta->mm_txnid_b.inconsistent = txnid; +} + +static __inline uint64_t mdbx_meta_sign(const MDBX_meta *meta) { + uint64_t sign = MDBX_DATASIGN_NONE; +#if 0 /* TODO */ + sign = hippeus_hash64(...); +#else + (void)meta; +#endif + /* LY: newer returns MDBX_DATASIGN_NONE or MDBX_DATASIGN_WEAK */ + return (sign > MDBX_DATASIGN_WEAK) ? sign : ~sign; +} + +enum meta_choise_mode { prefer_last, prefer_noweak, prefer_steady }; + +static __inline bool mdbx_meta_ot(const enum meta_choise_mode mode, + const MDBX_env *env, const MDBX_meta *a, + const MDBX_meta *b) { + mdbx_jitter4testing(true); + txnid_t txnid_a = mdbx_meta_txnid_fluid(env, a); + txnid_t txnid_b = mdbx_meta_txnid_fluid(env, b); + + mdbx_jitter4testing(true); + switch (mode) { + default: + assert(false); + __unreachable(); + /* fall through */ + __fallthrough; + case prefer_steady: + if (META_IS_STEADY(a) != META_IS_STEADY(b)) + return META_IS_STEADY(b); + /* fall through */ + __fallthrough; + case prefer_noweak: + if (META_IS_WEAK(a) != META_IS_WEAK(b)) + return !META_IS_WEAK(b); + /* fall through */ + __fallthrough; + case prefer_last: + mdbx_jitter4testing(true); + if (txnid_a == txnid_b) + return META_IS_STEADY(b) || (META_IS_WEAK(a) && !META_IS_WEAK(b)); + return txnid_a < txnid_b; + } +} + +static __inline bool mdbx_meta_eq(const MDBX_env *env, const MDBX_meta *a, + const MDBX_meta *b) { + mdbx_jitter4testing(true); + const txnid_t txnid = mdbx_meta_txnid_fluid(env, a); + if (!txnid || txnid != mdbx_meta_txnid_fluid(env, b)) + return false; + + mdbx_jitter4testing(true); + if (META_IS_STEADY(a) != META_IS_STEADY(b)) + return false; + + mdbx_jitter4testing(true); + return true; +} + +static int mdbx_meta_eq_mask(const MDBX_env *env) { + MDBX_meta *m0 = METAPAGE(env, 0); + MDBX_meta *m1 = METAPAGE(env, 1); + MDBX_meta *m2 = METAPAGE(env, 2); + + int rc = mdbx_meta_eq(env, m0, m1) ? 1 : 0; + if (mdbx_meta_eq(env, m1, m2)) + rc += 2; + if (mdbx_meta_eq(env, m2, m0)) + rc += 4; + return rc; +} + +static __inline MDBX_meta *mdbx_meta_recent(const enum meta_choise_mode mode, + const MDBX_env *env, MDBX_meta *a, + MDBX_meta *b) { + const bool a_older_that_b = mdbx_meta_ot(mode, env, a, b); + mdbx_assert(env, !mdbx_meta_eq(env, a, b)); + return a_older_that_b ? b : a; +} + +static __inline MDBX_meta *mdbx_meta_ancient(const enum meta_choise_mode mode, + const MDBX_env *env, MDBX_meta *a, + MDBX_meta *b) { + const bool a_older_that_b = mdbx_meta_ot(mode, env, a, b); + mdbx_assert(env, !mdbx_meta_eq(env, a, b)); + return a_older_that_b ? a : b; +} + +static __inline MDBX_meta * +mdbx_meta_mostrecent(const enum meta_choise_mode mode, const MDBX_env *env) { + MDBX_meta *m0 = METAPAGE(env, 0); + MDBX_meta *m1 = METAPAGE(env, 1); + MDBX_meta *m2 = METAPAGE(env, 2); + + MDBX_meta *head = mdbx_meta_recent(mode, env, m0, m1); + head = mdbx_meta_recent(mode, env, head, m2); + return head; +} + +static __hot MDBX_meta *mdbx_meta_steady(const MDBX_env *env) { + return mdbx_meta_mostrecent(prefer_steady, env); +} + +static __hot MDBX_meta *mdbx_meta_head(const MDBX_env *env) { + return mdbx_meta_mostrecent(prefer_last, env); +} + +static __hot txnid_t mdbx_recent_committed_txnid(const MDBX_env *env) { + while (true) { + const MDBX_meta *head = mdbx_meta_head(env); + const txnid_t recent = mdbx_meta_txnid_fluid(env, head); + mdbx_compiler_barrier(); + if (likely(head == mdbx_meta_head(env) && + recent == mdbx_meta_txnid_fluid(env, head))) + return recent; + } +} + +static __hot txnid_t mdbx_recent_steady_txnid(const MDBX_env *env) { + while (true) { + const MDBX_meta *head = mdbx_meta_steady(env); + const txnid_t recent = mdbx_meta_txnid_fluid(env, head); + mdbx_compiler_barrier(); + if (likely(head == mdbx_meta_steady(env) && + recent == mdbx_meta_txnid_fluid(env, head))) + return recent; + } +} + +static __hot txnid_t mdbx_reclaiming_detent(const MDBX_env *env) { + if (F_ISSET(env->me_flags, MDBX_UTTERLY_NOSYNC)) + return likely(env->me_txn0->mt_owner == mdbx_thread_self()) + ? env->me_txn0->mt_txnid - MDBX_TXNID_STEP + : mdbx_recent_committed_txnid(env); + + return mdbx_recent_steady_txnid(env); +} + +static const char *mdbx_durable_str(const MDBX_meta *const meta) { + if (META_IS_WEAK(meta)) + return "Weak"; + if (META_IS_STEADY(meta)) + return (meta->mm_datasync_sign == mdbx_meta_sign(meta)) ? "Steady" + : "Tainted"; + return "Legacy"; +} + +/*----------------------------------------------------------------------------*/ + +/* Find oldest txnid still referenced. */ +static txnid_t mdbx_find_oldest(MDBX_txn *txn) { + mdbx_tassert(txn, (txn->mt_flags & MDBX_RDONLY) == 0); + MDBX_env *env = txn->mt_env; + const txnid_t edge = mdbx_reclaiming_detent(env); + mdbx_tassert(txn, edge <= txn->mt_txnid); + + MDBX_lockinfo *const lck = env->me_lck; + if (unlikely(lck == NULL /* exclusive mode */)) + return env->me_lckless_stub.oldest = edge; + + const txnid_t last_oldest = lck->mti_oldest_reader; + mdbx_tassert(txn, edge >= last_oldest); + if (likely(last_oldest == edge)) + return edge; + + const uint32_t nothing_changed = MDBX_STRING_TETRAD("None"); + const uint32_t snap_readers_refresh_flag = lck->mti_readers_refresh_flag; + mdbx_jitter4testing(false); + if (snap_readers_refresh_flag == nothing_changed) + return last_oldest; + + txnid_t oldest = edge; + lck->mti_readers_refresh_flag = nothing_changed; + mdbx_flush_noncoherent_cpu_writeback(); + const unsigned snap_nreaders = lck->mti_numreaders; + for (unsigned i = 0; i < snap_nreaders; ++i) { + if (lck->mti_readers[i].mr_pid) { + /* mdbx_jitter4testing(true); */ + const txnid_t snap = safe64_read(&lck->mti_readers[i].mr_txnid); + if (oldest > snap && last_oldest <= /* ignore pending updates */ snap) { + oldest = snap; + if (oldest == last_oldest) + return oldest; + } + } + } + + if (oldest != last_oldest) { + mdbx_notice("update oldest %" PRIaTXN " -> %" PRIaTXN, last_oldest, oldest); + mdbx_tassert(txn, oldest >= lck->mti_oldest_reader); + lck->mti_oldest_reader = oldest; + } + return oldest; +} + +/* Find largest mvcc-snapshot still referenced. */ +static __cold pgno_t mdbx_find_largest(MDBX_env *env, pgno_t largest) { + MDBX_lockinfo *const lck = env->me_lck; + if (likely(lck != NULL /* exclusive mode */)) { + const unsigned snap_nreaders = lck->mti_numreaders; + for (unsigned i = 0; i < snap_nreaders; ++i) { + retry: + if (lck->mti_readers[i].mr_pid) { + /* mdbx_jitter4testing(true); */ + const pgno_t snap_pages = lck->mti_readers[i].mr_snapshot_pages_used; + const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid); + mdbx_memory_barrier(); + if (unlikely(snap_pages != lck->mti_readers[i].mr_snapshot_pages_used || + snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid))) + goto retry; + if (largest < snap_pages && + lck->mti_oldest_reader <= /* ignore pending updates */ snap_txnid && + snap_txnid <= env->me_txn0->mt_txnid) + largest = snap_pages; + } + } + } + + return largest; +} + +/* Add a page to the txn's dirty list */ +static int __must_check_result mdbx_page_dirty(MDBX_txn *txn, MDBX_page *mp) { + const int rc = mdbx_dpl_append(txn->tw.dirtylist, mp->mp_pgno, mp); + if (unlikely(rc != MDBX_SUCCESS)) { + txn->mt_flags |= MDBX_TXN_ERROR; + return rc; + } + txn->tw.dirtyroom--; + mdbx_tassert(txn, txn->mt_parent || + txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + return MDBX_SUCCESS; +} + +/* Turn on/off readahead. It's harmful when the DB is larger than RAM. */ +static int __cold mdbx_set_readahead(MDBX_env *env, const size_t offset, + const size_t length, const bool enable) { + assert(length > 0); + mdbx_notice("readahead %s %u..%u", enable ? "ON" : "OFF", + bytes2pgno(env, offset), bytes2pgno(env, offset + length)); + +#if defined(F_RDAHEAD) + if (unlikely(fcntl(env->me_fd, F_RDAHEAD, enable) == -1)) + return errno; +#endif /* F_RDAHEAD */ + + if (enable) { +#if defined(F_RDADVISE) + struct radvisory hint; + hint.ra_offset = offset; + hint.ra_count = length; + (void)/* Ignore ENOTTY for DB on the ram-disk and so on */ fcntl( + env->me_fd, F_RDADVISE, &hint); +#endif /* F_RDADVISE */ +#if defined(MADV_WILLNEED) + if (unlikely(madvise(env->me_map + offset, length, MADV_WILLNEED) != 0)) + return errno; +#elif defined(POSIX_MADV_WILLNEED) + rc = posix_madvise(env->me_map + offset, length, POSIX_MADV_WILLNEED); + if (unlikely(rc != 0)) + return errno; +#elif defined(_WIN32) || defined(_WIN64) + if (mdbx_PrefetchVirtualMemory) { + WIN32_MEMORY_RANGE_ENTRY hint; + hint.VirtualAddress = env->me_map + offset; + hint.NumberOfBytes = length; + (void)mdbx_PrefetchVirtualMemory(GetCurrentProcess(), 1, &hint, 0); + } +#elif defined(POSIX_FADV_WILLNEED) + int err = posix_fadvise(env->me_fd, offset, length, POSIX_FADV_WILLNEED); + if (unlikely(err != 0)) + return err; +#endif /* MADV_WILLNEED */ + } else { +#if defined(MADV_RANDOM) + if (unlikely(madvise(env->me_map + offset, length, MADV_RANDOM) != 0)) + return errno; +#elif defined(POSIX_MADV_RANDOM) + int err = posix_madvise(env->me_map + offset, length, POSIX_MADV_RANDOM); + if (unlikely(err != 0)) + return err; +#elif defined(POSIX_FADV_RANDOM) + int err = posix_fadvise(env->me_fd, offset, length, POSIX_FADV_RANDOM); + if (unlikely(err != 0)) + return err; +#endif /* MADV_RANDOM */ + } + return MDBX_SUCCESS; +} + +static __cold int mdbx_mapresize(MDBX_env *env, const pgno_t size_pgno, + const pgno_t limit_pgno) { + const size_t limit_bytes = pgno_align2os_bytes(env, limit_pgno); + const size_t size_bytes = pgno_align2os_bytes(env, size_pgno); + + mdbx_verbose("resize datafile/mapping: " + "present %" PRIuPTR " -> %" PRIuPTR ", " + "limit %" PRIuPTR " -> %" PRIuPTR, + env->me_dxb_mmap.current, size_bytes, env->me_dxb_mmap.limit, + limit_bytes); + + mdbx_assert(env, limit_bytes >= size_bytes); + mdbx_assert(env, bytes2pgno(env, size_bytes) >= size_pgno); + mdbx_assert(env, bytes2pgno(env, limit_bytes) >= limit_pgno); + +#if defined(_WIN32) || defined(_WIN64) + /* Acquire guard in exclusive mode for: + * - to avoid collision between read and write txns around env->me_dbgeo; + * - to avoid attachment of new reading threads (see mdbx_rdt_lock); */ + mdbx_srwlock_AcquireExclusive(&env->me_remap_guard); + mdbx_handle_array_t *suspended = NULL; + mdbx_handle_array_t array_onstack; + int rc = MDBX_SUCCESS; + if (limit_bytes == env->me_dxb_mmap.limit && + size_bytes == env->me_dxb_mmap.current && + size_bytes == env->me_dxb_mmap.filesize) + goto bailout; + + /* 1) Windows allows only extending a read-write section, but not a + * corresponing mapped view. Therefore in other cases we must suspend + * the local threads for safe remap. + * 2) At least on Windows 10 1803 the entire mapped section is unavailable + * for short time during NtExtendSection() or VirtualAlloc() execution. + * + * THEREFORE LOCAL THREADS SUSPENDING IS ALWAYS REQUIRED! */ + array_onstack.limit = ARRAY_LENGTH(array_onstack.handles); + array_onstack.count = 0; + suspended = &array_onstack; + rc = mdbx_suspend_threads_before_remap(env, &suspended); + if (rc != MDBX_SUCCESS) { + mdbx_error("failed suspend-for-remap: errcode %d", rc); + goto bailout; + } +#else + /* Acquire guard to avoid collision between read and write txns + * around env->me_dbgeo */ + int rc = mdbx_fastmutex_acquire(&env->me_remap_guard); + if (rc != MDBX_SUCCESS) + return rc; + if (limit_bytes == env->me_dxb_mmap.limit && + size_bytes == env->me_dxb_mmap.current) + goto bailout; +#endif /* Windows */ + + const size_t prev_limit = env->me_dxb_mmap.limit; + const void *const prev_addr = env->me_map; + const size_t prev_size = env->me_dxb_mmap.current; + if (size_bytes < prev_size) { + mdbx_notice("resize-MADV_%s %u..%u", + (env->me_flags & MDBX_WRITEMAP) ? "REMOVE" : "DONTNEED", + size_pgno, bytes2pgno(env, prev_size)); +#if defined(MADV_REMOVE) + if ((env->me_flags & MDBX_WRITEMAP) == 0 || + madvise(env->me_map + size_bytes, prev_size - size_bytes, + MADV_REMOVE) != 0) +#endif +#if defined(MADV_DONTNEED) + (void)madvise(env->me_map + size_bytes, prev_size - size_bytes, + MADV_DONTNEED); +#elif defined(POSIX_MADV_DONTNEED) + (void)posix_madvise(env->me_map + size_bytes, prev_size - size_bytes, + POSIX_MADV_DONTNEED); +#elif defined(POSIX_FADV_DONTNEED) + (void)posix_fadvise(env->me_fd, size_bytes, prev_size - size_bytes, + POSIX_FADV_DONTNEED); +#else + __noop(); +#endif /* MADV_DONTNEED */ + if (*env->me_discarded_tail > size_pgno) + *env->me_discarded_tail = size_pgno; + } + + rc = mdbx_mresize(env->me_flags, &env->me_dxb_mmap, size_bytes, limit_bytes); + if (rc == MDBX_SUCCESS && (env->me_flags & MDBX_NORDAHEAD) == 0) { + rc = mdbx_is_readahead_reasonable(size_bytes, 0); + if (rc == MDBX_RESULT_FALSE) + rc = mdbx_set_readahead( + env, 0, (size_bytes > prev_size) ? size_bytes : prev_size, false); + else if (rc == MDBX_RESULT_TRUE) { + rc = MDBX_SUCCESS; + const size_t readahead_pivot = + (limit_bytes != prev_limit || env->me_dxb_mmap.address != prev_addr +#if defined(_WIN32) || defined(_WIN64) + || prev_size > size_bytes +#endif /* Windows */ + ) + ? 0 /* reassign readahead to the entire map + because it was remapped */ + : prev_size; + if (size_bytes > readahead_pivot) { + *env->me_discarded_tail = size_pgno; + rc = mdbx_set_readahead(env, readahead_pivot, + size_bytes - readahead_pivot, true); + } + } + } + +bailout: + if (rc == MDBX_SUCCESS) { +#if defined(_WIN32) || defined(_WIN64) + mdbx_assert(env, size_bytes == env->me_dxb_mmap.current); + mdbx_assert(env, size_bytes <= env->me_dxb_mmap.filesize); + mdbx_assert(env, limit_bytes == env->me_dxb_mmap.limit); +#endif /* Windows */ +#ifdef MDBX_USE_VALGRIND + if (prev_limit != env->me_dxb_mmap.limit || prev_addr != env->me_map) { + VALGRIND_DISCARD(env->me_valgrind_handle); + env->me_valgrind_handle = 0; + if (env->me_dxb_mmap.limit) + env->me_valgrind_handle = + VALGRIND_CREATE_BLOCK(env->me_map, env->me_dxb_mmap.limit, "mdbx"); + } +#endif /* MDBX_USE_VALGRIND */ + } else { + if (rc != MDBX_RESULT_TRUE) { + mdbx_error("failed resize datafile/mapping: " + "present %" PRIuPTR " -> %" PRIuPTR ", " + "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d", + env->me_dxb_mmap.current, size_bytes, env->me_dxb_mmap.limit, + limit_bytes, rc); + } else { + mdbx_notice("unable resize datafile/mapping: " + "present %" PRIuPTR " -> %" PRIuPTR ", " + "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d", + env->me_dxb_mmap.current, size_bytes, env->me_dxb_mmap.limit, + limit_bytes, rc); + } + if (!env->me_dxb_mmap.address) { + env->me_flags |= MDBX_FATAL_ERROR; + if (env->me_txn) + env->me_txn->mt_flags |= MDBX_TXN_ERROR; + rc = MDBX_PANIC; + } + } + +#if defined(_WIN32) || defined(_WIN64) + int err = MDBX_SUCCESS; + mdbx_srwlock_ReleaseExclusive(&env->me_remap_guard); + if (suspended) { + err = mdbx_resume_threads_after_remap(suspended); + if (suspended != &array_onstack) + mdbx_free(suspended); + } +#else + int err = mdbx_fastmutex_release(&env->me_remap_guard); +#endif /* Windows */ + if (err != MDBX_SUCCESS) { + mdbx_fatal("failed resume-after-remap: errcode %d", err); + return MDBX_PANIC; + } + return rc; +} + +/* Allocate page numbers and memory for writing. Maintain mt_last_reclaimed, + * mt_reclaimed_pglist and mt_next_pgno. Set MDBX_TXN_ERROR on failure. + * + * If there are free pages available from older transactions, they + * are re-used first. Otherwise allocate a new page at mt_next_pgno. + * Do not modify the GC, just merge GC records into mt_reclaimed_pglist + * and move mt_last_reclaimed to say which records were consumed. Only this + * function can create mt_reclaimed_pglist and move + * mt_last_reclaimed/mt_next_pgno. + * + * [in] mc cursor A cursor handle identifying the transaction and + * database for which we are allocating. + * [in] num the number of pages to allocate. + * [out] mp Address of the allocated page(s). Requests for multiple pages + * will always be satisfied by a single contiguous chunk of memory. + * + * Returns 0 on success, non-zero on failure.*/ + +#define MDBX_ALLOC_CACHE 1 +#define MDBX_ALLOC_GC 2 +#define MDBX_ALLOC_NEW 4 +#define MDBX_ALLOC_KICK 8 +#define MDBX_ALLOC_ALL \ + (MDBX_ALLOC_CACHE | MDBX_ALLOC_GC | MDBX_ALLOC_NEW | MDBX_ALLOC_KICK) + +static int mdbx_page_alloc(MDBX_cursor *mc, unsigned num, MDBX_page **mp, + int flags) { + int rc; + MDBX_txn *txn = mc->mc_txn; + MDBX_env *env = txn->mt_env; + MDBX_page *np; + + if (likely(flags & MDBX_ALLOC_GC)) { + flags |= env->me_flags & (MDBX_COALESCE | MDBX_LIFORECLAIM); + if (unlikely(mc->mc_flags & C_RECLAIMING)) { + /* If mc is updating the GC, then the retired-list cannot play + * catch-up with itself by growing while trying to save it. */ + flags &= + ~(MDBX_ALLOC_GC | MDBX_ALLOC_KICK | MDBX_COALESCE | MDBX_LIFORECLAIM); + } else if (unlikely(txn->mt_dbs[FREE_DBI].md_entries == 0)) { + /* avoid (recursive) search inside empty tree and while tree is updating, + * https://github.com/leo-yuriev/libmdbx/issues/31 */ + flags &= ~MDBX_ALLOC_GC; + } + } + + if (likely(flags & MDBX_ALLOC_CACHE)) { + /* If there are any loose pages, just use them */ + mdbx_assert(env, mp && num); + if (likely(num == 1 && txn->tw.loose_pages)) { + if (txn->tw.loose_refund_wl > txn->mt_next_pgno) { + mdbx_refund(txn); + if (unlikely(!txn->tw.loose_pages)) + goto skip_cache; + } + + np = txn->tw.loose_pages; + txn->tw.loose_pages = np->mp_next; + txn->tw.loose_count--; + mdbx_debug("db %d use loose page %" PRIaPGNO, DDBI(mc), np->mp_pgno); + mdbx_tassert(txn, np->mp_pgno < txn->mt_next_pgno); + mdbx_ensure(env, np->mp_pgno >= NUM_METAS); + VALGRIND_MAKE_MEM_UNDEFINED(page_data(np), page_space(txn->mt_env)); + ASAN_UNPOISON_MEMORY_REGION(page_data(np), page_space(txn->mt_env)); + *mp = np; + return MDBX_SUCCESS; + } + } +skip_cache: + + mdbx_tassert( + txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, txn->mt_next_pgno)); + pgno_t pgno, *re_list = txn->tw.reclaimed_pglist; + unsigned range_begin = 0, re_len = MDBX_PNL_SIZE(re_list); + txnid_t oldest = 0, last = 0; + const unsigned wanna_range = num - 1; + + while (1) { /* oom-kick retry loop */ + /* If our dirty list is already full, we can't do anything */ + if (unlikely(txn->tw.dirtyroom == 0)) { + rc = MDBX_TXN_FULL; + goto fail; + } + + MDBX_cursor recur; + for (MDBX_cursor_op op = MDBX_FIRST;; + op = (flags & MDBX_LIFORECLAIM) ? MDBX_PREV : MDBX_NEXT) { + MDBX_val key, data; + + /* Seek a big enough contiguous page range. + * Prefer pages with lower pgno. */ + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + if (likely(flags & MDBX_ALLOC_CACHE) && re_len > wanna_range && + (!(flags & MDBX_COALESCE) || op == MDBX_FIRST)) { + mdbx_tassert(txn, MDBX_PNL_LAST(re_list) < txn->mt_next_pgno && + MDBX_PNL_FIRST(re_list) < txn->mt_next_pgno); + range_begin = MDBX_PNL_ASCENDING ? 1 : re_len; + pgno = MDBX_PNL_LEAST(re_list); + if (likely(wanna_range == 0)) + goto done; +#if MDBX_PNL_ASCENDING + mdbx_tassert(txn, pgno == re_list[1] && range_begin == 1); + while (true) { + unsigned range_end = range_begin + wanna_range; + if (re_list[range_end] - pgno == wanna_range) + goto done; + if (range_end == re_len) + break; + pgno = re_list[++range_begin]; + } +#else + mdbx_tassert(txn, pgno == re_list[re_len] && range_begin == re_len); + while (true) { + if (re_list[range_begin - wanna_range] - pgno == wanna_range) + goto done; + if (range_begin == wanna_range) + break; + pgno = re_list[--range_begin]; + } +#endif /* MDBX_PNL sort-order */ + } + + if (op == MDBX_FIRST) { /* 1st iteration, setup cursor, etc */ + if (unlikely(!(flags & MDBX_ALLOC_GC))) + break /* reclaiming is prohibited for now */; + + /* Prepare to fetch more and coalesce */ + oldest = (flags & MDBX_LIFORECLAIM) ? mdbx_find_oldest(txn) + : *env->me_oldest; + rc = mdbx_cursor_init(&recur, txn, FREE_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + if (flags & MDBX_LIFORECLAIM) { + /* Begin from oldest reader if any */ + if (oldest > 2) { + last = oldest - 1; + op = MDBX_SET_RANGE; + } + } else if (txn->tw.last_reclaimed) { + /* Continue lookup from txn->tw.last_reclaimed to oldest reader */ + last = txn->tw.last_reclaimed; + op = MDBX_SET_RANGE; + } + + key.iov_base = &last; + key.iov_len = sizeof(last); + } + + if (!(flags & MDBX_LIFORECLAIM)) { + /* Do not try fetch more if the record will be too recent */ + if (op != MDBX_FIRST && ++last >= oldest) { + oldest = mdbx_find_oldest(txn); + if (oldest <= last) + break; + } + } + + rc = mdbx_cursor_get(&recur, &key, NULL, op); + if (rc == MDBX_NOTFOUND && (flags & MDBX_LIFORECLAIM)) { + if (op == MDBX_SET_RANGE) + continue; + txnid_t snap = mdbx_find_oldest(txn); + if (oldest < snap) { + oldest = snap; + last = oldest - 1; + key.iov_base = &last; + key.iov_len = sizeof(last); + op = MDBX_SET_RANGE; + rc = mdbx_cursor_get(&recur, &key, NULL, op); + } + } + if (unlikely(rc)) { + if (rc == MDBX_NOTFOUND) + break; + goto fail; + } + + if (unlikely(key.iov_len != sizeof(txnid_t))) { + rc = MDBX_CORRUPTED; + goto fail; + } + last = unaligned_peek_u64(4, key.iov_base); + if (unlikely(last < 1 || last >= SAFE64_INVALID_THRESHOLD)) { + rc = MDBX_CORRUPTED; + goto fail; + } + if (oldest <= last) { + oldest = mdbx_find_oldest(txn); + if (oldest <= last) { + if (flags & MDBX_LIFORECLAIM) + continue; + break; + } + } + + if (flags & MDBX_LIFORECLAIM) { + /* skip IDs of records that already reclaimed */ + if (txn->tw.lifo_reclaimed) { + unsigned i; + for (i = (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed); i > 0; --i) + if (txn->tw.lifo_reclaimed[i] == last) + break; + if (i) + continue; + } + } + + /* Reading next GC record */ + np = recur.mc_pg[recur.mc_top]; + if (unlikely((rc = mdbx_node_read( + &recur, page_node(np, recur.mc_ki[recur.mc_top]), + &data)) != MDBX_SUCCESS)) + goto fail; + + if ((flags & MDBX_LIFORECLAIM) && !txn->tw.lifo_reclaimed) { + txn->tw.lifo_reclaimed = mdbx_txl_alloc(); + if (unlikely(!txn->tw.lifo_reclaimed)) { + rc = MDBX_ENOMEM; + goto fail; + } + } + + /* Append PNL from GC record to me_reclaimed_pglist */ + mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0); + pgno_t *gc_pnl = (pgno_t *)data.iov_base; + mdbx_tassert(txn, data.iov_len >= MDBX_PNL_SIZEOF(gc_pnl)); + if (unlikely(data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) || + !mdbx_pnl_check(gc_pnl, txn->mt_next_pgno))) { + rc = MDBX_CORRUPTED; + goto fail; + } + const unsigned gc_len = MDBX_PNL_SIZE(gc_pnl); + rc = mdbx_pnl_need(&txn->tw.reclaimed_pglist, gc_len); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + re_list = txn->tw.reclaimed_pglist; + + /* Remember ID of GC record */ + if (flags & MDBX_LIFORECLAIM) { + if ((rc = mdbx_txl_append(&txn->tw.lifo_reclaimed, last)) != 0) + goto fail; + } + txn->tw.last_reclaimed = last; + + if (mdbx_log_enabled(MDBX_LOG_EXTRA)) { + mdbx_debug_extra("PNL read txn %" PRIaTXN " root %" PRIaPGNO + " num %u, PNL", + last, txn->mt_dbs[FREE_DBI].md_root, gc_len); + unsigned i; + for (i = gc_len; i; i--) + mdbx_debug_extra_print(" %" PRIaPGNO, gc_pnl[i]); + mdbx_debug_extra_print("%s", "\n"); + } + + /* Merge in descending sorted order */ + const unsigned prev_re_len = MDBX_PNL_SIZE(re_list); + mdbx_pnl_xmerge(re_list, gc_pnl); + /* re-check to avoid duplicates */ + if (unlikely(!mdbx_pnl_check(re_list, txn->mt_next_pgno))) { + rc = MDBX_CORRUPTED; + goto fail; + } + + re_len = MDBX_PNL_SIZE(re_list); + mdbx_tassert(txn, re_len == 0 || re_list[re_len] < txn->mt_next_pgno); + if (re_len && unlikely(MDBX_PNL_MOST(re_list) == txn->mt_next_pgno - 1)) { + /* Refund suitable pages into "unallocated" space */ + mdbx_refund(txn); + re_list = txn->tw.reclaimed_pglist; + re_len = MDBX_PNL_SIZE(re_list); + } + + if (unlikely((flags & MDBX_ALLOC_CACHE) == 0)) { + /* Done for a kick-reclaim mode, actually no page needed */ + return MDBX_SUCCESS; + } + + /* Don't try to coalesce too much. */ + if (unlikely(re_len > MDBX_DPL_TXNFULL / 4)) + break; + if (re_len /* current size */ >= env->me_maxgc_ov1page || + (re_len > prev_re_len && re_len - prev_re_len /* delta from prev */ >= + env->me_maxgc_ov1page / 2)) + flags &= ~MDBX_COALESCE; + } + + if ((flags & (MDBX_COALESCE | MDBX_ALLOC_CACHE)) == + (MDBX_COALESCE | MDBX_ALLOC_CACHE) && + re_len > wanna_range) { + range_begin = MDBX_PNL_ASCENDING ? 1 : re_len; + pgno = MDBX_PNL_LEAST(re_list); + if (likely(wanna_range == 0)) + goto done; +#if MDBX_PNL_ASCENDING + mdbx_tassert(txn, pgno == re_list[1] && range_begin == 1); + while (true) { + unsigned range_end = range_begin + wanna_range; + if (re_list[range_end] - pgno == wanna_range) + goto done; + if (range_end == re_len) + break; + pgno = re_list[++range_begin]; + } +#else + mdbx_tassert(txn, pgno == re_list[re_len] && range_begin == re_len); + while (true) { + if (re_list[range_begin - wanna_range] - pgno == wanna_range) + goto done; + if (range_begin == wanna_range) + break; + pgno = re_list[--range_begin]; + } +#endif /* MDBX_PNL sort-order */ + } + + /* Use new pages from the map when nothing suitable in the GC */ + range_begin = 0; + pgno = txn->mt_next_pgno; + rc = MDBX_MAP_FULL; + const pgno_t next = pgno_add(pgno, num); + if (likely(next <= txn->mt_end_pgno)) { + rc = MDBX_NOTFOUND; + if (likely(flags & MDBX_ALLOC_NEW)) + goto done; + } + + const MDBX_meta *head = mdbx_meta_head(env); + if ((flags & MDBX_ALLOC_GC) && + ((flags & MDBX_ALLOC_KICK) || rc == MDBX_MAP_FULL)) { + MDBX_meta *steady = mdbx_meta_steady(env); + + if (oldest == mdbx_meta_txnid_stable(env, steady) && + !META_IS_STEADY(head) && META_IS_STEADY(steady)) { + /* LY: Here an oom was happened: + * - all pages had allocated; + * - reclaiming was stopped at the last steady-sync; + * - the head-sync is weak. + * Now we need make a sync to resume reclaiming. If both + * MDBX_NOSYNC and MDBX_MAPASYNC flags are set, then assume that + * utterly no-sync write mode was requested. In such case + * don't make a steady-sync, but only a legacy-mode checkpoint, + * just for resume reclaiming only, not for data consistency. */ + + mdbx_debug("kick-gc: head %" PRIaTXN "-%s, tail %" PRIaTXN + "-%s, oldest %" PRIaTXN, + mdbx_meta_txnid_stable(env, head), mdbx_durable_str(head), + mdbx_meta_txnid_stable(env, steady), + mdbx_durable_str(steady), oldest); + + const unsigned syncflags = F_ISSET(env->me_flags, MDBX_UTTERLY_NOSYNC) + ? env->me_flags + : env->me_flags & MDBX_WRITEMAP; + MDBX_meta meta = *head; + if (mdbx_sync_locked(env, syncflags, &meta) == MDBX_SUCCESS) { + txnid_t snap = mdbx_find_oldest(txn); + if (snap > oldest) + continue; + } + } + + if (rc == MDBX_MAP_FULL && oldest < txn->mt_txnid - MDBX_TXNID_STEP) { + if (mdbx_oomkick(env, oldest) > oldest) + continue; + } + } + + if (rc == MDBX_MAP_FULL && next < head->mm_geo.upper) { + mdbx_assert(env, next > txn->mt_end_pgno); + pgno_t aligned = pgno_align2os_pgno( + env, pgno_add(next, head->mm_geo.grow - next % head->mm_geo.grow)); + + if (aligned > head->mm_geo.upper) + aligned = head->mm_geo.upper; + mdbx_assert(env, aligned > txn->mt_end_pgno); + + mdbx_verbose("try growth datafile to %" PRIaPGNO " pages (+%" PRIaPGNO + ")", + aligned, aligned - txn->mt_end_pgno); + rc = mdbx_mapresize(env, aligned, head->mm_geo.upper); + if (rc == MDBX_SUCCESS) { + env->me_txn->mt_end_pgno = aligned; + if (!mp) + return rc; + goto done; + } + + mdbx_warning("unable growth datafile to %" PRIaPGNO "pages (+%" PRIaPGNO + "), errcode %d", + aligned, aligned - txn->mt_end_pgno, rc); + } + + fail: + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + if (mp) { + *mp = NULL; + txn->mt_flags |= MDBX_TXN_ERROR; + } + mdbx_assert(env, rc != MDBX_SUCCESS); + return rc; + } + +done: + mdbx_tassert(txn, mp && num); + mdbx_ensure(env, pgno >= NUM_METAS); + if (env->me_flags & MDBX_WRITEMAP) { + np = pgno2page(env, pgno); + /* LY: reset no-access flag from mdbx_loose_page() */ + VALGRIND_MAKE_MEM_UNDEFINED(np, pgno2bytes(env, num)); + ASAN_UNPOISON_MEMORY_REGION(np, pgno2bytes(env, num)); + } else { + if (unlikely(!(np = mdbx_page_malloc(txn, num)))) { + rc = MDBX_ENOMEM; + goto fail; + } + } + + if (range_begin) { + mdbx_cassert(mc, (mc->mc_flags & C_GCFREEZE) == 0); + mdbx_tassert(txn, pgno < txn->mt_next_pgno); + mdbx_tassert(txn, pgno == re_list[range_begin]); + /* Cutoff allocated pages from me_reclaimed_pglist */ +#if MDBX_PNL_ASCENDING + for (unsigned i = range_begin + num; i <= re_len;) + re_list[range_begin++] = re_list[i++]; + MDBX_PNL_SIZE(re_list) = re_len = range_begin - 1; +#else + MDBX_PNL_SIZE(re_list) = re_len -= num; + for (unsigned i = range_begin - num; i < re_len;) + re_list[++i] = re_list[++range_begin]; +#endif + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + } else { + txn->mt_next_pgno = pgno + num; + mdbx_assert(env, txn->mt_next_pgno <= txn->mt_end_pgno); + } + + if (unlikely(env->me_flags & MDBX_PAGEPERTURB)) + memset(np, -1, pgno2bytes(env, num)); + VALGRIND_MAKE_MEM_UNDEFINED(np, pgno2bytes(env, num)); + + np->mp_pgno = pgno; + np->mp_leaf2_ksize = 0; + np->mp_flags = 0; + np->mp_pages = num; + rc = mdbx_page_dirty(txn, np); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + *mp = np; + + mdbx_tassert( + txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, txn->mt_next_pgno)); + return MDBX_SUCCESS; +} + +/* Copy the used portions of a non-overflow page. + * [in] dst page to copy into + * [in] src page to copy from + * [in] psize size of a page */ +__hot static void mdbx_page_copy(MDBX_page *dst, MDBX_page *src, + unsigned psize) { + STATIC_ASSERT(UINT16_MAX > MAX_PAGESIZE - PAGEHDRSZ); + STATIC_ASSERT(MIN_PAGESIZE > PAGEHDRSZ + NODESIZE * 4); + if (!IS_LEAF2(src)) { + size_t upper = src->mp_upper, lower = src->mp_lower, unused = upper - lower; + + /* If page isn't full, just copy the used portion. Adjust + * alignment so memcpy may copy words instead of bytes. */ + if (unused > sizeof(void *) * 42) { + lower = roundup_powerof2(lower + PAGEHDRSZ, sizeof(void *)); + upper = (upper + PAGEHDRSZ) & ~(sizeof(void *) - 1); + memcpy(dst, src, lower); + memcpy((char *)dst + upper, (char *)src + upper, psize - upper); + return; + } + } + memcpy(dst, src, psize); +} + +/* Pull a page off the txn's spill list, if present. + * + * If a page being referenced was spilled to disk in this txn, bring + * it back and make it dirty/writable again. + * + * [in] txn the transaction handle. + * [in] mp the page being referenced. It must not be dirty. + * [out] ret the writable page, if any. + * ret is unchanged if mp wasn't spilled. */ +__hot static int __must_check_result mdbx_page_unspill(MDBX_txn *txn, + MDBX_page *mp, + MDBX_page **ret) { + MDBX_env *env = txn->mt_env; + pgno_t pgno = mp->mp_pgno, pn = pgno << 1; + + for (const MDBX_txn *tx2 = txn; tx2; tx2 = tx2->mt_parent) { + if (!tx2->tw.spill_pages) + continue; + unsigned i = mdbx_pnl_exist(tx2->tw.spill_pages, pn); + if (!i) + continue; + if (txn->tw.dirtyroom == 0) + return MDBX_TXN_FULL; + unsigned num = IS_OVERFLOW(mp) ? mp->mp_pages : 1; + MDBX_page *np = mp; + if ((env->me_flags & MDBX_WRITEMAP) == 0) { + np = mdbx_page_malloc(txn, num); + if (unlikely(!np)) + return MDBX_ENOMEM; + if (unlikely(num > 1)) + memcpy(np, mp, pgno2bytes(env, num)); + else + mdbx_page_copy(np, mp, env->me_psize); + } + mdbx_debug("unspill page %" PRIaPGNO, mp->mp_pgno); + if (tx2 == txn) { + /* If in current txn, this page is no longer spilled. + * If it happens to be the last page, truncate the spill list. + * Otherwise mark it as deleted by setting the LSB. */ + txn->tw.spill_pages[i] |= 1; + if (i == MDBX_PNL_SIZE(txn->tw.spill_pages)) + MDBX_PNL_SIZE(txn->tw.spill_pages) -= 1; + } /* otherwise, if belonging to a parent txn, the + * page remains spilled until child commits */ + + int rc = mdbx_page_dirty(txn, np); + if (likely(rc == MDBX_SUCCESS)) { + np->mp_flags |= P_DIRTY; + *ret = np; + } + return rc; + } + return MDBX_SUCCESS; +} + +/* Touch a page: make it dirty and re-insert into tree with updated pgno. + * Set MDBX_TXN_ERROR on failure. + * + * [in] mc cursor pointing to the page to be touched + * + * Returns 0 on success, non-zero on failure. */ +__hot static int mdbx_page_touch(MDBX_cursor *mc) { + MDBX_page *mp = mc->mc_pg[mc->mc_top], *np; + MDBX_txn *txn = mc->mc_txn; + MDBX_cursor *m2, *m3; + pgno_t pgno; + int rc; + + mdbx_cassert(mc, !IS_OVERFLOW(mp)); + if (!F_ISSET(mp->mp_flags, P_DIRTY)) { + if (txn->mt_flags & MDBX_TXN_SPILLS) { + np = NULL; + rc = mdbx_page_unspill(txn, mp, &np); + if (unlikely(rc)) + goto fail; + if (likely(np)) + goto done; + } + + if (unlikely((rc = mdbx_pnl_need(&txn->tw.retired_pages, 1)) || + (rc = mdbx_page_alloc(mc, 1, &np, MDBX_ALLOC_ALL)))) + goto fail; + pgno = np->mp_pgno; + mdbx_debug("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, DDBI(mc), + mp->mp_pgno, pgno); + mdbx_cassert(mc, mp->mp_pgno != pgno); + mdbx_pnl_xappend(txn->tw.retired_pages, mp->mp_pgno); + mdbx_tassert(txn, mdbx_dpl_find(txn->tw.dirtylist, mp->mp_pgno) == nullptr); + /* Update the parent page, if any, to point to the new page */ + if (mc->mc_top) { + MDBX_page *parent = mc->mc_pg[mc->mc_top - 1]; + MDBX_node *node = page_node(parent, mc->mc_ki[mc->mc_top - 1]); + node_set_pgno(node, pgno); + } else { + mc->mc_db->md_root = pgno; + } + } else if (txn->mt_parent && !IS_SUBP(mp)) { + mdbx_tassert(txn, (txn->mt_env->me_flags & MDBX_WRITEMAP) == 0); + pgno = mp->mp_pgno; + /* If txn has a parent, make sure the page is in our dirty list. */ + const MDBX_page *const dp = mdbx_dpl_find(txn->tw.dirtylist, pgno); + if (dp) { + if (unlikely(mp != dp)) { /* bad cursor? */ + mdbx_error("wrong page 0x%p #%" PRIaPGNO + " in the dirtylist, expecting %p", + __Wpedantic_format_voidptr(dp), pgno, + __Wpedantic_format_voidptr(mp)); + mc->mc_flags &= ~(C_INITIALIZED | C_EOF); + rc = MDBX_PROBLEM; + goto fail; + } + return MDBX_SUCCESS; + } + + mdbx_debug("clone db %d page %" PRIaPGNO, DDBI(mc), mp->mp_pgno); + mdbx_cassert(mc, txn->tw.dirtylist->length <= MDBX_DPL_TXNFULL); + /* No - copy it */ + np = mdbx_page_malloc(txn, 1); + if (unlikely(!np)) { + rc = MDBX_ENOMEM; + goto fail; + } + rc = mdbx_dpl_append(txn->tw.dirtylist, pgno, np); + if (unlikely(rc)) { + mdbx_dpage_free(txn->mt_env, np, 1); + goto fail; + } + } else { + return MDBX_SUCCESS; + } + + mdbx_page_copy(np, mp, txn->mt_env->me_psize); + np->mp_pgno = pgno; + np->mp_flags |= P_DIRTY; + +done: + /* Adjust cursors pointing to mp */ + mc->mc_pg[mc->mc_top] = np; + m2 = txn->mt_cursors[mc->mc_dbi]; + if (mc->mc_flags & C_SUB) { + for (; m2; m2 = m2->mc_next) { + m3 = &m2->mc_xcursor->mx_cursor; + if (m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) + m3->mc_pg[mc->mc_top] = np; + } + } else { + for (; m2; m2 = m2->mc_next) { + if (m2->mc_snum < mc->mc_snum) + continue; + if (m2 == mc) + continue; + if (m2->mc_pg[mc->mc_top] == mp) { + m2->mc_pg[mc->mc_top] = np; + if (XCURSOR_INITED(m2) && IS_LEAF(np)) + XCURSOR_REFRESH(m2, np, m2->mc_ki[mc->mc_top]); + } + } + } + return MDBX_SUCCESS; + +fail: + txn->mt_flags |= MDBX_TXN_ERROR; + return rc; +} + +__cold int mdbx_env_sync_ex(MDBX_env *env, int force, int nonblock) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + +#if MDBX_TXN_CHECKPID + if (unlikely(env->me_pid != mdbx_getpid())) { + env->me_flags |= MDBX_FATAL_ERROR; + return MDBX_PANIC; + } +#endif /* MDBX_TXN_CHECKPID */ + + unsigned flags = env->me_flags & ~MDBX_NOMETASYNC; + if (unlikely(flags & (MDBX_RDONLY | MDBX_FATAL_ERROR))) + return MDBX_EACCESS; + + if (unlikely(!env->me_map)) + return MDBX_EPERM; + + int rc = MDBX_RESULT_TRUE /* means "nothing to sync" */; + bool need_unlock = false; + if (nonblock && *env->me_unsynced_pages == 0) + goto fastpath; + + const bool outside_txn = (env->me_txn0->mt_owner != mdbx_thread_self()); + if (outside_txn) { + int err = mdbx_txn_lock(env, nonblock); + if (unlikely(err != MDBX_SUCCESS)) + return err; + need_unlock = true; + } + + const MDBX_meta *head = mdbx_meta_head(env); + pgno_t unsynced_pages = *env->me_unsynced_pages; + if (!META_IS_STEADY(head) || unsynced_pages) { + const pgno_t autosync_threshold = *env->me_autosync_threshold; + const uint64_t autosync_period = *env->me_autosync_period; + if (force || (autosync_threshold && unsynced_pages >= autosync_threshold) || + (autosync_period && + mdbx_osal_monotime() - *env->me_sync_timestamp >= autosync_period)) + flags &= MDBX_WRITEMAP /* clear flags for full steady sync */; + + if (outside_txn) { + if (unsynced_pages > /* FIXME: define threshold */ 16 && + (flags & (MDBX_NOSYNC | MDBX_MAPASYNC)) == 0) { + mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0); + const size_t usedbytes = pgno_align2os_bytes(env, head->mm_geo.next); + + mdbx_txn_unlock(env); + + /* LY: pre-sync without holding lock to reduce latency for writer(s) */ + int err = (flags & MDBX_WRITEMAP) + ? mdbx_msync(&env->me_dxb_mmap, 0, usedbytes, false) + : mdbx_filesync(env->me_fd, MDBX_SYNC_DATA); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + err = mdbx_txn_lock(env, nonblock); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + /* LY: head and unsynced_pages may be changed. */ + head = mdbx_meta_head(env); + unsynced_pages = *env->me_unsynced_pages; + } + env->me_txn0->mt_txnid = meta_txnid(env, head, false); + mdbx_find_oldest(env->me_txn0); + rc = MDBX_RESULT_FALSE /* means "some data was synced" */; + } + + if (!META_IS_STEADY(head) || + ((flags & (MDBX_NOSYNC | MDBX_MAPASYNC)) == 0 && unsynced_pages)) { + mdbx_debug("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIaPGNO, + data_page(head)->mp_pgno, mdbx_durable_str(head), + unsynced_pages); + MDBX_meta meta = *head; + int err = mdbx_sync_locked(env, flags | MDBX_SHRINK_ALLOWED, &meta); + if (unlikely(err != MDBX_SUCCESS)) { + if (need_unlock) + mdbx_txn_unlock(env); + return err; + } + rc = MDBX_RESULT_FALSE /* means "some data was synced" */; + } + } + +fastpath: + /* LY: sync meta-pages if MDBX_NOMETASYNC enabled + * and someone was not synced above. */ + if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_NOMETASYNC) != 0) { + const txnid_t head_txnid = mdbx_recent_committed_txnid(env); + if (*env->me_meta_sync_txnid != (uint32_t)head_txnid) { + rc = (flags & MDBX_WRITEMAP) + ? mdbx_msync(&env->me_dxb_mmap, 0, pgno2bytes(env, NUM_METAS), + false) + : mdbx_filesync(env->me_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + if (likely(rc == MDBX_SUCCESS)) + *env->me_meta_sync_txnid = (uint32_t)head_txnid; + } + } + if (need_unlock) + mdbx_txn_unlock(env); + return rc; +} + +__cold int mdbx_env_sync(MDBX_env *env) { + return mdbx_env_sync_ex(env, true, false); +} + +__cold int mdbx_env_sync_poll(MDBX_env *env) { + return mdbx_env_sync_ex(env, false, true); +} + +/* Back up parent txn's cursors, then grab the originals for tracking */ +static int mdbx_cursor_shadow(MDBX_txn *src, MDBX_txn *dst) { + MDBX_cursor *mc, *bk; + MDBX_xcursor *mx; + + for (int i = src->mt_numdbs; --i >= 0;) { + dst->mt_cursors[i] = NULL; + if ((mc = src->mt_cursors[i]) != NULL) { + size_t size = sizeof(MDBX_cursor); + if (mc->mc_xcursor) + size += sizeof(MDBX_xcursor); + for (; mc; mc = bk->mc_next) { + bk = mdbx_malloc(size); + if (unlikely(!bk)) + return MDBX_ENOMEM; + *bk = *mc; + mc->mc_backup = bk; + mc->mc_db = &dst->mt_dbs[i]; + /* Kill pointers into src to reduce abuse: The + * user may not use mc until dst ends. But we need a valid + * txn pointer here for cursor fixups to keep working. */ + mc->mc_txn = dst; + mc->mc_dbflag = &dst->mt_dbflags[i]; + if ((mx = mc->mc_xcursor) != NULL) { + *(MDBX_xcursor *)(bk + 1) = *mx; + mx->mx_cursor.mc_txn = dst; + } + mc->mc_next = dst->mt_cursors[i]; + dst->mt_cursors[i] = mc; + } + } + } + return MDBX_SUCCESS; +} + +/* Close this write txn's cursors, give parent txn's cursors back to parent. + * + * [in] txn the transaction handle. + * [in] merge true to keep changes to parent cursors, false to revert. + * + * Returns 0 on success, non-zero on failure. */ +static void mdbx_cursors_eot(MDBX_txn *txn, unsigned merge) { + MDBX_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; + MDBX_xcursor *mx; + int i; + + for (i = txn->mt_numdbs; --i >= 0;) { + for (mc = cursors[i]; mc; mc = next) { + unsigned stage = mc->mc_signature; + mdbx_ensure(txn->mt_env, + stage == MDBX_MC_SIGNATURE || stage == MDBX_MC_WAIT4EOT); + next = mc->mc_next; + mdbx_tassert(txn, !next || next->mc_signature == MDBX_MC_SIGNATURE || + next->mc_signature == MDBX_MC_WAIT4EOT); + if ((bk = mc->mc_backup) != NULL) { + if (merge) { + /* Commit changes to parent txn */ + mc->mc_next = bk->mc_next; + mc->mc_backup = bk->mc_backup; + mc->mc_txn = bk->mc_txn; + mc->mc_db = bk->mc_db; + mc->mc_dbflag = bk->mc_dbflag; + if ((mx = mc->mc_xcursor) != NULL) + mx->mx_cursor.mc_txn = bk->mc_txn; + } else { + /* Abort nested txn */ + *mc = *bk; + if ((mx = mc->mc_xcursor) != NULL) + *mx = *(MDBX_xcursor *)(bk + 1); + } + bk->mc_signature = 0; + mdbx_free(bk); + } + if (stage == MDBX_MC_WAIT4EOT) { + mc->mc_signature = 0; + mdbx_free(mc); + } else { + mc->mc_signature = MDBX_MC_READY4CLOSE; + mc->mc_flags = 0 /* reset C_UNTRACK */; + } + } + cursors[i] = NULL; + } +} + +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) +/* Find largest mvcc-snapshot still referenced by this process. */ +static pgno_t mdbx_find_largest_this(MDBX_env *env, pgno_t largest) { + MDBX_lockinfo *const lck = env->me_lck; + if (likely(lck != NULL /* exclusive mode */)) { + const unsigned snap_nreaders = lck->mti_numreaders; + for (unsigned i = 0; i < snap_nreaders; ++i) { + retry: + if (lck->mti_readers[i].mr_pid == env->me_pid) { + /* mdbx_jitter4testing(true); */ + const pgno_t snap_pages = lck->mti_readers[i].mr_snapshot_pages_used; + const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid); + mdbx_memory_barrier(); + if (unlikely(snap_pages != lck->mti_readers[i].mr_snapshot_pages_used || + snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid))) + goto retry; + if (largest < snap_pages && + lck->mti_oldest_reader <= /* ignore pending updates */ snap_txnid && + snap_txnid < SAFE64_INVALID_THRESHOLD) + largest = snap_pages; + } + } + } + return largest; +} + +static void mdbx_txn_valgrind(MDBX_env *env, MDBX_txn *txn) { +#if !defined(__SANITIZE_ADDRESS__) + if (!RUNNING_ON_VALGRIND) + return; +#endif + + if (txn) { /* transaction start */ + if (env->me_poison_edge < txn->mt_next_pgno) + env->me_poison_edge = txn->mt_next_pgno; + VALGRIND_MAKE_MEM_DEFINED(env->me_map, pgno2bytes(env, txn->mt_next_pgno)); + ASAN_UNPOISON_MEMORY_REGION(env->me_map, + pgno2bytes(env, txn->mt_next_pgno)); + /* don't touch more, it should be already poisoned */ + } else { /* transaction end */ + bool should_unlock = false; + pgno_t last = MAX_PAGENO; + if (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self()) { + /* inside write-txn */ + MDBX_meta *head = mdbx_meta_head(env); + last = head->mm_geo.next; + } else if (mdbx_txn_lock(env, true) == MDBX_SUCCESS) { + /* no write-txn */ + last = NUM_METAS; + should_unlock = true; + } else { + /* write txn is running, therefore shouldn't poison any memory range */ + return; + } + + last = mdbx_find_largest_this(env, last); + const pgno_t edge = env->me_poison_edge; + if (edge > last) { + mdbx_assert(env, last >= NUM_METAS); + env->me_poison_edge = last; + VALGRIND_MAKE_MEM_NOACCESS(env->me_map + pgno2bytes(env, last), + pgno2bytes(env, edge - last)); + ASAN_POISON_MEMORY_REGION(env->me_map + pgno2bytes(env, last), + pgno2bytes(env, edge - last)); + } + if (should_unlock) + mdbx_txn_unlock(env); + } +} +#endif /* MDBX_USE_VALGRIND */ + +/* Common code for mdbx_txn_begin() and mdbx_txn_renew(). */ +static int mdbx_txn_renew0(MDBX_txn *txn, unsigned flags) { + MDBX_env *env = txn->mt_env; + int rc; + +#if MDBX_TXN_CHECKPID + if (unlikely(env->me_pid != mdbx_getpid())) { + env->me_flags |= MDBX_FATAL_ERROR; + return MDBX_PANIC; + } +#endif /* MDBX_TXN_CHECKPID */ + + STATIC_ASSERT(sizeof(MDBX_reader) == 32); +#ifdef MDBX_OSAL_LOCK + STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_wmutex) % MDBX_CACHELINE_SIZE == 0); + STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_rmutex) % MDBX_CACHELINE_SIZE == 0); +#else + STATIC_ASSERT( + offsetof(MDBX_lockinfo, mti_oldest_reader) % MDBX_CACHELINE_SIZE == 0); + STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_numreaders) % MDBX_CACHELINE_SIZE == + 0); +#endif + STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE == + 0); + + if (flags & MDBX_RDONLY) { + txn->mt_flags = MDBX_RDONLY | (env->me_flags & MDBX_NOTLS); + MDBX_reader *r = txn->to.reader; + STATIC_ASSERT(sizeof(size_t) == sizeof(r->mr_tid)); + if (likely(env->me_flags & MDBX_ENV_TXKEY)) { + mdbx_assert(env, !(env->me_flags & MDBX_NOTLS)); + r = thread_rthc_get(env->me_txkey); + if (likely(r)) { + mdbx_assert(env, r->mr_pid == env->me_pid); + mdbx_assert(env, r->mr_tid == mdbx_thread_self()); + } + } else { + mdbx_assert(env, !env->me_lck || (env->me_flags & MDBX_NOTLS)); + } + + if (likely(r)) { + if (unlikely(r->mr_pid != env->me_pid || + r->mr_txnid.inconsistent < SAFE64_INVALID_THRESHOLD)) + return MDBX_BAD_RSLOT; + } else if (env->me_lck) { + unsigned slot, nreaders; + const size_t tid = mdbx_thread_self(); + mdbx_assert(env, env->me_lck->mti_magic_and_version == MDBX_LOCK_MAGIC); + mdbx_assert(env, env->me_lck->mti_os_and_format == MDBX_LOCK_FORMAT); + + rc = mdbx_rdt_lock(env); + if (unlikely(MDBX_IS_ERROR(rc))) + return rc; + if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { + mdbx_rdt_unlock(env); + return MDBX_PANIC; + } +#if defined(_WIN32) || defined(_WIN64) + if (unlikely(!env->me_map)) { + mdbx_rdt_unlock(env); + return MDBX_EPERM; + } +#endif /* Windows */ + rc = MDBX_SUCCESS; + + if (unlikely(env->me_live_reader != env->me_pid)) { + rc = mdbx_rpid_set(env); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_rdt_unlock(env); + return rc; + } + env->me_live_reader = env->me_pid; + } + + while (1) { + nreaders = env->me_lck->mti_numreaders; + for (slot = 0; slot < nreaders; slot++) + if (env->me_lck->mti_readers[slot].mr_pid == 0) + break; + + if (likely(slot < env->me_maxreaders)) + break; + + rc = mdbx_reader_check0(env, true, NULL); + if (rc != MDBX_RESULT_TRUE) { + mdbx_rdt_unlock(env); + return (rc == MDBX_SUCCESS) ? MDBX_READERS_FULL : rc; + } + } + + r = &env->me_lck->mti_readers[slot]; + /* Claim the reader slot, carefully since other code + * uses the reader table un-mutexed: First reset the + * slot, next publish it in lck->mti_numreaders. After + * that, it is safe for mdbx_env_close() to touch it. + * When it will be closed, we can finally claim it. */ + r->mr_pid = 0; + safe64_reset(&r->mr_txnid, true); + if (slot == nreaders) + env->me_lck->mti_numreaders = ++nreaders; + r->mr_tid = tid; + r->mr_pid = env->me_pid; + mdbx_rdt_unlock(env); + + if (likely(env->me_flags & MDBX_ENV_TXKEY)) { + mdbx_assert(env, env->me_live_reader == env->me_pid); + thread_rthc_set(env->me_txkey, r); + } + } + + while (1) { + MDBX_meta *const meta = mdbx_meta_head(env); + mdbx_jitter4testing(false); + const txnid_t snap = mdbx_meta_txnid_fluid(env, meta); + mdbx_jitter4testing(false); + if (likely(r)) { + safe64_reset(&r->mr_txnid, false); + r->mr_snapshot_pages_used = meta->mm_geo.next; + r->mr_snapshot_pages_retired = meta->mm_pages_retired; + safe64_write(&r->mr_txnid, snap); + mdbx_jitter4testing(false); + mdbx_assert(env, r->mr_pid == mdbx_getpid()); + mdbx_assert(env, r->mr_tid == mdbx_thread_self()); + mdbx_assert(env, r->mr_txnid.inconsistent == snap); + mdbx_compiler_barrier(); + env->me_lck->mti_readers_refresh_flag = true; + mdbx_flush_noncoherent_cpu_writeback(); + } + mdbx_jitter4testing(true); + + /* Snap the state from current meta-head */ + txn->mt_txnid = snap; + txn->mt_geo = meta->mm_geo; + memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db)); + txn->mt_canary = meta->mm_canary; + + /* LY: Retry on a race, ITS#7970. */ + mdbx_compiler_barrier(); + if (likely(meta == mdbx_meta_head(env) && + snap == mdbx_meta_txnid_fluid(env, meta) && + snap >= *env->me_oldest)) { + mdbx_jitter4testing(false); + break; + } + } + + if (unlikely(txn->mt_txnid == 0 || + txn->mt_txnid >= SAFE64_INVALID_THRESHOLD)) { + mdbx_error("%s", "environment corrupted by died writer, must shutdown!"); + rc = MDBX_WANNA_RECOVERY; + goto bailout; + } + mdbx_assert(env, txn->mt_txnid >= *env->me_oldest); + txn->to.reader = r; + txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ + mdbx_ensure(env, txn->mt_txnid >= + /* paranoia is appropriate here */ *env->me_oldest); + } else { + /* Not yet touching txn == env->me_txn0, it may be active */ + mdbx_jitter4testing(false); + rc = mdbx_txn_lock(env, F_ISSET(flags, MDBX_TRYTXN)); + if (unlikely(rc)) + return rc; + if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { + mdbx_txn_unlock(env); + return MDBX_PANIC; + } +#if defined(_WIN32) || defined(_WIN64) + if (unlikely(!env->me_map)) { + mdbx_txn_unlock(env); + return MDBX_EPERM; + } +#endif /* Windows */ + + mdbx_jitter4testing(false); + MDBX_meta *meta = mdbx_meta_head(env); + mdbx_jitter4testing(false); + txn->mt_canary = meta->mm_canary; + const txnid_t snap = mdbx_meta_txnid_stable(env, meta); + txn->mt_txnid = safe64_txnid_next(snap); + if (unlikely(txn->mt_txnid >= SAFE64_INVALID_THRESHOLD)) { + mdbx_debug("%s", "txnid overflow!"); + rc = MDBX_TXN_FULL; + goto bailout; + } + + txn->mt_flags = flags; + txn->mt_child = NULL; + txn->tw.loose_pages = NULL; + txn->tw.loose_count = 0; + txn->tw.dirtyroom = MDBX_DPL_TXNFULL; + txn->tw.dirtylist = env->me_dirtylist; + mdbx_dpl_clear(txn->tw.dirtylist); + MDBX_PNL_SIZE(txn->tw.retired_pages) = 0; + txn->tw.spill_pages = NULL; + txn->tw.last_reclaimed = 0; + if (txn->tw.lifo_reclaimed) + MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) = 0; + env->me_txn = txn; + memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned)); + /* Copy the DB info and flags */ + memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDBX_db)); + /* Moved to here to avoid a data race in read TXNs */ + txn->mt_geo = meta->mm_geo; + txn->tw.loose_refund_wl = txn->mt_next_pgno; + } + + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; + mdbx_compiler_barrier(); + for (unsigned i = CORE_DBS; i < txn->mt_numdbs; i++) { + unsigned x = env->me_dbflags[i]; + txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = + (x & MDBX_VALID) ? DB_VALID | DB_USRVALID | DB_STALE : 0; + } + txn->mt_dbflags[MAIN_DBI] = DB_VALID | DB_USRVALID; + txn->mt_dbflags[FREE_DBI] = DB_VALID; + + if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { + mdbx_warning("%s", "environment had fatal error, must shutdown!"); + rc = MDBX_PANIC; + } else { + const size_t size = pgno2bytes(env, txn->mt_end_pgno); + if (unlikely(size > env->me_dxb_mmap.limit)) { + if (txn->mt_geo.upper > MAX_PAGENO || + bytes2pgno(env, pgno2bytes(env, txn->mt_geo.upper)) != + txn->mt_geo.upper) { + rc = MDBX_MAP_RESIZED; + goto bailout; + } + rc = mdbx_mapresize(env, txn->mt_end_pgno, txn->mt_geo.upper); + if (rc != MDBX_SUCCESS) { + if (rc == MDBX_RESULT_TRUE) + rc = MDBX_MAP_RESIZED; + goto bailout; + } + } + if (txn->mt_flags & MDBX_RDONLY) { +#if defined(_WIN32) || defined(_WIN64) + if (size > env->me_dbgeo.lower && env->me_dbgeo.shrink) { + txn->mt_flags |= MDBX_SHRINK_ALLOWED; + mdbx_srwlock_AcquireShared(&env->me_remap_guard); + } +#endif + } else { + env->me_dxb_mmap.current = size; + } +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + mdbx_txn_valgrind(env, txn); +#endif + txn->mt_owner = mdbx_thread_self(); + return MDBX_SUCCESS; + } +bailout: + mdbx_tassert(txn, rc != MDBX_SUCCESS); + mdbx_txn_end(txn, MDBX_END_SLOT | MDBX_END_FAIL_BEGIN); + return rc; +} + +static __always_inline int check_txn(const MDBX_txn *txn, int bad_bits) { + if (unlikely(!txn)) + return MDBX_EINVAL; + + if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely(txn->mt_flags & bad_bits)) + return MDBX_BAD_TXN; + +#if MDBX_TXN_CHECKOWNER + if ((txn->mt_flags & MDBX_NOTLS) == 0 && + unlikely(txn->mt_owner != mdbx_thread_self())) + return txn->mt_owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN; +#endif /* MDBX_TXN_CHECKOWNER */ + + return MDBX_SUCCESS; +} + +static __always_inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) { + if (unlikely(!txn)) + return MDBX_EINVAL; + + if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely(txn->mt_flags & bad_bits)) + return MDBX_BAD_TXN; + + if (unlikely(F_ISSET(txn->mt_flags, MDBX_RDONLY))) + return MDBX_EACCESS; + +#if MDBX_TXN_CHECKOWNER + if (unlikely(txn->mt_owner != mdbx_thread_self())) + return txn->mt_owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN; +#endif /* MDBX_TXN_CHECKOWNER */ + + return MDBX_SUCCESS; +} + +int mdbx_txn_renew(MDBX_txn *txn) { + int rc; + + if (unlikely(!txn)) + return MDBX_EINVAL; + + if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely((txn->mt_flags & MDBX_RDONLY) == 0)) + return MDBX_EINVAL; + + if (unlikely(txn->mt_owner != 0)) + return MDBX_THREAD_MISMATCH; + + rc = mdbx_txn_renew0(txn, MDBX_RDONLY); + if (rc == MDBX_SUCCESS) { + txn->mt_owner = mdbx_thread_self(); + mdbx_debug("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO + "/%" PRIaPGNO, + txn->mt_txnid, (txn->mt_flags & MDBX_RDONLY) ? 'r' : 'w', + (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root, + txn->mt_dbs[FREE_DBI].md_root); + } + return rc; +} + +int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, unsigned flags, + MDBX_txn **ret) { + MDBX_txn *txn; + int rc; + unsigned size, tsize; + + if (unlikely(!ret)) + return MDBX_EINVAL; + *ret = NULL; + + if (unlikely(!env)) + return MDBX_EINVAL; + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + +#if MDBX_TXN_CHECKPID + if (unlikely(env->me_pid != mdbx_getpid())) + env->me_flags |= MDBX_FATAL_ERROR; +#endif /* MDBX_TXN_CHECKPID */ + + if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) + return MDBX_PANIC; + +#if !defined(_WIN32) && !defined(_WIN64) + /* Don't check env->me_map until lock to + * avoid race with re-mapping for shrinking */ + if (unlikely(!env->me_map)) + return MDBX_EPERM; +#endif /* Windows */ + + if (unlikely(flags & ~MDBX_TXN_BEGIN_FLAGS)) + return MDBX_EINVAL; + + if (unlikely(env->me_flags & MDBX_RDONLY & + ~flags)) /* write txn in RDONLY env */ + return MDBX_EACCESS; + + flags |= env->me_flags & MDBX_WRITEMAP; + + if (parent) { + /* Nested transactions: Max 1 child, write txns only, no writemap */ + rc = check_txn_rw(parent, MDBX_RDONLY | MDBX_WRITEMAP | MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + +#if defined(_WIN32) || defined(_WIN64) + if (unlikely(!env->me_map)) + return MDBX_EPERM; +#endif /* Windows */ + + flags |= parent->mt_flags & + (MDBX_TXN_BEGIN_FLAGS | MDBX_SHRINK_ALLOWED | MDBX_TXN_SPILLS); + /* Child txns save MDBX_pgstate and use own copy of cursors */ + size = env->me_maxdbs * (sizeof(MDBX_db) + sizeof(MDBX_cursor *) + 1); + size += tsize = sizeof(MDBX_txn); + } else if (flags & MDBX_RDONLY) { + if (env->me_txn0 && unlikely(env->me_txn0->mt_owner == mdbx_thread_self())) + return MDBX_BUSY; + size = env->me_maxdbs * (sizeof(MDBX_db) + 1); + size += tsize = sizeof(MDBX_txn); + } else { + /* Reuse preallocated write txn. However, do not touch it until + * mdbx_txn_renew0() succeeds, since it currently may be active. */ + txn = env->me_txn0; + if (unlikely(txn->mt_owner == mdbx_thread_self())) + return MDBX_BUSY; + goto renew; + } + if (unlikely((txn = mdbx_malloc(size)) == NULL)) { + mdbx_debug("calloc: %s", "failed"); + return MDBX_ENOMEM; + } + memset(txn, 0, tsize); + txn->mt_dbxs = env->me_dbxs; /* static */ + txn->mt_dbs = (MDBX_db *)((char *)txn + tsize); + txn->mt_dbflags = (uint8_t *)txn + size - env->me_maxdbs; + txn->mt_flags = flags; + txn->mt_env = env; + + if (parent) { + mdbx_tassert(txn, mdbx_dirtylist_check(parent)); + txn->mt_cursors = (MDBX_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = parent->mt_dbiseqs; + txn->tw.dirtylist = mdbx_malloc(sizeof(MDBX_DP) * (MDBX_DPL_TXNFULL + 1)); + txn->tw.reclaimed_pglist = + mdbx_pnl_alloc(MDBX_PNL_ALLOCLEN(parent->tw.reclaimed_pglist)); + if (!txn->tw.dirtylist || !txn->tw.reclaimed_pglist) { + mdbx_pnl_free(txn->tw.reclaimed_pglist); + mdbx_free(txn->tw.dirtylist); + mdbx_free(txn); + return MDBX_ENOMEM; + } + mdbx_dpl_clear(txn->tw.dirtylist); + memcpy(txn->tw.reclaimed_pglist, parent->tw.reclaimed_pglist, + MDBX_PNL_SIZEOF(parent->tw.reclaimed_pglist)); + mdbx_assert(env, mdbx_pnl_check4assert( + txn->tw.reclaimed_pglist, + (txn->mt_next_pgno /* LY: intentional assigment here, + only for assertion */ + = parent->mt_next_pgno))); + + txn->tw.last_reclaimed = parent->tw.last_reclaimed; + if (parent->tw.lifo_reclaimed) { + txn->tw.lifo_reclaimed = parent->tw.lifo_reclaimed; + parent->tw.lifo_reclaimed = + (void *)(intptr_t)MDBX_PNL_SIZE(parent->tw.lifo_reclaimed); + } + + txn->tw.retired_pages = parent->tw.retired_pages; + parent->tw.retired_pages = + (void *)(intptr_t)MDBX_PNL_SIZE(parent->tw.retired_pages); + + txn->mt_txnid = parent->mt_txnid; + txn->tw.dirtyroom = parent->tw.dirtyroom; + txn->mt_geo = parent->mt_geo; + txn->tw.loose_refund_wl = parent->tw.loose_refund_wl; + txn->mt_canary = parent->mt_canary; + parent->mt_flags |= MDBX_TXN_HAS_CHILD; + parent->mt_child = txn; + txn->mt_parent = parent; + txn->mt_numdbs = parent->mt_numdbs; + txn->mt_owner = parent->mt_owner; + memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDBX_db)); + /* Copy parent's mt_dbflags, but clear DB_NEW */ + for (unsigned i = 0; i < txn->mt_numdbs; i++) + txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~(DB_FRESH | DB_CREAT); + mdbx_tassert(parent, + parent->mt_parent || + parent->tw.dirtyroom + parent->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + env->me_txn = txn; + rc = mdbx_cursor_shadow(parent, txn); + if (unlikely(rc != MDBX_SUCCESS)) + mdbx_txn_end(txn, MDBX_END_FAIL_BEGINCHILD); + } else { /* MDBX_RDONLY */ + txn->mt_dbiseqs = env->me_dbiseqs; + renew: + rc = mdbx_txn_renew0(txn, flags); + } + + if (unlikely(rc != MDBX_SUCCESS)) { + if (txn != env->me_txn0) + mdbx_free(txn); + } else { + mdbx_assert(env, (txn->mt_flags & + ~(MDBX_RDONLY | MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED | + MDBX_NOMETASYNC | MDBX_NOSYNC | MDBX_MAPASYNC)) == 0); + txn->mt_signature = MDBX_MT_SIGNATURE; + *ret = txn; + mdbx_debug("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO + "/%" PRIaPGNO, + txn->mt_txnid, (flags & MDBX_RDONLY) ? 'r' : 'w', (void *)txn, + (void *)env, txn->mt_dbs[MAIN_DBI].md_root, + txn->mt_dbs[FREE_DBI].md_root); + } + + return rc; +} + +int mdbx_txn_info(MDBX_txn *txn, MDBX_txn_info *info, int scan_rlt) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!info)) + return MDBX_EINVAL; + + MDBX_env *const env = txn->mt_env; +#if MDBX_TXN_CHECKPID + if (unlikely(env->me_pid != mdbx_getpid())) { + env->me_flags |= MDBX_FATAL_ERROR; + return MDBX_PANIC; + } +#endif /* MDBX_TXN_CHECKPID */ + + info->txn_id = txn->mt_txnid; + info->txn_space_used = pgno2bytes(env, txn->mt_geo.next); + + if (txn->mt_flags & MDBX_RDONLY) { + const MDBX_meta *head_meta; + txnid_t head_txnid; + uint64_t head_retired; + do { + /* fetch info from volatile head */ + head_meta = mdbx_meta_head(env); + head_txnid = mdbx_meta_txnid_fluid(env, head_meta); + head_retired = head_meta->mm_pages_retired; + info->txn_space_limit_soft = pgno2bytes(env, head_meta->mm_geo.now); + info->txn_space_limit_hard = pgno2bytes(env, head_meta->mm_geo.upper); + info->txn_space_leftover = + pgno2bytes(env, head_meta->mm_geo.now - head_meta->mm_geo.next); + mdbx_compiler_barrier(); + } while (unlikely(head_meta != mdbx_meta_head(env) || + head_txnid != mdbx_meta_txnid_fluid(env, head_meta))); + + info->txn_reader_lag = head_txnid - info->txn_id; + info->txn_space_dirty = info->txn_space_retired = 0; + if (txn->to.reader && + head_retired > txn->to.reader->mr_snapshot_pages_retired) { + info->txn_space_dirty = info->txn_space_retired = + pgno2bytes(env, (pgno_t)(head_retired - + txn->to.reader->mr_snapshot_pages_retired)); + + size_t retired_next_reader = 0; + MDBX_lockinfo *const lck = env->me_lck; + if (scan_rlt && info->txn_reader_lag > 1 && lck) { + /* find next more recent reader */ + txnid_t next_reader = head_txnid; + const unsigned snap_nreaders = lck->mti_numreaders; + for (unsigned i = 0; i < snap_nreaders; ++i) { + retry: + if (lck->mti_readers[i].mr_pid) { + mdbx_jitter4testing(true); + const txnid_t snap_txnid = + safe64_read(&lck->mti_readers[i].mr_txnid); + const uint64_t snap_retired = + lck->mti_readers[i].mr_snapshot_pages_retired; + mdbx_compiler_barrier(); + if (unlikely(snap_retired != + lck->mti_readers[i].mr_snapshot_pages_retired) || + snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid)) + goto retry; + if (snap_txnid <= txn->mt_txnid) { + retired_next_reader = 0; + break; + } + if (snap_txnid < next_reader) { + next_reader = snap_txnid; + retired_next_reader = pgno2bytes( + env, (pgno_t)(snap_retired - + txn->to.reader->mr_snapshot_pages_retired)); + } + } + } + } + info->txn_space_dirty = retired_next_reader; + } + } else { + info->txn_space_limit_soft = pgno2bytes(env, txn->mt_geo.now); + info->txn_space_limit_hard = pgno2bytes(env, txn->mt_geo.upper); + info->txn_space_retired = pgno2bytes( + env, txn->mt_child ? (unsigned)(uintptr_t)txn->tw.retired_pages + : MDBX_PNL_SIZE(txn->tw.retired_pages)); + info->txn_space_leftover = pgno2bytes(env, txn->tw.dirtyroom); + info->txn_space_dirty = + pgno2bytes(env, MDBX_DPL_TXNFULL - txn->tw.dirtyroom); + info->txn_reader_lag = INT64_MAX; + MDBX_lockinfo *const lck = env->me_lck; + if (scan_rlt && lck) { + txnid_t oldest_snapshot = txn->mt_txnid; + const unsigned snap_nreaders = lck->mti_numreaders; + if (snap_nreaders) { + oldest_snapshot = mdbx_find_oldest(txn); + if (oldest_snapshot == txn->mt_txnid - 1) { + /* check if there is at least one reader */ + bool exists = false; + for (unsigned i = 0; i < snap_nreaders; ++i) { + if (lck->mti_readers[i].mr_pid && + txn->mt_txnid > safe64_read(&lck->mti_readers[i].mr_txnid)) { + exists = true; + break; + } + } + oldest_snapshot += !exists; + } + } + info->txn_reader_lag = txn->mt_txnid - oldest_snapshot; + } + } + + return MDBX_SUCCESS; +} + +MDBX_env *mdbx_txn_env(MDBX_txn *txn) { + if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE || + txn->mt_env->me_signature != MDBX_ME_SIGNATURE)) + return NULL; + return txn->mt_env; +} + +uint64_t mdbx_txn_id(MDBX_txn *txn) { + if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE)) + return 0; + return txn->mt_txnid; +} + +int mdbx_txn_flags(MDBX_txn *txn) { + if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE)) + return -1; + return txn->mt_flags; +} + +/* Export or close DBI handles opened in this txn. */ +static void mdbx_dbis_update(MDBX_txn *txn, int keep) { + MDBX_dbi n = txn->mt_numdbs; + if (n) { + bool locked = false; + MDBX_env *env = txn->mt_env; + uint8_t *tdbflags = txn->mt_dbflags; + + for (unsigned i = n; --i >= CORE_DBS;) { + if (likely((tdbflags[i] & DB_CREAT) == 0)) + continue; + if (!locked) { + mdbx_ensure(env, + mdbx_fastmutex_acquire(&env->me_dbi_lock) == MDBX_SUCCESS); + locked = true; + } + if (keep) { + env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDBX_VALID; + mdbx_compiler_barrier(); + if (env->me_numdbs <= i) + env->me_numdbs = i + 1; + } else { + char *ptr = env->me_dbxs[i].md_name.iov_base; + if (ptr) { + env->me_dbxs[i].md_name.iov_len = 0; + mdbx_compiler_barrier(); + mdbx_assert(env, env->me_dbflags[i] == 0); + env->me_dbiseqs[i]++; + env->me_dbxs[i].md_name.iov_base = NULL; + mdbx_free(ptr); + } + } + } + + if (unlikely(locked)) + mdbx_ensure(env, + mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); + } +} + +/* End a transaction, except successful commit of a nested transaction. + * May be called twice for readonly txns: First reset it, then abort. + * [in] txn the transaction handle to end + * [in] mode why and how to end the transaction */ +static int mdbx_txn_end(MDBX_txn *txn, unsigned mode) { + MDBX_env *env = txn->mt_env; + static const char *const names[] = MDBX_END_NAMES; + +#if MDBX_TXN_CHECKPID + if (unlikely(txn->mt_env->me_pid != mdbx_getpid())) { + env->me_flags |= MDBX_FATAL_ERROR; + return MDBX_PANIC; + } +#endif /* MDBX_TXN_CHECKPID */ + + mdbx_debug("%s txn %" PRIaTXN "%c %p on mdbenv %p, root page %" PRIaPGNO + "/%" PRIaPGNO, + names[mode & MDBX_END_OPMASK], txn->mt_txnid, + (txn->mt_flags & MDBX_RDONLY) ? 'r' : 'w', (void *)txn, + (void *)env, txn->mt_dbs[MAIN_DBI].md_root, + txn->mt_dbs[FREE_DBI].md_root); + + mdbx_ensure(env, txn->mt_txnid >= + /* paranoia is appropriate here */ *env->me_oldest); + + int rc = MDBX_SUCCESS; + if (F_ISSET(txn->mt_flags, MDBX_RDONLY)) { + if (txn->to.reader) { + MDBX_reader *slot = txn->to.reader; + mdbx_assert(env, slot->mr_pid == env->me_pid); + if (likely(!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED))) { + mdbx_assert(env, txn->mt_txnid == slot->mr_txnid.inconsistent && + slot->mr_txnid.inconsistent >= + env->me_lck->mti_oldest_reader); +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + mdbx_txn_valgrind(env, nullptr); +#endif + slot->mr_snapshot_pages_used = 0; + safe64_reset(&slot->mr_txnid, false); + env->me_lck->mti_readers_refresh_flag = true; + mdbx_flush_noncoherent_cpu_writeback(); + } else { + mdbx_assert(env, slot->mr_pid == env->me_pid); + mdbx_assert(env, + slot->mr_txnid.inconsistent >= SAFE64_INVALID_THRESHOLD); + } + if (mode & MDBX_END_SLOT) { + if ((env->me_flags & MDBX_ENV_TXKEY) == 0) + slot->mr_pid = 0; + txn->to.reader = NULL; + } + } +#if defined(_WIN32) || defined(_WIN64) + if (txn->mt_flags & MDBX_SHRINK_ALLOWED) + mdbx_srwlock_ReleaseShared(&env->me_remap_guard); +#endif + txn->mt_numdbs = 0; /* prevent further DBI activity */ + txn->mt_flags = MDBX_RDONLY | MDBX_TXN_FINISHED; + txn->mt_owner = 0; + } else if (!F_ISSET(txn->mt_flags, MDBX_TXN_FINISHED)) { +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + if (txn == env->me_txn0) + mdbx_txn_valgrind(env, nullptr); +#endif + /* Export or close DBI handles created in this txn */ + mdbx_dbis_update(txn, mode & MDBX_END_UPDATE); + if (!(mode & MDBX_END_EOTDONE)) /* !(already closed cursors) */ + mdbx_cursors_eot(txn, 0); + if (!(env->me_flags & MDBX_WRITEMAP)) + mdbx_dlist_free(txn); + + txn->mt_flags = MDBX_TXN_FINISHED; + txn->mt_owner = 0; + env->me_txn = txn->mt_parent; + if (txn == env->me_txn0) { + mdbx_assert(env, txn->mt_parent == NULL); + mdbx_pnl_shrink(&txn->tw.retired_pages); + mdbx_pnl_shrink(&txn->tw.reclaimed_pglist); + /* The writer mutex was locked in mdbx_txn_begin. */ + mdbx_txn_unlock(env); + } else { + mdbx_assert(env, txn->mt_parent != NULL); + mdbx_assert(env, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + MDBX_txn *const parent = txn->mt_parent; + env->me_txn->mt_child = NULL; + env->me_txn->mt_flags &= ~MDBX_TXN_HAS_CHILD; + mdbx_pnl_free(txn->tw.reclaimed_pglist); + mdbx_pnl_free(txn->tw.spill_pages); + + if (txn->tw.lifo_reclaimed) { + mdbx_assert(env, MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) >= + (unsigned)(uintptr_t)parent->tw.lifo_reclaimed); + MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) = + (unsigned)(uintptr_t)parent->tw.lifo_reclaimed; + parent->tw.lifo_reclaimed = txn->tw.lifo_reclaimed; + } + + if (txn->tw.retired_pages) { + mdbx_assert(env, MDBX_PNL_SIZE(txn->tw.retired_pages) >= + (unsigned)(uintptr_t)parent->tw.retired_pages); + MDBX_PNL_SIZE(txn->tw.retired_pages) = + (unsigned)(uintptr_t)parent->tw.retired_pages; + parent->tw.retired_pages = txn->tw.retired_pages; + } + + mdbx_free(txn->tw.dirtylist); + + if (parent->mt_geo.upper != txn->mt_geo.upper || + parent->mt_geo.now != txn->mt_geo.now) { + /* undo resize performed by child txn */ + rc = mdbx_mapresize(env, parent->mt_geo.now, parent->mt_geo.upper); + if (rc == MDBX_RESULT_TRUE) { + /* unable undo resize (it is regular for Windows), + * therefore promote size changes from child to the parent txn */ + mdbx_notice("unable undo resize performed by child txn, promote to " + "the parent (%u->%u, %u->%u)", + txn->mt_geo.now, parent->mt_geo.now, txn->mt_geo.upper, + parent->mt_geo.upper); + parent->mt_geo.now = txn->mt_geo.now; + parent->mt_geo.upper = txn->mt_geo.upper; + rc = MDBX_SUCCESS; + } else if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_error("error %d while undo resize performed by child txn, fail " + "the parent", + rc); + parent->mt_flags |= MDBX_TXN_ERROR; + if (!env->me_dxb_mmap.address) + env->me_flags |= MDBX_FATAL_ERROR; + } + } + } + } + + mdbx_assert(env, txn == env->me_txn0 || txn->mt_owner == 0); + if ((mode & MDBX_END_FREE) != 0 && txn != env->me_txn0) { + txn->mt_signature = 0; + mdbx_free(txn); + } + + return rc; +} + +int mdbx_txn_reset(MDBX_txn *txn) { + int rc = check_txn(txn, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + /* This call is only valid for read-only txns */ + if (unlikely((txn->mt_flags & MDBX_RDONLY) == 0)) + return MDBX_EINVAL; + + /* LY: don't close DBI-handles */ + rc = mdbx_txn_end(txn, MDBX_END_RESET | MDBX_END_UPDATE); + if (rc == MDBX_SUCCESS) { + mdbx_tassert(txn, txn->mt_signature == MDBX_MT_SIGNATURE); + mdbx_tassert(txn, txn->mt_owner == 0); + } + return rc; +} + +int mdbx_txn_abort(MDBX_txn *txn) { + int rc = check_txn(txn, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (F_ISSET(txn->mt_flags, MDBX_RDONLY)) + /* LY: don't close DBI-handles */ + return mdbx_txn_end(txn, MDBX_END_ABORT | MDBX_END_UPDATE | MDBX_END_SLOT | + MDBX_END_FREE); + + if (txn->mt_child) + mdbx_txn_abort(txn->mt_child); + + return mdbx_txn_end(txn, MDBX_END_ABORT | MDBX_END_SLOT | MDBX_END_FREE); +} + +static __inline int mdbx_backlog_size(MDBX_txn *txn) { + int reclaimed = MDBX_PNL_SIZE(txn->tw.reclaimed_pglist); + return reclaimed + txn->tw.loose_count; +} + +static __inline int mdbx_backlog_extragap(MDBX_env *env) { + /* LY: extra page(s) for b-tree rebalancing */ + return (env->me_flags & MDBX_LIFORECLAIM) ? 2 : 1; +} + +/* LY: Prepare a backlog of pages to modify GC itself, + * while reclaiming is prohibited. It should be enough to prevent search + * in mdbx_page_alloc() during a deleting, when GC tree is unbalanced. */ +static int mdbx_prep_backlog(MDBX_txn *txn, MDBX_cursor *mc) { + /* LY: extra page(s) for b-tree rebalancing */ + const int extra = + mdbx_backlog_extragap(txn->mt_env) + + MDBX_PNL_SIZEOF(txn->tw.retired_pages) / txn->mt_env->me_maxkey_limit; + + if (mdbx_backlog_size(txn) < mc->mc_db->md_depth + extra) { + mc->mc_flags &= ~C_RECLAIMING; + int rc = mdbx_cursor_touch(mc); + if (unlikely(rc)) + return rc; + + while (unlikely(mdbx_backlog_size(txn) < extra)) { + rc = mdbx_page_alloc(mc, 1, NULL, MDBX_ALLOC_GC); + if (unlikely(rc)) { + if (rc != MDBX_NOTFOUND) + return rc; + break; + } + } + mc->mc_flags |= C_RECLAIMING; + } + + return MDBX_SUCCESS; +} + +static void mdbx_prep_backlog_data(MDBX_txn *txn, MDBX_cursor *mc, + size_t bytes) { + const int wanna = (int)number_of_ovpages(txn->mt_env, bytes) + + mdbx_backlog_extragap(txn->mt_env); + if (unlikely(wanna > mdbx_backlog_size(txn))) { + mc->mc_flags &= ~C_RECLAIMING; + do { + if (mdbx_page_alloc(mc, 1, NULL, MDBX_ALLOC_GC) != MDBX_SUCCESS) + break; + } while (wanna > mdbx_backlog_size(txn)); + mc->mc_flags |= C_RECLAIMING; + } +} + +/* Count all the pages in each DB and in the freelist and make sure + * it matches the actual number of pages being used. + * All named DBs must be open for a correct count. */ +static __cold int mdbx_audit_ex(MDBX_txn *txn, unsigned retired_stored, + bool dont_filter_gc) { + pgno_t pending = 0; + if ((txn->mt_flags & MDBX_RDONLY) == 0) { + pending = txn->tw.loose_count + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) + + (MDBX_PNL_SIZE(txn->tw.retired_pages) - retired_stored) + + txn->tw.retired2parent_count; + for (MDBX_txn *parent = txn->mt_parent; parent; parent = parent->mt_parent) + pending += parent->tw.loose_count; + } + + MDBX_cursor_couple cx; + int rc = mdbx_cursor_init(&cx.outer, txn, FREE_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + pgno_t freecount = 0; + MDBX_val key, data; + while ((rc = mdbx_cursor_get(&cx.outer, &key, &data, MDBX_NEXT)) == 0) { + if (!dont_filter_gc) { + if (unlikely(key.iov_len != sizeof(txnid_t))) + return MDBX_CORRUPTED; + txnid_t id = unaligned_peek_u64(4, key.iov_base); + if (txn->tw.lifo_reclaimed) { + for (unsigned i = 1; i <= MDBX_PNL_SIZE(txn->tw.lifo_reclaimed); ++i) + if (id == txn->tw.lifo_reclaimed[i]) + goto skip; + } else if (id <= txn->tw.last_reclaimed) + goto skip; + } + + freecount += *(pgno_t *)data.iov_base; + skip:; + } + mdbx_tassert(txn, rc == MDBX_NOTFOUND); + + for (MDBX_dbi i = FREE_DBI; i < txn->mt_numdbs; i++) + txn->mt_dbflags[i] &= ~DB_AUDITED; + + pgno_t count = 0; + for (MDBX_dbi i = FREE_DBI; i <= MAIN_DBI; i++) { + if (!(txn->mt_dbflags[i] & DB_VALID)) + continue; + rc = mdbx_cursor_init(&cx.outer, txn, i); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + txn->mt_dbflags[i] |= DB_AUDITED; + if (txn->mt_dbs[i].md_root == P_INVALID) + continue; + count += txn->mt_dbs[i].md_branch_pages + txn->mt_dbs[i].md_leaf_pages + + txn->mt_dbs[i].md_overflow_pages; + + if (i != MAIN_DBI) + continue; + rc = mdbx_page_search(&cx.outer, NULL, MDBX_PS_FIRST); + while (rc == MDBX_SUCCESS) { + MDBX_page *mp = cx.outer.mc_pg[cx.outer.mc_top]; + for (unsigned j = 0; j < page_numkeys(mp); j++) { + MDBX_node *node = page_node(mp, j); + if (node_flags(node) == F_SUBDATA) { + if (unlikely(node_ds(node) < sizeof(MDBX_db))) + return MDBX_CORRUPTED; + MDBX_db db_copy, *db; + memcpy(db = &db_copy, node_data(node), sizeof(db_copy)); + if ((txn->mt_flags & MDBX_RDONLY) == 0) { + for (MDBX_dbi k = txn->mt_numdbs; --k > MAIN_DBI;) { + if ((txn->mt_dbflags[k] & DB_VALID) && + /* txn->mt_dbxs[k].md_name.iov_len > 0 && */ + node_ks(node) == txn->mt_dbxs[k].md_name.iov_len && + memcmp(node_key(node), txn->mt_dbxs[k].md_name.iov_base, + node_ks(node)) == 0) { + txn->mt_dbflags[k] |= DB_AUDITED; + if (txn->mt_dbflags[k] & DB_DIRTY) { + mdbx_tassert(txn, (txn->mt_dbflags[k] & DB_STALE) == 0); + db = txn->mt_dbs + k; + } + break; + } + } + } + count += + db->md_branch_pages + db->md_leaf_pages + db->md_overflow_pages; + } + } + rc = mdbx_cursor_sibling(&cx.outer, 1); + } + mdbx_tassert(txn, rc == MDBX_NOTFOUND); + } + + for (MDBX_dbi i = FREE_DBI; i < txn->mt_numdbs; i++) { + if ((txn->mt_dbflags[i] & (DB_VALID | DB_AUDITED | DB_STALE)) != DB_VALID) + continue; + if (F_ISSET(txn->mt_dbflags[i], DB_DIRTY | DB_CREAT)) { + count += txn->mt_dbs[i].md_branch_pages + txn->mt_dbs[i].md_leaf_pages + + txn->mt_dbs[i].md_overflow_pages; + } else { + mdbx_warning("audit %s@%" PRIaTXN + ": unable account dbi %d / \"%*s\", state 0x%02x", + txn->mt_parent ? "nested-" : "", txn->mt_txnid, i, + (int)txn->mt_dbxs[i].md_name.iov_len, + (const char *)txn->mt_dbxs[i].md_name.iov_base, + txn->mt_dbflags[i]); + } + } + + if (pending + freecount + count + NUM_METAS == txn->mt_next_pgno) + return MDBX_SUCCESS; + + if ((txn->mt_flags & MDBX_RDONLY) == 0) + mdbx_error("audit @%" PRIaTXN ": %u(pending) = %u(loose-count) + " + "%u(reclaimed-list) + %u(retired-pending) - %u(retired-stored) " + "+ %u(retired2parent)", + txn->mt_txnid, pending, txn->tw.loose_count, + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist), + txn->tw.retired_pages ? MDBX_PNL_SIZE(txn->tw.retired_pages) : 0, + retired_stored, txn->tw.retired2parent_count); + mdbx_error("audit @%" PRIaTXN ": %" PRIaPGNO "(pending) + %" PRIaPGNO + "(free) + %" PRIaPGNO "(count) = %" PRIaPGNO + "(total) <> %" PRIaPGNO "(next-pgno)", + txn->mt_txnid, pending, freecount, count + NUM_METAS, + pending + freecount + count + NUM_METAS, txn->mt_next_pgno); + return MDBX_PROBLEM; +} + +static __inline void clean_reserved_gc_pnl(MDBX_env *env, MDBX_val pnl) { + /* PNL is initially empty, zero out at least the length */ + memset(pnl.iov_base, 0, sizeof(pgno_t)); + if ((env->me_flags & (MDBX_WRITEMAP | MDBX_NOMEMINIT)) == 0) + /* zero out to avoid leaking values from uninitialized malloc'ed memory + * to the file in non-writemap mode if length of the saving page-list + * was changed during space reservation. */ + memset(pnl.iov_base, 0, pnl.iov_len); +} + +/* Cleanup reclaimed GC records, than save the retired-list as of this + * transaction to GC (aka freeDB). This recursive changes the reclaimed-list + * loose-list and retired-list. Keep trying until it stabilizes. */ +static int mdbx_update_gc(MDBX_txn *txn) { + /* txn->tw.reclaimed_pglist[] can grow and shrink during this call. + * txn->tw.last_reclaimed and txn->tw.retired_pages[] can only grow. + * Page numbers cannot disappear from txn->tw.retired_pages[]. */ + MDBX_env *const env = txn->mt_env; + const bool lifo = (env->me_flags & MDBX_LIFORECLAIM) != 0; + const char *dbg_prefix_mode = lifo ? " lifo" : " fifo"; + (void)dbg_prefix_mode; + mdbx_trace("\n>>> @%" PRIaTXN, txn->mt_txnid); + + unsigned retired_stored = 0, loop = 0; + MDBX_cursor mc; + int rc = mdbx_cursor_init(&mc, txn, FREE_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout_notracking; + + mc.mc_flags |= C_RECLAIMING; + mc.mc_next = txn->mt_cursors[FREE_DBI]; + txn->mt_cursors[FREE_DBI] = &mc; + +retry: + mdbx_trace("%s", " >> restart"); + mdbx_tassert( + txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, txn->mt_next_pgno)); + mdbx_tassert(txn, mdbx_dirtylist_check(txn)); + mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + if (unlikely(/* paranoia */ ++loop > 42)) { + mdbx_error("too more loops %u, bailout", loop); + rc = MDBX_PROBLEM; + goto bailout; + } + + unsigned settled = 0, cleaned_gc_slot = 0, reused_gc_slot = 0, + filled_gc_slot = ~0u; + txnid_t cleaned_gc_id = 0, gc_rid = txn->tw.last_reclaimed; + while (true) { + /* Come back here after each Put() in case retired-list changed */ + MDBX_val key, data; + mdbx_trace("%s", " >> continue"); + + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + if (txn->tw.lifo_reclaimed) { + if (cleaned_gc_slot < MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)) { + settled = 0; + cleaned_gc_slot = 0; + reused_gc_slot = 0; + filled_gc_slot = ~0u; + /* LY: cleanup reclaimed records. */ + do { + cleaned_gc_id = txn->tw.lifo_reclaimed[++cleaned_gc_slot]; + mdbx_tassert(txn, + cleaned_gc_slot > 0 && cleaned_gc_id < *env->me_oldest); + key.iov_base = &cleaned_gc_id; + key.iov_len = sizeof(cleaned_gc_id); + rc = mdbx_cursor_get(&mc, &key, NULL, MDBX_SET); + if (rc == MDBX_NOTFOUND) + continue; + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + rc = mdbx_prep_backlog(txn, &mc); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + mdbx_tassert(txn, cleaned_gc_id < *env->me_oldest); + mdbx_trace("%s.cleanup-reclaimed-id [%u]%" PRIaTXN, dbg_prefix_mode, + cleaned_gc_slot, cleaned_gc_id); + rc = mdbx_cursor_del(&mc, 0); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } while (cleaned_gc_slot < MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)); + mdbx_txl_sort(txn->tw.lifo_reclaimed); + gc_rid = MDBX_PNL_LAST(txn->tw.lifo_reclaimed); + } + } else { + /* If using records from GC which we have not yet deleted, + * now delete them and any we reserved for me_reclaimed_pglist. */ + while (cleaned_gc_id <= txn->tw.last_reclaimed) { + gc_rid = cleaned_gc_id; + settled = 0; + rc = mdbx_cursor_first(&mc, &key, NULL); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_NOTFOUND) + break; + goto bailout; + } + if (unlikely(key.iov_len != sizeof(txnid_t))) { + rc = MDBX_CORRUPTED; + goto bailout; + } + cleaned_gc_id = unaligned_peek_u64(4, key.iov_base); + if (unlikely(cleaned_gc_id < 1 || + cleaned_gc_id >= SAFE64_INVALID_THRESHOLD)) { + rc = MDBX_CORRUPTED; + goto bailout; + } + if (cleaned_gc_id > txn->tw.last_reclaimed) + break; + if (cleaned_gc_id < txn->tw.last_reclaimed) { + rc = mdbx_prep_backlog(txn, &mc); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + mdbx_tassert(txn, cleaned_gc_id <= txn->tw.last_reclaimed); + mdbx_tassert(txn, cleaned_gc_id < *env->me_oldest); + mdbx_trace("%s.cleanup-reclaimed-id %" PRIaTXN, dbg_prefix_mode, + cleaned_gc_id); + rc = mdbx_cursor_del(&mc, 0); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + } + + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + mdbx_tassert(txn, mdbx_dirtylist_check(txn)); + mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + if (mdbx_audit_enabled()) { + rc = mdbx_audit_ex(txn, retired_stored, false); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + + /* return suitable into unallocated space */ + if (mdbx_refund(txn)) { + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + if (mdbx_audit_enabled()) { + rc = mdbx_audit_ex(txn, retired_stored, false); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + } + + /* handle loose pages - put ones into the reclaimed- or retired-list */ + if (txn->tw.loose_pages) { + /* Return loose page numbers to me_reclaimed_pglist, + * though usually none are left at this point. + * The pages themselves remain in dirtylist. */ + if (unlikely(!txn->tw.lifo_reclaimed && txn->tw.last_reclaimed < 1)) { + if (txn->tw.loose_count > 0) { + /* Put loose page numbers in tw.retired_pages, + * since unable to return them to me_reclaimed_pglist. */ + if (unlikely((rc = mdbx_pnl_need(&txn->tw.retired_pages, + txn->tw.loose_count)) != 0)) + goto bailout; + for (MDBX_page *mp = txn->tw.loose_pages; mp; mp = mp->mp_next) + mdbx_pnl_xappend(txn->tw.retired_pages, mp->mp_pgno); + mdbx_trace("%s: append %u loose-pages to retired-pages", + dbg_prefix_mode, txn->tw.loose_count); + } + } else { + /* Room for loose pages + temp PNL with same */ + rc = mdbx_pnl_need(&txn->tw.reclaimed_pglist, + 2 * txn->tw.loose_count + 2); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + MDBX_PNL loose = txn->tw.reclaimed_pglist + + MDBX_PNL_ALLOCLEN(txn->tw.reclaimed_pglist) - + txn->tw.loose_count - 1; + unsigned count = 0; + for (MDBX_page *mp = txn->tw.loose_pages; mp; mp = mp->mp_next) { + mdbx_tassert(txn, mp->mp_flags == (P_LOOSE | P_DIRTY)); + loose[++count] = mp->mp_pgno; + } + mdbx_tassert(txn, count == txn->tw.loose_count); + MDBX_PNL_SIZE(loose) = count; + mdbx_pnl_sort(loose); + mdbx_pnl_xmerge(txn->tw.reclaimed_pglist, loose); + mdbx_trace("%s: append %u loose-pages to reclaimed-pages", + dbg_prefix_mode, txn->tw.loose_count); + } + + /* filter-out list of dirty-pages from loose-pages */ + const MDBX_DPL dl = txn->tw.dirtylist; + unsigned w = 0; + for (unsigned r = w; ++r <= dl->length;) { + MDBX_page *dp = dl[r].ptr; + mdbx_tassert(txn, (dp->mp_flags & P_DIRTY)); + mdbx_tassert(txn, dl[r].pgno + (IS_OVERFLOW(dp) ? dp->mp_pages : 1) <= + txn->mt_next_pgno); + if ((dp->mp_flags & P_LOOSE) == 0) { + if (++w != r) + dl[w] = dl[r]; + } else { + mdbx_tassert(txn, dp->mp_flags == (P_LOOSE | P_DIRTY)); + if ((env->me_flags & MDBX_WRITEMAP) == 0) + mdbx_dpage_free(env, dp, 1); + } + } + mdbx_trace("%s: filtered-out loose-pages from %u -> %u dirty-pages", + dbg_prefix_mode, dl->length, w); + mdbx_tassert(txn, txn->tw.loose_count == dl->length - w); + dl->length = w; + dl->sorted = 0; + txn->tw.dirtyroom += txn->tw.loose_count; + txn->tw.loose_pages = NULL; + txn->tw.loose_count = 0; + } + + /* handle retired-list - store ones into single gc-record */ + if (retired_stored < MDBX_PNL_SIZE(txn->tw.retired_pages)) { + if (unlikely(!retired_stored)) { + /* Make sure last page of GC is touched and on retired-list */ + mc.mc_flags &= ~C_RECLAIMING; + rc = mdbx_page_search(&mc, NULL, MDBX_PS_LAST | MDBX_PS_MODIFY); + mc.mc_flags |= C_RECLAIMING; + if (unlikely(rc != MDBX_SUCCESS) && rc != MDBX_NOTFOUND) + goto bailout; + } + /* Write to last page of GC */ + key.iov_len = sizeof(txn->mt_txnid); + key.iov_base = &txn->mt_txnid; + do { + data.iov_len = MDBX_PNL_SIZEOF(txn->tw.retired_pages); + mdbx_prep_backlog_data(txn, &mc, data.iov_len); + rc = mdbx_cursor_put(&mc, &key, &data, MDBX_RESERVE); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + /* Retry if tw.retired_pages[] grew during the Put() */ + } while (data.iov_len < MDBX_PNL_SIZEOF(txn->tw.retired_pages)); + + retired_stored = (unsigned)MDBX_PNL_SIZE(txn->tw.retired_pages); + mdbx_pnl_sort(txn->tw.retired_pages); + mdbx_assert(env, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages)); + memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len); + + mdbx_trace("%s.put-retired #%u @ %" PRIaTXN, dbg_prefix_mode, + retired_stored, txn->mt_txnid); + + if (mdbx_log_enabled(MDBX_LOG_EXTRA)) { + unsigned i = retired_stored; + mdbx_debug_extra("PNL write txn %" PRIaTXN " root %" PRIaPGNO + " num %u, PNL", + txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i); + for (; i; i--) + mdbx_debug_extra_print(" %" PRIaPGNO, txn->tw.retired_pages[i]); + mdbx_debug_extra_print("%s", "\n"); + } + continue; + } + + /* handle reclaimed and loost pages - merge and store both into gc */ + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + mdbx_tassert(txn, txn->tw.loose_count == 0); + + mdbx_trace("%s", " >> reserving"); + if (mdbx_audit_enabled()) { + rc = mdbx_audit_ex(txn, retired_stored, false); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + const unsigned amount = (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist); + const unsigned left = amount - settled; + mdbx_trace("%s: amount %u, settled %d, left %d, lifo-reclaimed-slots %u, " + "reused-gc-slots %u", + dbg_prefix_mode, amount, settled, (int)left, + txn->tw.lifo_reclaimed + ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) + : 0, + reused_gc_slot); + if (0 >= (int)left) + break; + + const unsigned prefer_max_scatter = 257; + txnid_t reservation_gc_id; + if (lifo) { + const unsigned lifo_len = + txn->tw.lifo_reclaimed + ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) + : 0; + if (lifo_len < prefer_max_scatter && + left > (lifo_len - reused_gc_slot) * env->me_maxgc_ov1page) { + /* LY: need just a txn-id for save page list. */ + mc.mc_flags &= ~C_RECLAIMING; + rc = mdbx_page_alloc(&mc, 0, NULL, MDBX_ALLOC_GC | MDBX_ALLOC_KICK); + mc.mc_flags |= C_RECLAIMING; + if (likely(rc == MDBX_SUCCESS)) { + /* LY: ok, reclaimed from GC. */ + mdbx_trace("%s: took @%" PRIaTXN " from GC, continue", + dbg_prefix_mode, MDBX_PNL_LAST(txn->tw.lifo_reclaimed)); + continue; + } + if (unlikely(rc != MDBX_NOTFOUND)) + /* LY: other troubles... */ + goto bailout; + + /* LY: GC is empty, will look any free txn-id in high2low order. */ + if (unlikely(gc_rid == 0)) { + mdbx_tassert(txn, txn->tw.last_reclaimed == 0); + txn->tw.last_reclaimed = gc_rid = mdbx_find_oldest(txn) - 1; + if (txn->tw.lifo_reclaimed == nullptr) { + txn->tw.lifo_reclaimed = mdbx_txl_alloc(); + if (unlikely(!txn->tw.lifo_reclaimed)) { + rc = MDBX_ENOMEM; + goto bailout; + } + } + } + + mdbx_tassert(txn, txn->tw.lifo_reclaimed != nullptr); + rc = MDBX_RESULT_TRUE; + do { + if (unlikely(gc_rid <= 1)) { + if (unlikely(MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) <= + reused_gc_slot)) { + mdbx_notice("** restart: reserve depleted (reused_gc_slot %u >= " + "lifo_reclaimed %u" PRIaTXN, + reused_gc_slot, + (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)); + goto retry; + } + break; + } + + mdbx_tassert(txn, gc_rid > 1 && gc_rid < SAFE64_INVALID_THRESHOLD); + rc = mdbx_txl_append(&txn->tw.lifo_reclaimed, --gc_rid); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + cleaned_gc_slot += 1 /* mark GC cleanup is not needed. */; + + mdbx_trace("%s: append @%" PRIaTXN + " to lifo-reclaimed, cleaned-gc-slot = %u", + dbg_prefix_mode, gc_rid, cleaned_gc_slot); + } while (gc_rid && + MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) < prefer_max_scatter && + left > ((unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) - + reused_gc_slot) * + env->me_maxgc_ov1page); + if (reused_gc_slot && rc != MDBX_RESULT_TRUE) { + mdbx_trace("%s: restart innter-loop since reused_gc_slot %u > 0", + dbg_prefix_mode, reused_gc_slot); + continue; + } + } + + const unsigned i = + (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) - reused_gc_slot; + mdbx_tassert(txn, i > 0 && i <= MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)); + reservation_gc_id = txn->tw.lifo_reclaimed[i]; + mdbx_trace("%s: take @%" PRIaTXN " from lifo-reclaimed[%u]", + dbg_prefix_mode, reservation_gc_id, i); + } else { + mdbx_tassert(txn, txn->tw.lifo_reclaimed == NULL); + if (unlikely(gc_rid == 0)) { + gc_rid = mdbx_find_oldest(txn) - 1; + rc = mdbx_cursor_get(&mc, &key, NULL, MDBX_FIRST); + if (rc == MDBX_SUCCESS) { + if (unlikely(key.iov_len != sizeof(txnid_t))) { + rc = MDBX_CORRUPTED; + goto bailout; + } + txnid_t gc_first = unaligned_peek_u64(4, key.iov_base); + if (unlikely(gc_first < 1 || gc_first >= SAFE64_INVALID_THRESHOLD)) { + rc = MDBX_CORRUPTED; + goto bailout; + } + if (gc_rid >= gc_first) + gc_rid = gc_first - 1; + if (unlikely(gc_rid == 0)) { + mdbx_error("%s", "** no GC tail-space to store"); + rc = MDBX_PROBLEM; + goto bailout; + } + } else if (rc != MDBX_NOTFOUND) + goto bailout; + mdbx_tassert(txn, txn->tw.last_reclaimed == 0); + txn->tw.last_reclaimed = gc_rid; + } + reservation_gc_id = gc_rid--; + mdbx_trace("%s: take @%" PRIaTXN " from head-gc-id", dbg_prefix_mode, + reservation_gc_id); + } + ++reused_gc_slot; + + unsigned chunk = left; + if (unlikely(chunk > env->me_maxgc_ov1page)) { + const unsigned avail_gc_slots = + txn->tw.lifo_reclaimed + ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) - + reused_gc_slot + 1 + : (gc_rid < INT16_MAX) ? (unsigned)gc_rid : INT16_MAX; + if (avail_gc_slots > 1) { + if (chunk < env->me_maxgc_ov1page * 2) + chunk /= 2; + else { + const unsigned threshold = + env->me_maxgc_ov1page * ((avail_gc_slots < prefer_max_scatter) + ? avail_gc_slots + : prefer_max_scatter); + if (left < threshold) + chunk = env->me_maxgc_ov1page; + else { + const unsigned tail = left - threshold + env->me_maxgc_ov1page + 1; + unsigned span = 1; + unsigned avail = (unsigned)((pgno2bytes(env, span) - PAGEHDRSZ) / + sizeof(pgno_t)) /*- 1 + span */; + if (tail > avail) { + for (unsigned i = amount - span; i > 0; --i) { + if (MDBX_PNL_ASCENDING + ? (txn->tw.reclaimed_pglist[i] + span) + : (txn->tw.reclaimed_pglist[i] - span) == + txn->tw.reclaimed_pglist[i + span]) { + span += 1; + avail = (unsigned)((pgno2bytes(env, span) - PAGEHDRSZ) / + sizeof(pgno_t)) - + 1 + span; + if (avail >= tail) + break; + } + } + } + + chunk = (avail >= tail) ? tail - span + : (avail_gc_slots > 3 && + reused_gc_slot < prefer_max_scatter - 3) + ? avail - span + : tail; + } + } + } + } + mdbx_tassert(txn, chunk > 0); + + mdbx_trace("%s: rc_rid %" PRIaTXN ", reused_gc_slot %u, reservation-id " + "%" PRIaTXN, + dbg_prefix_mode, gc_rid, reused_gc_slot, reservation_gc_id); + + mdbx_trace("%s: chunk %u, gc-per-ovpage %u", dbg_prefix_mode, chunk, + env->me_maxgc_ov1page); + + mdbx_tassert(txn, reservation_gc_id < *env->me_oldest); + if (unlikely(reservation_gc_id < 1 || + reservation_gc_id >= *env->me_oldest)) { + mdbx_error("%s", "** internal error (reservation_gc_id)"); + rc = MDBX_PROBLEM; + goto bailout; + } + + key.iov_len = sizeof(reservation_gc_id); + key.iov_base = &reservation_gc_id; + data.iov_len = (chunk + 1) * sizeof(pgno_t); + mdbx_trace("%s.reserve: %u [%u...%u] @%" PRIaTXN, dbg_prefix_mode, chunk, + settled + 1, settled + chunk + 1, reservation_gc_id); + mdbx_prep_backlog_data(txn, &mc, data.iov_len); + rc = mdbx_cursor_put(&mc, &key, &data, MDBX_RESERVE | MDBX_NOOVERWRITE); + mdbx_tassert(txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, + txn->mt_next_pgno)); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + clean_reserved_gc_pnl(env, data); + settled += chunk; + mdbx_trace("%s.settled %u (+%u), continue", dbg_prefix_mode, settled, + chunk); + + if (txn->tw.lifo_reclaimed && + unlikely(amount < MDBX_PNL_SIZE(txn->tw.reclaimed_pglist))) { + mdbx_notice("** restart: reclaimed-list growth %u -> %u", amount, + (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist)); + goto retry; + } + + continue; + } + + mdbx_tassert( + txn, + cleaned_gc_slot == + (txn->tw.lifo_reclaimed ? MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) : 0)); + + mdbx_trace("%s", " >> filling"); + /* Fill in the reserved records */ + filled_gc_slot = + txn->tw.lifo_reclaimed + ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) - reused_gc_slot + : reused_gc_slot; + rc = MDBX_SUCCESS; + mdbx_tassert( + txn, mdbx_pnl_check4assert(txn->tw.reclaimed_pglist, txn->mt_next_pgno)); + mdbx_tassert(txn, mdbx_dirtylist_check(txn)); + if (MDBX_PNL_SIZE(txn->tw.reclaimed_pglist)) { + MDBX_val key, data; + key.iov_len = data.iov_len = 0; /* avoid MSVC warning */ + key.iov_base = data.iov_base = NULL; + + const unsigned amount = MDBX_PNL_SIZE(txn->tw.reclaimed_pglist); + unsigned left = amount; + if (txn->tw.lifo_reclaimed == nullptr) { + mdbx_tassert(txn, lifo == 0); + rc = mdbx_cursor_first(&mc, &key, &data); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } else { + mdbx_tassert(txn, lifo != 0); + } + + while (true) { + txnid_t fill_gc_id; + mdbx_trace("%s: left %u of %u", dbg_prefix_mode, left, + (unsigned)MDBX_PNL_SIZE(txn->tw.reclaimed_pglist)); + if (txn->tw.lifo_reclaimed == nullptr) { + mdbx_tassert(txn, lifo == 0); + fill_gc_id = unaligned_peek_u64(4, key.iov_base); + if (filled_gc_slot-- == 0 || fill_gc_id > txn->tw.last_reclaimed) { + mdbx_notice( + "** restart: reserve depleted (filled_slot %u, fill_id %" PRIaTXN + " > last_reclaimed %" PRIaTXN, + filled_gc_slot, fill_gc_id, txn->tw.last_reclaimed); + goto retry; + } + } else { + mdbx_tassert(txn, lifo != 0); + if (++filled_gc_slot > + (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)) { + mdbx_notice("** restart: reserve depleted (filled_gc_slot %u > " + "lifo_reclaimed %u" PRIaTXN, + filled_gc_slot, + (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)); + goto retry; + } + fill_gc_id = txn->tw.lifo_reclaimed[filled_gc_slot]; + mdbx_trace("%s.seek-reservation @%" PRIaTXN " at lifo_reclaimed[%u]", + dbg_prefix_mode, fill_gc_id, filled_gc_slot); + key.iov_base = &fill_gc_id; + key.iov_len = sizeof(fill_gc_id); + rc = mdbx_cursor_get(&mc, &key, &data, MDBX_SET_KEY); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + mdbx_tassert(txn, cleaned_gc_slot == + (txn->tw.lifo_reclaimed + ? MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) + : 0)); + mdbx_tassert(txn, fill_gc_id > 0 && fill_gc_id < *env->me_oldest); + key.iov_base = &fill_gc_id; + key.iov_len = sizeof(fill_gc_id); + + mdbx_tassert(txn, data.iov_len >= sizeof(pgno_t) * 2); + mc.mc_flags |= C_GCFREEZE; + unsigned chunk = (unsigned)(data.iov_len / sizeof(pgno_t)) - 1; + if (unlikely(chunk > left)) { + mdbx_trace("%s: chunk %u > left %u, @%" PRIaTXN, dbg_prefix_mode, chunk, + left, fill_gc_id); + if ((loop < 5 && chunk - left > loop / 2) || + chunk - left > env->me_maxgc_ov1page) { + data.iov_len = (left + 1) * sizeof(pgno_t); + if (loop < 7) + mc.mc_flags &= ~C_GCFREEZE; + } + chunk = left; + } + rc = mdbx_cursor_put(&mc, &key, &data, MDBX_CURRENT | MDBX_RESERVE); + mc.mc_flags &= ~C_GCFREEZE; + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + clean_reserved_gc_pnl(env, data); + + if (unlikely(txn->tw.loose_count || + amount != MDBX_PNL_SIZE(txn->tw.reclaimed_pglist))) { + mdbx_notice("** restart: reclaimed-list changed (%u -> %u, %u)", amount, + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist), + txn->tw.loose_count); + goto retry; + } + if (unlikely(txn->tw.lifo_reclaimed + ? cleaned_gc_slot < MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) + : cleaned_gc_id < txn->tw.last_reclaimed)) { + mdbx_notice("%s", "** restart: reclaimed-slots changed"); + goto retry; + } + + pgno_t *dst = data.iov_base; + *dst++ = chunk; + pgno_t *src = MDBX_PNL_BEGIN(txn->tw.reclaimed_pglist) + left - chunk; + memcpy(dst, src, chunk * sizeof(pgno_t)); + pgno_t *from = src, *to = src + chunk; + mdbx_trace("%s.fill: %u [ %u:%" PRIaPGNO "...%u:%" PRIaPGNO + "] @%" PRIaTXN, + dbg_prefix_mode, chunk, + (unsigned)(from - txn->tw.reclaimed_pglist), from[0], + (unsigned)(to - txn->tw.reclaimed_pglist), to[-1], fill_gc_id); + + left -= chunk; + if (mdbx_audit_enabled()) { + rc = mdbx_audit_ex(txn, retired_stored + amount - left, true); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + if (left == 0) { + rc = MDBX_SUCCESS; + break; + } + + if (txn->tw.lifo_reclaimed == nullptr) { + mdbx_tassert(txn, lifo == 0); + rc = mdbx_cursor_next(&mc, &key, &data, MDBX_NEXT); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } else { + mdbx_tassert(txn, lifo != 0); + } + } + } + + mdbx_tassert(txn, rc == MDBX_SUCCESS); + if (unlikely(txn->tw.loose_count != 0 || + filled_gc_slot != + (txn->tw.lifo_reclaimed + ? (unsigned)MDBX_PNL_SIZE(txn->tw.lifo_reclaimed) + : 0))) { + mdbx_notice("** restart: reserve excess (filled-slot %u, loose-count %u)", + filled_gc_slot, txn->tw.loose_count); + goto retry; + } + + mdbx_tassert(txn, + txn->tw.lifo_reclaimed == NULL || + cleaned_gc_slot == MDBX_PNL_SIZE(txn->tw.lifo_reclaimed)); + +bailout: + txn->mt_cursors[FREE_DBI] = mc.mc_next; + +bailout_notracking: + MDBX_PNL_SIZE(txn->tw.reclaimed_pglist) = 0; + mdbx_trace("<<< %u loops, rc = %d", loop, rc); + return rc; +} + +static int mdbx_flush_iov(MDBX_txn *const txn, struct iovec *iov, + unsigned iov_items, size_t iov_off, + size_t iov_bytes) { + MDBX_env *const env = txn->mt_env; + int rc = mdbx_pwritev(env->me_fd, iov, iov_items, iov_off, iov_bytes); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_error("Write error: %s", mdbx_strerror(rc)); + txn->mt_flags |= MDBX_TXN_ERROR; + } + + for (unsigned i = 0; i < iov_items; i++) + mdbx_dpage_free(env, (MDBX_page *)iov[i].iov_base, + bytes2pgno(env, iov[i].iov_len)); + + return rc; +} + +/* Flush (some) dirty pages to the map, after clearing their dirty flag. + * [in] txn the transaction that's being committed + * [in] keep number of initial pages in dirtylist to keep dirty. + * Returns 0 on success, non-zero on failure. */ +static int mdbx_page_flush(MDBX_txn *txn, const unsigned keep) { + struct iovec iov[MDBX_COMMIT_PAGES]; + const MDBX_DPL dl = (keep || txn->tw.loose_count > 1) + ? mdbx_dpl_sort(txn->tw.dirtylist) + : txn->tw.dirtylist; + MDBX_env *const env = txn->mt_env; + pgno_t flush_begin = MAX_PAGENO; + pgno_t flush_end = MIN_PAGENO; + unsigned iov_items = 0; + size_t iov_bytes = 0; + size_t iov_off = 0; + unsigned r, w; + for (r = w = keep; ++r <= dl->length;) { + MDBX_page *dp = dl[r].ptr; + mdbx_tassert(txn, + dp->mp_pgno >= MIN_PAGENO && dp->mp_pgno < txn->mt_next_pgno); + mdbx_tassert(txn, dp->mp_flags & P_DIRTY); + + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE | P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[++w] = dl[r]; + continue; + } + + const unsigned npages = IS_OVERFLOW(dp) ? dp->mp_pages : 1; + flush_begin = (flush_begin < dp->mp_pgno) ? flush_begin : dp->mp_pgno; + flush_end = + (flush_end > dp->mp_pgno + npages) ? flush_end : dp->mp_pgno + npages; + *env->me_unsynced_pages += npages; + dp->mp_flags &= ~P_DIRTY; + dp->mp_validator = 0 /* TODO */; + + if ((env->me_flags & MDBX_WRITEMAP) == 0) { + const size_t size = pgno2bytes(env, npages); + if (iov_off + iov_bytes != pgno2bytes(env, dp->mp_pgno) || + iov_items == ARRAY_LENGTH(iov) || iov_bytes + size > MAX_WRITE) { + if (iov_items) { + int rc = mdbx_flush_iov(txn, iov, iov_items, iov_off, iov_bytes); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; +#if MDBX_CPU_CACHE_MMAP_NONCOHERENT +#if defined(__linux__) || defined(__gnu_linux__) + if (mdbx_linux_kernel_version >= 0x02060b00) + /* Linux kernels older than version 2.6.11 ignore the addr and nbytes + * arguments, making this function fairly expensive. Therefore, the + * whole cache is always flushed. */ +#endif /* Linux */ + mdbx_invalidate_mmap_noncoherent_cache(env->me_map + iov_off, + iov_bytes); +#endif /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */ + iov_items = 0; + iov_bytes = 0; + } + iov_off = pgno2bytes(env, dp->mp_pgno); + } + iov[iov_items].iov_base = dp; + iov[iov_items].iov_len = size; + iov_items += 1; + iov_bytes += size; + } + } + + if (iov_items) { + int rc = mdbx_flush_iov(txn, iov, iov_items, iov_off, iov_bytes); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + +#if MDBX_CPU_CACHE_MMAP_NONCOHERENT && \ + (defined(__linux__) || defined(__gnu_linux__)) + if ((env->me_flags & MDBX_WRITEMAP) == 0 && + mdbx_linux_kernel_version < 0x02060b00) + /* Linux kernels older than version 2.6.11 ignore the addr and nbytes + * arguments, making this function fairly expensive. Therefore, the + * whole cache is always flushed. */ + mdbx_invalidate_mmap_noncoherent_cache( + env->me_map + pgno2bytes(env, flush_begin), + pgno2bytes(env, flush_end - flush_begin)); +#endif /* MDBX_CPU_CACHE_MMAP_NONCOHERENT && Linux */ + + /* TODO: use flush_begin & flush_end for msync() & sync_file_range(). */ + (void)flush_begin; + (void)flush_end; + + txn->tw.dirtyroom += r - 1 - w; + dl->length = w; + mdbx_tassert(txn, txn->mt_parent || + txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + return MDBX_SUCCESS; +} + +/* Check for misused dbi handles */ +#define TXN_DBI_CHANGED(txn, dbi) \ + ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) + +/* Import DBI which opened after txn started into context */ +static __cold bool mdbx_txn_import_dbi(MDBX_txn *txn, MDBX_dbi dbi) { + MDBX_env *env = txn->mt_env; + if (dbi < CORE_DBS || dbi >= env->me_numdbs) + return false; + + mdbx_ensure(env, mdbx_fastmutex_acquire(&env->me_dbi_lock) == MDBX_SUCCESS); + const unsigned snap_numdbs = env->me_numdbs; + mdbx_compiler_barrier(); + for (unsigned i = CORE_DBS; i < snap_numdbs; ++i) { + if (i >= txn->mt_numdbs) + txn->mt_dbflags[i] = 0; + if (!(txn->mt_dbflags[i] & DB_USRVALID) && + (env->me_dbflags[i] & MDBX_VALID)) { + txn->mt_dbs[i].md_flags = env->me_dbflags[i] & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = DB_VALID | DB_USRVALID | DB_STALE; + mdbx_tassert(txn, txn->mt_dbxs[i].md_cmp != NULL); + } + } + txn->mt_numdbs = snap_numdbs; + + mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); + return txn->mt_dbflags[dbi] & DB_USRVALID; +} + +/* Check txn and dbi arguments to a function */ +static __inline bool TXN_DBI_EXIST(MDBX_txn *txn, MDBX_dbi dbi, + unsigned validity) { + if (likely(dbi < txn->mt_numdbs && (txn->mt_dbflags[dbi] & validity))) + return true; + + return mdbx_txn_import_dbi(txn, dbi); +} + +int mdbx_txn_commit(MDBX_txn *txn) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + MDBX_env *env = txn->mt_env; +#if MDBX_TXN_CHECKPID + if (unlikely(env->me_pid != mdbx_getpid())) { + env->me_flags |= MDBX_FATAL_ERROR; + return MDBX_PANIC; + } +#endif /* MDBX_TXN_CHECKPID */ + + /* mdbx_txn_end() mode for a commit which writes nothing */ + unsigned end_mode = + MDBX_END_EMPTY_COMMIT | MDBX_END_UPDATE | MDBX_END_SLOT | MDBX_END_FREE; + if (unlikely(F_ISSET(txn->mt_flags, MDBX_RDONLY))) + goto done; + + if (txn->mt_child) { + rc = mdbx_txn_commit(txn->mt_child); + mdbx_tassert(txn, txn->mt_child == NULL); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + + if (unlikely(txn != env->me_txn)) { + mdbx_debug("%s", "attempt to commit unknown transaction"); + rc = MDBX_EINVAL; + goto fail; + } + + if (txn->mt_parent) { + MDBX_txn *const parent = txn->mt_parent; + mdbx_tassert(txn, mdbx_dirtylist_check(txn)); + + /* Preserve space for spill list to avoid parent's state corruption + * if allocation fails. */ + if (txn->tw.spill_pages && parent->tw.spill_pages) { + rc = mdbx_pnl_need(&parent->tw.spill_pages, + MDBX_PNL_SIZE(txn->tw.spill_pages)); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + + //------------------------------------------------------------------------- + + parent->tw.lifo_reclaimed = txn->tw.lifo_reclaimed; + txn->tw.lifo_reclaimed = NULL; + + parent->tw.retired_pages = txn->tw.retired_pages; + txn->tw.retired_pages = NULL; + + mdbx_pnl_free(parent->tw.reclaimed_pglist); + parent->tw.reclaimed_pglist = txn->tw.reclaimed_pglist; + txn->tw.reclaimed_pglist = NULL; + parent->tw.last_reclaimed = txn->tw.last_reclaimed; + + parent->mt_geo = txn->mt_geo; + parent->mt_canary = txn->mt_canary; + parent->mt_flags = txn->mt_flags; + + /* Merge our cursors into parent's and close them */ + mdbx_cursors_eot(txn, 1); + + /* Update parent's DB table. */ + memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDBX_db)); + parent->mt_numdbs = txn->mt_numdbs; + parent->mt_dbflags[FREE_DBI] = txn->mt_dbflags[FREE_DBI]; + parent->mt_dbflags[MAIN_DBI] = txn->mt_dbflags[MAIN_DBI]; + for (unsigned i = CORE_DBS; i < txn->mt_numdbs; i++) { + /* preserve parent's DB_NEW status */ + parent->mt_dbflags[i] = + txn->mt_dbflags[i] | (parent->mt_dbflags[i] & (DB_CREAT | DB_FRESH)); + } + + /* Remove refunded pages from parent's dirty & spill lists */ + MDBX_DPL dst = mdbx_dpl_sort(parent->tw.dirtylist); + while (dst->length && dst[dst->length].pgno >= parent->mt_next_pgno) { + MDBX_page *mp = dst[dst->length].ptr; + if (mp && (txn->mt_env->me_flags & MDBX_WRITEMAP) == 0) + mdbx_dpage_free(txn->mt_env, mp, IS_OVERFLOW(mp) ? mp->mp_pages : 1); + dst->length -= 1; + } + parent->tw.dirtyroom += dst->sorted - dst->length; + dst->sorted = dst->length; + mdbx_tassert(parent, + parent->mt_parent || + parent->tw.dirtyroom + parent->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + + if (parent->tw.spill_pages && MDBX_PNL_SIZE(parent->tw.spill_pages) > 0 && + MDBX_PNL_MOST(parent->tw.spill_pages) >= parent->mt_next_pgno << 1) { + const MDBX_PNL ps = parent->tw.spill_pages; +#if MDBX_PNL_ASCENDING + unsigned i = MDBX_PNL_SIZE(ps); + assert(MDBX_PNL_MOST(ps) == MDBX_PNL_LAST(ps)); + do + i -= 1; + while (i && ps[i] >= parent->mt_next_pgno << 1); + MDBX_PNL_SIZE(ps) = i; +#else + assert(MDBX_PNL_MOST(ps) == MDBX_PNL_FIRST(ps)); + unsigned i = 1, len = MDBX_PNL_SIZE(ps); + while (i < len && ps[i + 1] >= parent->mt_next_pgno << 1) + ++i; + MDBX_PNL_SIZE(ps) = len -= i; + for (unsigned k = 1; k <= len; ++k) + ps[k] = ps[k + i]; +#endif + } + + /* Remove anything in our dirty list from parent's spill list */ + MDBX_DPL src = mdbx_dpl_sort(txn->tw.dirtylist); + if (likely(src->length > 0) && parent->tw.spill_pages && + MDBX_PNL_SIZE(parent->tw.spill_pages) > 0) { + MDBX_PNL sp = parent->tw.spill_pages; + assert(mdbx_pnl_check4assert(sp, txn->mt_next_pgno)); + + const unsigned len = MDBX_PNL_SIZE(parent->tw.spill_pages); + MDBX_PNL_SIZE(sp) = ~(pgno_t)0; + + /* Mark our dirty pages as deleted in parent spill list */ + unsigned r, w, i = 1; + w = r = len; + do { + pgno_t pn = src[i].pgno << 1; + while (pn > sp[r]) + r--; + if (pn == sp[r]) { + sp[r] = 1; + w = --r; + } + } while (++i <= src->length); + + /* Squash deleted pagenums if we deleted any */ + for (r = w; ++r <= len;) + if ((sp[r] & 1) == 0) + sp[++w] = sp[r]; + MDBX_PNL_SIZE(sp) = w; + assert(mdbx_pnl_check4assert(sp, txn->mt_next_pgno << 1)); + } + + /* Remove anything in our spill list from parent's dirty list */ + if (txn->tw.spill_pages && MDBX_PNL_SIZE(txn->tw.spill_pages) > 0) { + const MDBX_PNL sp = txn->tw.spill_pages; + mdbx_pnl_sort(sp); + /* Scanning in ascend order */ + const int step = MDBX_PNL_ASCENDING ? 1 : -1; + const int begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_SIZE(sp); + const int end = MDBX_PNL_ASCENDING ? MDBX_PNL_SIZE(sp) + 1 : 0; + mdbx_tassert(txn, sp[begin] <= sp[end - step]); + + unsigned r, w = r = mdbx_dpl_search(dst, sp[begin] >> 1); + mdbx_tassert(txn, dst->sorted == dst->length); + for (int i = begin; r <= dst->length;) { + mdbx_tassert(txn, (sp[i] & 1) == 0); + const pgno_t pgno = sp[i] >> 1; + if (dst[r].pgno < pgno) { + dst[w++] = dst[r++]; + } else if (dst[r].pgno > pgno) { + i += step; + if (i == end) + while (r <= dst->length) + dst[w++] = dst[r++]; + } else { + MDBX_page *dp = dst[r++].ptr; + if ((env->me_flags & MDBX_WRITEMAP) == 0) + mdbx_dpage_free(env, dp, IS_OVERFLOW(dp) ? dp->mp_pages : 1); + } + } + mdbx_tassert(txn, r == dst->length + 1); + dst->length = w; + parent->tw.dirtyroom += r - w; + } + assert(dst->sorted == dst->length); + mdbx_tassert(parent, + parent->mt_parent || + parent->tw.dirtyroom + parent->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + + unsigned d, s, l; + /* Find length of merging our dirty list with parent's */ + for (l = 0, d = dst->length, s = src->length; d > 0 && s > 0; ++l) { + const pgno_t s_pgno = src[s].pgno; + const pgno_t d_pgno = dst[d].pgno; + d -= d_pgno >= s_pgno; + s -= d_pgno <= s_pgno; + } + assert(dst->sorted == dst->length); + dst->sorted = l += d + s; + assert(dst->sorted >= dst->length); + parent->tw.dirtyroom -= dst->sorted - dst->length; + + /* Merge our dirty list into parent's */ + for (d = dst->length, s = src->length; d > 0 && s > 0; --l) { + if (dst[d].pgno > src[s].pgno) + dst[l] = dst[d--]; + else if (dst[d].pgno < src[s].pgno) + dst[l] = src[s--]; + else { + MDBX_page *dp = dst[d--].ptr; + if (dp && (env->me_flags & MDBX_WRITEMAP) == 0) + mdbx_dpage_free(env, dp, IS_OVERFLOW(dp) ? dp->mp_pgno : 1); + dst[l] = src[s--]; + } + } + if (s) { + do + dst[l--] = src[s--]; + while (s > 0); + } else if (d) { + do + dst[l--] = dst[d--]; + while (d > 0); + } + assert(l == 0); + dst->length = dst->sorted; + mdbx_free(txn->tw.dirtylist); + txn->tw.dirtylist = nullptr; + mdbx_tassert(parent, + parent->mt_parent || + parent->tw.dirtyroom + parent->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + + if (txn->tw.spill_pages) { + if (parent->tw.spill_pages) { + /* Must not fail since space was preserved above. */ + rc = mdbx_pnl_append_list(&parent->tw.spill_pages, txn->tw.spill_pages); + mdbx_assert(env, rc == MDBX_SUCCESS); + (void)rc; + mdbx_pnl_free(txn->tw.spill_pages); + mdbx_pnl_sort(parent->tw.spill_pages); + } else { + parent->tw.spill_pages = txn->tw.spill_pages; + } + } + if (parent->tw.spill_pages) + assert(mdbx_pnl_check4assert(parent->tw.spill_pages, + parent->mt_next_pgno << 1)); + + /* Append our loose page list to parent's */ + if (txn->tw.loose_pages) { + MDBX_page **lp = &parent->tw.loose_pages; + while (*lp) + lp = &(*lp)->mp_next; + *lp = txn->tw.loose_pages; + parent->tw.loose_count += txn->tw.loose_count; + } + if (txn->tw.retired2parent_pages) { + MDBX_page *mp = txn->tw.retired2parent_pages; + do { + MDBX_page *next = mp->mp_next; + rc = mdbx_page_loose(parent, mp); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + mp = next; + } while (mp); + } + + env->me_txn = parent; + parent->mt_child = NULL; + txn->mt_signature = 0; + mdbx_free(txn); + mdbx_tassert(parent, mdbx_dirtylist_check(parent)); + + /* Scan parent's loose page for suitable for refund */ + for (MDBX_page *mp = parent->tw.loose_pages; mp; mp = mp->mp_next) { + if (mp->mp_pgno == parent->mt_next_pgno - 1) { + mdbx_refund(parent); + break; + } + } + mdbx_tassert(parent, mdbx_dirtylist_check(parent)); + return MDBX_SUCCESS; + } + + mdbx_tassert(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + mdbx_cursors_eot(txn, 0); + end_mode |= MDBX_END_EOTDONE; + + if (txn->tw.dirtylist->length == 0 && + (txn->mt_flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS)) == 0) + goto done; + + mdbx_debug("committing txn %" PRIaTXN " %p on mdbenv %p, root page %" PRIaPGNO + "/%" PRIaPGNO, + txn->mt_txnid, (void *)txn, (void *)env, + txn->mt_dbs[MAIN_DBI].md_root, txn->mt_dbs[FREE_DBI].md_root); + + /* Update DB root pointers */ + if (txn->mt_numdbs > CORE_DBS) { + MDBX_cursor mc; + MDBX_val data; + data.iov_len = sizeof(MDBX_db); + + rc = mdbx_cursor_init(&mc, txn, MAIN_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + for (MDBX_dbi i = CORE_DBS; i < txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + if (unlikely(TXN_DBI_CHANGED(txn, i))) { + rc = MDBX_BAD_DBI; + goto fail; + } + MDBX_db *db = &txn->mt_dbs[i]; + db->md_mod_txnid = txn->mt_txnid; + data.iov_base = db; + WITH_CURSOR_TRACKING(mc, + rc = mdbx_cursor_put(&mc, &txn->mt_dbxs[i].md_name, + &data, F_SUBDATA)); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + } + } + + rc = mdbx_update_gc(txn); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + + if (mdbx_audit_enabled()) { + rc = mdbx_audit_ex(txn, MDBX_PNL_SIZE(txn->tw.retired_pages), true); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + + rc = mdbx_page_flush(txn, 0); + if (likely(rc == MDBX_SUCCESS)) { + if (txn->mt_dbs[MAIN_DBI].md_flags & DB_DIRTY) + txn->mt_dbs[MAIN_DBI].md_mod_txnid = txn->mt_txnid; + + MDBX_meta meta, *head = mdbx_meta_head(env); + meta.mm_magic_and_version = head->mm_magic_and_version; + meta.mm_extra_flags = head->mm_extra_flags; + meta.mm_validator_id = head->mm_validator_id; + meta.mm_extra_pagehdr = head->mm_extra_pagehdr; + meta.mm_pages_retired = + head->mm_pages_retired + MDBX_PNL_SIZE(txn->tw.retired_pages); + + meta.mm_geo = txn->mt_geo; + meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI]; + meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; + meta.mm_canary = txn->mt_canary; + mdbx_meta_set_txnid(env, &meta, txn->mt_txnid); + + rc = mdbx_sync_locked( + env, env->me_flags | txn->mt_flags | MDBX_SHRINK_ALLOWED, &meta); + } + if (unlikely(rc != MDBX_SUCCESS)) { + env->me_flags |= MDBX_FATAL_ERROR; + goto fail; + } + + if (likely(env->me_lck)) + env->me_lck->mti_readers_refresh_flag = false; + end_mode = MDBX_END_COMMITTED | MDBX_END_UPDATE | MDBX_END_EOTDONE; + +done: + return mdbx_txn_end(txn, end_mode); + +fail: + mdbx_txn_abort(txn); + return rc; +} + +/* Read the environment parameters of a DB environment + * before mapping it into memory. */ +static int __cold mdbx_read_header(MDBX_env *env, MDBX_meta *dest, + uint64_t *filesize) { + int rc = mdbx_filesize(env->me_fd, filesize); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + memset(dest, 0, sizeof(MDBX_meta)); + dest->mm_datasync_sign = MDBX_DATASIGN_WEAK; + rc = MDBX_CORRUPTED; + + /* Read twice all meta pages so we can find the latest one. */ + unsigned loop_limit = NUM_METAS * 2; + for (unsigned loop_count = 0; loop_count < loop_limit; ++loop_count) { + /* We don't know the page size on first time. + * So, just guess it. */ + unsigned guess_pagesize = dest->mm_psize; + if (guess_pagesize == 0) + guess_pagesize = + (loop_count > NUM_METAS) ? env->me_psize : env->me_os_psize; + + const unsigned meta_number = loop_count % NUM_METAS; + const unsigned offset = guess_pagesize * meta_number; + + char buffer[MIN_PAGESIZE]; + unsigned retryleft = 42; + while (1) { + mdbx_trace("reading meta[%d]: offset %u, bytes %u, retry-left %u", + meta_number, offset, MIN_PAGESIZE, retryleft); + int err = mdbx_pread(env->me_fd, buffer, MIN_PAGESIZE, offset); + if (err != MDBX_SUCCESS) { + if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 && + *filesize == 0 && (env->me_flags & MDBX_RDONLY) == 0) + mdbx_notice("read meta: empty file (%d, %s)", err, + mdbx_strerror(err)); + else + mdbx_error("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err, + mdbx_strerror(err)); + return err; + } + + char again[MIN_PAGESIZE]; + err = mdbx_pread(env->me_fd, again, MIN_PAGESIZE, offset); + if (err != MDBX_SUCCESS) { + mdbx_error("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err, + mdbx_strerror(err)); + return err; + } + + if (memcmp(buffer, again, MIN_PAGESIZE) == 0 || --retryleft == 0) + break; + + mdbx_verbose("meta[%u] was updated, re-read it", meta_number); + } + + MDBX_page *const page = (MDBX_page *)buffer; + MDBX_meta *const meta = page_meta(page); + if (meta->mm_magic_and_version != MDBX_DATA_MAGIC && + meta->mm_magic_and_version != MDBX_DATA_MAGIC_DEVEL) { + mdbx_error("meta[%u] has invalid magic/version %" PRIx64, meta_number, + meta->mm_magic_and_version); + return ((meta->mm_magic_and_version >> 8) != MDBX_MAGIC) + ? MDBX_INVALID + : MDBX_VERSION_MISMATCH; + } + + if (page->mp_pgno != meta_number) { + mdbx_error("meta[%u] has invalid pageno %" PRIaPGNO, meta_number, + page->mp_pgno); + return MDBX_INVALID; + } + + if (page->mp_flags != P_META) { + mdbx_error("page #%u not a meta-page", meta_number); + return MDBX_INVALID; + } + + if (!retryleft) { + mdbx_error("meta[%u] is too volatile, skip it", meta_number); + continue; + } + + /* LY: check pagesize */ + if (!is_powerof2(meta->mm_psize) || meta->mm_psize < MIN_PAGESIZE || + meta->mm_psize > MAX_PAGESIZE) { + mdbx_notice("meta[%u] has invalid pagesize (%u), skip it", meta_number, + meta->mm_psize); + rc = is_powerof2(meta->mm_psize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID; + continue; + } + + if (meta_number == 0 && guess_pagesize != meta->mm_psize) { + dest->mm_psize = meta->mm_psize; + mdbx_verbose("meta[%u] took pagesize %u", meta_number, meta->mm_psize); + } + + if (safe64_read(&meta->mm_txnid_a) != safe64_read(&meta->mm_txnid_b)) { + mdbx_warning("meta[%u] not completely updated, skip it", meta_number); + continue; + } + + /* LY: check signature as a checksum */ + if (META_IS_STEADY(meta) && + meta->mm_datasync_sign != mdbx_meta_sign(meta)) { + mdbx_notice("meta[%u] has invalid steady-checksum (0x%" PRIx64 + " != 0x%" PRIx64 "), skip it", + meta_number, meta->mm_datasync_sign, mdbx_meta_sign(meta)); + continue; + } + + mdbx_debug("read meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO + ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO + " +%u -%u, txn_id %" PRIaTXN ", %s", + page->mp_pgno, meta->mm_dbs[MAIN_DBI].md_root, + meta->mm_dbs[FREE_DBI].md_root, meta->mm_geo.lower, + meta->mm_geo.next, meta->mm_geo.now, meta->mm_geo.upper, + meta->mm_geo.grow, meta->mm_geo.shrink, + meta->mm_txnid_a.inconsistent, mdbx_durable_str(meta)); + + /* LY: check min-pages value */ + if (meta->mm_geo.lower < MIN_PAGENO || meta->mm_geo.lower > MAX_PAGENO) { + mdbx_notice("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", + meta_number, meta->mm_geo.lower); + rc = MDBX_INVALID; + continue; + } + + /* LY: check max-pages value */ + if (meta->mm_geo.upper < MIN_PAGENO || meta->mm_geo.upper > MAX_PAGENO || + meta->mm_geo.upper < meta->mm_geo.lower) { + mdbx_notice("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", + meta_number, meta->mm_geo.upper); + rc = MDBX_INVALID; + continue; + } + + /* LY: check last_pgno */ + if (meta->mm_geo.next < MIN_PAGENO || meta->mm_geo.next - 1 > MAX_PAGENO) { + mdbx_notice("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", + meta_number, meta->mm_geo.next); + rc = MDBX_CORRUPTED; + continue; + } + + /* LY: check filesize & used_bytes */ + const uint64_t used_bytes = meta->mm_geo.next * (uint64_t)meta->mm_psize; + if (used_bytes > *filesize) { + /* Here could be a race with DB-shrinking performed by other process */ + rc = mdbx_filesize(env->me_fd, filesize); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + if (used_bytes > *filesize) { + mdbx_notice("meta[%u] used-bytes (%" PRIu64 + ") beyond filesize (%" PRIu64 "), skip it", + meta_number, used_bytes, *filesize); + rc = MDBX_CORRUPTED; + continue; + } + } + + /* LY: check mapsize limits */ + const uint64_t mapsize_min = meta->mm_geo.lower * (uint64_t)meta->mm_psize; + STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MAX_PAGESIZE); + STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); + if (mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE) { + mdbx_notice("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", + meta_number, mapsize_min); + rc = MDBX_VERSION_MISMATCH; + continue; + } + + const uint64_t mapsize_max = meta->mm_geo.upper * (uint64_t)meta->mm_psize; + STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); + if (mapsize_max > MAX_MAPSIZE || + MAX_PAGENO < roundup_powerof2((size_t)mapsize_max, env->me_os_psize) / + (size_t)meta->mm_psize) { + if (meta->mm_geo.next - 1 > MAX_PAGENO || used_bytes > MAX_MAPSIZE) { + mdbx_notice("meta[%u] has too large max-mapsize (%" PRIu64 "), skip it", + meta_number, mapsize_max); + rc = MDBX_TOO_LARGE; + continue; + } + + /* allow to open large DB from a 32-bit environment */ + mdbx_notice("meta[%u] has too large max-mapsize (%" PRIu64 "), " + "but size of used space still acceptable (%" PRIu64 ")", + meta_number, mapsize_max, used_bytes); + meta->mm_geo.upper = (pgno_t)(MAX_MAPSIZE / meta->mm_psize); + } + + /* LY: check and silently put mm_geo.now into [geo.lower...geo.upper]. + * + * Copy-with-compaction by previous version of libmdbx could produce DB-file + * less than meta.geo.lower bound, in case actual filling is low or no data + * at all. This is not a problem as there is no damage or loss of data. + * Therefore it is better not to consider such situation as an error, but + * silently correct it. */ + if (meta->mm_geo.now < meta->mm_geo.lower) + meta->mm_geo.now = meta->mm_geo.lower; + if (meta->mm_geo.now > meta->mm_geo.upper && + meta->mm_geo.next <= meta->mm_geo.upper) + meta->mm_geo.now = meta->mm_geo.upper; + + if (meta->mm_geo.next > meta->mm_geo.now) { + mdbx_notice("meta[%u] next-pageno (%" PRIaPGNO + ") is beyond end-pgno (%" PRIaPGNO "), skip it", + meta_number, meta->mm_geo.next, meta->mm_geo.now); + rc = MDBX_CORRUPTED; + continue; + } + + /* LY: GC root */ + if (meta->mm_dbs[FREE_DBI].md_root == P_INVALID) { + if (meta->mm_dbs[FREE_DBI].md_branch_pages || + meta->mm_dbs[FREE_DBI].md_depth || + meta->mm_dbs[FREE_DBI].md_entries || + meta->mm_dbs[FREE_DBI].md_leaf_pages || + meta->mm_dbs[FREE_DBI].md_overflow_pages) { + mdbx_notice("meta[%u] has false-empty GC, skip it", meta_number); + rc = MDBX_CORRUPTED; + continue; + } + } else if (meta->mm_dbs[FREE_DBI].md_root >= meta->mm_geo.next) { + mdbx_notice("meta[%u] has invalid GC-root %" PRIaPGNO ", skip it", + meta_number, meta->mm_dbs[FREE_DBI].md_root); + rc = MDBX_CORRUPTED; + continue; + } + + /* LY: MainDB root */ + if (meta->mm_dbs[MAIN_DBI].md_root == P_INVALID) { + if (meta->mm_dbs[MAIN_DBI].md_branch_pages || + meta->mm_dbs[MAIN_DBI].md_depth || + meta->mm_dbs[MAIN_DBI].md_entries || + meta->mm_dbs[MAIN_DBI].md_leaf_pages || + meta->mm_dbs[MAIN_DBI].md_overflow_pages) { + mdbx_notice("meta[%u] has false-empty maindb", meta_number); + rc = MDBX_CORRUPTED; + continue; + } + } else if (meta->mm_dbs[MAIN_DBI].md_root >= meta->mm_geo.next) { + mdbx_notice("meta[%u] has invalid maindb-root %" PRIaPGNO ", skip it", + meta_number, meta->mm_dbs[MAIN_DBI].md_root); + rc = MDBX_CORRUPTED; + continue; + } + + if (safe64_read(&meta->mm_txnid_a) == 0) { + mdbx_warning("meta[%u] has zero txnid, skip it", meta_number); + continue; + } + + if (mdbx_meta_ot(prefer_noweak, env, dest, meta)) { + *dest = *meta; + if (META_IS_WEAK(dest)) + loop_limit += 1; /* LY: should re-read to hush race with update */ + mdbx_verbose("latch meta[%u]", meta_number); + } + } + + if (META_IS_WEAK(dest)) { + mdbx_error("%s", "no usable meta-pages, database is corrupted"); + return rc; + } + + return MDBX_SUCCESS; +} + +static MDBX_page *__cold mdbx_meta_model(const MDBX_env *env, MDBX_page *model, + unsigned num) { + + mdbx_ensure(env, is_powerof2(env->me_psize)); + mdbx_ensure(env, env->me_psize >= MIN_PAGESIZE); + mdbx_ensure(env, env->me_psize <= MAX_PAGESIZE); + mdbx_ensure(env, env->me_dbgeo.lower >= MIN_MAPSIZE); + mdbx_ensure(env, env->me_dbgeo.upper <= MAX_MAPSIZE); + mdbx_ensure(env, env->me_dbgeo.now >= env->me_dbgeo.lower); + mdbx_ensure(env, env->me_dbgeo.now <= env->me_dbgeo.upper); + + memset(model, 0, sizeof(*model)); + model->mp_pgno = num; + model->mp_flags = P_META; + MDBX_meta *const model_meta = page_meta(model); + model_meta->mm_magic_and_version = MDBX_DATA_MAGIC; + + model_meta->mm_geo.lower = bytes2pgno(env, env->me_dbgeo.lower); + model_meta->mm_geo.upper = bytes2pgno(env, env->me_dbgeo.upper); + model_meta->mm_geo.grow = (uint16_t)bytes2pgno(env, env->me_dbgeo.grow); + model_meta->mm_geo.shrink = (uint16_t)bytes2pgno(env, env->me_dbgeo.shrink); + model_meta->mm_geo.now = bytes2pgno(env, env->me_dbgeo.now); + model_meta->mm_geo.next = NUM_METAS; + + mdbx_ensure(env, model_meta->mm_geo.lower >= MIN_PAGENO); + mdbx_ensure(env, model_meta->mm_geo.upper <= MAX_PAGENO); + mdbx_ensure(env, model_meta->mm_geo.now >= model_meta->mm_geo.lower); + mdbx_ensure(env, model_meta->mm_geo.now <= model_meta->mm_geo.upper); + mdbx_ensure(env, model_meta->mm_geo.next >= MIN_PAGENO); + mdbx_ensure(env, model_meta->mm_geo.next <= model_meta->mm_geo.now); + mdbx_ensure(env, + model_meta->mm_geo.grow == bytes2pgno(env, env->me_dbgeo.grow)); + mdbx_ensure(env, model_meta->mm_geo.shrink == + bytes2pgno(env, env->me_dbgeo.shrink)); + + model_meta->mm_psize = env->me_psize; + model_meta->mm_flags = (uint16_t)env->me_flags; + model_meta->mm_flags |= + MDBX_INTEGERKEY; /* this is mm_dbs[FREE_DBI].md_flags */ + model_meta->mm_dbs[FREE_DBI].md_root = P_INVALID; + model_meta->mm_dbs[MAIN_DBI].md_root = P_INVALID; + mdbx_meta_set_txnid(env, model_meta, MIN_TXNID + num); + model_meta->mm_datasync_sign = mdbx_meta_sign(model_meta); + return (MDBX_page *)((uint8_t *)model + env->me_psize); +} + +/* Fill in most of the zeroed meta-pages for an empty database environment. + * Return pointer to recenly (head) meta-page. */ +static MDBX_meta *__cold mdbx_init_metas(const MDBX_env *env, void *buffer) { + MDBX_page *page0 = (MDBX_page *)buffer; + MDBX_page *page1 = mdbx_meta_model(env, page0, 0); + MDBX_page *page2 = mdbx_meta_model(env, page1, 1); + mdbx_meta_model(env, page2, 2); + mdbx_assert(env, !mdbx_meta_eq(env, page_meta(page0), page_meta(page1))); + mdbx_assert(env, !mdbx_meta_eq(env, page_meta(page1), page_meta(page2))); + mdbx_assert(env, !mdbx_meta_eq(env, page_meta(page2), page_meta(page0))); + return page_meta(page2); +} + +static int mdbx_sync_locked(MDBX_env *env, unsigned flags, + MDBX_meta *const pending) { + mdbx_assert(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0); + MDBX_meta *const meta0 = METAPAGE(env, 0); + MDBX_meta *const meta1 = METAPAGE(env, 1); + MDBX_meta *const meta2 = METAPAGE(env, 2); + MDBX_meta *const head = mdbx_meta_head(env); + + mdbx_assert(env, mdbx_meta_eq_mask(env) == 0); + mdbx_assert(env, + pending < METAPAGE(env, 0) || pending > METAPAGE(env, NUM_METAS)); + mdbx_assert(env, (env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0); + mdbx_assert(env, !META_IS_STEADY(head) || *env->me_unsynced_pages != 0); + mdbx_assert(env, pending->mm_geo.next <= pending->mm_geo.now); + + if (flags & (MDBX_NOSYNC | MDBX_MAPASYNC)) { + /* Check auto-sync conditions */ + const pgno_t autosync_threshold = *env->me_autosync_threshold; + const uint64_t autosync_period = *env->me_autosync_period; + if ((autosync_threshold && *env->me_unsynced_pages >= autosync_threshold) || + (autosync_period && + mdbx_osal_monotime() - *env->me_sync_timestamp >= autosync_period)) + flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED; /* force steady */ + } + + pgno_t shrink = 0; + if (flags & MDBX_SHRINK_ALLOWED) { + /* LY: check conditions to discard unused pages */ + const pgno_t largest_pgno = mdbx_find_largest( + env, (head->mm_geo.next > pending->mm_geo.next) ? head->mm_geo.next + : pending->mm_geo.next); + mdbx_assert(env, largest_pgno >= NUM_METAS); +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + const pgno_t edge = env->me_poison_edge; + if (edge > largest_pgno) { + env->me_poison_edge = largest_pgno; + VALGRIND_MAKE_MEM_NOACCESS(env->me_map + pgno2bytes(env, largest_pgno), + pgno2bytes(env, edge - largest_pgno)); + ASAN_POISON_MEMORY_REGION(env->me_map + pgno2bytes(env, largest_pgno), + pgno2bytes(env, edge - largest_pgno)); + } +#endif /* MDBX_USE_VALGRIND */ +#if defined(MADV_DONTNEED) + const size_t largest_aligned2os_bytes = + pgno_align2os_bytes(env, largest_pgno); + const pgno_t largest_aligned2os_pgno = + bytes2pgno(env, largest_aligned2os_bytes); + const pgno_t prev_discarded_pgno = *env->me_discarded_tail; + if (prev_discarded_pgno > + largest_aligned2os_pgno + + /* 1M threshold to avoid unreasonable madvise() call */ + bytes2pgno(env, MEGABYTE)) { + mdbx_notice("open-MADV_%s %u..%u", "DONTNEED", *env->me_discarded_tail, + largest_pgno); + *env->me_discarded_tail = largest_aligned2os_pgno; + const size_t prev_discarded_bytes = + pgno2bytes(env, prev_discarded_pgno) & ~(env->me_os_psize - 1); + mdbx_ensure(env, prev_discarded_bytes > largest_aligned2os_bytes); + int advise = MADV_DONTNEED; +#if defined(MADV_FREE) && \ + 0 /* MADV_FREE works for only anon vma at the moment */ + if ((env->me_flags & MDBX_WRITEMAP) && + mdbx_linux_kernel_version > 0x04050000) + advise = MADV_FREE; +#endif /* MADV_FREE */ + int err = madvise(env->me_map + largest_aligned2os_bytes, + prev_discarded_bytes - largest_aligned2os_bytes, advise) + ? errno + : MDBX_SUCCESS; + mdbx_assert(env, err == MDBX_SUCCESS); + (void)err; + } +#endif /* MADV_FREE || MADV_DONTNEED */ + + /* LY: check conditions to shrink datafile */ + const pgno_t backlog_gap = + pending->mm_dbs[FREE_DBI].md_depth + mdbx_backlog_extragap(env); + if (pending->mm_geo.shrink && pending->mm_geo.now - pending->mm_geo.next > + pending->mm_geo.shrink + backlog_gap) { + if (pending->mm_geo.now > largest_pgno && + pending->mm_geo.now - largest_pgno > + pending->mm_geo.shrink + backlog_gap) { + const pgno_t aligner = pending->mm_geo.grow ? pending->mm_geo.grow + : pending->mm_geo.shrink; + const pgno_t with_backlog_gap = largest_pgno + backlog_gap; + const pgno_t aligned = pgno_align2os_pgno( + env, with_backlog_gap + aligner - with_backlog_gap % aligner); + const pgno_t bottom = + (aligned > pending->mm_geo.lower) ? aligned : pending->mm_geo.lower; + if (pending->mm_geo.now > bottom) { + flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED; /* force steady */ + shrink = pending->mm_geo.now - bottom; + pending->mm_geo.now = bottom; + if (mdbx_meta_txnid_stable(env, head) == + pending->mm_txnid_a.inconsistent) + mdbx_meta_set_txnid( + env, pending, + safe64_txnid_next(pending->mm_txnid_a.inconsistent)); + } + } + } + } + + /* LY: step#1 - sync previously written/updated data-pages */ + int rc = *env->me_unsynced_pages ? MDBX_RESULT_TRUE /* carry non-steady */ + : MDBX_RESULT_FALSE /* carry steady */; + if (rc != MDBX_RESULT_FALSE && (flags & MDBX_NOSYNC) == 0) { + mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0); + MDBX_meta *const recent_steady_meta = mdbx_meta_steady(env); + if (flags & MDBX_WRITEMAP) { + const size_t usedbytes = pgno_align2os_bytes(env, pending->mm_geo.next); + rc = mdbx_msync(&env->me_dxb_mmap, 0, usedbytes, flags & MDBX_MAPASYNC); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + rc = MDBX_RESULT_TRUE /* carry non-steady */; + if ((flags & MDBX_MAPASYNC) == 0) { + if (unlikely(pending->mm_geo.next > recent_steady_meta->mm_geo.now)) { + rc = mdbx_filesync(env->me_fd, MDBX_SYNC_SIZE); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + rc = MDBX_RESULT_FALSE /* carry steady */; + } + } else { + rc = mdbx_filesync(env->me_fd, + (pending->mm_geo.next > recent_steady_meta->mm_geo.now) + ? MDBX_SYNC_DATA | MDBX_SYNC_SIZE + : MDBX_SYNC_DATA); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + } + + /* Steady or Weak */ + if (rc == MDBX_RESULT_FALSE /* carry steady */) { + pending->mm_datasync_sign = mdbx_meta_sign(pending); + *env->me_unsynced_pages = 0; + *env->me_sync_timestamp = mdbx_osal_monotime(); + } else { + assert(rc == MDBX_RESULT_TRUE /* carry non-steady */); + pending->mm_datasync_sign = F_ISSET(env->me_flags, MDBX_UTTERLY_NOSYNC) + ? MDBX_DATASIGN_NONE + : MDBX_DATASIGN_WEAK; + } + + MDBX_meta *target = nullptr; + if (mdbx_meta_txnid_stable(env, head) == pending->mm_txnid_a.inconsistent) { + mdbx_assert(env, memcmp(&head->mm_dbs, &pending->mm_dbs, + sizeof(head->mm_dbs)) == 0); + mdbx_assert(env, memcmp(&head->mm_canary, &pending->mm_canary, + sizeof(head->mm_canary)) == 0); + mdbx_assert(env, memcmp(&head->mm_geo, &pending->mm_geo, + sizeof(pending->mm_geo)) == 0); + if (!META_IS_STEADY(head) && META_IS_STEADY(pending)) + target = head; + else { + mdbx_ensure(env, mdbx_meta_eq(env, head, pending)); + mdbx_debug("%s", "skip update meta"); + return MDBX_SUCCESS; + } + } else if (head == meta0) + target = mdbx_meta_ancient(prefer_steady, env, meta1, meta2); + else if (head == meta1) + target = mdbx_meta_ancient(prefer_steady, env, meta0, meta2); + else { + mdbx_assert(env, head == meta2); + target = mdbx_meta_ancient(prefer_steady, env, meta0, meta1); + } + + /* LY: step#2 - update meta-page. */ + mdbx_debug("writing meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO + ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO + " +%u -%u, txn_id %" PRIaTXN ", %s", + data_page(target)->mp_pgno, pending->mm_dbs[MAIN_DBI].md_root, + pending->mm_dbs[FREE_DBI].md_root, pending->mm_geo.lower, + pending->mm_geo.next, pending->mm_geo.now, pending->mm_geo.upper, + pending->mm_geo.grow, pending->mm_geo.shrink, + pending->mm_txnid_a.inconsistent, mdbx_durable_str(pending)); + + mdbx_debug("meta0: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO + "/%" PRIaPGNO, + (meta0 == head) ? "head" : (meta0 == target) ? "tail" : "stay", + mdbx_durable_str(meta0), mdbx_meta_txnid_fluid(env, meta0), + meta0->mm_dbs[MAIN_DBI].md_root, meta0->mm_dbs[FREE_DBI].md_root); + mdbx_debug("meta1: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO + "/%" PRIaPGNO, + (meta1 == head) ? "head" : (meta1 == target) ? "tail" : "stay", + mdbx_durable_str(meta1), mdbx_meta_txnid_fluid(env, meta1), + meta1->mm_dbs[MAIN_DBI].md_root, meta1->mm_dbs[FREE_DBI].md_root); + mdbx_debug("meta2: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO + "/%" PRIaPGNO, + (meta2 == head) ? "head" : (meta2 == target) ? "tail" : "stay", + mdbx_durable_str(meta2), mdbx_meta_txnid_fluid(env, meta2), + meta2->mm_dbs[MAIN_DBI].md_root, meta2->mm_dbs[FREE_DBI].md_root); + + mdbx_assert(env, !mdbx_meta_eq(env, pending, meta0)); + mdbx_assert(env, !mdbx_meta_eq(env, pending, meta1)); + mdbx_assert(env, !mdbx_meta_eq(env, pending, meta2)); + + mdbx_assert(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0); + mdbx_ensure(env, target == head || mdbx_meta_txnid_stable(env, target) < + pending->mm_txnid_a.inconsistent); + if (env->me_flags & MDBX_WRITEMAP) { + mdbx_jitter4testing(true); + if (likely(target != head)) { + /* LY: 'invalidate' the meta. */ + mdbx_meta_update_begin(env, target, pending->mm_txnid_a.inconsistent); + target->mm_datasync_sign = MDBX_DATASIGN_WEAK; +#ifndef NDEBUG + /* debug: provoke failure to catch a violators, but don't touch mm_psize + * and mm_flags to allow readers catch actual pagesize. */ + uint8_t *provoke_begin = (uint8_t *)&target->mm_dbs[FREE_DBI].md_root; + uint8_t *provoke_end = (uint8_t *)&target->mm_datasync_sign; + memset(provoke_begin, 0xCC, provoke_end - provoke_begin); + mdbx_jitter4testing(false); +#endif + + /* LY: update info */ + target->mm_geo = pending->mm_geo; + target->mm_dbs[FREE_DBI] = pending->mm_dbs[FREE_DBI]; + target->mm_dbs[MAIN_DBI] = pending->mm_dbs[MAIN_DBI]; + target->mm_canary = pending->mm_canary; + target->mm_pages_retired = pending->mm_pages_retired; + mdbx_jitter4testing(true); + mdbx_flush_noncoherent_cpu_writeback(); + + /* LY: 'commit' the meta */ + mdbx_meta_update_end(env, target, pending->mm_txnid_b.inconsistent); + mdbx_jitter4testing(true); + } else { + /* dangerous case (target == head), only mm_datasync_sign could + * me updated, check assertions once again */ + mdbx_ensure(env, mdbx_meta_txnid_stable(env, head) == + pending->mm_txnid_a.inconsistent && + !META_IS_STEADY(head) && META_IS_STEADY(pending)); + mdbx_ensure(env, memcmp(&head->mm_geo, &pending->mm_geo, + sizeof(head->mm_geo)) == 0); + mdbx_ensure(env, memcmp(&head->mm_dbs, &pending->mm_dbs, + sizeof(head->mm_dbs)) == 0); + mdbx_ensure(env, memcmp(&head->mm_canary, &pending->mm_canary, + sizeof(head->mm_canary)) == 0); + } + target->mm_datasync_sign = pending->mm_datasync_sign; + mdbx_flush_noncoherent_cpu_writeback(); + mdbx_jitter4testing(true); + } else { + rc = mdbx_pwrite(env->me_fd, pending, sizeof(MDBX_meta), + (uint8_t *)target - env->me_map); + if (unlikely(rc != MDBX_SUCCESS)) { + undo: + mdbx_debug("%s", "write failed, disk error?"); + /* On a failure, the pagecache still contains the new data. + * Try write some old data back, to prevent it from being used. */ + mdbx_pwrite(env->me_fd, (void *)target, sizeof(MDBX_meta), + (uint8_t *)target - env->me_map); + goto fail; + } + mdbx_invalidate_mmap_noncoherent_cache(target, sizeof(MDBX_meta)); + } + + /* LY: step#3 - sync meta-pages. */ + mdbx_assert(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0); + if ((flags & (MDBX_NOSYNC | MDBX_NOMETASYNC)) == 0) { + mdbx_assert(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0); + if (flags & MDBX_WRITEMAP) { + const size_t offset = (uint8_t *)data_page(head) - env->me_dxb_mmap.dxb; + const size_t paged_offset = offset & ~(env->me_os_psize - 1); + const size_t paged_length = roundup_powerof2( + env->me_psize + offset - paged_offset, env->me_os_psize); + rc = mdbx_msync(&env->me_dxb_mmap, paged_offset, paged_length, + flags & MDBX_MAPASYNC); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } else { + rc = mdbx_filesync(env->me_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + if (rc != MDBX_SUCCESS) + goto undo; + } + *env->me_meta_sync_txnid = (uint32_t)pending->mm_txnid_a.inconsistent; + } + + /* LY: shrink datafile if needed */ + if (unlikely(shrink)) { + mdbx_verbose("shrink to %" PRIaPGNO " pages (-%" PRIaPGNO ")", + pending->mm_geo.now, shrink); + rc = mdbx_mapresize(env, pending->mm_geo.now, pending->mm_geo.upper); + if (MDBX_IS_ERROR(rc)) + goto fail; + } + + return MDBX_SUCCESS; + +fail: + env->me_flags |= MDBX_FATAL_ERROR; + return rc; +} + +int __cold mdbx_env_get_maxkeysize(MDBX_env *env) { + if (!env || env->me_signature != MDBX_ME_SIGNATURE || !env->me_maxkey_limit) + return (MDBX_EINVAL > 0) ? -MDBX_EINVAL : MDBX_EINVAL; + return env->me_maxkey_limit; +} + +#define mdbx_nodemax(pagesize) \ + (((((pagesize)-PAGEHDRSZ) / MDBX_MINKEYS) & ~(uintptr_t)1) - sizeof(indx_t)) + +#define mdbx_maxkey(nodemax) ((nodemax)-NODESIZE - sizeof(MDBX_db)) + +#define mdbx_maxgc_ov1page(pagesize) \ + (((pagesize)-PAGEHDRSZ) / sizeof(pgno_t) - 1) + +static void __cold mdbx_setup_pagesize(MDBX_env *env, const size_t pagesize) { + STATIC_ASSERT(PTRDIFF_MAX > MAX_MAPSIZE); + STATIC_ASSERT(MIN_PAGESIZE > sizeof(MDBX_page) + sizeof(MDBX_meta)); + mdbx_ensure(env, is_powerof2(pagesize)); + mdbx_ensure(env, pagesize >= MIN_PAGESIZE); + mdbx_ensure(env, pagesize <= MAX_PAGESIZE); + env->me_psize = (unsigned)pagesize; + + STATIC_ASSERT(mdbx_maxgc_ov1page(MIN_PAGESIZE) > 4); + STATIC_ASSERT(mdbx_maxgc_ov1page(MAX_PAGESIZE) < MDBX_DPL_TXNFULL); + const intptr_t maxgc_ov1page = (pagesize - PAGEHDRSZ) / sizeof(pgno_t) - 1; + mdbx_ensure(env, + maxgc_ov1page > 42 && maxgc_ov1page < (intptr_t)MDBX_DPL_TXNFULL); + env->me_maxgc_ov1page = (unsigned)maxgc_ov1page; + + STATIC_ASSERT(mdbx_nodemax(MIN_PAGESIZE) > 42); + STATIC_ASSERT(mdbx_nodemax(MAX_PAGESIZE) < UINT16_MAX); + const intptr_t nodemax = mdbx_nodemax(pagesize); + mdbx_ensure(env, nodemax > 42 && nodemax < UINT16_MAX && nodemax % 2 == 0); + env->me_nodemax = (unsigned)nodemax; + + STATIC_ASSERT(mdbx_maxkey(MIN_PAGESIZE) > 42); + STATIC_ASSERT(mdbx_maxkey(MIN_PAGESIZE) < MIN_PAGESIZE); + STATIC_ASSERT(mdbx_maxkey(MAX_PAGESIZE) > 42); + STATIC_ASSERT(mdbx_maxkey(MAX_PAGESIZE) < MAX_PAGESIZE); + const intptr_t maxkey_limit = mdbx_maxkey(env->me_nodemax); + mdbx_ensure(env, maxkey_limit > 42 && (size_t)maxkey_limit < pagesize && + maxkey_limit % 2 == 0); + env->me_maxkey_limit = (unsigned)maxkey_limit; + + env->me_psize2log = log2n(pagesize); + mdbx_assert(env, pgno2bytes(env, 1) == pagesize); + mdbx_assert(env, bytes2pgno(env, pagesize + pagesize) == 2); +} + +int __cold mdbx_env_create(MDBX_env **penv) { + MDBX_env *env = mdbx_calloc(1, sizeof(MDBX_env)); + if (unlikely(!env)) + return MDBX_ENOMEM; + + env->me_maxreaders = DEFAULT_READERS; + env->me_maxdbs = env->me_numdbs = CORE_DBS; + env->me_fd = INVALID_HANDLE_VALUE; + env->me_lfd = INVALID_HANDLE_VALUE; + env->me_pid = mdbx_getpid(); + + int rc; + const size_t os_psize = mdbx_syspagesize(); + if (unlikely(!is_powerof2(os_psize) || os_psize < MIN_PAGESIZE)) { + mdbx_error("unsuitable system pagesize %" PRIuPTR, os_psize); + rc = MDBX_INCOMPATIBLE; + goto bailout; + } + env->me_os_psize = (unsigned)os_psize; + mdbx_setup_pagesize(env, env->me_os_psize); + + rc = mdbx_fastmutex_init(&env->me_dbi_lock); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + +#if defined(_WIN32) || defined(_WIN64) + mdbx_srwlock_Init(&env->me_remap_guard); + InitializeCriticalSection(&env->me_windowsbug_lock); +#else + rc = mdbx_fastmutex_init(&env->me_remap_guard); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_fastmutex_destroy(&env->me_dbi_lock); + goto bailout; + } + rc = mdbx_fastmutex_init(&env->me_lckless_stub.wmutex); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_fastmutex_destroy(&env->me_remap_guard); + mdbx_fastmutex_destroy(&env->me_dbi_lock); + goto bailout; + } +#endif /* Windows */ + + VALGRIND_CREATE_MEMPOOL(env, 0, 0); + env->me_signature = MDBX_ME_SIGNATURE; + *penv = env; + return MDBX_SUCCESS; + +bailout: + mdbx_free(env); + *penv = nullptr; + return rc; +} + +__cold LIBMDBX_API int +mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, + intptr_t size_upper, intptr_t growth_step, + intptr_t shrink_threshold, intptr_t pagesize) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + +#if MDBX_TXN_CHECKPID + if (unlikely(env->me_pid != mdbx_getpid())) + env->me_flags |= MDBX_FATAL_ERROR; +#endif /* MDBX_TXN_CHECKPID */ + + if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) + return MDBX_PANIC; + + const bool inside_txn = + (env->me_txn0 && env->me_txn0->mt_owner == mdbx_thread_self()); + +#if MDBX_DEBUG + if (growth_step < 0) + growth_step = 1; + if (shrink_threshold < 0) + shrink_threshold = 1; +#endif + + bool need_unlock = false; + int rc = MDBX_PROBLEM; + if (env->me_map) { + /* env already mapped */ + if (unlikely(env->me_flags & MDBX_RDONLY)) + return MDBX_EACCESS; + + if (!inside_txn) { + int err = mdbx_txn_lock(env, false); + if (unlikely(err != MDBX_SUCCESS)) + return err; + need_unlock = true; + } + MDBX_meta *head = mdbx_meta_head(env); + if (!inside_txn) { + env->me_txn0->mt_txnid = meta_txnid(env, head, false); + mdbx_find_oldest(env->me_txn0); + } + + /* get untouched params from DB */ + if (pagesize < 0) + pagesize = env->me_psize; + if (size_lower < 0) + size_lower = pgno2bytes(env, head->mm_geo.lower); + if (size_now < 0) + size_now = pgno2bytes(env, head->mm_geo.now); + if (size_upper < 0) + size_upper = pgno2bytes(env, head->mm_geo.upper); + if (growth_step < 0) + growth_step = pgno2bytes(env, head->mm_geo.grow); + if (shrink_threshold < 0) + shrink_threshold = pgno2bytes(env, head->mm_geo.shrink); + + if (pagesize != (intptr_t)env->me_psize) { + rc = MDBX_EINVAL; + goto bailout; + } + const size_t usedbytes = + pgno2bytes(env, mdbx_find_largest(env, head->mm_geo.next)); + if ((size_t)size_upper < usedbytes) { + rc = MDBX_MAP_FULL; + goto bailout; + } + if ((size_t)size_now < usedbytes) + size_now = usedbytes; + } else { + /* env NOT yet mapped */ + if (unlikely(inside_txn)) + return MDBX_PANIC; + + if (pagesize < 0) { + pagesize = env->me_os_psize; + if ((uintptr_t)pagesize > MAX_PAGESIZE) + pagesize = MAX_PAGESIZE; + mdbx_assert(env, (uintptr_t)pagesize >= MIN_PAGESIZE); + } + } + + if (pagesize == 0) + pagesize = MIN_PAGESIZE; + else if (pagesize == INTPTR_MAX) + pagesize = MAX_PAGESIZE; + + if (pagesize < (intptr_t)MIN_PAGESIZE || pagesize > (intptr_t)MAX_PAGESIZE || + !is_powerof2(pagesize)) { + rc = MDBX_EINVAL; + goto bailout; + } + + if (size_lower <= 0) { + size_lower = MIN_MAPSIZE; + if (MIN_MAPSIZE / pagesize < MIN_PAGENO) + size_lower = MIN_PAGENO * pagesize; + } + + if (size_now <= 0) { + size_now = DEFAULT_MAPSIZE; + if (size_now < size_lower) + size_now = size_lower; + if (size_upper >= size_lower && size_now > size_upper) + size_now = size_upper; + } + + if (size_upper <= 0) { + if ((size_t)size_now >= MAX_MAPSIZE / 2) + size_upper = MAX_MAPSIZE; + else if (MAX_MAPSIZE != MAX_MAPSIZE32 && + (size_t)size_now >= MAX_MAPSIZE32 / 2 && + (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3) + size_upper = MAX_MAPSIZE32; + else { + size_upper = size_now + size_now; + if ((size_t)size_upper < DEFAULT_MAPSIZE * 2) + size_upper = DEFAULT_MAPSIZE * 2; + } + if ((size_t)size_upper / pagesize > MAX_PAGENO) + size_upper = pagesize * MAX_PAGENO; + } + + if (unlikely(size_lower < (intptr_t)MIN_MAPSIZE || size_lower > size_upper)) { + rc = MDBX_EINVAL; + goto bailout; + } + + if ((uint64_t)size_lower / pagesize < MIN_PAGENO) { + rc = MDBX_EINVAL; + goto bailout; + } + + if (unlikely((size_t)size_upper > MAX_MAPSIZE || + (uint64_t)size_upper / pagesize > MAX_PAGENO)) { + rc = MDBX_TOO_LARGE; + goto bailout; + } + + size_lower = roundup_powerof2(size_lower, env->me_os_psize); + size_upper = roundup_powerof2(size_upper, env->me_os_psize); + size_now = roundup_powerof2(size_now, env->me_os_psize); + + /* LY: подбираем значение size_upper: + * - кратное размеру ÑиÑтемной Ñтраницы + * - без Ð½Ð°Ñ€ÑƒÑˆÐµÐ½Ð¸Ñ MAX_MAPSIZE и MAX_PAGENO */ + while (unlikely((size_t)size_upper > MAX_MAPSIZE || + (uint64_t)size_upper / pagesize > MAX_PAGENO)) { + if ((size_t)size_upper < env->me_os_psize + MIN_MAPSIZE || + (size_t)size_upper < env->me_os_psize * (MIN_PAGENO + 1)) { + /* паранойа на Ñлучай Ð¿ÐµÑ€ÐµÐ¿Ð¾Ð»Ð½ÐµÐ½Ð¸Ñ Ð¿Ñ€Ð¸ невероÑтных значениÑÑ… */ + rc = MDBX_EINVAL; + goto bailout; + } + size_upper -= env->me_os_psize; + if ((size_t)size_upper < (size_t)size_lower) + size_lower = size_upper; + } + mdbx_assert(env, (size_upper - size_lower) % env->me_os_psize == 0); + + if (size_now < size_lower) + size_now = size_lower; + if (size_now > size_upper) + size_now = size_upper; + + if (growth_step < 0) { + growth_step = ((size_t)(size_upper - size_lower)) / 42; + if (growth_step > size_lower) + growth_step = size_lower; + if (growth_step < 65536) + growth_step = 65536; + if ((size_t)growth_step > MEGABYTE * 16) + growth_step = MEGABYTE * 16; + } + if (growth_step == 0 && shrink_threshold > 0) + growth_step = 1; + growth_step = roundup_powerof2(growth_step, env->me_os_psize); + if (bytes2pgno(env, growth_step) > UINT16_MAX) + growth_step = pgno2bytes(env, UINT16_MAX); + + if (shrink_threshold < 0) + shrink_threshold = growth_step + growth_step; + shrink_threshold = roundup_powerof2(shrink_threshold, env->me_os_psize); + if (bytes2pgno(env, shrink_threshold) > UINT16_MAX) + shrink_threshold = pgno2bytes(env, UINT16_MAX); + + /* save user's geo-params for future open/create */ + env->me_dbgeo.lower = size_lower; + env->me_dbgeo.now = size_now; + env->me_dbgeo.upper = size_upper; + env->me_dbgeo.grow = growth_step; + env->me_dbgeo.shrink = shrink_threshold; + rc = MDBX_SUCCESS; + + mdbx_ensure(env, pagesize >= MIN_PAGESIZE); + mdbx_ensure(env, pagesize <= MAX_PAGESIZE); + mdbx_ensure(env, is_powerof2(pagesize)); + mdbx_ensure(env, is_powerof2(env->me_os_psize)); + + mdbx_ensure(env, env->me_dbgeo.lower >= MIN_MAPSIZE); + mdbx_ensure(env, env->me_dbgeo.lower / pagesize >= MIN_PAGENO); + mdbx_ensure(env, env->me_dbgeo.lower % pagesize == 0); + mdbx_ensure(env, env->me_dbgeo.lower % env->me_os_psize == 0); + + mdbx_ensure(env, env->me_dbgeo.upper <= MAX_MAPSIZE); + mdbx_ensure(env, env->me_dbgeo.upper / pagesize <= MAX_PAGENO); + mdbx_ensure(env, env->me_dbgeo.upper % pagesize == 0); + mdbx_ensure(env, env->me_dbgeo.upper % env->me_os_psize == 0); + + mdbx_ensure(env, env->me_dbgeo.now >= env->me_dbgeo.lower); + mdbx_ensure(env, env->me_dbgeo.now <= env->me_dbgeo.upper); + mdbx_ensure(env, env->me_dbgeo.now % pagesize == 0); + mdbx_ensure(env, env->me_dbgeo.now % env->me_os_psize == 0); + + mdbx_ensure(env, env->me_dbgeo.grow % pagesize == 0); + mdbx_ensure(env, env->me_dbgeo.grow % env->me_os_psize == 0); + mdbx_ensure(env, env->me_dbgeo.shrink % pagesize == 0); + mdbx_ensure(env, env->me_dbgeo.shrink % env->me_os_psize == 0); + + if (env->me_map) { + /* apply new params to opened environment */ + mdbx_ensure(env, pagesize == (intptr_t)env->me_psize); + MDBX_meta meta; + MDBX_meta *head = nullptr; + const mdbx_geo_t *current_geo; + if (inside_txn) { + current_geo = &env->me_txn->mt_geo; + } else { + head = mdbx_meta_head(env); + meta = *head; + current_geo = &meta.mm_geo; + } + + mdbx_geo_t new_geo; + new_geo.lower = bytes2pgno(env, env->me_dbgeo.lower); + new_geo.now = bytes2pgno(env, env->me_dbgeo.now); + new_geo.upper = bytes2pgno(env, env->me_dbgeo.upper); + new_geo.grow = (uint16_t)bytes2pgno(env, env->me_dbgeo.grow); + new_geo.shrink = (uint16_t)bytes2pgno(env, env->me_dbgeo.shrink); + new_geo.next = current_geo->next; + + mdbx_ensure(env, + pgno_align2os_bytes(env, new_geo.lower) == env->me_dbgeo.lower); + mdbx_ensure(env, + pgno_align2os_bytes(env, new_geo.upper) == env->me_dbgeo.upper); + mdbx_ensure(env, + pgno_align2os_bytes(env, new_geo.now) == env->me_dbgeo.now); + mdbx_ensure(env, + pgno_align2os_bytes(env, new_geo.grow) == env->me_dbgeo.grow); + mdbx_ensure(env, pgno_align2os_bytes(env, new_geo.shrink) == + env->me_dbgeo.shrink); + + mdbx_ensure(env, env->me_dbgeo.lower >= MIN_MAPSIZE); + mdbx_ensure(env, new_geo.lower >= MIN_PAGENO); + mdbx_ensure(env, env->me_dbgeo.upper <= MAX_MAPSIZE); + mdbx_ensure(env, new_geo.upper <= MAX_PAGENO); + mdbx_ensure(env, new_geo.now >= new_geo.next); + mdbx_ensure(env, new_geo.upper >= new_geo.now); + mdbx_ensure(env, new_geo.now >= new_geo.lower); + + if (memcmp(current_geo, &new_geo, sizeof(mdbx_geo_t)) != 0) { +#if defined(_WIN32) || defined(_WIN64) + /* Was DB shrinking disabled before and now it will be enabled? */ + if (new_geo.lower < new_geo.upper && new_geo.shrink && + !(current_geo->lower < current_geo->upper && current_geo->shrink)) { + if (!env->me_lck) { + rc = MDBX_EPERM; + goto bailout; + } + rc = mdbx_rdt_lock(env); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + /* Check if there are any reading threads that do not use the SRWL */ + const size_t CurrentTid = GetCurrentThreadId(); + const MDBX_reader *const begin = env->me_lck->mti_readers; + const MDBX_reader *const end = begin + env->me_lck->mti_numreaders; + for (const MDBX_reader *reader = begin; reader < end; ++reader) { + if (reader->mr_pid == env->me_pid && reader->mr_tid && + reader->mr_tid != CurrentTid) { + /* At least one thread may don't use SRWL */ + rc = MDBX_EPERM; + break; + } + } + + mdbx_rdt_unlock(env); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } +#endif + + if (new_geo.now != current_geo->now || + new_geo.upper != current_geo->upper) { + rc = mdbx_mapresize(env, new_geo.now, new_geo.upper); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + mdbx_assert(env, (head == nullptr) == inside_txn); + if (head) + head = /* base address could be changed */ mdbx_meta_head(env); + } + if (inside_txn) { + env->me_txn->mt_geo = new_geo; + if ((env->me_txn->mt_flags & MDBX_TXN_DIRTY) == 0) { + env->me_txn->mt_flags |= MDBX_TXN_DIRTY; + *env->me_unsynced_pages += 1; + } + } else { + *env->me_unsynced_pages += 1; + mdbx_meta_set_txnid( + env, &meta, safe64_txnid_next(mdbx_meta_txnid_stable(env, head))); + rc = mdbx_sync_locked(env, env->me_flags, &meta); + } + } + } else if (pagesize != (intptr_t)env->me_psize) { + mdbx_setup_pagesize(env, pagesize); + } + +bailout: + if (need_unlock) + mdbx_txn_unlock(env); + return rc; +} + +int __cold mdbx_env_set_mapsize(MDBX_env *env, size_t size) { + return mdbx_env_set_geometry(env, size, size, size, -1, -1, -1); +} + +int __cold mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) { + if (unlikely(dbs > MAX_DBI)) + return MDBX_EINVAL; + + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely(env->me_map)) + return MDBX_EPERM; + + env->me_maxdbs = dbs + CORE_DBS; + return MDBX_SUCCESS; +} + +int __cold mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) { + if (unlikely(readers < 1 || readers > MDBX_READERS_LIMIT)) + return MDBX_EINVAL; + + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely(env->me_map)) + return MDBX_EPERM; + + env->me_maxreaders = readers; + return MDBX_SUCCESS; +} + +int __cold mdbx_env_get_maxreaders(MDBX_env *env, unsigned *readers) { + if (!env || !readers) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + *readers = env->me_maxreaders; + return MDBX_SUCCESS; +} + +/* Further setup required for opening an MDBX environment */ +static int __cold mdbx_setup_dxb(MDBX_env *env, const int lck_rc) { + uint64_t filesize_before; + MDBX_meta meta; + int rc = MDBX_RESULT_FALSE; + int err = mdbx_read_header(env, &meta, &filesize_before); + if (unlikely(err != MDBX_SUCCESS)) { + if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA || + (env->me_flags & MDBX_RDONLY) != 0) + return err; + + mdbx_debug("%s", "create new database"); + rc = /* new database */ MDBX_RESULT_TRUE; + + if (!env->me_dbgeo.now) { + /* set defaults if not configured */ + err = mdbx_env_set_mapsize(env, DEFAULT_MAPSIZE); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + + void *buffer = mdbx_calloc(NUM_METAS, env->me_psize); + if (!buffer) + return MDBX_ENOMEM; + + meta = *mdbx_init_metas(env, buffer); + err = mdbx_pwrite(env->me_fd, buffer, env->me_psize * NUM_METAS, 0); + mdbx_free(buffer); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + err = mdbx_ftruncate(env->me_fd, filesize_before = env->me_dbgeo.now); + if (unlikely(err != MDBX_SUCCESS)) + return err; + +#ifndef NDEBUG /* just for checking */ + err = mdbx_read_header(env, &meta, &filesize_before); + if (unlikely(err != MDBX_SUCCESS)) + return err; +#endif + } + + mdbx_verbose("header: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO + "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO + " +%u -%u, txn_id %" PRIaTXN ", %s", + meta.mm_dbs[MAIN_DBI].md_root, meta.mm_dbs[FREE_DBI].md_root, + meta.mm_geo.lower, meta.mm_geo.next, meta.mm_geo.now, + meta.mm_geo.upper, meta.mm_geo.grow, meta.mm_geo.shrink, + meta.mm_txnid_a.inconsistent, mdbx_durable_str(&meta)); + + mdbx_setup_pagesize(env, meta.mm_psize); + const size_t used_bytes = pgno2bytes(env, meta.mm_geo.next); + const size_t used_aligned2os_bytes = + roundup_powerof2(used_bytes, env->me_os_psize); + if ((env->me_flags & MDBX_RDONLY) /* readonly */ + || lck_rc != MDBX_RESULT_TRUE /* not exclusive */) { + /* use present params from db */ + const size_t pagesize = meta.mm_psize; + err = mdbx_env_set_geometry( + env, meta.mm_geo.lower * pagesize, meta.mm_geo.now * pagesize, + meta.mm_geo.upper * pagesize, meta.mm_geo.grow * pagesize, + meta.mm_geo.shrink * pagesize, meta.mm_psize); + if (unlikely(err != MDBX_SUCCESS)) { + mdbx_error("%s", "could not use present dbsize-params from db"); + return MDBX_INCOMPATIBLE; + } + } else if (env->me_dbgeo.now) { + /* silently growth to last used page */ + if (env->me_dbgeo.now < used_aligned2os_bytes) + env->me_dbgeo.now = used_aligned2os_bytes; + if (env->me_dbgeo.upper < used_aligned2os_bytes) + env->me_dbgeo.upper = used_aligned2os_bytes; + + /* apply preconfigured params, but only if substantial changes: + * - upper or lower limit changes + * - shrink threshold or growth step + * But ignore change just a 'now/current' size. */ + if (bytes_align2os_bytes(env, env->me_dbgeo.upper) != + pgno2bytes(env, meta.mm_geo.upper) || + bytes_align2os_bytes(env, env->me_dbgeo.lower) != + pgno2bytes(env, meta.mm_geo.lower) || + bytes_align2os_bytes(env, env->me_dbgeo.shrink) != + pgno2bytes(env, meta.mm_geo.shrink) || + bytes_align2os_bytes(env, env->me_dbgeo.grow) != + pgno2bytes(env, meta.mm_geo.grow)) { + + if (env->me_dbgeo.shrink && env->me_dbgeo.now > used_bytes) + /* pre-shrink if enabled */ + env->me_dbgeo.now = used_bytes + env->me_dbgeo.shrink - + used_bytes % env->me_dbgeo.shrink; + + err = mdbx_env_set_geometry(env, env->me_dbgeo.lower, env->me_dbgeo.now, + env->me_dbgeo.upper, env->me_dbgeo.grow, + env->me_dbgeo.shrink, meta.mm_psize); + if (unlikely(err != MDBX_SUCCESS)) { + mdbx_error("%s", "could not apply preconfigured dbsize-params to db"); + return MDBX_INCOMPATIBLE; + } + + /* update meta fields */ + meta.mm_geo.now = bytes2pgno(env, env->me_dbgeo.now); + meta.mm_geo.lower = bytes2pgno(env, env->me_dbgeo.lower); + meta.mm_geo.upper = bytes2pgno(env, env->me_dbgeo.upper); + meta.mm_geo.grow = (uint16_t)bytes2pgno(env, env->me_dbgeo.grow); + meta.mm_geo.shrink = (uint16_t)bytes2pgno(env, env->me_dbgeo.shrink); + + mdbx_verbose("amended: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO + "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO + " +%u -%u, txn_id %" PRIaTXN ", %s", + meta.mm_dbs[MAIN_DBI].md_root, meta.mm_dbs[FREE_DBI].md_root, + meta.mm_geo.lower, meta.mm_geo.next, meta.mm_geo.now, + meta.mm_geo.upper, meta.mm_geo.grow, meta.mm_geo.shrink, + meta.mm_txnid_a.inconsistent, mdbx_durable_str(&meta)); + } else { + /* fetch back 'now/current' size, since it was ignored during comparison + * and may differ. */ + env->me_dbgeo.now = pgno_align2os_bytes(env, meta.mm_geo.now); + } + mdbx_ensure(env, meta.mm_geo.now >= meta.mm_geo.next); + } else { + /* geo-params are not pre-configured by user, + * get current values from the meta. */ + env->me_dbgeo.now = pgno2bytes(env, meta.mm_geo.now); + env->me_dbgeo.lower = pgno2bytes(env, meta.mm_geo.lower); + env->me_dbgeo.upper = pgno2bytes(env, meta.mm_geo.upper); + env->me_dbgeo.grow = pgno2bytes(env, meta.mm_geo.grow); + env->me_dbgeo.shrink = pgno2bytes(env, meta.mm_geo.shrink); + } + + mdbx_ensure(env, + pgno_align2os_bytes(env, meta.mm_geo.now) == env->me_dbgeo.now); + mdbx_ensure(env, env->me_dbgeo.now >= used_bytes); + if (unlikely(filesize_before != env->me_dbgeo.now)) { + if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) { + mdbx_verbose("filesize mismatch (expect %" PRIuPTR "b/%" PRIaPGNO + "p, have %" PRIu64 "b/%" PRIaPGNO "p), " + "assume other process working", + env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now), + filesize_before, bytes2pgno(env, (size_t)filesize_before)); + } else { + mdbx_notice("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO + "p, have %" PRIu64 "b/%" PRIaPGNO "p)", + env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now), + filesize_before, bytes2pgno(env, (size_t)filesize_before)); + if (filesize_before < used_bytes) { + mdbx_error("last-page beyond end-of-file (last %" PRIaPGNO + ", have %" PRIaPGNO ")", + meta.mm_geo.next, bytes2pgno(env, (size_t)filesize_before)); + return MDBX_CORRUPTED; + } + + if (env->me_flags & MDBX_RDONLY) { + if (filesize_before & (env->me_os_psize - 1)) { + mdbx_error("%s", "filesize should be rounded-up to system page"); + return MDBX_WANNA_RECOVERY; + } + mdbx_warning("%s", "ignore filesize mismatch in readonly-mode"); + } else { + mdbx_verbose("will resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO + " pages", + env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now)); + } + } + } + + err = mdbx_mmap(env->me_flags, &env->me_dxb_mmap, env->me_dbgeo.now, + env->me_dbgeo.upper, lck_rc); + if (unlikely(err != MDBX_SUCCESS)) + return err; + +#if defined(MADV_DODUMP) && defined(MADV_DONTDUMP) + const size_t meta_length = pgno2bytes(env, NUM_METAS); + (void)madvise(env->me_map, meta_length, MADV_DODUMP); + (void)madvise(env->me_map + meta_length, env->me_dxb_mmap.limit - meta_length, + (mdbx_runtime_flags & MDBX_DBG_DUMP) ? MADV_DODUMP + : MADV_DONTDUMP); +#endif + + *env->me_discarded_tail = bytes2pgno(env, used_aligned2os_bytes); + if (used_aligned2os_bytes < env->me_dxb_mmap.current) { +#if defined(MADV_REMOVE) + if (lck_rc && (env->me_flags & MDBX_WRITEMAP) != 0) { + mdbx_notice("open-MADV_%s %u..%u", "REMOVE", *env->me_discarded_tail, + bytes2pgno(env, env->me_dxb_mmap.current)); + (void)madvise(env->me_map + used_aligned2os_bytes, + env->me_dxb_mmap.current - used_aligned2os_bytes, + MADV_REMOVE); + } +#endif /* MADV_REMOVE */ +#if defined(MADV_DONTNEED) + mdbx_notice("open-MADV_%s %u..%u", "DONTNEED", *env->me_discarded_tail, + bytes2pgno(env, env->me_dxb_mmap.current)); + (void)madvise(env->me_map + used_aligned2os_bytes, + env->me_dxb_mmap.current - used_aligned2os_bytes, + MADV_DONTNEED); +#elif defined(POSIX_MADV_DONTNEED) + (void)madvise(env->me_map + used_aligned2os_bytes, + env->me_dxb_mmap.current - used_aligned2os_bytes, + POSIX_MADV_DONTNEED); +#elif defined(POSIX_FADV_DONTNEED) + (void)posix_fadvise(env->me_fd, used_aligned2os_bytes, + env->me_dxb_mmap.current - used_aligned2os_bytes, + POSIX_FADV_DONTNEED); +#endif /* MADV_DONTNEED */ + } + +#ifdef MDBX_USE_VALGRIND + env->me_valgrind_handle = + VALGRIND_CREATE_BLOCK(env->me_map, env->me_dxb_mmap.limit, "mdbx"); +#endif + + const bool readahead = (env->me_flags & MDBX_NORDAHEAD) == 0 && + mdbx_is_readahead_reasonable(env->me_dxb_mmap.current, + 0) == MDBX_RESULT_TRUE; + err = mdbx_set_readahead(env, 0, env->me_dxb_mmap.current, readahead); + if (err != MDBX_SUCCESS) + return err; + + mdbx_assert(env, used_bytes >= pgno2bytes(env, NUM_METAS) && + used_bytes <= env->me_dxb_mmap.limit); +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + VALGRIND_MAKE_MEM_NOACCESS(env->me_map + used_bytes, + env->me_dxb_mmap.limit - used_bytes); + ASAN_POISON_MEMORY_REGION(env->me_map + used_bytes, + env->me_dxb_mmap.limit - used_bytes); + env->me_poison_edge = bytes2pgno(env, env->me_dxb_mmap.limit); +#endif /* MDBX_USE_VALGRIND */ + + /* NOTE: AddressSanitizer (at least GCC 7.x, 8.x) could generate + * false-positive alarm here. I have no other explanation for this + * except due to an internal ASAN error, as the problem is reproduced + * in a single-threaded application under the active assert() above. */ + const unsigned meta_clash_mask = mdbx_meta_eq_mask(env); + if (meta_clash_mask) { + mdbx_error("meta-pages are clashed: mask 0x%d", meta_clash_mask); + return MDBX_WANNA_RECOVERY; + } + + while (1) { + MDBX_meta *head = mdbx_meta_head(env); + const txnid_t head_txnid = mdbx_meta_txnid_fluid(env, head); + if (head_txnid == meta.mm_txnid_a.inconsistent) + break; + + if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) { + mdbx_assert(env, META_IS_STEADY(&meta) && !META_IS_STEADY(head)); + if (env->me_flags & MDBX_RDONLY) { + mdbx_error("rollback needed: (from head %" PRIaTXN + " to steady %" PRIaTXN "), but unable in read-only mode", + head_txnid, meta.mm_txnid_a.inconsistent); + return MDBX_WANNA_RECOVERY /* LY: could not recovery/rollback */; + } + + const MDBX_meta *const meta0 = METAPAGE(env, 0); + const MDBX_meta *const meta1 = METAPAGE(env, 1); + const MDBX_meta *const meta2 = METAPAGE(env, 2); + txnid_t undo_txnid = 0 /* zero means undo is unneeded */; + while ( + (head != meta0 && mdbx_meta_txnid_fluid(env, meta0) == undo_txnid) || + (head != meta1 && mdbx_meta_txnid_fluid(env, meta1) == undo_txnid) || + (head != meta2 && mdbx_meta_txnid_fluid(env, meta2) == undo_txnid)) + undo_txnid = safe64_txnid_next(undo_txnid); + if (unlikely(undo_txnid >= meta.mm_txnid_a.inconsistent)) { + mdbx_fatal("rollback failed: no suitable txnid (0,1,2) < %" PRIaTXN, + meta.mm_txnid_a.inconsistent); + return MDBX_PANIC /* LY: could not recovery/rollback */; + } + + /* LY: rollback weak checkpoint */ + mdbx_trace("rollback: from %" PRIaTXN ", to %" PRIaTXN " as %" PRIaTXN, + head_txnid, meta.mm_txnid_a.inconsistent, undo_txnid); + mdbx_ensure(env, head_txnid == mdbx_meta_txnid_stable(env, head)); + + if (env->me_flags & MDBX_WRITEMAP) { + /* It is possible to update txnid without safe64_write(), + * since DB opened exclusive for now */ + head->mm_txnid_a.inconsistent = undo_txnid; + head->mm_datasync_sign = MDBX_DATASIGN_WEAK; + head->mm_txnid_b.inconsistent = undo_txnid; + const size_t offset = (uint8_t *)data_page(head) - env->me_dxb_mmap.dxb; + const size_t paged_offset = offset & ~(env->me_os_psize - 1); + const size_t paged_length = roundup_powerof2( + env->me_psize + offset - paged_offset, env->me_os_psize); + err = mdbx_msync(&env->me_dxb_mmap, paged_offset, paged_length, false); + } else { + MDBX_meta rollback = *head; + mdbx_meta_set_txnid(env, &rollback, undo_txnid); + rollback.mm_datasync_sign = MDBX_DATASIGN_WEAK; + err = mdbx_pwrite(env->me_fd, &rollback, sizeof(MDBX_meta), + (uint8_t *)head - (uint8_t *)env->me_map); + } + if (err) { + mdbx_error("error %d rollback from %" PRIaTXN ", to %" PRIaTXN + " as %" PRIaTXN, + err, head_txnid, meta.mm_txnid_a.inconsistent, undo_txnid); + return err; + } + + mdbx_invalidate_mmap_noncoherent_cache(env->me_map, + pgno2bytes(env, NUM_METAS)); + mdbx_ensure(env, undo_txnid == mdbx_meta_txnid_fluid(env, head)); + mdbx_ensure(env, 0 == mdbx_meta_eq_mask(env)); + continue; + } + + if (!env->me_lck) { + /* LY: without-lck (read-only) mode, so it is imposible that other + * process made weak checkpoint. */ + mdbx_error("%s", "without-lck, unable recovery/rollback"); + return MDBX_WANNA_RECOVERY; + } + + /* LY: assume just have a collision with other running process, + * or someone make a weak checkpoint */ + mdbx_verbose("%s", "assume collision or online weak checkpoint"); + break; + } + + const MDBX_meta *head = mdbx_meta_head(env); + if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) { + /* re-check size after mmap */ + if ((env->me_dxb_mmap.current & (env->me_os_psize - 1)) != 0 || + env->me_dxb_mmap.current < used_bytes) { + mdbx_error("unacceptable/unexpected datafile size %" PRIuPTR, + env->me_dxb_mmap.current); + return MDBX_PROBLEM; + } + if (env->me_dxb_mmap.current != env->me_dbgeo.now && + (env->me_flags & MDBX_RDONLY) == 0) { + meta.mm_geo.now = bytes2pgno(env, env->me_dxb_mmap.current); + mdbx_verbose("update meta-geo to filesize %" PRIuPTR " bytes, %" PRIaPGNO + " pages", + env->me_dxb_mmap.current, meta.mm_geo.now); + } + + if (memcmp(&meta.mm_geo, &head->mm_geo, sizeof(meta.mm_geo))) { + const txnid_t txnid = mdbx_meta_txnid_stable(env, head); + const txnid_t next_txnid = safe64_txnid_next(txnid); + mdbx_verbose("updating meta.geo: " + "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u (txn#%" PRIaTXN "), " + "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u (txn#%" PRIaTXN ")", + head->mm_geo.lower, head->mm_geo.now, head->mm_geo.upper, + head->mm_geo.shrink, head->mm_geo.grow, txnid, + meta.mm_geo.lower, meta.mm_geo.now, meta.mm_geo.upper, + meta.mm_geo.shrink, meta.mm_geo.grow, next_txnid); + + mdbx_ensure(env, mdbx_meta_eq(env, &meta, head)); + mdbx_meta_set_txnid(env, &meta, next_txnid); + *env->me_unsynced_pages += 1; + err = mdbx_sync_locked(env, env->me_flags | MDBX_SHRINK_ALLOWED, &meta); + if (err) { + mdbx_error("error %d, while updating meta.geo: " + "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u (txn#%" PRIaTXN "), " + "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u (txn#%" PRIaTXN ")", + err, head->mm_geo.lower, head->mm_geo.now, + head->mm_geo.upper, head->mm_geo.shrink, head->mm_geo.grow, + txnid, meta.mm_geo.lower, meta.mm_geo.now, meta.mm_geo.upper, + meta.mm_geo.shrink, meta.mm_geo.grow, next_txnid); + return err; + } + } + } + + return rc; +} + +/******************************************************************************/ + +/* Open and/or initialize the lock region for the environment. */ +static int __cold mdbx_setup_lck(MDBX_env *env, char *lck_pathname, + mode_t mode) { + mdbx_assert(env, env->me_fd != INVALID_HANDLE_VALUE); + mdbx_assert(env, env->me_lfd == INVALID_HANDLE_VALUE); + + int err = mdbx_openfile(lck_pathname, O_RDWR | O_CREAT, mode, &env->me_lfd, + (env->me_flags & MDBX_EXCLUSIVE) ? true : false); + if (err != MDBX_SUCCESS) { + if (!(err == MDBX_ENOFILE && (env->me_flags & MDBX_EXCLUSIVE)) && + !((err == MDBX_EROFS || err == MDBX_EACCESS || err == MDBX_EPERM) && + (env->me_flags & MDBX_RDONLY))) + return err; + + /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ + /* beginning of a locked section ---------------------------------------- */ + lcklist_lock(); + mdbx_assert(env, env->me_lcklist_next == nullptr); + env->me_lfd = INVALID_HANDLE_VALUE; + const int rc = mdbx_lck_seize(env); + if (MDBX_IS_ERROR(rc)) { + /* Calling lcklist_detach_locked() is required to restore POSIX-filelock + * and this job will be done by mdbx_env_close0(). */ + lcklist_unlock(); + return rc; + } + /* insert into inprocess lck-list */ + env->me_lcklist_next = inprocess_lcklist_head; + inprocess_lcklist_head = env; + lcklist_unlock(); + /* end of a locked section ---------------------------------------------- */ + + env->me_oldest = &env->me_lckless_stub.oldest; + env->me_sync_timestamp = &env->me_lckless_stub.sync_timestamp; + env->me_autosync_period = &env->me_lckless_stub.autosync_period; + env->me_unsynced_pages = &env->me_lckless_stub.autosync_pending; + env->me_autosync_threshold = &env->me_lckless_stub.autosync_threshold; + env->me_discarded_tail = &env->me_lckless_stub.discarded_tail; + env->me_meta_sync_txnid = &env->me_lckless_stub.meta_sync_txnid; + env->me_maxreaders = UINT_MAX; +#ifdef MDBX_OSAL_LOCK + env->me_wmutex = &env->me_lckless_stub.wmutex; +#endif + mdbx_debug("lck-setup:%s%s%s", " lck-less", + (env->me_flags & MDBX_RDONLY) ? " readonly" : "", + (rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative"); + return rc; + } + + /* beginning of a locked section ------------------------------------------ */ + lcklist_lock(); + mdbx_assert(env, env->me_lcklist_next == nullptr); + + /* Try to get exclusive lock. If we succeed, then + * nobody is using the lock region and we should initialize it. */ + err = mdbx_lck_seize(env); + if (MDBX_IS_ERROR(err)) { + bailout: + /* Calling lcklist_detach_locked() is required to restore POSIX-filelock + * and this job will be done by mdbx_env_close0(). */ + lcklist_unlock(); + return err; + } + + MDBX_env *inprocess_neighbor = nullptr; + if (err == MDBX_RESULT_TRUE) { + err = uniq_check(&env->me_lck_mmap, &inprocess_neighbor); + if (MDBX_IS_ERROR(err)) + goto bailout; + if (inprocess_neighbor && + ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 || + (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) != 0)) { + err = MDBX_BUSY; + goto bailout; + } + } + const int lck_seize_rc = err; + + mdbx_debug("lck-setup:%s%s%s", " with-lck", + (env->me_flags & MDBX_RDONLY) ? " readonly" : "", + (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" + : " cooperative"); + + uint64_t size; + err = mdbx_filesize(env->me_lfd, &size); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; + + if (lck_seize_rc == MDBX_RESULT_TRUE) { + size = roundup_powerof2(env->me_maxreaders * sizeof(MDBX_reader) + + sizeof(MDBX_lockinfo), + env->me_os_psize); +#ifndef NDEBUG + err = mdbx_ftruncate(env->me_lfd, 0); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; +#endif + mdbx_jitter4testing(false); + } else { + if (env->me_flags & MDBX_EXCLUSIVE) { + err = MDBX_BUSY; + goto bailout; + } + if (size > INT_MAX || (size & (env->me_os_psize - 1)) != 0 || + size < env->me_os_psize) { + mdbx_error("lck-file has invalid size %" PRIu64 " bytes", size); + err = MDBX_PROBLEM; + goto bailout; + } + } + + const size_t maxreaders = + ((size_t)size - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader); + if (size > 65536 || maxreaders < 2 || maxreaders > MDBX_READERS_LIMIT) { + mdbx_error("lck-size too big (up to %" PRIuPTR " readers)", maxreaders); + err = MDBX_PROBLEM; + goto bailout; + } + env->me_maxreaders = (unsigned)maxreaders; + + err = mdbx_mmap(MDBX_WRITEMAP, &env->me_lck_mmap, (size_t)size, (size_t)size, + lck_seize_rc); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; + +#ifdef MADV_DODUMP + (void)madvise(env->me_lck, size, MADV_DODUMP); +#endif + +#ifdef MADV_WILLNEED + if (madvise(env->me_lck, size, MADV_WILLNEED) < 0) + goto bailout; +#endif + + if (lck_seize_rc == MDBX_RESULT_TRUE) { + /* LY: exlcusive mode, reset lck */ + memset(env->me_lck, 0, (size_t)size); + mdbx_jitter4testing(false); + env->me_lck->mti_magic_and_version = MDBX_LOCK_MAGIC; + env->me_lck->mti_os_and_format = MDBX_LOCK_FORMAT; + } else { + if (env->me_lck->mti_magic_and_version != MDBX_LOCK_MAGIC) { + mdbx_error("%s", "lock region has invalid magic/version"); + err = ((env->me_lck->mti_magic_and_version >> 8) != MDBX_MAGIC) + ? MDBX_INVALID + : MDBX_VERSION_MISMATCH; + goto bailout; + } + if (env->me_lck->mti_os_and_format != MDBX_LOCK_FORMAT) { + mdbx_error("lock region has os/format 0x%" PRIx32 ", expected 0x%" PRIx32, + env->me_lck->mti_os_and_format, MDBX_LOCK_FORMAT); + err = MDBX_VERSION_MISMATCH; + goto bailout; + } + } + + err = mdbx_lck_init(env, inprocess_neighbor, lck_seize_rc); + if (MDBX_IS_ERROR(err)) + goto bailout; + + mdbx_ensure(env, env->me_lcklist_next == nullptr); + /* insert into inprocess lck-list */ + env->me_lcklist_next = inprocess_lcklist_head; + inprocess_lcklist_head = env; + lcklist_unlock(); + /* end of a locked section ------------------------------------------------ */ + + mdbx_assert(env, !MDBX_IS_ERROR(lck_seize_rc)); + env->me_oldest = &env->me_lck->mti_oldest_reader; + env->me_sync_timestamp = &env->me_lck->mti_sync_timestamp; + env->me_autosync_period = &env->me_lck->mti_autosync_period; + env->me_unsynced_pages = &env->me_lck->mti_unsynced_pages; + env->me_autosync_threshold = &env->me_lck->mti_autosync_threshold; + env->me_discarded_tail = &env->me_lck->mti_discarded_tail; + env->me_meta_sync_txnid = &env->me_lck->mti_meta_sync_txnid; +#ifdef MDBX_OSAL_LOCK + env->me_wmutex = &env->me_lck->mti_wmutex; +#endif + return lck_seize_rc; +} + +__cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) { + if (volume <= 1024 * 1024 * 4ul) + return MDBX_RESULT_TRUE; + + const intptr_t pagesize = mdbx_syspagesize(); + if (unlikely(pagesize < MIN_PAGESIZE || !is_powerof2(pagesize))) + return MDBX_INCOMPATIBLE; + +#if defined(_WIN32) || defined(_WIN64) + MEMORYSTATUSEX info; + memset(&info, 0, sizeof(info)); + info.dwLength = sizeof(info); + if (!GlobalMemoryStatusEx(&info)) + return GetLastError(); +#endif + const int log2page = log2n(pagesize); + +#if defined(_WIN32) || defined(_WIN64) + const intptr_t total_ram_pages = (intptr_t)(info.ullTotalPhys >> log2page); +#elif defined(_SC_PHYS_PAGES) + const intptr_t total_ram_pages = sysconf(_SC_PHYS_PAGES); + if (total_ram_pages == -1) + return errno; +#elif defined(_SC_AIX_REALMEM) + const intptr_t total_ram_Kb = sysconf(_SC_AIX_REALMEM); + if (total_ram_Kb == -1) + return errno; + const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page; +#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || \ + defined(HW_PHYSMEM) + size_t ram, len = sizeof(ram); + static const int mib[] = { + CTL_HW, +#if defined(HW_USERMEM) + HW_USERMEM +#elif defined(HW_PHYSMEM64) + HW_PHYSMEM64 +#elif defined(HW_MEMSIZE) + HW_MEMSIZE +#else + HW_PHYSMEM +#endif + }; + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) +#endif + mib, + ARRAY_LENGTH(mib), &ram, &len, NULL, 0) != 0) + return errno; + if (len != sizeof(ram)) + return MDBX_ENOSYS; + const intptr_t total_ram_pages = (intptr_t)(ram >> log2page); +#else +#error "FIXME: Get User-accessible or physical RAM" +#endif + if (total_ram_pages < 1) + return MDBX_ENOSYS; + + const intptr_t volume_pages = (volume + pagesize - 1) >> log2page; + const intptr_t redundancy_pages = + (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page) + : (intptr_t)(redundancy + pagesize - 1) >> log2page; + if (volume_pages >= total_ram_pages || + volume_pages + redundancy_pages >= total_ram_pages) + return MDBX_RESULT_FALSE; + +#if defined(_WIN32) || defined(_WIN64) + const intptr_t avail_ram_pages = (intptr_t)(info.ullAvailPhys >> log2page); +#elif defined(_SC_AVPHYS_PAGES) + const intptr_t avail_ram_pages = sysconf(_SC_AVPHYS_PAGES); + if (avail_ram_pages == -1) + return errno; +#elif defined(__MACH__) + mach_msg_type_number_t count = HOST_VM_INFO_COUNT; + vm_statistics_data_t vmstat; + mach_port_t mport = mach_host_self(); + kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO, + (host_info_t)&vmstat, &count); + mach_port_deallocate(mach_task_self(), mport); + if (unlikely(kerr != KERN_SUCCESS)) + return MDBX_ENOSYS; + const intptr_t avail_ram_pages = vmstat.free_count; +#elif defined(VM_TOTAL) || defined(VM_METER) + struct vmtotal info; + size_t len = sizeof(info); + static const int mib[] = { + CTL_VM, +#if defined(VM_TOTAL) + VM_TOTAL +#elif defined(VM_METER) + VM_METER +#endif + }; + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) +#endif + mib, + ARRAY_LENGTH(mib), &info, &len, NULL, 0) != 0) + return errno; + if (len != sizeof(info)) + return MDBX_ENOSYS; + const intptr_t avail_ram_pages = info.t_free; +#else +#error "FIXME: Get Available RAM" +#endif + if (avail_ram_pages < 1) + return MDBX_ENOSYS; + + return (volume_pages + redundancy_pages >= avail_ram_pages) + ? MDBX_RESULT_FALSE + : MDBX_RESULT_TRUE; +} + +/* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ +#define CHANGEABLE \ + (MDBX_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC | MDBX_NOMEMINIT | \ + MDBX_COALESCE | MDBX_PAGEPERTURB) +#define CHANGELESS \ + (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) + +#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE | CHANGELESS) +#error "Persistent DB flags & env flags overlap, but both go in mm_flags" +#endif + +int __cold mdbx_env_open(MDBX_env *env, const char *path, unsigned flags, + mode_t mode) { + if (unlikely(!env || !path)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + if (flags & ~(CHANGEABLE | CHANGELESS)) + return MDBX_EINVAL; + + if (env->me_fd != INVALID_HANDLE_VALUE || + (env->me_flags & MDBX_ENV_ACTIVE) != 0) + return MDBX_EPERM; + + size_t len_full, len = strlen(path); + if (flags & MDBX_NOSUBDIR) { + len_full = len + sizeof(MDBX_LOCK_SUFFIX) + len + 1; + } else { + len_full = len + sizeof(MDBX_LOCKNAME) + len + sizeof(MDBX_DATANAME); + } + char *lck_pathname = mdbx_malloc(len_full); + if (!lck_pathname) + return MDBX_ENOMEM; + + char *dxb_pathname; + if (flags & MDBX_NOSUBDIR) { + dxb_pathname = lck_pathname + len + sizeof(MDBX_LOCK_SUFFIX); + sprintf(lck_pathname, "%s" MDBX_LOCK_SUFFIX, path); + strcpy(dxb_pathname, path); + } else { + dxb_pathname = lck_pathname + len + sizeof(MDBX_LOCKNAME); + sprintf(lck_pathname, "%s" MDBX_LOCKNAME, path); + sprintf(dxb_pathname, "%s" MDBX_DATANAME, path); + } + + int rc = MDBX_SUCCESS; + flags |= env->me_flags; + if (flags & MDBX_RDONLY) { + /* LY: silently ignore irrelevant flags when + * we're only getting read access */ + flags &= ~(MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NOSYNC | MDBX_NOMETASYNC | + MDBX_COALESCE | MDBX_LIFORECLAIM | MDBX_NOMEMINIT); + } else { + env->me_dirtylist = mdbx_calloc(MDBX_DPL_TXNFULL + 1, sizeof(MDBX_DP)); + if (!env->me_dirtylist) + rc = MDBX_ENOMEM; + } + + const uint32_t saved_me_flags = env->me_flags; + env->me_flags = (flags & ~MDBX_FATAL_ERROR) | MDBX_ENV_ACTIVE; + if (rc) + goto bailout; + + env->me_path = mdbx_strdup(path); + env->me_dbxs = mdbx_calloc(env->me_maxdbs, sizeof(MDBX_dbx)); + env->me_dbflags = mdbx_calloc(env->me_maxdbs, sizeof(env->me_dbflags[0])); + env->me_dbiseqs = mdbx_calloc(env->me_maxdbs, sizeof(env->me_dbiseqs[0])); + if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { + rc = MDBX_ENOMEM; + goto bailout; + } + env->me_dbxs[FREE_DBI].md_cmp = + mdbx_cmp_int_align4; /* aligned MDBX_INTEGERKEY */ + + int oflags; + if (F_ISSET(flags, MDBX_RDONLY)) + oflags = O_RDONLY; + else if (mode != 0) { + if ((flags & MDBX_NOSUBDIR) == 0) { +#if defined(_WIN32) || defined(_WIN64) + if (!CreateDirectoryA(path, nullptr)) { + rc = GetLastError(); + if (rc != ERROR_ALREADY_EXISTS) + goto bailout; + } +#else + const mode_t dir_mode = + (/* inherit read/write permissions for group and others */ mode & + (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | + /* always add read/write/search for owner */ S_IRWXU | + ((mode & S_IRGRP) ? /* +search if readable by group */ S_IXGRP : 0) | + ((mode & S_IROTH) ? /* +search if readable by others */ S_IXOTH : 0); + if (mkdir(path, dir_mode)) { + rc = errno; + if (rc != EEXIST) + goto bailout; + } +#endif + } + oflags = O_RDWR | O_CREAT; + } else + oflags = O_RDWR; + + rc = mdbx_openfile(dxb_pathname, oflags, mode, &env->me_fd, + (env->me_flags & MDBX_EXCLUSIVE) ? true : false); + if (rc != MDBX_SUCCESS) + goto bailout; + + const int lck_rc = mdbx_setup_lck(env, lck_pathname, mode); + if (MDBX_IS_ERROR(lck_rc)) { + rc = lck_rc; + goto bailout; + } + + const int dxb_rc = mdbx_setup_dxb(env, lck_rc); + if (MDBX_IS_ERROR(dxb_rc)) { + rc = dxb_rc; + goto bailout; + } + + mdbx_debug("opened dbenv %p", (void *)env); + if (env->me_lck) { + const unsigned mode_flags = + MDBX_WRITEMAP | MDBX_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC; + if (lck_rc == MDBX_RESULT_TRUE) { + env->me_lck->mti_envmode = env->me_flags & (mode_flags | MDBX_RDONLY); + rc = mdbx_lck_downgrade(env); + mdbx_debug("lck-downgrade-%s: rc %i", + (env->me_flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc); + if (rc != MDBX_SUCCESS) + goto bailout; + } else { + rc = mdbx_reader_check0(env, false, NULL); + if (MDBX_IS_ERROR(rc)) + goto bailout; + if ((env->me_flags & MDBX_RDONLY) == 0) { + while (env->me_lck->mti_envmode == MDBX_RDONLY) { + if (atomic_cas32(&env->me_lck->mti_envmode, MDBX_RDONLY, + env->me_flags & mode_flags)) + break; + atomic_yield(); + } + if ((env->me_lck->mti_envmode ^ env->me_flags) & mode_flags) { + mdbx_error("%s", "current mode/flags incompatible with requested"); + rc = MDBX_INCOMPATIBLE; + goto bailout; + } + } + } + + if ((env->me_flags & MDBX_NOTLS) == 0) { + rc = mdbx_rthc_alloc(&env->me_txkey, &env->me_lck->mti_readers[0], + &env->me_lck->mti_readers[env->me_maxreaders]); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + env->me_flags |= MDBX_ENV_TXKEY; + } + } + + if ((flags & MDBX_RDONLY) == 0) { + rc = MDBX_ENOMEM; + MDBX_txn *txn; + int tsize = sizeof(MDBX_txn), + size = + tsize + env->me_maxdbs * (sizeof(MDBX_db) + sizeof(MDBX_cursor *) + + sizeof(unsigned) + 1); + if ((env->me_pbuf = mdbx_calloc( + 1 /* page buffer */ + 1 /* page killer bufer */, env->me_psize)) && + (txn = mdbx_calloc(1, size))) { + txn->mt_dbs = (MDBX_db *)((char *)txn + tsize); + txn->mt_cursors = (MDBX_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = (unsigned *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (uint8_t *)(txn->mt_dbiseqs + env->me_maxdbs); + txn->mt_env = env; + txn->mt_dbxs = env->me_dbxs; + txn->mt_flags = MDBX_TXN_FINISHED; + env->me_txn0 = txn; + txn->tw.retired_pages = mdbx_pnl_alloc(MDBX_PNL_INITIAL); + txn->tw.reclaimed_pglist = mdbx_pnl_alloc(MDBX_PNL_INITIAL); + if (txn->tw.retired_pages && txn->tw.reclaimed_pglist) + rc = MDBX_SUCCESS; + } + } + +#if MDBX_DEBUG + if (rc == MDBX_SUCCESS) { + MDBX_meta *meta = mdbx_meta_head(env); + MDBX_db *db = &meta->mm_dbs[MAIN_DBI]; + + mdbx_debug("opened database version %u, pagesize %u", + (uint8_t)meta->mm_magic_and_version, env->me_psize); + mdbx_debug("using meta page %" PRIaPGNO ", txn %" PRIaTXN, + data_page(meta)->mp_pgno, mdbx_meta_txnid_fluid(env, meta)); + mdbx_debug("depth: %u", db->md_depth); + mdbx_debug("entries: %" PRIu64, db->md_entries); + mdbx_debug("branch pages: %" PRIaPGNO, db->md_branch_pages); + mdbx_debug("leaf pages: %" PRIaPGNO, db->md_leaf_pages); + mdbx_debug("overflow pages: %" PRIaPGNO, db->md_overflow_pages); + mdbx_debug("root: %" PRIaPGNO, db->md_root); + mdbx_debug("schema_altered: %" PRIaTXN, db->md_mod_txnid); + } +#endif + +bailout: + if (rc) { + rc = mdbx_env_close0(env) ? MDBX_PANIC : rc; + env->me_flags = saved_me_flags | MDBX_FATAL_ERROR; + } + mdbx_free(lck_pathname); + return rc; +} + +/* Destroy resources from mdbx_env_open(), clear our readers & DBIs */ +static int __cold mdbx_env_close0(MDBX_env *env) { + if (!(env->me_flags & MDBX_ENV_ACTIVE)) { + mdbx_ensure(env, env->me_lcklist_next == nullptr); + return MDBX_SUCCESS; + } + + env->me_flags &= ~MDBX_ENV_ACTIVE; + env->me_oldest = nullptr; + env->me_sync_timestamp = nullptr; + env->me_autosync_period = nullptr; + env->me_unsynced_pages = nullptr; + env->me_autosync_threshold = nullptr; + env->me_discarded_tail = nullptr; + env->me_meta_sync_txnid = nullptr; + if (env->me_flags & MDBX_ENV_TXKEY) + mdbx_rthc_remove(env->me_txkey); + + lcklist_lock(); + const int rc = lcklist_detach_locked(env); + lcklist_unlock(); + + if (env->me_map) { + mdbx_munmap(&env->me_dxb_mmap); +#ifdef MDBX_USE_VALGRIND + VALGRIND_DISCARD(env->me_valgrind_handle); + env->me_valgrind_handle = -1; +#endif + } + if (env->me_fd != INVALID_HANDLE_VALUE) { + (void)mdbx_closefile(env->me_fd); + env->me_fd = INVALID_HANDLE_VALUE; + } + + if (env->me_lck) + mdbx_munmap(&env->me_lck_mmap); + + if (env->me_lfd != INVALID_HANDLE_VALUE) { + (void)mdbx_closefile(env->me_lfd); + env->me_lfd = INVALID_HANDLE_VALUE; + } + + if (env->me_dbxs) { + for (unsigned i = env->me_maxdbs; --i >= CORE_DBS;) + mdbx_free(env->me_dbxs[i].md_name.iov_base); + mdbx_free(env->me_dbxs); + } + mdbx_free(env->me_pbuf); + mdbx_free(env->me_dbiseqs); + mdbx_free(env->me_dbflags); + mdbx_free(env->me_path); + mdbx_free(env->me_dirtylist); + if (env->me_txn0) { + mdbx_txl_free(env->me_txn0->tw.lifo_reclaimed); + mdbx_pnl_free(env->me_txn0->tw.retired_pages); + mdbx_pnl_free(env->me_txn0->tw.spill_pages); + mdbx_pnl_free(env->me_txn0->tw.reclaimed_pglist); + mdbx_free(env->me_txn0); + } + env->me_flags = 0; + return rc; +} + +int __cold mdbx_env_close_ex(MDBX_env *env, int dont_sync) { + MDBX_page *dp; + int rc = MDBX_SUCCESS; + + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + +#if MDBX_TXN_CHECKPID || !(defined(_WIN32) || defined(_WIN64)) + /* Check the PID even if MDBX_TXN_CHECKPID=0 on non-Windows + * platforms (i.e. where fork() is available). + * This is required to legitimize a call after fork() + * from a child process, that should be allowed to free resources. */ + if (unlikely(env->me_pid != mdbx_getpid())) + env->me_flags |= MDBX_FATAL_ERROR; +#endif /* MDBX_TXN_CHECKPID */ + + if ((env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0) { + if (env->me_txn0 && env->me_txn0->mt_owner && + env->me_txn0->mt_owner != mdbx_thread_self()) + return MDBX_BUSY; + if (!dont_sync) { +#if defined(_WIN32) || defined(_WIN64) + /* On windows, without blocking is impossible to determine whether another + * process is running a writing transaction or not. + * Because in the "owner died" condition kernel don't release + * file lock immediately. */ + rc = mdbx_env_sync_ex(env, true, false); + rc = (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; +#else + rc = mdbx_env_sync_ex(env, true, true); + rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY || + rc == EWOULDBLOCK || rc == MDBX_RESULT_TRUE) + ? MDBX_SUCCESS + : rc; +#endif + } + } + + while ((dp = env->me_dpages) != NULL) { + ASAN_UNPOISON_MEMORY_REGION(&dp->mp_next, sizeof(dp->mp_next)); + VALGRIND_MAKE_MEM_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); + env->me_dpages = dp->mp_next; + mdbx_free(dp); + } + VALGRIND_DESTROY_MEMPOOL(env); + + rc = mdbx_env_close0(env) ? MDBX_PANIC : rc; + mdbx_ensure(env, mdbx_fastmutex_destroy(&env->me_dbi_lock) == MDBX_SUCCESS); +#if defined(_WIN32) || defined(_WIN64) + /* me_remap_guard don't have destructor (Slim Reader/Writer Lock) */ + DeleteCriticalSection(&env->me_windowsbug_lock); +#else + mdbx_ensure(env, + mdbx_fastmutex_destroy(&env->me_remap_guard) == MDBX_SUCCESS); +#endif /* Windows */ + +#ifdef MDBX_OSAL_LOCK + mdbx_ensure(env, mdbx_fastmutex_destroy(&env->me_lckless_stub.wmutex) == + MDBX_SUCCESS); +#endif + + mdbx_ensure(env, env->me_lcklist_next == nullptr); + env->me_pid = 0; + env->me_signature = 0; + mdbx_free(env); + + return rc; +} + +__cold int mdbx_env_close(MDBX_env *env) { + return mdbx_env_close_ex(env, false); +} + +/* Compare two items pointing at aligned unsigned int's. */ +static int __hot mdbx_cmp_int_align4(const MDBX_val *a, const MDBX_val *b) { + mdbx_assert(NULL, a->iov_len == b->iov_len); + switch (a->iov_len) { + case 4: + return CMP2INT(unaligned_peek_u32(4, a->iov_base), + unaligned_peek_u32(4, b->iov_base)); + case 8: + return CMP2INT(unaligned_peek_u64(4, a->iov_base), + unaligned_peek_u64(4, b->iov_base)); + default: + mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__, + __LINE__); + return 0; + } +} + +/* Compare two items pointing at 2-byte aligned unsigned int's. */ +static int __hot mdbx_cmp_int_align2(const MDBX_val *a, const MDBX_val *b) { + mdbx_assert(NULL, a->iov_len == b->iov_len); + switch (a->iov_len) { + case 4: + return CMP2INT(unaligned_peek_u32(2, a->iov_base), + unaligned_peek_u32(2, b->iov_base)); + case 8: + return CMP2INT(unaligned_peek_u64(2, a->iov_base), + unaligned_peek_u64(2, b->iov_base)); + default: + mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__, + __LINE__); + return 0; + } +} + +/* Compare two items pointing at unsigneds of unknown alignment. + * + * This is also set as MDBX_INTEGERDUP|MDBX_DUPFIXED's MDBX_dbx.md_dcmp. */ +static int __hot mdbx_cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b) { + mdbx_assert(NULL, a->iov_len == b->iov_len); + switch (a->iov_len) { + case 4: + return CMP2INT(unaligned_peek_u32(1, a->iov_base), + unaligned_peek_u32(1, b->iov_base)); + case 8: + return CMP2INT(unaligned_peek_u64(1, a->iov_base), + unaligned_peek_u64(1, b->iov_base)); + default: + mdbx_assert_fail(NULL, "invalid size for INTEGERKEY/INTEGERDUP", __func__, + __LINE__); + return 0; + } +} + +/* Compare two items lexically */ +static int __hot mdbx_cmp_memn(const MDBX_val *a, const MDBX_val *b) { + if (a->iov_len == b->iov_len) + return memcmp(a->iov_base, b->iov_base, a->iov_len); + + const int diff_len = (a->iov_len < b->iov_len) ? -1 : 1; + const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len; + int diff_data = memcmp(a->iov_base, b->iov_base, shortest); + return likely(diff_data) ? diff_data : diff_len; +} + +/* Compare two items in reverse byte order */ +static int __hot mdbx_cmp_memnr(const MDBX_val *a, const MDBX_val *b) { + const uint8_t *pa = (const uint8_t *)a->iov_base + a->iov_len; + const uint8_t *pb = (const uint8_t *)b->iov_base + b->iov_len; + const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len; + + const uint8_t *const end = pa - shortest; + while (pa != end) { + int diff = *--pa - *--pb; + if (likely(diff)) + return diff; + } + return CMP2INT(a->iov_len, b->iov_len); +} + +/* Search for key within a page, using binary search. + * Returns the smallest entry larger or equal to the key. + * If exactp is non-null, stores whether the found entry was an exact match + * in *exactp (1 or 0). + * Updates the cursor index with the index of the found entry. + * If no entry larger or equal to the key is found, returns NULL. */ +static MDBX_node *__hot mdbx_node_search(MDBX_cursor *mc, MDBX_val *key, + int *exactp) { + int low, high; + int rc = 0; + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + MDBX_node *node = nullptr; + MDBX_val nodekey; + MDBX_cmp_func *cmp; + DKBUF; + + const unsigned nkeys = page_numkeys(mp); + + mdbx_debug("searching %u keys in %s %spage %" PRIaPGNO, nkeys, + IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", + mp->mp_pgno); + + low = IS_LEAF(mp) ? 0 : 1; + high = nkeys - 1; + cmp = mc->mc_dbx->md_cmp; + + /* Branch pages have no data, so if using integer keys, + * alignment is guaranteed. Use faster mdbx_cmp_int_ai. + */ + if (cmp == mdbx_cmp_int_align2 && IS_BRANCH(mp)) + cmp = mdbx_cmp_int_align4; + + unsigned i = 0; + if (IS_LEAF2(mp)) { + mdbx_cassert(mc, mp->mp_leaf2_ksize == mc->mc_db->md_xsize); + nodekey.iov_len = mp->mp_leaf2_ksize; + node = (MDBX_node *)(intptr_t)-1; /* fake */ + while (low <= high) { + i = (low + high) >> 1; + nodekey.iov_base = page_leaf2key(mp, i, nodekey.iov_len); + mdbx_cassert(mc, (char *)mp + mc->mc_txn->mt_env->me_psize >= + (char *)nodekey.iov_base + nodekey.iov_len); + rc = cmp(key, &nodekey); + mdbx_debug("found leaf index %u [%s], rc = %i", i, DKEY(&nodekey), rc); + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } else { + while (low <= high) { + i = (low + high) >> 1; + + node = page_node(mp, i); + nodekey.iov_len = node_ks(node); + nodekey.iov_base = node_key(node); + mdbx_cassert(mc, (char *)mp + mc->mc_txn->mt_env->me_psize >= + (char *)nodekey.iov_base + nodekey.iov_len); + + rc = cmp(key, &nodekey); + if (IS_LEAF(mp)) + mdbx_debug("found leaf index %u [%s], rc = %i", i, DKEY(&nodekey), rc); + else + mdbx_debug("found branch index %u [%s -> %" PRIaPGNO "], rc = %i", i, + DKEY(&nodekey), node_pgno(node), rc); + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } + + if (rc > 0) /* Found entry is less than the key. */ + i++; /* Skip to get the smallest entry larger than key. */ + + if (exactp) + *exactp = (rc == 0 && nkeys > 0); + /* store the key index */ + mdbx_cassert(mc, i <= UINT16_MAX); + mc->mc_ki[mc->mc_top] = (indx_t)i; + if (i >= nkeys) + /* There is no entry larger or equal to the key. */ + return NULL; + + /* page_node is fake for LEAF2 */ + return IS_LEAF2(mp) ? node : page_node(mp, i); +} + +#if 0 /* unused for now */ +static void mdbx_cursor_adjust(MDBX_cursor *mc, func) { + MDBX_cursor *m2; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2 = m2->mc_next) { + if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { + func(mc, m2); + } + } +} +#endif + +/* Pop a page off the top of the cursor's stack. */ +static void mdbx_cursor_pop(MDBX_cursor *mc) { + if (mc->mc_snum) { + mdbx_debug("popped page %" PRIaPGNO " off db %d cursor %p", + mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *)mc); + + mc->mc_snum--; + if (mc->mc_snum) { + mc->mc_top--; + } else { + mc->mc_flags &= ~C_INITIALIZED; + } + } +} + +/* Push a page onto the top of the cursor's stack. + * Set MDBX_TXN_ERROR on failure. */ +static int mdbx_cursor_push(MDBX_cursor *mc, MDBX_page *mp) { + mdbx_debug("pushing page %" PRIaPGNO " on db %d cursor %p", mp->mp_pgno, + DDBI(mc), (void *)mc); + + if (unlikely(mc->mc_snum >= CURSOR_STACK)) { + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_CURSOR_FULL; + } + + mdbx_cassert(mc, mc->mc_snum < UINT16_MAX); + mc->mc_top = mc->mc_snum++; + mc->mc_pg[mc->mc_top] = mp; + mc->mc_ki[mc->mc_top] = 0; + + return MDBX_SUCCESS; +} + +/* Find the address of the page corresponding to a given page number. + * Set MDBX_TXN_ERROR on failure. + * + * [in] mc the cursor accessing the page. + * [in] pgno the page number for the page to retrieve. + * [out] ret address of a pointer where the page's address will be + * stored. + * [out] lvl dirtylist inheritance level of found page. 1=current txn, + * 0=mapped page. + * + * Returns 0 on success, non-zero on failure. */ +__hot static int mdbx_page_get(MDBX_cursor *mc, pgno_t pgno, MDBX_page **ret, + int *lvl) { + MDBX_txn *txn = mc->mc_txn; + if (unlikely(pgno >= txn->mt_next_pgno)) { + mdbx_debug("page %" PRIaPGNO " not found", pgno); + txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_PAGE_NOTFOUND; + } + + MDBX_env *env = txn->mt_env; + MDBX_page *p = NULL; + int level; + if ((txn->mt_flags & (MDBX_RDONLY | MDBX_WRITEMAP)) == 0) { + level = 1; + do { + /* Spilled pages were dirtied in this txn and flushed + * because the dirty list got full. Bring this page + * back in from the map (but don't unspill it here, + * leave that unless page_touch happens again). */ + if (txn->tw.spill_pages && mdbx_pnl_exist(txn->tw.spill_pages, pgno << 1)) + goto mapped; + p = mdbx_dpl_find(txn->tw.dirtylist, pgno); + if (p) + goto done; + level++; + } while ((txn = txn->mt_parent) != NULL); + } + level = 0; + +mapped: + p = pgno2page(env, pgno); + +done: + txn = nullptr /* avoid future use */; + if (unlikely(p->mp_pgno != pgno)) { + mdbx_error("mismatch pgno %" PRIaPGNO " (actual) != %" PRIaPGNO + " (expected)", + p->mp_pgno, pgno); + return MDBX_CORRUPTED; + } + + if (unlikely(p->mp_upper < p->mp_lower || ((p->mp_lower | p->mp_upper) & 1) || + PAGEHDRSZ + p->mp_upper > env->me_psize) && + !IS_OVERFLOW(p)) { + mdbx_error("invalid page lower(%u)/upper(%u), pg-limit %u", p->mp_lower, + p->mp_upper, page_space(env)); + return MDBX_CORRUPTED; + } + /* TODO: more checks here, including p->mp_validator */ + + if (mdbx_audit_enabled()) { + int err = mdbx_page_check(env, p, true); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + + *ret = p; + if (lvl) + *lvl = level; + return MDBX_SUCCESS; +} + +/* Finish mdbx_page_search() / mdbx_page_search_lowest(). + * The cursor is at the root page, set up the rest of it. */ +__hot static int mdbx_page_search_root(MDBX_cursor *mc, MDBX_val *key, + int flags) { + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + int rc; + DKBUF; + + while (IS_BRANCH(mp)) { + MDBX_node *node; + int i; + + mdbx_debug("branch page %" PRIaPGNO " has %u keys", mp->mp_pgno, + page_numkeys(mp)); + /* Don't assert on branch pages in the GC. We can get here + * while in the process of rebalancing a GC branch page; we must + * let that proceed. ITS#8336 */ + mdbx_cassert(mc, !mc->mc_dbi || page_numkeys(mp) > 1); + mdbx_debug("found index 0 to page %" PRIaPGNO, node_pgno(page_node(mp, 0))); + + if (flags & (MDBX_PS_FIRST | MDBX_PS_LAST)) { + i = 0; + if (flags & MDBX_PS_LAST) { + i = page_numkeys(mp) - 1; + /* if already init'd, see if we're already in right place */ + if (mc->mc_flags & C_INITIALIZED) { + if (mc->mc_ki[mc->mc_top] == i) { + mc->mc_top = mc->mc_snum++; + mp = mc->mc_pg[mc->mc_top]; + goto ready; + } + } + } + } else { + int exact; + node = mdbx_node_search(mc, key, &exact); + if (node == NULL) + i = page_numkeys(mp) - 1; + else { + i = mc->mc_ki[mc->mc_top]; + if (!exact) { + mdbx_cassert(mc, i > 0); + i--; + } + } + mdbx_debug("following index %u for key [%s]", i, DKEY(key)); + } + + mdbx_cassert(mc, i < (int)page_numkeys(mp)); + node = page_node(mp, i); + + if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp, NULL)) != 0)) + return rc; + + mc->mc_ki[mc->mc_top] = (indx_t)i; + if (unlikely(rc = mdbx_cursor_push(mc, mp))) + return rc; + + ready: + if (flags & MDBX_PS_MODIFY) { + if (unlikely((rc = mdbx_page_touch(mc)) != 0)) + return rc; + mp = mc->mc_pg[mc->mc_top]; + } + } + + if (unlikely(!IS_LEAF(mp))) { + mdbx_debug("internal error, index points to a page with 0x%02x flags!?", + mp->mp_flags); + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_CORRUPTED; + } + + mdbx_debug("found leaf page %" PRIaPGNO " for key [%s]", mp->mp_pgno, + DKEY(key)); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + return MDBX_SUCCESS; +} + +static int mdbx_fetch_sdb(MDBX_txn *txn, MDBX_dbi dbi) { + MDBX_cursor mc; + if (unlikely(TXN_DBI_CHANGED(txn, dbi))) + return MDBX_BAD_DBI; + int rc = mdbx_cursor_init(&mc, txn, MAIN_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_page_search(&mc, &txn->mt_dbxs[dbi].md_name, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_NOTFOUND) ? MDBX_BAD_DBI : rc; + + MDBX_val data; + int exact = 0; + MDBX_node *node = mdbx_node_search(&mc, &txn->mt_dbxs[dbi].md_name, &exact); + if (unlikely(!exact)) + return MDBX_BAD_DBI; + if (unlikely((node_flags(node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA)) + return MDBX_INCOMPATIBLE; /* not a named DB */ + rc = mdbx_node_read(&mc, node, &data); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(data.iov_len < sizeof(MDBX_db))) + return MDBX_INCOMPATIBLE; /* not a named DB */ + + uint16_t md_flags = UNALIGNED_PEEK_16(data.iov_base, MDBX_db, md_flags); + /* The txn may not know this DBI, or another process may + * have dropped and recreated the DB with other flags. */ + if (unlikely((txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS) != md_flags)) + return MDBX_INCOMPATIBLE; + + memcpy(&txn->mt_dbs[dbi], data.iov_base, sizeof(MDBX_db)); + txn->mt_dbflags[dbi] &= ~DB_STALE; + return MDBX_SUCCESS; +} + +/* Search for the lowest key under the current branch page. + * This just bypasses a numkeys check in the current page + * before calling mdbx_page_search_root(), because the callers + * are all in situations where the current page is known to + * be underfilled. */ +__hot static int mdbx_page_search_lowest(MDBX_cursor *mc) { + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + mdbx_cassert(mc, IS_BRANCH(mp)); + MDBX_node *node = page_node(mp, 0); + int rc; + + if (unlikely((rc = mdbx_page_get(mc, node_pgno(node), &mp, NULL)) != 0)) + return rc; + + mc->mc_ki[mc->mc_top] = 0; + if (unlikely(rc = mdbx_cursor_push(mc, mp))) + return rc; + return mdbx_page_search_root(mc, NULL, MDBX_PS_FIRST); +} + +/* Search for the page a given key should be in. + * Push it and its parent pages on the cursor stack. + * + * [in,out] mc the cursor for this operation. + * [in] key the key to search for, or NULL for first/last page. + * [in] flags If MDBX_PS_MODIFY is set, visited pages in the DB + * are touched (updated with new page numbers). + * If MDBX_PS_FIRST or MDBX_PS_LAST is set, find first or last + * leaf. + * This is used by mdbx_cursor_first() and mdbx_cursor_last(). + * If MDBX_PS_ROOTONLY set, just fetch root node, no further + * lookups. + * + * Returns 0 on success, non-zero on failure. */ +__hot static int mdbx_page_search(MDBX_cursor *mc, MDBX_val *key, int flags) { + int rc; + pgno_t root; + + /* Make sure the txn is still viable, then find the root from + * the txn's db table and set it as the root of the cursor's stack. */ + if (unlikely(mc->mc_txn->mt_flags & MDBX_TXN_BLOCKED)) { + mdbx_debug("%s", "transaction has failed, must abort"); + return MDBX_BAD_TXN; + } + + /* Make sure we're using an up-to-date root */ + if (unlikely(*mc->mc_dbflag & DB_STALE)) { + rc = mdbx_fetch_sdb(mc->mc_txn, mc->mc_dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + root = mc->mc_db->md_root; + + if (unlikely(root == P_INVALID)) { /* Tree is empty. */ + mdbx_debug("%s", "tree is empty"); + return MDBX_NOTFOUND; + } + + mdbx_cassert(mc, root >= NUM_METAS); + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) + if (unlikely((rc = mdbx_page_get(mc, root, &mc->mc_pg[0], NULL)) != 0)) + return rc; + + mc->mc_snum = 1; + mc->mc_top = 0; + + mdbx_debug("db %d root page %" PRIaPGNO " has flags 0x%X", DDBI(mc), root, + mc->mc_pg[0]->mp_flags); + + if (flags & MDBX_PS_MODIFY) { + if (unlikely(rc = mdbx_page_touch(mc))) + return rc; + } + + if (flags & MDBX_PS_ROOTONLY) + return MDBX_SUCCESS; + + return mdbx_page_search_root(mc, key, flags); +} + +/* Return the data associated with a given node. + * + * [in] mc The cursor for this operation. + * [in] leaf The node being read. + * [out] data Updated to point to the node's data. + * + * Returns 0 on success, non-zero on failure. */ +static __inline int mdbx_node_read(MDBX_cursor *mc, MDBX_node *node, + MDBX_val *data) { + data->iov_len = node_ds(node); + data->iov_base = node_data(node); + if (unlikely(F_ISSET(node_flags(node), F_BIGDATA))) { + /* Read overflow data. */ + MDBX_page *omp; /* overflow page */ + int rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL); + if (unlikely((rc != MDBX_SUCCESS))) { + mdbx_debug("read overflow page %" PRIaPGNO " failed", + node_largedata_pgno(node)); + return rc; + } + data->iov_base = page_data(omp); + } + return MDBX_SUCCESS; +} + +int mdbx_get(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data) { + DKBUF; + mdbx_debug("===> get db %u key [%s]", dbi, DKEY(key)); + + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!key || !data)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + MDBX_cursor_couple cx; + rc = mdbx_cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + int exact = 0; + return mdbx_cursor_set(&cx.outer, key, data, MDBX_SET, &exact); +} + +int mdbx_get_nearest(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, + MDBX_val *data) { + DKBUF; + mdbx_debug("===> get db %u key [%s]", dbi, DKEY(key)); + + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!key || !data)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED)) + return MDBX_BAD_TXN; + + MDBX_cursor_couple cx; + rc = mdbx_cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + MDBX_val save_data = *data; + int exact = 0; + rc = mdbx_cursor_set(&cx.outer, key, data, MDBX_SET_RANGE, &exact); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (exact && (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) != 0) { + *data = save_data; + exact = 0; + rc = mdbx_cursor_set(&cx.outer, key, data, MDBX_GET_BOTH_RANGE, &exact); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + + return exact ? MDBX_SUCCESS : MDBX_RESULT_TRUE; +} + +int mdbx_get_ex(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, + size_t *values_count) { + DKBUF; + mdbx_debug("===> get db %u key [%s]", dbi, DKEY(key)); + + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!key || !data)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + MDBX_cursor_couple cx; + rc = mdbx_cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + int exact = 0; + rc = mdbx_cursor_set(&cx.outer, key, data, MDBX_SET_KEY, &exact); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_NOTFOUND && values_count) + *values_count = 0; + return rc; + } + + if (values_count) { + *values_count = 1; + if (cx.outer.mc_xcursor != NULL) { + MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top], + cx.outer.mc_ki[cx.outer.mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + mdbx_tassert(txn, cx.outer.mc_xcursor == &cx.inner && + (cx.inner.mx_cursor.mc_flags & C_INITIALIZED)); + *values_count = + (sizeof(*values_count) >= sizeof(cx.inner.mx_db.md_entries) || + cx.inner.mx_db.md_entries <= PTRDIFF_MAX) + ? (size_t)cx.inner.mx_db.md_entries + : PTRDIFF_MAX; + } + } + } + return MDBX_SUCCESS; +} + +/* Find a sibling for a page. + * Replaces the page at the top of the cursor's stack with the specified + * sibling, if one exists. + * + * [in] mc The cursor for this operation. + * [in] move_right Non-zero if the right sibling is requested, + * otherwise the left sibling. + * + * Returns 0 on success, non-zero on failure. */ +static int mdbx_cursor_sibling(MDBX_cursor *mc, int move_right) { + int rc; + MDBX_node *indx; + MDBX_page *mp; + + if (unlikely(mc->mc_snum < 2)) + return MDBX_NOTFOUND; /* root has no siblings */ + + mdbx_cursor_pop(mc); + mdbx_debug("parent page is page %" PRIaPGNO ", index %u", + mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]); + + if (move_right + ? (mc->mc_ki[mc->mc_top] + 1u >= page_numkeys(mc->mc_pg[mc->mc_top])) + : (mc->mc_ki[mc->mc_top] == 0)) { + mdbx_debug("no more keys left, moving to %s sibling", + move_right ? "right" : "left"); + if (unlikely((rc = mdbx_cursor_sibling(mc, move_right)) != MDBX_SUCCESS)) { + /* undo cursor_pop before returning */ + mc->mc_top++; + mc->mc_snum++; + return rc; + } + } else { + if (move_right) + mc->mc_ki[mc->mc_top]++; + else + mc->mc_ki[mc->mc_top]--; + mdbx_debug("just moving to %s index key %u", move_right ? "right" : "left", + mc->mc_ki[mc->mc_top]); + } + mdbx_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); + + indx = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (unlikely((rc = mdbx_page_get(mc, node_pgno(indx), &mp, NULL)) != 0)) { + /* mc will be inconsistent if caller does mc_snum++ as above */ + mc->mc_flags &= ~(C_INITIALIZED | C_EOF); + return rc; + } + + rc = mdbx_cursor_push(mc, mp); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + if (!move_right) + mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mp) - 1; + + return MDBX_SUCCESS; +} + +/* Move the cursor to the next data item. */ +static int mdbx_cursor_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, + MDBX_cursor_op op) { + MDBX_page *mp; + MDBX_node *node; + int rc; + + if ((mc->mc_flags & C_DEL) && op == MDBX_NEXT_DUP) + return MDBX_NOTFOUND; + + if (!(mc->mc_flags & C_INITIALIZED)) + return mdbx_cursor_first(mc, key, data); + + mp = mc->mc_pg[mc->mc_top]; + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] + 1u >= page_numkeys(mp)) + return MDBX_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + + if (mc->mc_db->md_flags & MDBX_DUPSORT) { + node = page_node(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + if (op == MDBX_NEXT || op == MDBX_NEXT_DUP) { + rc = + mdbx_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_NEXT); + if (op != MDBX_NEXT || rc != MDBX_NOTFOUND) { + if (likely(rc == MDBX_SUCCESS)) + get_key_optional(node, key); + return rc; + } + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); + if (op == MDBX_NEXT_DUP) + return MDBX_NOTFOUND; + } + } + + mdbx_debug("cursor_next: top page is %" PRIaPGNO " in cursor %p", mp->mp_pgno, + (void *)mc); + if (mc->mc_flags & C_DEL) { + mc->mc_flags ^= C_DEL; + goto skip; + } + + if (mc->mc_ki[mc->mc_top] + 1u >= page_numkeys(mp)) { + mdbx_debug("%s", "=====> move to next sibling page"); + if (unlikely((rc = mdbx_cursor_sibling(mc, 1)) != MDBX_SUCCESS)) { + mc->mc_flags |= C_EOF; + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + mdbx_debug("next page is %" PRIaPGNO ", key index %u", mp->mp_pgno, + mc->mc_ki[mc->mc_top]); + } else + mc->mc_ki[mc->mc_top]++; + +skip: + mdbx_debug("==> cursor points to page %" PRIaPGNO + " with %u keys, key index %u", + mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mp)) { + if (likely(key)) { + key->iov_len = mc->mc_db->md_xsize; + key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); + } + return MDBX_SUCCESS; + } + + mdbx_cassert(mc, IS_LEAF(mp)); + node = page_node(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(node_flags(node), F_DUPDATA)) { + rc = mdbx_xcursor_init1(mc, node); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + if (data) { + if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + return rc; + + if (F_ISSET(node_flags(node), F_DUPDATA)) { + rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + } + + get_key_optional(node, key); + return MDBX_SUCCESS; +} + +/* Move the cursor to the previous data item. */ +static int mdbx_cursor_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, + MDBX_cursor_op op) { + MDBX_page *mp; + MDBX_node *node; + int rc; + + if ((mc->mc_flags & C_DEL) && op == MDBX_PREV_DUP) + return MDBX_NOTFOUND; + + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdbx_cursor_last(mc, key, data); + if (unlikely(rc)) + return rc; + mc->mc_ki[mc->mc_top]++; + } + + mp = mc->mc_pg[mc->mc_top]; + if ((mc->mc_db->md_flags & MDBX_DUPSORT) && + mc->mc_ki[mc->mc_top] < page_numkeys(mp)) { + node = page_node(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + if (op == MDBX_PREV || op == MDBX_PREV_DUP) { + rc = + mdbx_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_PREV); + if (op != MDBX_PREV || rc != MDBX_NOTFOUND) { + if (likely(rc == MDBX_SUCCESS)) { + get_key_optional(node, key); + mc->mc_flags &= ~C_EOF; + } + return rc; + } + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); + if (op == MDBX_PREV_DUP) + return MDBX_NOTFOUND; + } + } + + mdbx_debug("cursor_prev: top page is %" PRIaPGNO " in cursor %p", mp->mp_pgno, + (void *)mc); + + mc->mc_flags &= ~(C_EOF | C_DEL); + + if (mc->mc_ki[mc->mc_top] == 0) { + mdbx_debug("%s", "=====> move to prev sibling page"); + if ((rc = mdbx_cursor_sibling(mc, 0)) != MDBX_SUCCESS) { + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mp) - 1; + mdbx_debug("prev page is %" PRIaPGNO ", key index %u", mp->mp_pgno, + mc->mc_ki[mc->mc_top]); + } else + mc->mc_ki[mc->mc_top]--; + + mdbx_debug("==> cursor points to page %" PRIaPGNO + " with %u keys, key index %u", + mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mp)) { + if (likely(key)) { + key->iov_len = mc->mc_db->md_xsize; + key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); + } + return MDBX_SUCCESS; + } + + mdbx_cassert(mc, IS_LEAF(mp)); + node = page_node(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(node_flags(node), F_DUPDATA)) { + rc = mdbx_xcursor_init1(mc, node); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + if (data) { + if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + return rc; + + if (F_ISSET(node_flags(node), F_DUPDATA)) { + rc = mdbx_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + } + + get_key_optional(node, key); + return MDBX_SUCCESS; +} + +/* Set the cursor on a specific data item. */ +__hot static int mdbx_cursor_set(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, + MDBX_cursor_op op, int *exactp) { + int rc; + MDBX_page *mp; + MDBX_node *node = NULL; + DKBUF; + + if ((mc->mc_db->md_flags & MDBX_INTEGERKEY) && + unlikely(key->iov_len != sizeof(uint32_t) && + key->iov_len != sizeof(uint64_t))) { + mdbx_cassert(mc, !"key-size is invalid for MDBX_INTEGERKEY"); + return MDBX_BAD_VALSIZE; + } + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); + + /* See if we're already on the right page */ + if (mc->mc_flags & C_INITIALIZED) { + MDBX_val nodekey; + + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + mp = mc->mc_pg[mc->mc_top]; + if (!page_numkeys(mp)) { + mc->mc_ki[mc->mc_top] = 0; + return MDBX_NOTFOUND; + } + if (IS_LEAF2(mp)) { + nodekey.iov_len = mc->mc_db->md_xsize; + nodekey.iov_base = page_leaf2key(mp, 0, nodekey.iov_len); + } else { + node = page_node(mp, 0); + get_key(node, &nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (unlikely(rc == 0)) { + /* Probably happens rarely, but first node on the page + * was the one we wanted. */ + mc->mc_ki[mc->mc_top] = 0; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc > 0) { + const unsigned nkeys = page_numkeys(mp); + unsigned i; + if (nkeys > 1) { + if (IS_LEAF2(mp)) { + nodekey.iov_base = page_leaf2key(mp, nkeys - 1, nodekey.iov_len); + } else { + node = page_node(mp, nkeys - 1); + get_key(node, &nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* last node was the one we wanted */ + mdbx_cassert(mc, nkeys >= 1 && nkeys <= UINT16_MAX + 1); + mc->mc_ki[mc->mc_top] = (indx_t)(nkeys - 1); + if (exactp) + *exactp = 1; + goto set1; + } + if (rc < 0) { + if (mc->mc_ki[mc->mc_top] < page_numkeys(mp)) { + /* This is definitely the right page, skip search_page */ + if (IS_LEAF2(mp)) { + nodekey.iov_base = + page_leaf2key(mp, mc->mc_ki[mc->mc_top], nodekey.iov_len); + } else { + node = page_node(mp, mc->mc_ki[mc->mc_top]); + get_key(node, &nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* current node was the one we wanted */ + if (exactp) + *exactp = 1; + goto set1; + } + } + rc = 0; + mc->mc_flags &= ~C_EOF; + goto set2; + } + } + /* If any parents have right-sibs, search. + * Otherwise, there's nothing further. */ + for (i = 0; i < mc->mc_top; i++) + if (mc->mc_ki[i] < page_numkeys(mc->mc_pg[i]) - 1) + break; + if (i == mc->mc_top) { + /* There are no other pages */ + mdbx_cassert(mc, nkeys <= UINT16_MAX); + mc->mc_ki[mc->mc_top] = (uint16_t)nkeys; + return MDBX_NOTFOUND; + } + } + if (!mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = 0; + if (op == MDBX_SET_RANGE && !exactp) { + rc = 0; + goto set1; + } else + return MDBX_NOTFOUND; + } + } else { + mc->mc_pg[0] = 0; + } + + rc = mdbx_page_search(mc, key, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + mdbx_cassert(mc, IS_LEAF(mp)); + +set2: + node = mdbx_node_search(mc, key, exactp); + if (exactp != NULL && !*exactp) { + /* MDBX_SET specified and not an exact match. */ + return MDBX_NOTFOUND; + } + + if (node == NULL) { + mdbx_debug("%s", "===> inexact leaf not found, goto sibling"); + if (unlikely((rc = mdbx_cursor_sibling(mc, 1)) != MDBX_SUCCESS)) { + mc->mc_flags |= C_EOF; + return rc; /* no entries matched */ + } + mp = mc->mc_pg[mc->mc_top]; + mdbx_cassert(mc, IS_LEAF(mp)); + node = page_node(mp, 0); + } + +set1: + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + if (IS_LEAF2(mp)) { + if (op == MDBX_SET_RANGE || op == MDBX_SET_KEY) { + key->iov_len = mc->mc_db->md_xsize; + key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); + } + return MDBX_SUCCESS; + } + + if (F_ISSET(node_flags(node), F_DUPDATA)) { + rc = mdbx_xcursor_init1(mc, node); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + if (likely(data)) { + if (F_ISSET(node_flags(node), F_DUPDATA)) { + if (op == MDBX_SET || op == MDBX_SET_KEY || op == MDBX_SET_RANGE) { + rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + } else { + int ex2, *ex2p; + if (op == MDBX_GET_BOTH) { + ex2p = &ex2; + ex2 = 0; + } else { + ex2p = NULL; + } + rc = mdbx_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, + MDBX_SET_RANGE, ex2p); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + } else if (op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE) { + MDBX_val olddata; + if (unlikely((rc = mdbx_node_read(mc, node, &olddata)) != MDBX_SUCCESS)) + return rc; + if (unlikely(mc->mc_dbx->md_dcmp == NULL)) + return MDBX_EINVAL; + rc = mc->mc_dbx->md_dcmp(data, &olddata); + if (rc) { + if (op != MDBX_GET_BOTH_RANGE || rc > 0) + return MDBX_NOTFOUND; + rc = 0; + } + *data = olddata; + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); + if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + return rc; + } + } + + /* The key already matches in all other cases */ + if (op == MDBX_SET_RANGE || op == MDBX_SET_KEY) + get_key_optional(node, key); + + mdbx_debug("==> cursor placed on key [%s], data [%s]", DKEY(key), DVAL(data)); + return rc; +} + +/* Move the cursor to the first item in the database. */ +static int mdbx_cursor_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { + int rc; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdbx_page_search(mc, NULL, MDBX_PS_FIRST); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + mc->mc_ki[mc->mc_top] = 0; + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->iov_len = mc->mc_db->md_xsize; + key->iov_base = page_leaf2key(mc->mc_pg[mc->mc_top], 0, key->iov_len); + return MDBX_SUCCESS; + } + + MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], 0); + if (likely(data)) { + if (F_ISSET(node_flags(node), F_DUPDATA)) { + rc = mdbx_xcursor_init1(mc, node); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (unlikely(rc)) + return rc; + } else { + if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + return rc; + } + } + get_key_optional(node, key); + return MDBX_SUCCESS; +} + +/* Move the cursor to the last item in the database. */ +static int mdbx_cursor_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { + int rc; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); + + if (likely((mc->mc_flags & (C_EOF | C_DEL)) != C_EOF)) { + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdbx_page_search(mc, NULL, MDBX_PS_LAST); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + } + + mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]) - 1; + mc->mc_flags |= C_INITIALIZED | C_EOF; + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->iov_len = mc->mc_db->md_xsize; + key->iov_base = page_leaf2key(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], + key->iov_len); + return MDBX_SUCCESS; + } + + MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (likely(data)) { + if (F_ISSET(node_flags(node), F_DUPDATA)) { + rc = mdbx_xcursor_init1(mc, node); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (unlikely(rc)) + return rc; + } else { + if (unlikely((rc = mdbx_node_read(mc, node, data)) != MDBX_SUCCESS)) + return rc; + } + } + + get_key_optional(node, key); + return MDBX_SUCCESS; +} + +int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, + MDBX_cursor_op op) { + if (unlikely(mc == NULL)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + int exact = 0; + int (*mfunc)(MDBX_cursor * mc, MDBX_val * key, MDBX_val * data); + switch (op) { + case MDBX_GET_CURRENT: { + if (unlikely(!(mc->mc_flags & C_INITIALIZED))) + return MDBX_EINVAL; + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + const unsigned nkeys = page_numkeys(mp); + if (mc->mc_ki[mc->mc_top] >= nkeys) { + mdbx_cassert(mc, nkeys <= UINT16_MAX); + mc->mc_ki[mc->mc_top] = (uint16_t)nkeys; + return MDBX_NOTFOUND; + } + mdbx_cassert(mc, nkeys > 0); + + rc = MDBX_SUCCESS; + if (IS_LEAF2(mp)) { + key->iov_len = mc->mc_db->md_xsize; + key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); + } else { + MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); + get_key_optional(node, key); + if (data) { + if (F_ISSET(node_flags(node), F_DUPDATA)) { + if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))) { + rc = mdbx_xcursor_init1(mc, node); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (unlikely(rc)) + return rc; + } + rc = mdbx_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, + MDBX_GET_CURRENT); + } else { + rc = mdbx_node_read(mc, node, data); + } + if (unlikely(rc)) + return rc; + } + } + break; + } + case MDBX_GET_BOTH: + case MDBX_GET_BOTH_RANGE: + if (unlikely(data == NULL)) + return MDBX_EINVAL; + if (unlikely(mc->mc_xcursor == NULL)) + return MDBX_INCOMPATIBLE; + /* fall through */ + __fallthrough; + case MDBX_SET: + case MDBX_SET_KEY: + case MDBX_SET_RANGE: + if (unlikely(key == NULL)) + return MDBX_EINVAL; + rc = mdbx_cursor_set(mc, key, data, op, + op == MDBX_SET_RANGE ? NULL : &exact); + break; + case MDBX_GET_MULTIPLE: + if (unlikely(data == NULL || !(mc->mc_flags & C_INITIALIZED))) + return MDBX_EINVAL; + if (unlikely(!(mc->mc_db->md_flags & MDBX_DUPFIXED))) + return MDBX_INCOMPATIBLE; + rc = MDBX_SUCCESS; + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || + (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) + break; + goto fetchm; + case MDBX_NEXT_MULTIPLE: + if (unlikely(data == NULL)) + return MDBX_EINVAL; + if (unlikely(!(mc->mc_db->md_flags & MDBX_DUPFIXED))) + return MDBX_INCOMPATIBLE; + rc = mdbx_cursor_next(mc, key, data, MDBX_NEXT_DUP); + if (rc == MDBX_SUCCESS) { + if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + MDBX_cursor *mx; + fetchm: + mx = &mc->mc_xcursor->mx_cursor; + data->iov_len = + page_numkeys(mx->mc_pg[mx->mc_top]) * mx->mc_db->md_xsize; + data->iov_base = page_data(mx->mc_pg[mx->mc_top]); + mx->mc_ki[mx->mc_top] = (indx_t)page_numkeys(mx->mc_pg[mx->mc_top]) - 1; + } else { + rc = MDBX_NOTFOUND; + } + } + break; + case MDBX_PREV_MULTIPLE: + if (data == NULL) + return MDBX_EINVAL; + if (!(mc->mc_db->md_flags & MDBX_DUPFIXED)) + return MDBX_INCOMPATIBLE; + rc = MDBX_SUCCESS; + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdbx_cursor_last(mc, key, data); + if (rc == MDBX_SUCCESS) { + MDBX_cursor *mx = &mc->mc_xcursor->mx_cursor; + if (mx->mc_flags & C_INITIALIZED) { + rc = mdbx_cursor_sibling(mx, 0); + if (rc == MDBX_SUCCESS) + goto fetchm; + } else { + rc = MDBX_NOTFOUND; + } + } + break; + case MDBX_NEXT: + case MDBX_NEXT_DUP: + case MDBX_NEXT_NODUP: + rc = mdbx_cursor_next(mc, key, data, op); + break; + case MDBX_PREV: + case MDBX_PREV_DUP: + case MDBX_PREV_NODUP: + rc = mdbx_cursor_prev(mc, key, data, op); + break; + case MDBX_FIRST: + rc = mdbx_cursor_first(mc, key, data); + break; + case MDBX_FIRST_DUP: + mfunc = mdbx_cursor_first; + mmove: + if (unlikely(data == NULL || !(mc->mc_flags & C_INITIALIZED))) + return MDBX_EINVAL; + if (unlikely(mc->mc_xcursor == NULL)) + return MDBX_INCOMPATIBLE; + if (mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top])) { + mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]); + return MDBX_NOTFOUND; + } + { + MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(node_flags(node), F_DUPDATA)) { + get_key_optional(node, key); + rc = mdbx_node_read(mc, node, data); + break; + } + } + if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))) + return MDBX_EINVAL; + rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); + break; + case MDBX_LAST: + rc = mdbx_cursor_last(mc, key, data); + break; + case MDBX_LAST_DUP: + mfunc = mdbx_cursor_last; + goto mmove; + default: + mdbx_debug("unhandled/unimplemented cursor operation %u", op); + return MDBX_EINVAL; + } + + mc->mc_flags &= ~C_DEL; + return rc; +} + +/* Touch all the pages in the cursor stack. Set mc_top. + * Makes sure all the pages are writable, before attempting a write operation. + * [in] mc The cursor to operate on. */ +static int mdbx_cursor_touch(MDBX_cursor *mc) { + int rc = MDBX_SUCCESS; + + if (mc->mc_dbi >= CORE_DBS && + (*mc->mc_dbflag & (DB_DIRTY | DB_DUPDATA)) == 0) { + mdbx_cassert(mc, (mc->mc_flags & C_RECLAIMING) == 0); + /* Touch DB record of named DB */ + MDBX_cursor_couple cx; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDBX_BAD_DBI; + rc = mdbx_cursor_init(&cx.outer, mc->mc_txn, MAIN_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_page_search(&cx.outer, &mc->mc_dbx->md_name, MDBX_PS_MODIFY); + if (unlikely(rc)) + return rc; + *mc->mc_dbflag |= DB_DIRTY; + } + mc->mc_top = 0; + if (mc->mc_snum) { + do { + rc = mdbx_page_touch(mc); + } while (!rc && ++(mc->mc_top) < mc->mc_snum); + mc->mc_top = mc->mc_snum - 1; + } + return rc; +} + +int mdbx_cursor_put(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, + unsigned flags) { + MDBX_env *env; + MDBX_page *fp, *sub_root = NULL; + uint16_t fp_flags; + MDBX_val xdata, *rdata, dkey, olddata; + MDBX_db nested_dupdb; + unsigned mcount = 0, dcount = 0, nospill; + size_t nsize; + int rc2; + unsigned nflags; + DKBUF; + + if (unlikely(mc == NULL || key == NULL)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + env = mc->mc_txn->mt_env; + + /* Check this first so counter will always be zero on any early failures. */ + if (flags & MDBX_MULTIPLE) { + if (unlikely(!F_ISSET(mc->mc_db->md_flags, MDBX_DUPFIXED))) + return MDBX_INCOMPATIBLE; + if (unlikely(data[1].iov_len >= INT_MAX)) + return MDBX_EINVAL; + dcount = (unsigned)data[1].iov_len; + data[1].iov_len = 0; + } + + if (flags & MDBX_RESERVE) { + if (unlikely(mc->mc_db->md_flags & (MDBX_DUPSORT | MDBX_REVERSEDUP))) + return MDBX_INCOMPATIBLE; + data->iov_base = nullptr; + } + + nospill = flags & MDBX_NOSPILL; + flags &= ~MDBX_NOSPILL; + + if (unlikely(mc->mc_txn->mt_flags & (MDBX_RDONLY | MDBX_TXN_BLOCKED))) + return (mc->mc_txn->mt_flags & MDBX_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + + if (unlikely(key->iov_len > env->me_maxkey_limit)) + return MDBX_BAD_VALSIZE; + + if (unlikely(data->iov_len > ((mc->mc_db->md_flags & MDBX_DUPSORT) + ? env->me_maxkey_limit + : MDBX_MAXDATASIZE))) + return MDBX_BAD_VALSIZE; + + if ((mc->mc_db->md_flags & MDBX_INTEGERKEY) && + unlikely(key->iov_len != sizeof(uint32_t) && + key->iov_len != sizeof(uint64_t))) { + mdbx_cassert(mc, !"key-size is invalid for MDBX_INTEGERKEY"); + return MDBX_BAD_VALSIZE; + } + + if ((mc->mc_db->md_flags & MDBX_INTEGERDUP) && + unlikely(data->iov_len != sizeof(uint32_t) && + data->iov_len != sizeof(uint64_t))) { + mdbx_cassert(mc, !"data-size is invalid MDBX_INTEGERDUP"); + return MDBX_BAD_VALSIZE; + } + + mdbx_debug("==> put db %d key [%s], size %" PRIuPTR + ", data [%s] size %" PRIuPTR, + DDBI(mc), DKEY(key), key ? key->iov_len : 0, + DVAL((flags & MDBX_RESERVE) ? nullptr : data), data->iov_len); + + int dupdata_flag = 0; + if ((flags & MDBX_CURRENT) != 0 && (mc->mc_flags & C_SUB) == 0) { + /* ÐžÐ¿Ñ†Ð¸Ñ MDBX_CURRENT означает, что запрошено обновление текущей запиÑи, + * на которой ÑÐµÐ¹Ñ‡Ð°Ñ Ñтоит курÑор. ПроверÑем что переданный ключ Ñовпадает + * Ñо значением в текущей позиции курÑора. + * ЗдеÑÑŒ проще вызвать mdbx_cursor_get(), так как Ð´Ð»Ñ Ð¾Ð±ÑÐ»ÑƒÐ¶Ð¸Ð²Ð°Ð½Ð¸Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† + * Ñ MDBX_DUPSORT также требуетÑÑ Ñ‚ÐµÐºÑƒÑ‰Ð¸Ð¹ размер данных. */ + MDBX_val current_key, current_data; + rc = mdbx_cursor_get(mc, ¤t_key, ¤t_data, MDBX_GET_CURRENT); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + if (mc->mc_dbx->md_cmp(key, ¤t_key) != 0) + return MDBX_EKEYMISMATCH; + + if (F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)) { + MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + mdbx_cassert(mc, + mc->mc_xcursor != NULL && + (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)); + /* ЕÑли за ключом более одного значениÑ, либо еÑли размер данных + * отличаетÑÑ, то вмеÑто inplace Ð¾Ð±Ð½Ð¾Ð²Ð»ÐµÐ½Ð¸Ñ Ñ‚Ñ€ÐµÐ±ÑƒÐµÑ‚ÑÑ ÑƒÐ´Ð°Ð»ÐµÐ½Ð¸Ðµ и + * поÑÐ»ÐµÐ´ÑƒÑŽÑ‰Ð°Ñ Ð²Ñтавка. */ + if (mc->mc_xcursor->mx_db.md_entries > 1 || + current_data.iov_len != data->iov_len) { + rc = mdbx_cursor_del(mc, 0); + if (rc != MDBX_SUCCESS) + return rc; + flags -= MDBX_CURRENT; + } + } else if (unlikely(node_size(key, data) > env->me_nodemax)) { + rc = mdbx_cursor_del(mc, 0); + if (rc != MDBX_SUCCESS) + return rc; + flags -= MDBX_CURRENT; + } + } + } + + if (mc->mc_db->md_root == P_INVALID) { + /* new database, cursor has nothing to point to */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + rc = MDBX_NO_ROOT; + } else if ((flags & MDBX_CURRENT) == 0) { + int exact = 0; + MDBX_val d2; + if (flags & MDBX_APPEND) { + MDBX_val k2; + rc = mdbx_cursor_last(mc, &k2, &d2); + if (rc == 0) { + rc = mc->mc_dbx->md_cmp(key, &k2); + if (rc > 0) { + rc = MDBX_NOTFOUND; + mc->mc_ki[mc->mc_top]++; + } else if (unlikely(rc < 0 || (flags & MDBX_APPENDDUP) == 0)) { + /* new key is <= last key */ + rc = MDBX_EKEYMISMATCH; + } + } + } else { + rc = mdbx_cursor_set(mc, key, &d2, MDBX_SET, &exact); + } + if ((flags & MDBX_NOOVERWRITE) && + (rc == MDBX_SUCCESS || rc == MDBX_EKEYMISMATCH)) { + mdbx_debug("duplicate key [%s]", DKEY(key)); + *data = d2; + return MDBX_KEYEXIST; + } + if (rc && unlikely(rc != MDBX_NOTFOUND)) + return rc; + } + + mc->mc_flags &= ~C_DEL; + + /* Cursor is positioned, check for room in the dirty list */ + if (!nospill) { + if (flags & MDBX_MULTIPLE) { + rdata = &xdata; + xdata.iov_len = data->iov_len * dcount; + } else { + rdata = data; + } + if (unlikely(rc2 = mdbx_page_spill(mc, key, rdata))) + return rc2; + } + + if (rc == MDBX_NO_ROOT) { + MDBX_page *np; + /* new database, write a root leaf page */ + mdbx_debug("%s", "allocating new root leaf page"); + if (unlikely(rc2 = mdbx_page_new(mc, P_LEAF, 1, &np))) { + return rc2; + } + rc2 = mdbx_cursor_push(mc, np); + if (unlikely(rc2 != MDBX_SUCCESS)) + return rc2; + mc->mc_db->md_root = np->mp_pgno; + mc->mc_db->md_depth++; + *mc->mc_dbflag |= DB_DIRTY; + if ((mc->mc_db->md_flags & (MDBX_DUPSORT | MDBX_DUPFIXED)) == MDBX_DUPFIXED) + np->mp_flags |= P_LEAF2; + mc->mc_flags |= C_INITIALIZED; + } else { + /* make sure all cursor pages are writable */ + rc2 = mdbx_cursor_touch(mc); + if (unlikely(rc2)) + return rc2; + } + + bool insert_key, insert_data, do_sub = false; + insert_key = insert_data = (rc != MDBX_SUCCESS); + if (insert_key) { + /* The key does not exist */ + mdbx_debug("inserting key at index %i", mc->mc_ki[mc->mc_top]); + if ((mc->mc_db->md_flags & MDBX_DUPSORT) && + node_size(key, data) > env->me_nodemax) { + /* Too big for a node, insert in sub-DB. Set up an empty + * "old sub-page" for prep_subDB to expand to a full page. */ + fp_flags = P_LEAF | P_DIRTY; + fp = env->me_pbuf; + fp->mp_leaf2_ksize = (uint16_t)data->iov_len; /* used if MDBX_DUPFIXED */ + fp->mp_lower = fp->mp_upper = 0; + olddata.iov_len = PAGEHDRSZ; + goto prep_subDB; + } + } else { + /* there's only a key anyway, so this is a no-op */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + char *ptr; + unsigned ksize = mc->mc_db->md_xsize; + if (key->iov_len != ksize) + return MDBX_BAD_VALSIZE; + ptr = page_leaf2key(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); + memcpy(ptr, key->iov_base, ksize); + fix_parent: + /* if overwriting slot 0 of leaf, need to + * update branch key if there is a parent page */ + if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + unsigned dtop = 1; + mc->mc_top--; + /* slot 0 is always an empty key, find real slot */ + while (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + mc->mc_top--; + dtop++; + } + if (mc->mc_ki[mc->mc_top]) + rc2 = mdbx_update_key(mc, key); + else + rc2 = MDBX_SUCCESS; + mdbx_cassert(mc, mc->mc_top + dtop < UINT16_MAX); + mc->mc_top += (uint16_t)dtop; + if (rc2) + return rc2; + } + + if (mdbx_audit_enabled()) { + int err = mdbx_cursor_check(mc, false); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + return MDBX_SUCCESS; + } + + more:; + if (mdbx_audit_enabled()) { + int err = mdbx_cursor_check(mc, false); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + /* overflow page overwrites need special handling */ + if (unlikely(F_ISSET(node_flags(node), F_BIGDATA))) { + int level, ovpages, + dpages = (node_size(key, data) > env->me_nodemax) + ? number_of_ovpages(env, data->iov_len) + : 0; + + const pgno_t pg = node_largedata_pgno(node); + MDBX_page *omp; + if (unlikely((rc2 = mdbx_page_get(mc, pg, &omp, &level)) != 0)) + return rc2; + ovpages = omp->mp_pages; + + /* Is the ov page large enough? */ + if (unlikely(mc->mc_flags & C_GCFREEZE) + ? ovpages >= dpages + : ovpages == + /* LY: add configurable threshold to keep reserve space */ + dpages) { + if (!IS_DIRTY(omp) && (level || (env->me_flags & MDBX_WRITEMAP))) { + rc = mdbx_page_unspill(mc->mc_txn, omp, &omp); + if (unlikely(rc)) + return rc; + level = 0; /* dirty in this txn or clean */ + } + /* Is it dirty? */ + if (IS_DIRTY(omp)) { + /* yes, overwrite it. Note in this case we don't + * bother to try shrinking the page if the new data + * is smaller than the overflow threshold. */ + if (unlikely(level > 1)) { + /* It is writable only in a parent txn */ + MDBX_page *np = mdbx_page_malloc(mc->mc_txn, ovpages); + if (unlikely(!np)) + return MDBX_ENOMEM; + /* Note - this page is already counted in parent's dirtyroom */ + rc2 = mdbx_dpl_append(mc->mc_txn->tw.dirtylist, pg, np); + if (unlikely(rc2 != MDBX_SUCCESS)) { + rc = rc2; + mdbx_dpage_free(env, np, ovpages); + goto fail; + } + + /* Currently we make the page look as with put() in the + * parent txn, in case the user peeks at MDBX_RESERVEd + * or unused parts. Some users treat ovpages specially. */ + const size_t whole = pgno2bytes(env, ovpages); + /* Skip the part where MDBX will put *data. + * Copy end of page, adjusting alignment so + * compiler may copy words instead of bytes. */ + const size_t off = + (PAGEHDRSZ + data->iov_len) & -(intptr_t)sizeof(size_t); + memcpy((size_t *)((char *)np + off), (size_t *)((char *)omp + off), + whole - off); + memcpy(np, omp, PAGEHDRSZ); /* Copy header of page */ + omp = np; + } + node_set_ds(node, data->iov_len); + if (F_ISSET(flags, MDBX_RESERVE)) + data->iov_base = page_data(omp); + else + memcpy(page_data(omp), data->iov_base, data->iov_len); + + if (mdbx_audit_enabled()) { + int err = mdbx_cursor_check(mc, false); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + return MDBX_SUCCESS; + } + } + if ((rc2 = mdbx_page_retire(mc, omp)) != MDBX_SUCCESS) + return rc2; + } else { + olddata.iov_len = node_ds(node); + olddata.iov_base = node_data(node); + mdbx_cassert(mc, (char *)olddata.iov_base + olddata.iov_len <= + (char *)(mc->mc_pg[mc->mc_top]) + env->me_psize); + + /* DB has dups? */ + if (F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)) { + /* Prepare (sub-)page/sub-DB to accept the new item, if needed. + * fp: old sub-page or a header faking it. + * mp: new (sub-)page. offset: growth in page size. + * xdata: node data with new page or DB. */ + unsigned i; + size_t offset = 0; + MDBX_page *mp = fp = xdata.iov_base = env->me_pbuf; + mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + + /* Was a single item before, must convert now */ + if (!F_ISSET(node_flags(node), F_DUPDATA)) { + + /* Just overwrite the current item */ + if (flags & MDBX_CURRENT) { + mdbx_cassert(mc, node_size(key, data) <= env->me_nodemax); + goto current; + } + + /* does data match? */ + if (!mc->mc_dbx->md_dcmp(data, &olddata)) { + if (unlikely(flags & (MDBX_NODUPDATA | MDBX_APPENDDUP))) + return MDBX_KEYEXIST; + /* overwrite it */ + mdbx_cassert(mc, node_size(key, data) <= env->me_nodemax); + goto current; + } + + /* Back up original data item */ + dupdata_flag = 1; + dkey.iov_len = olddata.iov_len; + dkey.iov_base = memcpy(fp + 1, olddata.iov_base, olddata.iov_len); + + /* Make sub-page header for the dup items, with dummy body */ + fp->mp_flags = P_LEAF | P_DIRTY | P_SUBP; + fp->mp_lower = 0; + xdata.iov_len = PAGEHDRSZ + dkey.iov_len + data->iov_len; + if (mc->mc_db->md_flags & MDBX_DUPFIXED) { + fp->mp_flags |= P_LEAF2; + fp->mp_leaf2_ksize = (uint16_t)data->iov_len; + xdata.iov_len += 2 * data->iov_len; /* leave space for 2 more */ + mdbx_cassert(mc, xdata.iov_len <= env->me_psize); + } else { + xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) + + (dkey.iov_len & 1) + (data->iov_len & 1); + mdbx_cassert(mc, xdata.iov_len <= env->me_psize); + } + fp->mp_upper = (uint16_t)(xdata.iov_len - PAGEHDRSZ); + olddata.iov_len = xdata.iov_len; /* pretend olddata is fp */ + } else if (node_flags(node) & F_SUBDATA) { + /* Data is on sub-DB, just store it */ + flags |= F_DUPDATA | F_SUBDATA; + goto put_sub; + } else { + /* Data is on sub-page */ + fp = olddata.iov_base; + switch (flags) { + default: + if (!(mc->mc_db->md_flags & MDBX_DUPFIXED)) { + offset = node_size(data, nullptr) + sizeof(indx_t); + break; + } + offset = fp->mp_leaf2_ksize; + if (page_room(fp) < offset) { + offset *= 4; /* space for 4 more */ + break; + } + /* FALLTHRU: Big enough MDBX_DUPFIXED sub-page */ + __fallthrough; + case MDBX_CURRENT | MDBX_NODUPDATA: + case MDBX_CURRENT: + fp->mp_flags |= P_DIRTY; + fp->mp_pgno = mp->mp_pgno; + mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; + flags |= F_DUPDATA; + goto put_sub; + } + xdata.iov_len = olddata.iov_len + offset; + } + + fp_flags = fp->mp_flags; + if (NODESIZE + node_ks(node) + xdata.iov_len > env->me_nodemax) { + /* Too big for a sub-page, convert to sub-DB */ + fp_flags &= ~P_SUBP; + prep_subDB: + nested_dupdb.md_xsize = 0; + nested_dupdb.md_flags = 0; + if (mc->mc_db->md_flags & MDBX_DUPFIXED) { + fp_flags |= P_LEAF2; + nested_dupdb.md_xsize = fp->mp_leaf2_ksize; + nested_dupdb.md_flags = MDBX_DUPFIXED; + if (mc->mc_db->md_flags & MDBX_INTEGERDUP) + nested_dupdb.md_flags |= MDBX_INTEGERKEY; + } + nested_dupdb.md_depth = 1; + nested_dupdb.md_branch_pages = 0; + nested_dupdb.md_leaf_pages = 1; + nested_dupdb.md_overflow_pages = 0; + nested_dupdb.md_entries = page_numkeys(fp); + xdata.iov_len = sizeof(nested_dupdb); + xdata.iov_base = &nested_dupdb; + if ((rc = mdbx_page_alloc(mc, 1, &mp, MDBX_ALLOC_ALL))) + return rc; + mc->mc_db->md_leaf_pages += 1; + mdbx_cassert(mc, env->me_psize > olddata.iov_len); + offset = env->me_psize - (unsigned)olddata.iov_len; + flags |= F_DUPDATA | F_SUBDATA; + nested_dupdb.md_root = mp->mp_pgno; + nested_dupdb.md_seq = nested_dupdb.md_mod_txnid = 0; + sub_root = mp; + } + if (mp != fp) { + mp->mp_flags = fp_flags | P_DIRTY; + mp->mp_leaf2_ksize = fp->mp_leaf2_ksize; + mp->mp_lower = fp->mp_lower; + mdbx_cassert(mc, fp->mp_upper + offset <= UINT16_MAX); + mp->mp_upper = (indx_t)(fp->mp_upper + offset); + if (unlikely(fp_flags & P_LEAF2)) { + memcpy(page_data(mp), page_data(fp), + page_numkeys(fp) * fp->mp_leaf2_ksize); + } else { + memcpy((char *)mp + mp->mp_upper + PAGEHDRSZ, + (char *)fp + fp->mp_upper + PAGEHDRSZ, + olddata.iov_len - fp->mp_upper - PAGEHDRSZ); + memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs), + page_numkeys(fp) * sizeof(mp->mp_ptrs[0])); + for (i = 0; i < page_numkeys(fp); i++) { + mdbx_cassert(mc, mp->mp_ptrs[i] + offset <= UINT16_MAX); + mp->mp_ptrs[i] += (indx_t)offset; + } + } + } + + rdata = &xdata; + flags |= F_DUPDATA; + do_sub = true; + if (!insert_key) + mdbx_node_del(mc, 0); + goto new_sub; + } + + /* MDBX passes F_SUBDATA in 'flags' to write a DB record */ + if (unlikely((node_flags(node) ^ flags) & F_SUBDATA)) + return MDBX_INCOMPATIBLE; + + current: + if (data->iov_len == olddata.iov_len) { + mdbx_cassert(mc, EVEN(key->iov_len) == EVEN(node_ks(node))); + /* same size, just replace it. Note that we could + * also reuse this node if the new data is smaller, + * but instead we opt to shrink the node in that case. */ + if (F_ISSET(flags, MDBX_RESERVE)) + data->iov_base = olddata.iov_base; + else if (!(mc->mc_flags & C_SUB)) + memcpy(olddata.iov_base, data->iov_base, data->iov_len); + else { + mdbx_cassert(mc, page_numkeys(mc->mc_pg[mc->mc_top]) == 1); + mdbx_cassert(mc, PAGETYPE(mc->mc_pg[mc->mc_top]) == P_LEAF); + mdbx_cassert(mc, node_ds(node) == 0); + mdbx_cassert(mc, node_flags(node) == 0); + mdbx_cassert(mc, key->iov_len < UINT16_MAX); + node_set_ks(node, key->iov_len); + memcpy(node_key(node), key->iov_base, key->iov_len); + mdbx_cassert(mc, (char *)node_key(node) + node_ds(node) < + (char *)(mc->mc_pg[mc->mc_top]) + env->me_psize); + goto fix_parent; + } + + if (mdbx_audit_enabled()) { + int err = mdbx_cursor_check(mc, false); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + return MDBX_SUCCESS; + } + } + mdbx_node_del(mc, 0); + } + + rdata = data; + +new_sub: + nflags = flags & NODE_ADD_FLAGS; + nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->iov_len + : leaf_size(env, key, rdata); + if (page_room(mc->mc_pg[mc->mc_top]) < nsize) { + if ((flags & (F_DUPDATA | F_SUBDATA)) == F_DUPDATA) + nflags &= ~MDBX_APPEND; /* sub-page may need room to grow */ + if (!insert_key) + nflags |= MDBX_SPLIT_REPLACE; + rc = mdbx_page_split(mc, key, rdata, P_INVALID, nflags); + if (rc == MDBX_SUCCESS && mdbx_audit_enabled()) + rc = mdbx_cursor_check(mc, false); + } else { + /* There is room already in this leaf page. */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0 && + rdata->iov_len == 0); + rc = mdbx_node_add_leaf2(mc, mc->mc_ki[mc->mc_top], key); + } else + rc = mdbx_node_add_leaf(mc, mc->mc_ki[mc->mc_top], key, rdata, nflags); + if (likely(rc == 0)) { + /* Adjust other cursors pointing to mp */ + MDBX_cursor *m2, *m3; + MDBX_dbi dbi = mc->mc_dbi; + unsigned i = mc->mc_top; + MDBX_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp) + continue; + if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) { + m3->mc_ki[i]++; + } + if (XCURSOR_INITED(m3)) + XCURSOR_REFRESH(m3, mp, m3->mc_ki[i]); + } + } + } + + if (likely(rc == MDBX_SUCCESS)) { + /* Now store the actual data in the child DB. Note that we're + * storing the user data in the keys field, so there are strict + * size limits on dupdata. The actual data fields of the child + * DB are all zero size. */ + if (do_sub) { + int xflags; + size_t ecount; + put_sub: + xdata.iov_len = 0; + xdata.iov_base = nullptr; + MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (flags & MDBX_CURRENT) { + xflags = (flags & MDBX_NODUPDATA) + ? MDBX_CURRENT | MDBX_NOOVERWRITE | MDBX_NOSPILL + : MDBX_CURRENT | MDBX_NOSPILL; + } else { + rc2 = mdbx_xcursor_init1(mc, node); + if (unlikely(rc2 != MDBX_SUCCESS)) + return rc2; + xflags = (flags & MDBX_NODUPDATA) ? MDBX_NOOVERWRITE | MDBX_NOSPILL + : MDBX_NOSPILL; + } + if (sub_root) + mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root; + /* converted, write the original data first */ + if (dupdata_flag) { + rc = mdbx_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); + if (unlikely(rc)) + goto bad_sub; + /* we've done our job */ + dkey.iov_len = 0; + } + if (!(node_flags(node) & F_SUBDATA) || sub_root) { + /* Adjust other cursors pointing to mp */ + MDBX_cursor *m2; + MDBX_xcursor *mx = mc->mc_xcursor; + unsigned i = mc->mc_top; + MDBX_page *mp = mc->mc_pg[i]; + const int nkeys = page_numkeys(mp); + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2 = m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) + continue; + if (!(m2->mc_flags & C_INITIALIZED)) + continue; + if (m2->mc_pg[i] == mp) { + if (m2->mc_ki[i] == mc->mc_ki[i]) { + rc2 = mdbx_xcursor_init2(m2, mx, dupdata_flag); + if (unlikely(rc2 != MDBX_SUCCESS)) + return rc2; + } else if (!insert_key && m2->mc_ki[i] < nkeys) { + XCURSOR_REFRESH(m2, mp, m2->mc_ki[i]); + } + } + } + } + mdbx_cassert(mc, mc->mc_xcursor->mx_db.md_entries < PTRDIFF_MAX); + ecount = (size_t)mc->mc_xcursor->mx_db.md_entries; + if (flags & MDBX_APPENDDUP) + xflags |= MDBX_APPEND; + rc = mdbx_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); + if (flags & F_SUBDATA) { + void *db = node_data(node); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db)); + } + insert_data = (ecount != (size_t)mc->mc_xcursor->mx_db.md_entries); + } + /* Increment count unless we just replaced an existing item. */ + if (insert_data) + mc->mc_db->md_entries++; + if (insert_key) { + /* Invalidate txn if we created an empty sub-DB */ + if (unlikely(rc)) + goto bad_sub; + /* If we succeeded and the key didn't exist before, + * make sure the cursor is marked valid. */ + mc->mc_flags |= C_INITIALIZED; + } + if (flags & MDBX_MULTIPLE) { + if (!rc) { + mcount++; + /* let caller know how many succeeded, if any */ + data[1].iov_len = mcount; + if (mcount < dcount) { + data[0].iov_base = (char *)data[0].iov_base + data[0].iov_len; + insert_key = insert_data = false; + goto more; + } + } + } + if (rc == MDBX_SUCCESS && mdbx_audit_enabled()) + rc = mdbx_cursor_check(mc, false); + return rc; + bad_sub: + if (unlikely(rc == MDBX_KEYEXIST)) + mdbx_error("unexpected %s", "MDBX_KEYEXIST"); + /* should not happen, we deleted that item */ + rc = MDBX_PROBLEM; + } +fail: + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return rc; +} + +int mdbx_cursor_del(MDBX_cursor *mc, unsigned flags) { + if (unlikely(!mc)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!(mc->mc_flags & C_INITIALIZED))) + return MDBX_EINVAL; + + if (unlikely(mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top]))) + return MDBX_NOTFOUND; + + if (unlikely(!(flags & MDBX_NOSPILL) && + (rc = mdbx_page_spill(mc, NULL, NULL)))) + return rc; + + rc = mdbx_cursor_touch(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + if (IS_LEAF2(mp)) + goto del_key; + + MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + if (flags & MDBX_NODUPDATA) { + /* mdbx_cursor_del0() will subtract the final entry */ + mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; + mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + } else { + if (!F_ISSET(node_flags(node), F_SUBDATA)) { + mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); + } + rc = mdbx_cursor_del(&mc->mc_xcursor->mx_cursor, MDBX_NOSPILL); + if (unlikely(rc)) + return rc; + /* If sub-DB still has entries, we're done */ + if (mc->mc_xcursor->mx_db.md_entries) { + if (node_flags(node) & F_SUBDATA) { + /* update subDB info */ + void *db = node_data(node); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db)); + } else { + MDBX_cursor *m2; + /* shrink fake page */ + mdbx_node_shrink(mp, mc->mc_ki[mc->mc_top]); + node = page_node(mp, mc->mc_ki[mc->mc_top]); + mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); + /* fix other sub-DB cursors pointed at fake pages on this page */ + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2 = m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) + continue; + if (!(m2->mc_flags & C_INITIALIZED)) + continue; + if (m2->mc_pg[mc->mc_top] == mp) { + MDBX_node *inner = node; + if (m2->mc_ki[mc->mc_top] >= page_numkeys(mp)) + continue; + if (m2->mc_ki[mc->mc_top] != mc->mc_ki[mc->mc_top]) { + inner = page_node(mp, m2->mc_ki[mc->mc_top]); + if (node_flags(inner) & F_SUBDATA) + continue; + } + m2->mc_xcursor->mx_cursor.mc_pg[0] = node_data(inner); + } + } + } + mc->mc_db->md_entries--; + mdbx_cassert(mc, mc->mc_db->md_entries > 0 && mc->mc_db->md_depth > 0 && + mc->mc_db->md_root != P_INVALID); + return rc; + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + } + /* otherwise fall thru and delete the sub-DB */ + } + + if (node_flags(node) & F_SUBDATA) { + /* add all the child DB's pages to the free list */ + rc = mdbx_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (unlikely(rc)) + goto fail; + } + } + /* MDBX passes F_SUBDATA in 'flags' to delete a DB record */ + else if (unlikely((node_flags(node) ^ flags) & F_SUBDATA)) { + rc = MDBX_INCOMPATIBLE; + goto fail; + } + + /* add overflow pages to free list */ + if (F_ISSET(node_flags(node), F_BIGDATA)) { + MDBX_page *omp; + if (unlikely( + (rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL)) || + (rc = mdbx_page_retire(mc, omp)))) + goto fail; + } + +del_key: + return mdbx_cursor_del0(mc); + +fail: + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return rc; +} + +/* Allocate and initialize new pages for a database. + * Set MDBX_TXN_ERROR on failure. + * + * [in] mc a cursor on the database being added to. + * [in] flags flags defining what type of page is being allocated. + * [in] num the number of pages to allocate. This is usually 1, + * unless allocating overflow pages for a large record. + * [out] mp Address of a page, or NULL on failure. + * + * Returns 0 on success, non-zero on failure. */ +static int mdbx_page_new(MDBX_cursor *mc, unsigned flags, unsigned num, + MDBX_page **mp) { + MDBX_page *np; + int rc; + + if (unlikely((rc = mdbx_page_alloc(mc, num, &np, MDBX_ALLOC_ALL)))) + return rc; + *mp = np; + mdbx_debug("allocated new page #%" PRIaPGNO ", size %u", np->mp_pgno, + mc->mc_txn->mt_env->me_psize); + np->mp_flags = (uint16_t)(flags | P_DIRTY); + np->mp_lower = 0; + np->mp_upper = (indx_t)(mc->mc_txn->mt_env->me_psize - PAGEHDRSZ); + + mc->mc_db->md_branch_pages += IS_BRANCH(np); + mc->mc_db->md_leaf_pages += IS_LEAF(np); + if (unlikely(IS_OVERFLOW(np))) { + mc->mc_db->md_overflow_pages += num; + np->mp_pages = num; + mdbx_cassert(mc, !(mc->mc_flags & C_SUB)); + } else if (unlikely(mc->mc_flags & C_SUB)) { + MDBX_db *outer = mdbx_outer_db(mc); + outer->md_branch_pages += IS_BRANCH(np); + outer->md_leaf_pages += IS_LEAF(np); + } + + return MDBX_SUCCESS; +} + +static int __must_check_result mdbx_node_add_leaf2(MDBX_cursor *mc, + unsigned indx, + const MDBX_val *key) { + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + DKBUF; + mdbx_debug("add to leaf2-%spage %" PRIaPGNO " index %i, " + " key size %" PRIuPTR " [%s]", + IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx, + key ? key->iov_len : 0, DKEY(key)); + + mdbx_cassert(mc, key); + mdbx_cassert(mc, PAGETYPE(mp) == (P_LEAF | P_LEAF2)); + const unsigned ksize = mc->mc_db->md_xsize; + mdbx_cassert(mc, ksize == key->iov_len); + const unsigned nkeys = page_numkeys(mp); + + /* Just using these for counting */ + const intptr_t lower = mp->mp_lower + sizeof(indx_t); + const intptr_t upper = mp->mp_upper - (ksize - sizeof(indx_t)); + if (unlikely(lower > upper)) { + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_PAGE_FULL; + } + mp->mp_lower = (indx_t)lower; + mp->mp_upper = (indx_t)upper; + + char *const ptr = page_leaf2key(mp, indx, ksize); + mdbx_cassert(mc, nkeys >= indx); + const unsigned diff = nkeys - indx; + if (likely(diff > 0)) + /* Move higher keys up one slot. */ + memmove(ptr + ksize, ptr, diff * ksize); + /* insert new key */ + memcpy(ptr, key->iov_base, ksize); + return MDBX_SUCCESS; +} + +static int __must_check_result mdbx_node_add_branch(MDBX_cursor *mc, + unsigned indx, + const MDBX_val *key, + pgno_t pgno) { + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + DKBUF; + mdbx_debug("add to branch-%spage %" PRIaPGNO " index %i, node-pgno %" PRIaPGNO + " key size %" PRIuPTR " [%s]", + IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx, pgno, + key ? key->iov_len : 0, DKEY(key)); + + mdbx_cassert(mc, PAGETYPE(mp) == P_BRANCH); + STATIC_ASSERT(NODESIZE % 2 == 0); + + /* Move higher pointers up one slot. */ + const unsigned nkeys = page_numkeys(mp); + mdbx_cassert(mc, nkeys >= indx); + for (unsigned i = nkeys; i > indx; --i) + mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; + + /* Adjust free space offsets. */ + const size_t branch_bytes = branch_size(mc->mc_txn->mt_env, key); + const intptr_t lower = mp->mp_lower + sizeof(indx_t); + const intptr_t upper = mp->mp_upper - (branch_bytes - sizeof(indx_t)); + if (unlikely(lower > upper)) { + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_PAGE_FULL; + } + mp->mp_lower = (indx_t)lower; + mp->mp_ptrs[indx] = mp->mp_upper = (indx_t)upper; + + /* Write the node data. */ + MDBX_node *node = page_node(mp, indx); + node_set_pgno(node, pgno); + node_set_flags(node, 0); + UNALIGNED_POKE_8(node, MDBX_node, mn_extra, 0); + node_set_ks(node, 0); + if (likely(key != NULL)) { + node_set_ks(node, key->iov_len); + memcpy(node_key(node), key->iov_base, key->iov_len); + } + return MDBX_SUCCESS; +} + +static int __must_check_result mdbx_node_add_leaf(MDBX_cursor *mc, + unsigned indx, + const MDBX_val *key, + MDBX_val *data, + unsigned flags) { + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + DKBUF; + mdbx_debug("add to leaf-%spage %" PRIaPGNO " index %i, data size %" PRIuPTR + " key size %" PRIuPTR " [%s]", + IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx, + data ? data->iov_len : 0, key ? key->iov_len : 0, DKEY(key)); + mdbx_cassert(mc, key != NULL && data != NULL); + mdbx_cassert(mc, PAGETYPE(mp) == P_LEAF); + MDBX_page *largepage = NULL; + + size_t leaf_bytes = 0; + if (unlikely(flags & F_BIGDATA)) { + /* Data already on overflow page. */ + STATIC_ASSERT(sizeof(pgno_t) % 2 == 0); + leaf_bytes = node_size(key, nullptr) + sizeof(pgno_t) + sizeof(indx_t); + } else if (unlikely(node_size(key, data) > mc->mc_txn->mt_env->me_nodemax)) { + /* Put data on overflow page. */ + mdbx_cassert(mc, !F_ISSET(mc->mc_db->md_flags, MDBX_DUPSORT)); + const pgno_t ovpages = number_of_ovpages(mc->mc_txn->mt_env, data->iov_len); + int rc = mdbx_page_new(mc, P_OVERFLOW, ovpages, &largepage); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + mdbx_debug("allocated %u overflow page(s) %" PRIaPGNO "for %" PRIuPTR + " data bytes", + largepage->mp_pages, largepage->mp_pgno, data->iov_len); + flags |= F_BIGDATA; + leaf_bytes = node_size(key, nullptr) + sizeof(pgno_t) + sizeof(indx_t); + } else { + leaf_bytes = node_size(key, data) + sizeof(indx_t); + } + mdbx_cassert(mc, leaf_bytes == leaf_size(mc->mc_txn->mt_env, key, data)); + + /* Move higher pointers up one slot. */ + const unsigned nkeys = page_numkeys(mp); + mdbx_cassert(mc, nkeys >= indx); + for (unsigned i = nkeys; i > indx; --i) + mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; + + /* Adjust free space offsets. */ + const intptr_t lower = mp->mp_lower + sizeof(indx_t); + const intptr_t upper = mp->mp_upper - (leaf_bytes - sizeof(indx_t)); + if (unlikely(lower > upper)) { + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_PAGE_FULL; + } + mp->mp_lower = (indx_t)lower; + mp->mp_ptrs[indx] = mp->mp_upper = (indx_t)upper; + + /* Write the node data. */ + MDBX_node *node = page_node(mp, indx); + node_set_ks(node, key->iov_len); + node_set_flags(node, (uint8_t)flags); + UNALIGNED_POKE_8(node, MDBX_node, mn_extra, 0); + node_set_ds(node, data->iov_len); + memcpy(node_key(node), key->iov_base, key->iov_len); + + void *nodedata = node_data(node); + if (likely(largepage == NULL)) { + if (unlikely(flags & F_BIGDATA)) + memcpy(nodedata, data->iov_base, sizeof(pgno_t)); + else if (unlikely(flags & MDBX_RESERVE)) + data->iov_base = nodedata; + else if (likely(nodedata != data->iov_base)) + memcpy(nodedata, data->iov_base, data->iov_len); + } else { + poke_pgno(nodedata, largepage->mp_pgno); + nodedata = page_data(largepage); + if (unlikely(flags & MDBX_RESERVE)) + data->iov_base = nodedata; + else if (likely(nodedata != data->iov_base)) + memcpy(nodedata, data->iov_base, data->iov_len); + } + return MDBX_SUCCESS; +} + +/* Delete the specified node from a page. + * [in] mc Cursor pointing to the node to delete. + * [in] ksize The size of a node. Only used if the page is + * part of a MDBX_DUPFIXED database. */ +static void mdbx_node_del(MDBX_cursor *mc, size_t ksize) { + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + int indx = mc->mc_ki[mc->mc_top]; + int i, j, nkeys, ptr; + MDBX_node *node; + char *base; + + mdbx_debug("delete node %u on %s page %" PRIaPGNO, indx, + IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno); + nkeys = page_numkeys(mp); + mdbx_cassert(mc, indx < nkeys); + + if (IS_LEAF2(mp)) { + mdbx_cassert(mc, ksize >= sizeof(indx_t)); + unsigned diff = nkeys - 1 - indx; + base = page_leaf2key(mp, indx, ksize); + if (diff) + memmove(base, base + ksize, diff * ksize); + mdbx_cassert(mc, mp->mp_lower >= sizeof(indx_t)); + mp->mp_lower -= sizeof(indx_t); + mdbx_cassert(mc, + (size_t)UINT16_MAX - mp->mp_upper >= ksize - sizeof(indx_t)); + mp->mp_upper += (indx_t)(ksize - sizeof(indx_t)); + return; + } + + node = page_node(mp, indx); + size_t sz = NODESIZE + node_ks(node); + if (IS_LEAF(mp)) { + if (F_ISSET(node_flags(node), F_BIGDATA)) + sz += sizeof(pgno_t); + else + sz += node_ds(node); + } + sz = EVEN(sz); + + ptr = mp->mp_ptrs[indx]; + for (i = j = 0; i < nkeys; i++) { + if (i != indx) { + mp->mp_ptrs[j] = mp->mp_ptrs[i]; + if (mp->mp_ptrs[i] < ptr) { + mdbx_cassert(mc, (size_t)UINT16_MAX - mp->mp_ptrs[j] >= sz); + mp->mp_ptrs[j] += (indx_t)sz; + } + j++; + } + } + + base = (char *)mp + mp->mp_upper + PAGEHDRSZ; + memmove(base + sz, base, ptr - mp->mp_upper); + + mdbx_cassert(mc, mp->mp_lower >= sizeof(indx_t)); + mp->mp_lower -= sizeof(indx_t); + mdbx_cassert(mc, (size_t)UINT16_MAX - mp->mp_upper >= sz); + mp->mp_upper += (indx_t)sz; +} + +/* Compact the main page after deleting a node on a subpage. + * [in] mp The main page to operate on. + * [in] indx The index of the subpage on the main page. */ +static void mdbx_node_shrink(MDBX_page *mp, unsigned indx) { + MDBX_node *node; + MDBX_page *sp, *xp; + char *base; + size_t nsize, delta, len, ptr; + int i; + + node = page_node(mp, indx); + sp = (MDBX_page *)node_data(node); + delta = page_room(sp); + assert(delta > 0); + + /* Prepare to shift upward, set len = length(subpage part to shift) */ + if (IS_LEAF2(sp)) { + delta &= /* do not make the node uneven-sized */ ~1u; + if (unlikely(delta) == 0) + return; + nsize = node_ds(node) - delta; + assert(nsize % 1 == 0); + len = nsize; + } else { + xp = (MDBX_page *)((char *)sp + delta); /* destination subpage */ + for (i = page_numkeys(sp); --i >= 0;) { + assert(sp->mp_ptrs[i] >= delta); + xp->mp_ptrs[i] = (indx_t)(sp->mp_ptrs[i] - delta); + } + nsize = node_ds(node) - delta; + len = PAGEHDRSZ; + } + sp->mp_upper = sp->mp_lower; + sp->mp_pgno = mp->mp_pgno; + node_set_ds(node, nsize); + + /* Shift upward */ + base = (char *)mp + mp->mp_upper + PAGEHDRSZ; + memmove(base + delta, base, (char *)sp + len - base); + + ptr = mp->mp_ptrs[indx]; + for (i = page_numkeys(mp); --i >= 0;) { + if (mp->mp_ptrs[i] <= ptr) { + assert((size_t)UINT16_MAX - mp->mp_ptrs[i] >= delta); + mp->mp_ptrs[i] += (indx_t)delta; + } + } + assert((size_t)UINT16_MAX - mp->mp_upper >= delta); + mp->mp_upper += (indx_t)delta; +} + +/* Initial setup of a sorted-dups cursor. + * + * Sorted duplicates are implemented as a sub-database for the given key. + * The duplicate data items are actually keys of the sub-database. + * Operations on the duplicate data items are performed using a sub-cursor + * initialized when the sub-database is first accessed. This function does + * the preliminary setup of the sub-cursor, filling in the fields that + * depend only on the parent DB. + * + * [in] mc The main cursor whose sorted-dups cursor is to be initialized. */ +static int mdbx_xcursor_init0(MDBX_cursor *mc) { + MDBX_xcursor *mx = mc->mc_xcursor; + if (unlikely(mx == nullptr)) + return MDBX_CORRUPTED; + + mx->mx_cursor.mc_xcursor = NULL; + mx->mx_cursor.mc_txn = mc->mc_txn; + mx->mx_cursor.mc_db = &mx->mx_db; + mx->mx_cursor.mc_dbx = &mx->mx_dbx; + mx->mx_cursor.mc_dbi = mc->mc_dbi; + mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + mx->mx_dbx.md_name.iov_len = 0; + mx->mx_dbx.md_name.iov_base = NULL; + mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; + mx->mx_dbx.md_dcmp = NULL; + return MDBX_SUCCESS; +} + +/* Final setup of a sorted-dups cursor. + * Sets up the fields that depend on the data from the main cursor. + * [in] mc The main cursor whose sorted-dups cursor is to be initialized. + * [in] node The data containing the MDBX_db record for the sorted-dup database. + */ +static int mdbx_xcursor_init1(MDBX_cursor *mc, MDBX_node *node) { + MDBX_xcursor *mx = mc->mc_xcursor; + if (unlikely(mx == nullptr)) + return MDBX_CORRUPTED; + + if (node_flags(node) & F_SUBDATA) { + if (unlikely(node_ds(node) != sizeof(MDBX_db))) + return MDBX_CORRUPTED; + memcpy(&mx->mx_db, node_data(node), sizeof(MDBX_db)); + mx->mx_cursor.mc_pg[0] = 0; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + } else { + if (unlikely(node_ds(node) <= PAGEHDRSZ)) + return MDBX_CORRUPTED; + MDBX_page *fp = node_data(node); + mx->mx_db.md_xsize = 0; + mx->mx_db.md_flags = 0; + mx->mx_db.md_depth = 1; + mx->mx_db.md_branch_pages = 0; + mx->mx_db.md_leaf_pages = 1; + mx->mx_db.md_overflow_pages = 0; + mx->mx_db.md_entries = page_numkeys(fp); + mx->mx_db.md_root = fp->mp_pgno; + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_INITIALIZED | C_SUB; + mx->mx_cursor.mc_pg[0] = fp; + mx->mx_cursor.mc_ki[0] = 0; + if (mc->mc_db->md_flags & MDBX_DUPFIXED) { + mx->mx_db.md_flags = MDBX_DUPFIXED; + mx->mx_db.md_xsize = fp->mp_leaf2_ksize; + if (mc->mc_db->md_flags & MDBX_INTEGERDUP) + mx->mx_db.md_flags |= MDBX_INTEGERKEY; + } + } + mdbx_debug("Sub-db -%u root page %" PRIaPGNO, mx->mx_cursor.mc_dbi, + mx->mx_db.md_root); + mx->mx_dbflag = DB_VALID | DB_USRVALID | DB_DUPDATA; + return MDBX_SUCCESS; +} + +/* Fixup a sorted-dups cursor due to underlying update. + * Sets up some fields that depend on the data from the main cursor. + * Almost the same as init1, but skips initialization steps if the + * xcursor had already been used. + * [in] mc The main cursor whose sorted-dups cursor is to be fixed up. + * [in] src_mx The xcursor of an up-to-date cursor. + * [in] new_dupdata True if converting from a non-F_DUPDATA item. */ +static int mdbx_xcursor_init2(MDBX_cursor *mc, MDBX_xcursor *src_mx, + int new_dupdata) { + MDBX_xcursor *mx = mc->mc_xcursor; + if (unlikely(mx == nullptr)) + return MDBX_CORRUPTED; + + if (new_dupdata) { + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags |= C_INITIALIZED; + mx->mx_cursor.mc_ki[0] = 0; + mx->mx_dbflag = DB_VALID | DB_USRVALID | DB_DUPDATA; + mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; + } else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { + return MDBX_SUCCESS; + } + mx->mx_db = src_mx->mx_db; + mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; + mdbx_debug("Sub-db -%u root page %" PRIaPGNO, mx->mx_cursor.mc_dbi, + mx->mx_db.md_root); + return MDBX_SUCCESS; +} + +/* Initialize a cursor for a given transaction and database. */ +static int mdbx_cursor_init(MDBX_cursor *mc, MDBX_txn *txn, MDBX_dbi dbi) { + mc->mc_signature = MDBX_MC_SIGNATURE; + mc->mc_next = NULL; + mc->mc_backup = NULL; + mc->mc_dbi = dbi; + mc->mc_txn = txn; + mc->mc_db = &txn->mt_dbs[dbi]; + mc->mc_dbx = &txn->mt_dbxs[dbi]; + mc->mc_dbflag = &txn->mt_dbflags[dbi]; + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_pg[0] = 0; + mc->mc_flags = 0; + mc->mc_ki[0] = 0; + mc->mc_xcursor = NULL; + + if (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) { + STATIC_ASSERT(offsetof(MDBX_cursor_couple, outer) == 0); + MDBX_xcursor *mx = &container_of(mc, MDBX_cursor_couple, outer)->inner; + mdbx_tassert(txn, mx != NULL); + mx->mx_cursor.mc_signature = MDBX_MC_SIGNATURE; + mc->mc_xcursor = mx; + int rc = mdbx_xcursor_init0(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + + int rc = MDBX_SUCCESS; + if (unlikely(*mc->mc_dbflag & DB_STALE)) { + rc = mdbx_page_search(mc, NULL, MDBX_PS_ROOTONLY); + rc = (rc != MDBX_NOTFOUND) ? rc : MDBX_SUCCESS; + } + return rc; +} + +int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) { + if (unlikely(!ret)) + return MDBX_EINVAL; + *ret = NULL; + + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_VALID))) + return MDBX_EINVAL; + + if (unlikely(dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDBX_RDONLY))) + return MDBX_EACCESS; + + const size_t size = (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) + ? sizeof(MDBX_cursor_couple) + : sizeof(MDBX_cursor); + + MDBX_cursor *mc; + if (likely((mc = mdbx_malloc(size)) != NULL)) { + rc = mdbx_cursor_init(mc, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_free(mc); + return rc; + } + if (txn->mt_cursors) { + mc->mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = mc; + mc->mc_flags |= C_UNTRACK; + } + } else { + return MDBX_ENOMEM; + } + + *ret = mc; + return MDBX_SUCCESS; +} + +int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *mc) { + if (unlikely(!mc)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE && + mc->mc_signature != MDBX_MC_READY4CLOSE)) + return MDBX_EINVAL; + + int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!TXN_DBI_EXIST(txn, mc->mc_dbi, DB_VALID))) + return MDBX_EINVAL; + + if (unlikely(mc->mc_backup)) + return MDBX_EINVAL; + + if (unlikely((mc->mc_flags & C_UNTRACK) || txn->mt_cursors)) { + MDBX_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) + prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + mc->mc_signature = MDBX_MC_READY4CLOSE; + } + + if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED)) + return MDBX_BAD_TXN; + + return mdbx_cursor_init(mc, txn, mc->mc_dbi); +} + +/* Return the count of duplicate data items for the current key */ +int mdbx_cursor_count(MDBX_cursor *mc, size_t *countp) { + if (unlikely(mc == NULL)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(countp == NULL || !(mc->mc_flags & C_INITIALIZED))) + return MDBX_EINVAL; + + if (!mc->mc_snum) { + *countp = 0; + return MDBX_NOTFOUND; + } + + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + if ((mc->mc_flags & C_EOF) && mc->mc_ki[mc->mc_top] >= page_numkeys(mp)) { + *countp = 0; + return MDBX_NOTFOUND; + } + + *countp = 1; + if (mc->mc_xcursor != NULL) { + MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + mdbx_cassert(mc, mc->mc_xcursor && (mc->mc_xcursor->mx_cursor.mc_flags & + C_INITIALIZED)); + *countp = unlikely(mc->mc_xcursor->mx_db.md_entries > PTRDIFF_MAX) + ? PTRDIFF_MAX + : (size_t)mc->mc_xcursor->mx_db.md_entries; + } + } + return MDBX_SUCCESS; +} + +void mdbx_cursor_close(MDBX_cursor *mc) { + if (mc) { + mdbx_ensure(NULL, mc->mc_signature == MDBX_MC_SIGNATURE || + mc->mc_signature == MDBX_MC_READY4CLOSE); + if (!mc->mc_backup) { + /* Remove from txn, if tracked. + * A read-only txn (!C_UNTRACK) may have been freed already, + * so do not peek inside it. Only write txns track cursors. */ + if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { + MDBX_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) + prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + } + mc->mc_signature = 0; + mdbx_free(mc); + } else { + /* cursor closed before nested txn ends */ + mdbx_cassert(mc, mc->mc_signature == MDBX_MC_SIGNATURE); + mc->mc_signature = MDBX_MC_WAIT4EOT; + } + } +} + +MDBX_txn *mdbx_cursor_txn(MDBX_cursor *mc) { + if (unlikely(!mc || mc->mc_signature != MDBX_MC_SIGNATURE)) + return NULL; + MDBX_txn *txn = mc->mc_txn; + if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE)) + return NULL; + if (unlikely(txn->mt_flags & MDBX_TXN_FINISHED)) + return NULL; + return txn; +} + +MDBX_dbi mdbx_cursor_dbi(MDBX_cursor *mc) { + if (unlikely(!mc || mc->mc_signature != MDBX_MC_SIGNATURE)) + return UINT_MAX; + return mc->mc_dbi; +} + +/* Replace the key for a branch node with a new key. + * Set MDBX_TXN_ERROR on failure. + * [in] mc Cursor pointing to the node to operate on. + * [in] key The new key to use. + * Returns 0 on success, non-zero on failure. */ +static int mdbx_update_key(MDBX_cursor *mc, const MDBX_val *key) { + MDBX_page *mp; + MDBX_node *node; + char *base; + size_t len; + int delta, ksize, oksize; + int ptr, i, nkeys, indx; + DKBUF; + + indx = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + node = page_node(mp, indx); + ptr = mp->mp_ptrs[indx]; + if (MDBX_DEBUG) { + MDBX_val k2; + char kbuf2[DKBUF_MAXKEYSIZE * 2 + 1]; + k2.iov_base = node_key(node); + k2.iov_len = node_ks(node); + mdbx_debug("update key %u (ofs %u) [%s] to [%s] on page %" PRIaPGNO, indx, + ptr, mdbx_dump_val(&k2, kbuf2, sizeof(kbuf2)), DKEY(key), + mp->mp_pgno); + } + + /* Sizes must be 2-byte aligned. */ + ksize = EVEN(key->iov_len); + oksize = EVEN(node_ks(node)); + delta = ksize - oksize; + + /* Shift node contents if EVEN(key length) changed. */ + if (delta) { + if (delta > (int)page_room(mp)) { + /* not enough space left, do a delete and split */ + mdbx_debug("Not enough room, delta = %d, splitting...", delta); + pgno_t pgno = node_pgno(node); + mdbx_node_del(mc, 0); + int rc = mdbx_page_split(mc, key, NULL, pgno, MDBX_SPLIT_REPLACE); + if (rc == MDBX_SUCCESS && mdbx_audit_enabled()) + rc = mdbx_cursor_check(mc, true); + return rc; + } + + nkeys = page_numkeys(mp); + for (i = 0; i < nkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) { + mdbx_cassert(mc, mp->mp_ptrs[i] >= delta); + mp->mp_ptrs[i] -= (indx_t)delta; + } + } + + base = (char *)mp + mp->mp_upper + PAGEHDRSZ; + len = ptr - mp->mp_upper + NODESIZE; + memmove(base - delta, base, len); + mdbx_cassert(mc, mp->mp_upper >= delta); + mp->mp_upper -= (indx_t)delta; + + node = page_node(mp, indx); + } + + /* But even if no shift was needed, update ksize */ + node_set_ks(node, key->iov_len); + + if (key->iov_len) + memcpy(node_key(node), key->iov_base, key->iov_len); + return MDBX_SUCCESS; +} + +/* Move a node from csrc to cdst. */ +static int mdbx_node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, int fromleft) { + int rc; + DKBUF; + + MDBX_page *psrc = csrc->mc_pg[csrc->mc_top]; + MDBX_page *pdst = cdst->mc_pg[cdst->mc_top]; + mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst)); + mdbx_cassert(csrc, csrc->mc_dbi == cdst->mc_dbi); + mdbx_cassert(csrc, csrc->mc_top == cdst->mc_top); + if (unlikely(PAGETYPE(psrc) != PAGETYPE(pdst))) { + bailout: + csrc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return MDBX_PROBLEM; + } + + MDBX_val key4move; + switch (PAGETYPE(psrc)) { + case P_BRANCH: { + const MDBX_node *srcnode = page_node(psrc, csrc->mc_ki[csrc->mc_top]); + mdbx_cassert(csrc, node_flags(srcnode) == 0); + const pgno_t srcpg = node_pgno(srcnode); + key4move.iov_len = node_ks(srcnode); + key4move.iov_base = node_key(srcnode); + + if (csrc->mc_ki[csrc->mc_top] == 0) { + const uint16_t snum = csrc->mc_snum; + mdbx_cassert(csrc, snum > 0); + /* must find the lowest key below src */ + rc = mdbx_page_search_lowest(csrc); + MDBX_page *lowest_page = csrc->mc_pg[csrc->mc_top]; + if (unlikely(rc)) + return rc; + mdbx_cassert(csrc, IS_LEAF(lowest_page)); + if (unlikely(!IS_LEAF(lowest_page))) + goto bailout; + if (IS_LEAF2(lowest_page)) { + key4move.iov_len = csrc->mc_db->md_xsize; + key4move.iov_base = page_leaf2key(lowest_page, 0, key4move.iov_len); + } else { + const MDBX_node *lowest_node = page_node(lowest_page, 0); + key4move.iov_len = node_ks(lowest_node); + key4move.iov_base = node_key(lowest_node); + } + + /* restore cursor after mdbx_page_search_lowest() */ + csrc->mc_snum = snum; + csrc->mc_top = snum - 1; + csrc->mc_ki[csrc->mc_top] = 0; + + /* paranoia */ + mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]); + mdbx_cassert(csrc, IS_BRANCH(psrc)); + if (unlikely(!IS_BRANCH(psrc))) + goto bailout; + } + + if (cdst->mc_ki[cdst->mc_top] == 0) { + const uint16_t snum = cdst->mc_snum; + mdbx_cassert(csrc, snum > 0); + MDBX_cursor mn; + mdbx_cursor_copy(cdst, &mn); + mn.mc_xcursor = NULL; + /* must find the lowest key below dst */ + rc = mdbx_page_search_lowest(&mn); + if (unlikely(rc)) + return rc; + MDBX_page *const lowest_page = mn.mc_pg[mn.mc_top]; + mdbx_cassert(cdst, IS_LEAF(lowest_page)); + if (unlikely(!IS_LEAF(lowest_page))) + goto bailout; + MDBX_val key; + if (IS_LEAF2(lowest_page)) { + key.iov_len = mn.mc_db->md_xsize; + key.iov_base = page_leaf2key(lowest_page, 0, key.iov_len); + } else { + MDBX_node *lowest_node = page_node(lowest_page, 0); + key.iov_len = node_ks(lowest_node); + key.iov_base = node_key(lowest_node); + } + + /* restore cursor after mdbx_page_search_lowest() */ + mn.mc_snum = snum; + mn.mc_top = snum - 1; + mn.mc_ki[mn.mc_top] = 0; + + const intptr_t delta = + EVEN(key.iov_len) - EVEN(node_ks(page_node(mn.mc_pg[mn.mc_top], 0))); + const intptr_t needed = + branch_size(cdst->mc_txn->mt_env, &key4move) + delta; + const intptr_t have = page_room(pdst); + if (unlikely(needed > have)) + return MDBX_RESULT_TRUE; + + if (unlikely((rc = mdbx_page_touch(csrc)) || + (rc = mdbx_page_touch(cdst)))) + return rc; + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + + rc = mdbx_update_key(&mn, &key); + if (unlikely(rc)) + return rc; + } else { + const size_t needed = branch_size(cdst->mc_txn->mt_env, &key4move); + const size_t have = page_room(pdst); + if (unlikely(needed > have)) + return MDBX_RESULT_TRUE; + + if (unlikely((rc = mdbx_page_touch(csrc)) || + (rc = mdbx_page_touch(cdst)))) + return rc; + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + } + + mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO + " to node %u on page %" PRIaPGNO, + "branch", csrc->mc_ki[csrc->mc_top], DKEY(&key4move), + psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno); + /* Add the node to the destination page. */ + rc = + mdbx_node_add_branch(cdst, cdst->mc_ki[cdst->mc_top], &key4move, srcpg); + } break; + + case P_LEAF: { + /* Mark src and dst as dirty. */ + if (unlikely((rc = mdbx_page_touch(csrc)) || (rc = mdbx_page_touch(cdst)))) + return rc; + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + const MDBX_node *srcnode = page_node(psrc, csrc->mc_ki[csrc->mc_top]); + MDBX_val data; + data.iov_len = node_ds(srcnode); + data.iov_base = node_data(srcnode); + key4move.iov_len = node_ks(srcnode); + key4move.iov_base = node_key(srcnode); + mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO + " to node %u on page %" PRIaPGNO, + "leaf", csrc->mc_ki[csrc->mc_top], DKEY(&key4move), + psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno); + /* Add the node to the destination page. */ + rc = mdbx_node_add_leaf(cdst, cdst->mc_ki[cdst->mc_top], &key4move, &data, + node_flags(srcnode)); + } break; + + case P_LEAF | P_LEAF2: { + /* Mark src and dst as dirty. */ + if (unlikely((rc = mdbx_page_touch(csrc)) || (rc = mdbx_page_touch(cdst)))) + return rc; + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + key4move.iov_len = csrc->mc_db->md_xsize; + key4move.iov_base = + page_leaf2key(psrc, csrc->mc_ki[csrc->mc_top], key4move.iov_len); + mdbx_debug("moving %s-node %u [%s] on page %" PRIaPGNO + " to node %u on page %" PRIaPGNO, + "leaf2", csrc->mc_ki[csrc->mc_top], DKEY(&key4move), + psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno); + /* Add the node to the destination page. */ + rc = mdbx_node_add_leaf2(cdst, cdst->mc_ki[cdst->mc_top], &key4move); + } break; + + default: + goto bailout; + } + + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + /* Delete the node from the source page. */ + mdbx_node_del(csrc, key4move.iov_len); + + mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]); + mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]); + mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst)); + + { + /* Adjust other cursors pointing to mp */ + MDBX_cursor *m2, *m3; + const MDBX_dbi dbi = csrc->mc_dbi; + mdbx_cassert(csrc, csrc->mc_top == cdst->mc_top); + if (fromleft) { + /* If we're adding on the left, bump others up */ + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { + m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) + continue; + if (m3 != cdst && m3->mc_pg[csrc->mc_top] == pdst && + m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) { + m3->mc_ki[csrc->mc_top]++; + } + if (m3 != csrc && m3->mc_pg[csrc->mc_top] == psrc && + m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = pdst; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + mdbx_cassert(csrc, csrc->mc_top > 0); + m3->mc_ki[csrc->mc_top - 1]++; + } + if (XCURSOR_INITED(m3) && IS_LEAF(psrc)) + XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]); + } + } else { + /* Adding on the right, bump others down */ + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { + m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (m3 == csrc) + continue; + if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) + continue; + if (m3->mc_pg[csrc->mc_top] == psrc) { + if (!m3->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = pdst; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + mdbx_cassert(csrc, csrc->mc_top > 0); + m3->mc_ki[csrc->mc_top - 1]--; + } else { + m3->mc_ki[csrc->mc_top]--; + } + if (XCURSOR_INITED(m3) && IS_LEAF(psrc)) + XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], + m3->mc_ki[csrc->mc_top]); + } + } + } + } + + /* Update the parent separators. */ + if (csrc->mc_ki[csrc->mc_top] == 0) { + mdbx_cassert(csrc, csrc->mc_top > 0); + if (csrc->mc_ki[csrc->mc_top - 1] != 0) { + MDBX_val key; + if (IS_LEAF2(psrc)) { + key.iov_len = psrc->mp_leaf2_ksize; + key.iov_base = page_leaf2key(psrc, 0, key.iov_len); + } else { + MDBX_node *srcnode = page_node(psrc, 0); + key.iov_len = node_ks(srcnode); + key.iov_base = node_key(srcnode); + } + mdbx_debug("update separator for source page %" PRIaPGNO " to [%s]", + psrc->mp_pgno, DKEY(&key)); + MDBX_cursor mn; + mdbx_cursor_copy(csrc, &mn); + mn.mc_xcursor = NULL; + mdbx_cassert(csrc, mn.mc_snum > 0); + mn.mc_snum--; + mn.mc_top--; + /* We want mdbx_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, rc = mdbx_update_key(&mn, &key)); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + if (IS_BRANCH(psrc)) { + const MDBX_val nullkey = {0, 0}; + const indx_t ix = csrc->mc_ki[csrc->mc_top]; + csrc->mc_ki[csrc->mc_top] = 0; + rc = mdbx_update_key(csrc, &nullkey); + csrc->mc_ki[csrc->mc_top] = ix; + mdbx_cassert(csrc, rc == MDBX_SUCCESS); + } + } + + if (cdst->mc_ki[cdst->mc_top] == 0) { + mdbx_cassert(cdst, cdst->mc_top > 0); + if (cdst->mc_ki[cdst->mc_top - 1] != 0) { + MDBX_val key; + if (IS_LEAF2(pdst)) { + key.iov_len = pdst->mp_leaf2_ksize; + key.iov_base = page_leaf2key(pdst, 0, key.iov_len); + } else { + MDBX_node *srcnode = page_node(pdst, 0); + key.iov_len = node_ks(srcnode); + key.iov_base = node_key(srcnode); + } + mdbx_debug("update separator for destination page %" PRIaPGNO " to [%s]", + pdst->mp_pgno, DKEY(&key)); + MDBX_cursor mn; + mdbx_cursor_copy(cdst, &mn); + mn.mc_xcursor = NULL; + mdbx_cassert(cdst, mn.mc_snum > 0); + mn.mc_snum--; + mn.mc_top--; + /* We want mdbx_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, rc = mdbx_update_key(&mn, &key)); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + if (IS_BRANCH(pdst)) { + const MDBX_val nullkey = {0, 0}; + const indx_t ix = cdst->mc_ki[cdst->mc_top]; + cdst->mc_ki[cdst->mc_top] = 0; + rc = mdbx_update_key(cdst, &nullkey); + cdst->mc_ki[cdst->mc_top] = ix; + mdbx_cassert(cdst, rc == MDBX_SUCCESS); + } + } + + return MDBX_SUCCESS; +} + +/* Merge one page into another. + * + * The nodes from the page pointed to by csrc will be copied to the page + * pointed to by cdst and then the csrc page will be freed. + * + * [in] csrc Cursor pointing to the source page. + * [in] cdst Cursor pointing to the destination page. + * + * Returns 0 on success, non-zero on failure. */ +static int mdbx_page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { + MDBX_val key; + int rc; + + mdbx_cassert(csrc, csrc != cdst); + const MDBX_page *const psrc = csrc->mc_pg[csrc->mc_top]; + MDBX_page *pdst = cdst->mc_pg[cdst->mc_top]; + mdbx_debug("merging page %" PRIaPGNO " into %" PRIaPGNO, psrc->mp_pgno, + pdst->mp_pgno); + + mdbx_cassert(csrc, PAGETYPE(psrc) == PAGETYPE(pdst)); + mdbx_cassert(csrc, + csrc->mc_dbi == cdst->mc_dbi && csrc->mc_db == cdst->mc_db); + mdbx_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ + mdbx_cassert(cdst, cdst->mc_snum > 1); + mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth || + IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1])); + mdbx_cassert(csrc, csrc->mc_snum < csrc->mc_db->md_depth || + IS_LEAF(csrc->mc_pg[csrc->mc_db->md_depth - 1])); + mdbx_cassert(cdst, page_room(pdst) >= page_used(cdst->mc_txn->mt_env, psrc)); + const int pagetype = PAGETYPE(psrc); + + /* Move all nodes from src to dst */ + const unsigned dst_nkeys = page_numkeys(pdst); + const unsigned src_nkeys = page_numkeys(psrc); + if (likely(src_nkeys)) { + unsigned j = dst_nkeys; + if (unlikely(pagetype & P_LEAF2)) { + /* Mark dst as dirty. */ + if (unlikely(rc = mdbx_page_touch(cdst))) + return rc; + + key.iov_len = csrc->mc_db->md_xsize; + key.iov_base = page_data(psrc); + unsigned i = 0; + do { + rc = mdbx_node_add_leaf2(cdst, j++, &key); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + key.iov_base = (char *)key.iov_base + key.iov_len; + } while (++i != src_nkeys); + } else { + MDBX_node *srcnode = page_node(psrc, 0); + key.iov_len = node_ks(srcnode); + key.iov_base = node_key(srcnode); + if (pagetype & P_BRANCH) { + MDBX_cursor mn; + mdbx_cursor_copy(csrc, &mn); + mn.mc_xcursor = NULL; + /* must find the lowest key below src */ + rc = mdbx_page_search_lowest(&mn); + if (unlikely(rc)) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + key.iov_len = mn.mc_db->md_xsize; + key.iov_base = page_leaf2key(mn.mc_pg[mn.mc_top], 0, key.iov_len); + } else { + MDBX_node *lowest = page_node(mn.mc_pg[mn.mc_top], 0); + key.iov_len = node_ks(lowest); + key.iov_base = node_key(lowest); + + const size_t dst_room = page_room(pdst); + const size_t src_used = page_used(cdst->mc_txn->mt_env, psrc); + const size_t space_needed = src_used - node_ks(srcnode) + key.iov_len; + if (unlikely(space_needed > dst_room)) + return MDBX_RESULT_TRUE; + } + } + + /* Mark dst as dirty. */ + if (unlikely(rc = mdbx_page_touch(cdst))) + return rc; + + unsigned i = 0; + while (true) { + if (pagetype & P_LEAF) { + MDBX_val data; + data.iov_len = node_ds(srcnode); + data.iov_base = node_data(srcnode); + rc = mdbx_node_add_leaf(cdst, j++, &key, &data, node_flags(srcnode)); + } else { + mdbx_cassert(csrc, node_flags(srcnode) == 0); + rc = mdbx_node_add_branch(cdst, j++, &key, node_pgno(srcnode)); + } + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (++i == src_nkeys) + break; + srcnode = page_node(psrc, i); + key.iov_len = node_ks(srcnode); + key.iov_base = node_key(srcnode); + } + } + + pdst = cdst->mc_pg[cdst->mc_top]; + mdbx_debug("dst page %" PRIaPGNO " now has %u keys (%.1f%% filled)", + pdst->mp_pgno, page_numkeys(pdst), + page_fill(cdst->mc_txn->mt_env, pdst)); + + mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]); + mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]); + } + + /* Unlink the src page from parent and add to free list. */ + csrc->mc_top--; + mdbx_node_del(csrc, 0); + if (csrc->mc_ki[csrc->mc_top] == 0) { + const MDBX_val nullkey = {0, 0}; + rc = mdbx_update_key(csrc, &nullkey); + if (unlikely(rc)) { + csrc->mc_top++; + return rc; + } + } + csrc->mc_top++; + + mdbx_cassert(csrc, psrc == csrc->mc_pg[csrc->mc_top]); + mdbx_cassert(cdst, pdst == cdst->mc_pg[cdst->mc_top]); + + { + /* Adjust other cursors pointing to mp */ + MDBX_cursor *m2, *m3; + const MDBX_dbi dbi = csrc->mc_dbi; + const unsigned top = csrc->mc_top; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { + m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (m3 == csrc || top >= m3->mc_snum) + continue; + if (m3->mc_pg[top] == psrc) { + m3->mc_pg[top] = pdst; + mdbx_cassert(m3, dst_nkeys + m3->mc_ki[top] <= UINT16_MAX); + m3->mc_ki[top] += (indx_t)dst_nkeys; + m3->mc_ki[top - 1] = cdst->mc_ki[top - 1]; + } else if (m3->mc_pg[top - 1] == csrc->mc_pg[top - 1] && + m3->mc_ki[top - 1] > csrc->mc_ki[top - 1]) { + m3->mc_ki[top - 1]--; + } + if (XCURSOR_INITED(m3) && IS_LEAF(psrc)) + XCURSOR_REFRESH(m3, m3->mc_pg[top], m3->mc_ki[top]); + } + } + + /* If not operating on GC, allow this page to be reused + * in this txn. Otherwise just add to free list. */ + rc = mdbx_page_retire(csrc, (MDBX_page *)psrc); + if (unlikely(rc)) + return rc; + + mdbx_cassert(cdst, cdst->mc_db->md_entries > 0); + mdbx_cassert(cdst, cdst->mc_snum <= cdst->mc_db->md_depth); + mdbx_cassert(cdst, cdst->mc_top > 0); + mdbx_cassert(cdst, cdst->mc_snum == cdst->mc_top + 1); + MDBX_page *const top_page = cdst->mc_pg[cdst->mc_top]; + const indx_t top_indx = cdst->mc_ki[cdst->mc_top]; + const uint16_t save_snum = cdst->mc_snum; + const uint16_t save_depth = cdst->mc_db->md_depth; + mdbx_cursor_pop(cdst); + rc = mdbx_rebalance(cdst); + if (unlikely(rc)) + return rc; + + mdbx_cassert(cdst, cdst->mc_db->md_entries > 0); + mdbx_cassert(cdst, cdst->mc_snum <= cdst->mc_db->md_depth); + mdbx_cassert(cdst, cdst->mc_snum == cdst->mc_top + 1); + + if (IS_LEAF(cdst->mc_pg[cdst->mc_top])) { + /* LY: don't touch cursor if top-page is a LEAF */ + mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) || + PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype); + return MDBX_SUCCESS; + } + + if (pagetype != PAGETYPE(top_page)) { + /* LY: LEAF-page becomes BRANCH, unable restore cursor's stack */ + goto bailout; + } + + if (top_page == cdst->mc_pg[cdst->mc_top]) { + /* LY: don't touch cursor if prev top-page already on the top */ + mdbx_cassert(cdst, cdst->mc_ki[cdst->mc_top] == top_indx); + mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) || + PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype); + return MDBX_SUCCESS; + } + + const int new_snum = save_snum - save_depth + cdst->mc_db->md_depth; + if (unlikely(new_snum < 1 || new_snum > cdst->mc_db->md_depth)) { + /* LY: out of range, unable restore cursor's stack */ + goto bailout; + } + + if (top_page == cdst->mc_pg[new_snum - 1]) { + mdbx_cassert(cdst, cdst->mc_ki[new_snum - 1] == top_indx); + /* LY: restore cursor stack */ + cdst->mc_snum = (uint16_t)new_snum; + cdst->mc_top = (uint16_t)new_snum - 1; + mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth || + IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1])); + mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) || + PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype); + return MDBX_SUCCESS; + } + + MDBX_page *const stub_page = (MDBX_page *)(~(uintptr_t)top_page); + const indx_t stub_indx = top_indx; + if (save_depth > cdst->mc_db->md_depth && + ((cdst->mc_pg[save_snum - 1] == top_page && + cdst->mc_ki[save_snum - 1] == top_indx) || + (cdst->mc_pg[save_snum - 1] == stub_page && + cdst->mc_ki[save_snum - 1] == stub_indx))) { + /* LY: restore cursor stack */ + cdst->mc_pg[new_snum - 1] = top_page; + cdst->mc_ki[new_snum - 1] = top_indx; + cdst->mc_pg[new_snum] = (MDBX_page *)(~(uintptr_t)cdst->mc_pg[new_snum]); + cdst->mc_ki[new_snum] = ~cdst->mc_ki[new_snum]; + cdst->mc_snum = (uint16_t)new_snum; + cdst->mc_top = (uint16_t)new_snum - 1; + mdbx_cassert(cdst, cdst->mc_snum < cdst->mc_db->md_depth || + IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1])); + mdbx_cassert(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) || + PAGETYPE(cdst->mc_pg[cdst->mc_top]) == pagetype); + return MDBX_SUCCESS; + } + +bailout: + /* LY: unable restore cursor's stack */ + cdst->mc_flags &= ~C_INITIALIZED; + return MDBX_CURSOR_FULL; +} + +/* Copy the contents of a cursor. + * [in] csrc The cursor to copy from. + * [out] cdst The cursor to copy to. */ +static void mdbx_cursor_copy(const MDBX_cursor *csrc, MDBX_cursor *cdst) { + unsigned i; + + mdbx_cassert(csrc, + csrc->mc_txn->mt_txnid >= *csrc->mc_txn->mt_env->me_oldest); + cdst->mc_txn = csrc->mc_txn; + cdst->mc_dbi = csrc->mc_dbi; + cdst->mc_db = csrc->mc_db; + cdst->mc_dbx = csrc->mc_dbx; + cdst->mc_snum = csrc->mc_snum; + cdst->mc_top = csrc->mc_top; + cdst->mc_flags = csrc->mc_flags; + + for (i = 0; i < csrc->mc_snum; i++) { + cdst->mc_pg[i] = csrc->mc_pg[i]; + cdst->mc_ki[i] = csrc->mc_ki[i]; + } +} + +/* Rebalance the tree after a delete operation. + * [in] mc Cursor pointing to the page where rebalancing should begin. + * Returns 0 on success, non-zero on failure. */ +static int mdbx_rebalance(MDBX_cursor *mc) { + int rc; + + mdbx_cassert(mc, mc->mc_snum > 0); + mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth || + IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1])); + const int pagetype = PAGETYPE(mc->mc_pg[mc->mc_top]); + + const unsigned minkeys = (P_BRANCH == 1) ? (pagetype & P_BRANCH) + 1 + : (pagetype & P_BRANCH) ? 2 : 1; + + /* The threshold of minimum page fill factor, in form of a negative binary + * exponent, i.e. 2 means 1/(2**3) == 1/4 == 25%. Pages emptier than this + * are candidates for merging. */ + const unsigned threshold_fill_exp2 = 2; + + /* The threshold of minimum page fill factor, as a number of free bytes on a + * page. Pages emptier than this are candidates for merging. */ + const unsigned spaceleft_threshold = + page_space(mc->mc_txn->mt_env) - + (page_space(mc->mc_txn->mt_env) >> threshold_fill_exp2); + + mdbx_debug("rebalancing %s page %" PRIaPGNO " (has %u keys, %.1f%% full)", + (pagetype & P_LEAF) ? "leaf" : "branch", + mc->mc_pg[mc->mc_top]->mp_pgno, + page_numkeys(mc->mc_pg[mc->mc_top]), + page_fill(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top])); + + if (page_fill_enough(mc->mc_pg[mc->mc_top], spaceleft_threshold, minkeys)) { + mdbx_debug("no need to rebalance page %" PRIaPGNO ", above fill threshold", + mc->mc_pg[mc->mc_top]->mp_pgno); + mdbx_cassert(mc, mc->mc_db->md_entries > 0); + return MDBX_SUCCESS; + } + + if (mc->mc_snum < 2) { + MDBX_page *const mp = mc->mc_pg[0]; + const unsigned nkeys = page_numkeys(mp); + mdbx_cassert(mc, (mc->mc_db->md_entries == 0) == (nkeys == 0)); + if (IS_SUBP(mp)) { + mdbx_debug("%s", "Can't rebalance a subpage, ignoring"); + mdbx_cassert(mc, pagetype & P_LEAF); + return MDBX_SUCCESS; + } + if (nkeys == 0) { + mdbx_cassert(mc, IS_LEAF(mp)); + mdbx_debug("%s", "tree is completely empty"); + mc->mc_db->md_root = P_INVALID; + mc->mc_db->md_depth = 0; + mdbx_cassert(mc, mc->mc_db->md_branch_pages == 0 && + mc->mc_db->md_overflow_pages == 0 && + mc->mc_db->md_leaf_pages == 1); + /* Adjust cursors pointing to mp */ + const MDBX_dbi dbi = mc->mc_dbi; + for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2; + m2 = m2->mc_next) { + MDBX_cursor *m3 = + (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (m3 == mc || !(m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum = 0; + m3->mc_top = 0; + m3->mc_flags &= ~C_INITIALIZED; + } + } + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + + rc = mdbx_page_retire(mc, mp); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } else if (IS_BRANCH(mp) && nkeys == 1) { + mdbx_debug("%s", "collapsing root page!"); + mc->mc_db->md_root = node_pgno(page_node(mp, 0)); + rc = mdbx_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], NULL); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + mc->mc_db->md_depth--; + mc->mc_ki[0] = mc->mc_ki[1]; + for (int i = 1; i < mc->mc_db->md_depth; i++) { + mc->mc_pg[i] = mc->mc_pg[i + 1]; + mc->mc_ki[i] = mc->mc_ki[i + 1]; + } + + /* Adjust other cursors pointing to mp */ + MDBX_cursor *m2, *m3; + MDBX_dbi dbi = mc->mc_dbi; + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (m3 == mc || !(m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_pg[0] == mp) { + for (int i = 0; i < mc->mc_db->md_depth; i++) { + m3->mc_pg[i] = m3->mc_pg[i + 1]; + m3->mc_ki[i] = m3->mc_ki[i + 1]; + } + m3->mc_snum--; + m3->mc_top--; + } + } + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]) || + PAGETYPE(mc->mc_pg[mc->mc_top]) == pagetype); + mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth || + IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1])); + + rc = mdbx_page_retire(mc, mp); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } else { + mdbx_debug("root page %" PRIaPGNO + " doesn't need rebalancing (flags 0x%x)", + mp->mp_pgno, mp->mp_flags); + } + return MDBX_SUCCESS; + } + + /* The parent (branch page) must have at least 2 pointers, + * otherwise the tree is invalid. */ + const unsigned pre_top = mc->mc_top - 1; + mdbx_cassert(mc, IS_BRANCH(mc->mc_pg[pre_top])); + mdbx_cassert(mc, !IS_SUBP(mc->mc_pg[0])); + mdbx_cassert(mc, page_numkeys(mc->mc_pg[pre_top]) > 1); + + /* Leaf page fill factor is below the threshold. + * Try to move keys from left or right neighbor, or + * merge with a neighbor page. */ + + /* Find neighbors. */ + MDBX_cursor mn; + mdbx_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + + MDBX_page *left = nullptr, *right = nullptr; + if (mn.mc_ki[pre_top] > 0) { + rc = mdbx_page_get( + &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] - 1)), + &left, NULL); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + mdbx_cassert(mc, PAGETYPE(left) == PAGETYPE(mc->mc_pg[mc->mc_top])); + } + if (mn.mc_ki[pre_top] + 1u < page_numkeys(mn.mc_pg[pre_top])) { + rc = mdbx_page_get( + &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] + 1)), + &right, NULL); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + mdbx_cassert(mc, PAGETYPE(right) == PAGETYPE(mc->mc_pg[mc->mc_top])); + } + + int ki = mc->mc_ki[mc->mc_top]; + bool fromleft; + if (!left || (right && page_room(left) < page_room(right))) { + mdbx_debug("merging %s neighbor", "right"); + mn.mc_pg[mn.mc_top] = right; + mn.mc_ki[pre_top] += 1; + mn.mc_ki[mn.mc_top] = 0; + mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]); + fromleft = false; + } else { + mdbx_debug("merging %s neighbor", "left"); + mn.mc_pg[mn.mc_top] = left; + mn.mc_ki[pre_top] -= 1; + mn.mc_ki[mn.mc_top] = (indx_t)page_numkeys(mn.mc_pg[mn.mc_top]) - 1; + mc->mc_ki[mc->mc_top] = 0; + fromleft = true; + } + + mdbx_debug("found neighbor page %" PRIaPGNO " (%u keys, %.1f%% full)", + mn.mc_pg[mn.mc_top]->mp_pgno, page_numkeys(mn.mc_pg[mn.mc_top]), + page_fill(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top])); + + /* If the neighbor page is above threshold and has enough keys, + * move one key from it. Otherwise we should try to merge them. + * (A branch page must never have less than 2 keys.) */ + if (page_fill_enough(mn.mc_pg[mn.mc_top], spaceleft_threshold, minkeys + 1)) { + rc = mdbx_node_move(&mn, mc, fromleft); + if (likely(rc == MDBX_SUCCESS)) + ki += fromleft /* if we inserted on left, bump position up */; + else if (unlikely(rc != MDBX_RESULT_TRUE)) + return rc; + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]) || + PAGETYPE(mc->mc_pg[mc->mc_top]) == pagetype); + mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth || + IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1])); + } else { + if (!fromleft) { + rc = mdbx_page_merge(&mn, mc); + if (unlikely(MDBX_IS_ERROR(rc))) + return rc; + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]) || + PAGETYPE(mc->mc_pg[mc->mc_top]) == pagetype); + mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth || + IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1])); + } else { + int new_ki = ki + page_numkeys(mn.mc_pg[mn.mc_top]); + mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; + /* We want mdbx_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, rc = mdbx_page_merge(mc, &mn)); + if (likely(rc == MDBX_SUCCESS)) { + ki = new_ki; + mdbx_cursor_copy(&mn, mc); + } else if (unlikely(rc != MDBX_RESULT_TRUE)) + return rc; + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]) || + PAGETYPE(mc->mc_pg[mc->mc_top]) == pagetype); + mdbx_cassert(mc, mc->mc_snum < mc->mc_db->md_depth || + IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1])); + } + } + mc->mc_ki[mc->mc_top] = (indx_t)ki; + return MDBX_SUCCESS; +} + +static __cold int mdbx_page_check(MDBX_env *env, const MDBX_page *const mp, + bool maybe_unfinished) { + const unsigned nkeys = page_numkeys(mp); + char *const end_of_page = (char *)mp + env->me_psize; + mdbx_assert(env, mp->mp_pgno >= MIN_PAGENO && mp->mp_pgno <= MAX_PAGENO); + if (unlikely(mp->mp_pgno < MIN_PAGENO || mp->mp_pgno > MAX_PAGENO)) + return MDBX_CORRUPTED; + if (IS_OVERFLOW(mp)) { + mdbx_assert(env, mp->mp_pages >= 1 && mp->mp_pages < MAX_PAGENO / 2); + if (unlikely(mp->mp_pages < 1 && mp->mp_pages >= MAX_PAGENO / 2)) + return MDBX_CORRUPTED; + mdbx_assert(env, mp->mp_pgno <= MAX_PAGENO - mp->mp_pages); + if (unlikely(mp->mp_pgno > MAX_PAGENO - mp->mp_pages)) + return MDBX_CORRUPTED; + return MDBX_SUCCESS; + } + if (!(IS_DIRTY(mp) && maybe_unfinished)) { + mdbx_assert(env, nkeys >= 2 || !IS_BRANCH(mp)); + if (unlikely(nkeys < 2 && IS_BRANCH(mp))) + return MDBX_CORRUPTED; + } + + for (unsigned i = IS_LEAF(mp) ? 0 : 1; i < nkeys; ++i) { + if (IS_LEAF2(mp)) { + const size_t ksize = mp->mp_leaf2_ksize; + const char *const key = page_leaf2key(mp, i, ksize); + mdbx_assert(env, key + ksize <= end_of_page); + if (unlikely(end_of_page < key + ksize)) + return MDBX_CORRUPTED; + } else { + const MDBX_node *const node = page_node(mp, i); + const char *node_end = (char *)node + NODESIZE; + mdbx_assert(env, node_end <= end_of_page); + if (unlikely(node_end > end_of_page)) + return MDBX_CORRUPTED; + if (IS_LEAF(mp) || i > 0) { + size_t ksize = node_ks(node); + char *key = node_key(node); + mdbx_assert(env, key + ksize <= end_of_page); + if (unlikely(end_of_page < key + ksize)) + return MDBX_CORRUPTED; + } + if (IS_BRANCH(mp)) + continue; + if (node_flags(node) == F_BIGDATA /* data on large-page */) { + continue; + } + const size_t dsize = node_ds(node); + const char *const data = node_data(node); + mdbx_assert(env, data + dsize <= end_of_page); + if (unlikely(end_of_page < data + dsize)) + return MDBX_CORRUPTED; + + switch (node_flags(node)) { + default: + mdbx_assert(env, false); + return MDBX_CORRUPTED; + case 0 /* usual */: + break; + case F_SUBDATA /* sub-db */: + mdbx_assert(env, dsize >= sizeof(MDBX_db)); + if (unlikely(dsize < sizeof(MDBX_db))) + return MDBX_CORRUPTED; + break; + case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: + mdbx_assert(env, dsize == sizeof(MDBX_db)); + if (unlikely(dsize != sizeof(MDBX_db))) + return MDBX_CORRUPTED; + break; + case F_DUPDATA /* short sub-page */: + mdbx_assert(env, dsize > PAGEHDRSZ); + if (unlikely(dsize <= PAGEHDRSZ)) + return MDBX_CORRUPTED; + else { + const MDBX_page *const sp = (MDBX_page *)data; + const char *const end_of_subpage = data + dsize; + const int nsubkeys = page_numkeys(sp); + switch (sp->mp_flags & ~P_DIRTY /* ignore for sub-pages */) { + case P_LEAF | P_SUBP: + case P_LEAF | P_LEAF2 | P_SUBP: + break; + default: + mdbx_assert(env, false); + return MDBX_CORRUPTED; + } + + for (int j = 0; j < nsubkeys; j++) { + if (IS_LEAF2(sp)) { + /* LEAF2 pages have no mp_ptrs[] or node headers */ + size_t sub_ksize = sp->mp_leaf2_ksize; + char *sub_key = page_leaf2key(sp, j, sub_ksize); + mdbx_assert(env, sub_key + sub_ksize <= end_of_subpage); + if (unlikely(end_of_subpage < sub_key + sub_ksize)) + return MDBX_CORRUPTED; + } else { + mdbx_assert(env, IS_LEAF(sp)); + if (unlikely(!IS_LEAF(sp))) + return MDBX_CORRUPTED; + const MDBX_node *const sub_node = page_node(sp, j); + const char *sub_node_end = (char *)sub_node + NODESIZE; + mdbx_assert(env, sub_node_end <= end_of_subpage); + if (unlikely(sub_node_end > end_of_subpage)) + return MDBX_CORRUPTED; + mdbx_assert(env, node_flags(sub_node) == 0); + if (unlikely(node_flags(sub_node) != 0)) + return MDBX_CORRUPTED; + + size_t sub_ksize = node_ks(sub_node); + char *sub_key = node_key(sub_node); + size_t sub_dsize = node_ds(sub_node); + char *sub_data = node_data(sub_node); + mdbx_assert(env, sub_key + sub_ksize <= end_of_subpage); + if (unlikely(end_of_subpage < sub_key + sub_ksize)) + return MDBX_CORRUPTED; + mdbx_assert(env, sub_data + sub_dsize <= end_of_subpage); + if (unlikely(end_of_subpage < sub_data + sub_dsize)) + return MDBX_CORRUPTED; + } + } + } + break; + } + } + } + return MDBX_SUCCESS; +} + +static __cold int mdbx_cursor_check(MDBX_cursor *mc, bool pending) { + mdbx_tassert(mc->mc_txn, mc->mc_txn->mt_parent || + mc->mc_txn->tw.dirtyroom + + mc->mc_txn->tw.dirtylist->length == + MDBX_DPL_TXNFULL); + mdbx_cassert(mc, mc->mc_top == mc->mc_snum - 1); + if (unlikely(mc->mc_top != mc->mc_snum - 1)) + return MDBX_CURSOR_FULL; + mdbx_cassert(mc, pending ? mc->mc_snum <= mc->mc_db->md_depth + : mc->mc_snum == mc->mc_db->md_depth); + if (unlikely(pending ? mc->mc_snum > mc->mc_db->md_depth + : mc->mc_snum != mc->mc_db->md_depth)) + return MDBX_CURSOR_FULL; + + for (int n = 0; n < mc->mc_snum; ++n) { + MDBX_page *mp = mc->mc_pg[n]; + const unsigned nkeys = page_numkeys(mp); + const bool expect_branch = (n < mc->mc_db->md_depth - 1) ? true : false; + const bool expect_nested_leaf = + (n + 1 == mc->mc_db->md_depth - 1) ? true : false; + const bool branch = IS_BRANCH(mp) ? true : false; + mdbx_cassert(mc, branch == expect_branch); + if (unlikely(branch != expect_branch)) + return MDBX_CURSOR_FULL; + if (!pending) { + mdbx_cassert(mc, + nkeys > mc->mc_ki[n] || (!branch && nkeys == mc->mc_ki[n] && + (mc->mc_flags & C_EOF) != 0)); + if (unlikely(nkeys <= mc->mc_ki[n] && + !(!branch && nkeys == mc->mc_ki[n] && + (mc->mc_flags & C_EOF) != 0))) + return MDBX_CURSOR_FULL; + } else { + mdbx_cassert(mc, nkeys + 1 >= mc->mc_ki[n]); + if (unlikely(nkeys + 1 < mc->mc_ki[n])) + return MDBX_CURSOR_FULL; + } + + int err = mdbx_page_check(mc->mc_txn->mt_env, mp, pending); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + for (unsigned i = 0; i < nkeys; ++i) { + if (branch) { + MDBX_node *node = page_node(mp, i); + mdbx_cassert(mc, node_flags(node) == 0); + if (unlikely(node_flags(node) != 0)) + return MDBX_CURSOR_FULL; + pgno_t pgno = node_pgno(node); + MDBX_page *np; + int rc = mdbx_page_get(mc, pgno, &np, NULL); + mdbx_cassert(mc, rc == MDBX_SUCCESS); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + const bool nested_leaf = IS_LEAF(np) ? true : false; + mdbx_cassert(mc, nested_leaf == expect_nested_leaf); + if (unlikely(nested_leaf != expect_nested_leaf)) + return MDBX_CURSOR_FULL; + err = mdbx_page_check(mc->mc_txn->mt_env, np, pending); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + } + } + return MDBX_SUCCESS; +} + +/* Complete a delete operation started by mdbx_cursor_del(). */ +static int mdbx_cursor_del0(MDBX_cursor *mc) { + int rc; + MDBX_page *mp; + indx_t ki; + unsigned nkeys; + MDBX_cursor *m2, *m3; + MDBX_dbi dbi = mc->mc_dbi; + + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + ki = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + mdbx_node_del(mc, mc->mc_db->md_xsize); + mc->mc_db->md_entries--; + { + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (m3 == mc || !(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] == ki) { + m3->mc_flags |= C_DEL; + if (mc->mc_db->md_flags & MDBX_DUPSORT) { + /* Sub-cursor referred into dataset which is gone */ + m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); + } + continue; + } else if (m3->mc_ki[mc->mc_top] > ki) { + m3->mc_ki[mc->mc_top]--; + } + if (XCURSOR_INITED(m3)) + XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); + } + } + } + rc = mdbx_rebalance(mc); + + if (likely(rc == MDBX_SUCCESS)) { + /* DB is totally empty now, just bail out. + * Other cursors adjustments were already done + * by mdbx_rebalance and aren't needed here. */ + if (!mc->mc_snum) { + mdbx_cassert(mc, mc->mc_db->md_entries == 0 && mc->mc_db->md_depth == 0 && + mc->mc_db->md_root == P_INVALID); + mc->mc_flags |= C_DEL | C_EOF; + return rc; + } + + ki = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + mdbx_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + nkeys = page_numkeys(mp); + mdbx_cassert(mc, (mc->mc_db->md_entries > 0 && nkeys > 0) || + ((mc->mc_flags & C_SUB) && + mc->mc_db->md_entries == 0 && nkeys == 0)); + + /* Adjust THIS and other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (m3 == mc || !(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + /* if m3 points past last node in page, find next sibling */ + if (m3->mc_ki[mc->mc_top] >= nkeys) { + rc = mdbx_cursor_sibling(m3, true); + if (rc == MDBX_NOTFOUND) { + m3->mc_flags |= C_EOF; + rc = MDBX_SUCCESS; + continue; + } else if (unlikely(rc != MDBX_SUCCESS)) + break; + } + if (m3->mc_ki[mc->mc_top] >= ki || m3->mc_pg[mc->mc_top] != mp) { + if ((mc->mc_db->md_flags & MDBX_DUPSORT) != 0 && + (m3->mc_flags & C_EOF) == 0) { + MDBX_node *node = + page_node(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]); + /* If this node has dupdata, it may need to be reinited + * because its data has moved. + * If the xcursor was not initd it must be reinited. + * Else if node points to a subDB, nothing is needed. */ + if (node_flags(node) & F_DUPDATA) { + if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + if (!(node_flags(node) & F_SUBDATA)) + m3->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); + } else { + rc = mdbx_xcursor_init1(m3, node); + if (unlikely(rc != MDBX_SUCCESS)) + break; + m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL; + } + } + } + } + } + } + + if (mc->mc_ki[mc->mc_top] >= nkeys) { + rc = mdbx_cursor_sibling(mc, true); + if (rc == MDBX_NOTFOUND) { + mc->mc_flags |= C_EOF; + rc = MDBX_SUCCESS; + } + } + if ((mc->mc_db->md_flags & MDBX_DUPSORT) != 0 && + (mc->mc_flags & C_EOF) == 0) { + MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + /* If this node has dupdata, it may need to be reinited + * because its data has moved. + * If the xcursor was not initd it must be reinited. + * Else if node points to a subDB, nothing is needed. */ + if (node_flags(node) & F_DUPDATA) { + if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + if (!(node_flags(node) & F_SUBDATA)) + mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); + } else { + rc = mdbx_xcursor_init1(mc, node); + if (likely(rc != MDBX_SUCCESS)) + mc->mc_xcursor->mx_cursor.mc_flags |= C_DEL; + } + } + } + mc->mc_flags |= C_DEL; + } + + if (unlikely(rc)) + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + else if (mdbx_audit_enabled()) + rc = mdbx_cursor_check(mc, false); + + return rc; +} + +int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data) { + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!key)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + if (unlikely(txn->mt_flags & (MDBX_RDONLY | MDBX_TXN_BLOCKED))) + return (txn->mt_flags & MDBX_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + + return mdbx_del0(txn, dbi, key, data, 0); +} + +static int mdbx_del0(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, + unsigned flags) { + MDBX_cursor_couple cx; + MDBX_cursor_op op; + MDBX_val rdata; + int rc, exact = 0; + DKBUF; + + mdbx_debug("====> delete db %u key [%s], data [%s]", dbi, DKEY(key), + DVAL(data)); + + rc = mdbx_cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (data) { + op = MDBX_GET_BOTH; + rdata = *data; + data = &rdata; + } else { + op = MDBX_SET; + flags |= MDBX_NODUPDATA; + } + rc = mdbx_cursor_set(&cx.outer, key, data, op, &exact); + if (likely(rc == MDBX_SUCCESS)) { + /* let mdbx_page_split know about this cursor if needed: + * delete will trigger a rebalance; if it needs to move + * a node from one page to another, it will have to + * update the parent's separator key(s). If the new sepkey + * is larger than the current one, the parent page may + * run out of space, triggering a split. We need this + * cursor to be consistent until the end of the rebalance. */ + cx.outer.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &cx.outer; + rc = mdbx_cursor_del(&cx.outer, flags); + txn->mt_cursors[dbi] = cx.outer.mc_next; + } + return rc; +} + +/* Split a page and insert a new node. + * Set MDBX_TXN_ERROR on failure. + * [in,out] mc Cursor pointing to the page and desired insertion index. + * The cursor will be updated to point to the actual page and index where + * the node got inserted after the split. + * [in] newkey The key for the newly inserted node. + * [in] newdata The data for the newly inserted node. + * [in] newpgno The page number, if the new node is a branch node. + * [in] nflags The NODE_ADD_FLAGS for the new node. + * Returns 0 on success, non-zero on failure. */ +static int mdbx_page_split(MDBX_cursor *mc, const MDBX_val *newkey, + MDBX_val *newdata, pgno_t newpgno, unsigned nflags) { + unsigned flags; + int rc = MDBX_SUCCESS, foliage = 0, did_split = 0; + pgno_t pgno = 0; + unsigned i, ptop; + MDBX_env *env = mc->mc_txn->mt_env; + MDBX_node *node; + MDBX_val sepkey, rkey, xdata; + MDBX_page *copy = NULL; + MDBX_page *rp, *pp; + MDBX_cursor mn; + DKBUF; + + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + unsigned newindx = mc->mc_ki[mc->mc_top]; + unsigned nkeys = page_numkeys(mp); + if (mdbx_audit_enabled()) { + rc = mdbx_cursor_check(mc, true); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + + mdbx_debug("-----> splitting %s page %" PRIaPGNO + " and adding [%s] at index %i/%i", + IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, DKEY(newkey), + mc->mc_ki[mc->mc_top], nkeys); + + /* Create a right sibling. */ + if ((rc = mdbx_page_new(mc, mp->mp_flags, 1, &rp))) + return rc; + rp->mp_leaf2_ksize = mp->mp_leaf2_ksize; + mdbx_debug("new right sibling: page %" PRIaPGNO, rp->mp_pgno); + + /* Usually when splitting the root page, the cursor + * height is 1. But when called from mdbx_update_key, + * the cursor height may be greater because it walks + * up the stack while finding the branch slot to update. */ + if (mc->mc_top < 1) { + if ((rc = mdbx_page_new(mc, P_BRANCH, 1, &pp))) + goto done; + /* shift current top to make room for new parent */ + mdbx_cassert(mc, mc->mc_snum < 2 && mc->mc_db->md_depth > 0); + mc->mc_pg[2] = mc->mc_pg[1]; + mc->mc_ki[2] = mc->mc_ki[1]; + mc->mc_pg[1] = mc->mc_pg[0]; + mc->mc_ki[1] = mc->mc_ki[0]; + mc->mc_pg[0] = pp; + mc->mc_ki[0] = 0; + mc->mc_db->md_root = pp->mp_pgno; + mdbx_debug("root split! new root = %" PRIaPGNO, pp->mp_pgno); + foliage = mc->mc_db->md_depth++; + + /* Add left (implicit) pointer. */ + if (unlikely((rc = mdbx_node_add_branch(mc, 0, NULL, mp->mp_pgno)) != + MDBX_SUCCESS)) { + /* undo the pre-push */ + mc->mc_pg[0] = mc->mc_pg[1]; + mc->mc_ki[0] = mc->mc_ki[1]; + mc->mc_db->md_root = mp->mp_pgno; + mc->mc_db->md_depth--; + goto done; + } + mc->mc_snum++; + mc->mc_top++; + ptop = 0; + } else { + ptop = mc->mc_top - 1; + mdbx_debug("parent branch page is %" PRIaPGNO, mc->mc_pg[ptop]->mp_pgno); + } + + mdbx_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + mn.mc_pg[mn.mc_top] = rp; + mn.mc_ki[mn.mc_top] = 0; + mn.mc_ki[ptop] = mc->mc_ki[ptop] + 1; + + unsigned split_indx; + if (nflags & MDBX_APPEND) { + mn.mc_ki[mn.mc_top] = 0; + sepkey = *newkey; + split_indx = newindx; + nkeys = 0; + } else { + split_indx = (nkeys + 1) / 2; + if (IS_LEAF2(rp)) { + char *split, *ins; + int x; + unsigned lsize, rsize, ksize; + /* Move half of the keys to the right sibling */ + x = mc->mc_ki[mc->mc_top] - split_indx; + ksize = mc->mc_db->md_xsize; + split = page_leaf2key(mp, split_indx, ksize); + rsize = (nkeys - split_indx) * ksize; + lsize = (nkeys - split_indx) * sizeof(indx_t); + mdbx_cassert(mc, mp->mp_lower >= lsize); + mp->mp_lower -= (indx_t)lsize; + mdbx_cassert(mc, rp->mp_lower + lsize <= UINT16_MAX); + rp->mp_lower += (indx_t)lsize; + mdbx_cassert(mc, mp->mp_upper + rsize - lsize <= UINT16_MAX); + mp->mp_upper += (indx_t)(rsize - lsize); + mdbx_cassert(mc, rp->mp_upper >= rsize - lsize); + rp->mp_upper -= (indx_t)(rsize - lsize); + sepkey.iov_len = ksize; + if (newindx == split_indx) { + sepkey.iov_base = newkey->iov_base; + } else { + sepkey.iov_base = split; + } + if (x < 0) { + mdbx_cassert(mc, ksize >= sizeof(indx_t)); + ins = page_leaf2key(mp, mc->mc_ki[mc->mc_top], ksize); + memcpy(rp->mp_ptrs, split, rsize); + sepkey.iov_base = rp->mp_ptrs; + memmove(ins + ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); + memcpy(ins, newkey->iov_base, ksize); + mdbx_cassert(mc, UINT16_MAX - mp->mp_lower >= (int)sizeof(indx_t)); + mp->mp_lower += sizeof(indx_t); + mdbx_cassert(mc, mp->mp_upper >= ksize - sizeof(indx_t)); + mp->mp_upper -= (indx_t)(ksize - sizeof(indx_t)); + } else { + if (x) + memcpy(rp->mp_ptrs, split, x * ksize); + ins = page_leaf2key(rp, x, ksize); + memcpy(ins, newkey->iov_base, ksize); + memcpy(ins + ksize, split + x * ksize, rsize - x * ksize); + mdbx_cassert(mc, UINT16_MAX - rp->mp_lower >= (int)sizeof(indx_t)); + rp->mp_lower += sizeof(indx_t); + mdbx_cassert(mc, rp->mp_upper >= ksize - sizeof(indx_t)); + rp->mp_upper -= (indx_t)(ksize - sizeof(indx_t)); + mdbx_cassert(mc, x <= UINT16_MAX); + mc->mc_ki[mc->mc_top] = (indx_t)x; + } + } else { + size_t psize, nsize, k; + /* Maximum free space in an empty page */ + const unsigned pmax = page_space(env); + nsize = IS_LEAF(mp) ? leaf_size(env, newkey, newdata) + : branch_size(env, newkey); + + /* grab a page to hold a temporary copy */ + copy = mdbx_page_malloc(mc->mc_txn, 1); + if (unlikely(copy == NULL)) { + rc = MDBX_ENOMEM; + goto done; + } + copy->mp_pgno = mp->mp_pgno; + copy->mp_flags = mp->mp_flags; + copy->mp_lower = 0; + copy->mp_upper = (indx_t)page_space(env); + + /* prepare to insert */ + for (unsigned j = i = 0; i < nkeys; i++) { + if (i == newindx) + copy->mp_ptrs[j++] = 0; + copy->mp_ptrs[j++] = mp->mp_ptrs[i]; + } + + /* When items are relatively large the split point needs + * to be checked, because being off-by-one will make the + * difference between success or failure in mdbx_node_add. + * + * It's also relevant if a page happens to be laid out + * such that one half of its nodes are all "small" and + * the other half of its nodes are "large." If the new + * item is also "large" and falls on the half with + * "large" nodes, it also may not fit. + * + * As a final tweak, if the new item goes on the last + * spot on the page (and thus, onto the new page), bias + * the split so the new page is emptier than the old page. + * This yields better packing during sequential inserts. + */ + int dir; + if (nkeys < 32 || nsize > pmax / 16 || newindx >= nkeys) { + /* Find split point */ + psize = 0; + if (newindx <= split_indx || newindx >= nkeys) { + i = 0; + dir = 1; + k = (newindx >= nkeys) ? nkeys : split_indx + 1 + IS_LEAF(mp); + } else { + i = nkeys; + dir = -1; + k = split_indx - 1; + } + for (; i != k; i += dir) { + if (i == newindx) { + psize += nsize; + node = NULL; + } else { + node = (MDBX_node *)((char *)mp + copy->mp_ptrs[i] + PAGEHDRSZ); + psize += NODESIZE + node_ks(node) + sizeof(indx_t); + if (IS_LEAF(mp)) { + if (F_ISSET(node_flags(node), F_BIGDATA)) + psize += sizeof(pgno_t); + else + psize += node_ds(node); + } + psize = EVEN(psize); + } + if (psize > pmax || i == k - dir) { + split_indx = i + (dir < 0); + break; + } + } + } + if (split_indx == newindx) { + sepkey.iov_len = newkey->iov_len; + sepkey.iov_base = newkey->iov_base; + } else { + node = + (MDBX_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEHDRSZ); + sepkey.iov_len = node_ks(node); + sepkey.iov_base = node_key(node); + } + } + } + + mdbx_debug("separator is %d [%s]", split_indx, DKEY(&sepkey)); + if (mdbx_audit_enabled()) { + rc = mdbx_cursor_check(mc, true); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + rc = mdbx_cursor_check(&mn, true); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + } + + /* Copy separator key to the parent. */ + if (page_room(mn.mc_pg[ptop]) < branch_size(env, &sepkey)) { + const int snum = mc->mc_snum; + const int depth = mc->mc_db->md_depth; + mn.mc_snum--; + mn.mc_top--; + did_split = 1; + /* We want other splits to find mn when doing fixups */ + WITH_CURSOR_TRACKING( + mn, rc = mdbx_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0)); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + mdbx_cassert(mc, mc->mc_snum - snum == mc->mc_db->md_depth - depth); + if (mdbx_audit_enabled()) { + rc = mdbx_cursor_check(mc, true); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + } + + /* root split? */ + ptop += mc->mc_snum - snum; + + /* Right page might now have changed parent. + * Check if left page also changed parent. */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) { + for (i = 0; i < ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + mc->mc_pg[ptop] = mn.mc_pg[ptop]; + if (mn.mc_ki[ptop]) { + mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; + } else { + /* find right page's left sibling */ + mc->mc_ki[ptop] = mn.mc_ki[ptop]; + rc = mdbx_cursor_sibling(mc, false); + } + } + } else { + mn.mc_top--; + rc = mdbx_node_add_branch(&mn, mn.mc_ki[ptop], &sepkey, rp->mp_pgno); + mn.mc_top++; + } + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_NOTFOUND) /* improper mdbx_cursor_sibling() result */ { + mdbx_error("unexpected %s", "MDBX_NOTFOUND"); + rc = MDBX_PROBLEM; + } + goto done; + } + + if (nflags & MDBX_APPEND) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[mc->mc_top] = 0; + switch (PAGETYPE(rp)) { + case P_BRANCH: { + mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0); + mdbx_cassert(mc, newpgno != 0 && newpgno != P_INVALID); + rc = mdbx_node_add_branch(mc, 0, newkey, newpgno); + } break; + case P_LEAF: { + mdbx_cassert(mc, newpgno == 0 || newpgno == P_INVALID); + rc = mdbx_node_add_leaf(mc, 0, newkey, newdata, nflags); + } break; + case P_LEAF | P_LEAF2: { + mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0); + mdbx_cassert(mc, newpgno == 0 || newpgno == P_INVALID); + rc = mdbx_node_add_leaf2(mc, 0, newkey); + } break; + default: + rc = MDBX_CORRUPTED; + } + if (rc) + goto done; + for (i = 0; i < mc->mc_top; i++) + mc->mc_ki[i] = mn.mc_ki[i]; + } else if (!IS_LEAF2(mp)) { + /* Move nodes */ + mc->mc_pg[mc->mc_top] = rp; + i = split_indx; + indx_t n = 0; + do { + MDBX_val *rdata = NULL; + if (i == newindx) { + rkey.iov_base = newkey->iov_base; + rkey.iov_len = newkey->iov_len; + if (IS_LEAF(mp)) { + rdata = newdata; + } else + pgno = newpgno; + flags = nflags; + /* Update index for the new key. */ + mc->mc_ki[mc->mc_top] = n; + } else { + node = (MDBX_node *)((char *)mp + copy->mp_ptrs[i] + PAGEHDRSZ); + rkey.iov_base = node_key(node); + rkey.iov_len = node_ks(node); + if (IS_LEAF(mp)) { + xdata.iov_base = node_data(node); + xdata.iov_len = node_ds(node); + rdata = &xdata; + } else + pgno = node_pgno(node); + flags = node_flags(node); + } + + switch (PAGETYPE(rp)) { + case P_BRANCH: { + mdbx_cassert(mc, 0 == (uint16_t)flags); + if (n == 0) { + /* First branch index doesn't need key data. */ + rkey.iov_len = 0; + } + rc = mdbx_node_add_branch(mc, n, &rkey, pgno); + } break; + case P_LEAF: { + mdbx_cassert(mc, pgno == 0); + mdbx_cassert(mc, rdata != NULL); + rc = mdbx_node_add_leaf(mc, n, &rkey, rdata, flags); + } break; + /* case P_LEAF | P_LEAF2: { + mdbx_cassert(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0); + mdbx_cassert(mc, gno == 0); + rc = mdbx_node_add_leaf2(mc, n, &rkey); + } break; */ + default: + rc = MDBX_CORRUPTED; + } + if (rc) + goto done; + + if (i == nkeys) { + i = 0; + n = 0; + mc->mc_pg[mc->mc_top] = copy; + } else { + i++; + n++; + } + } while (i != split_indx); + + nkeys = page_numkeys(copy); + for (i = 0; i < nkeys; i++) + mp->mp_ptrs[i] = copy->mp_ptrs[i]; + mp->mp_lower = copy->mp_lower; + mp->mp_upper = copy->mp_upper; + memcpy(page_node(mp, nkeys - 1), page_node(copy, nkeys - 1), + env->me_psize - copy->mp_upper - PAGEHDRSZ); + + /* reset back to original page */ + if (newindx < split_indx) { + mc->mc_pg[mc->mc_top] = mp; + } else { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) { + for (i = 0; i <= ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + if (nflags & MDBX_RESERVE) { + node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!(node_flags(node) & F_BIGDATA)) + newdata->iov_base = node_data(node); + } + } else { + if (newindx >= split_indx) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) { + for (i = 0; i <= ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + } + + { + /* Adjust other cursors pointing to mp */ + MDBX_cursor *m2, *m3; + MDBX_dbi dbi = mc->mc_dbi; + nkeys = page_numkeys(mp); + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (m3 == mc) + continue; + if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (foliage) { + int k; + /* sub cursors may be on different DB */ + if (m3->mc_pg[0] != mp) + continue; + /* root split */ + for (k = foliage; k >= 0; k--) { + m3->mc_ki[k + 1] = m3->mc_ki[k]; + m3->mc_pg[k + 1] = m3->mc_pg[k]; + } + m3->mc_ki[0] = (m3->mc_ki[0] >= nkeys) ? 1 : 0; + m3->mc_pg[0] = mc->mc_pg[0]; + m3->mc_snum++; + m3->mc_top++; + } + if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDBX_SPLIT_REPLACE)) + m3->mc_ki[mc->mc_top]++; + if (m3->mc_ki[mc->mc_top] >= nkeys) { + m3->mc_pg[mc->mc_top] = rp; + mdbx_cassert(mc, m3->mc_ki[mc->mc_top] >= nkeys); + m3->mc_ki[mc->mc_top] -= (indx_t)nkeys; + for (i = 0; i < mc->mc_top; i++) { + m3->mc_ki[i] = mn.mc_ki[i]; + m3->mc_pg[i] = mn.mc_pg[i]; + } + } + } else if (!did_split && m3->mc_top >= ptop && + m3->mc_pg[ptop] == mc->mc_pg[ptop] && + m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { + m3->mc_ki[ptop]++; + } + if (XCURSOR_INITED(m3) && IS_LEAF(mp)) + XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); + } + } + mdbx_debug("mp left: %d, rp left: %d", page_room(mp), page_room(rp)); + +done: + if (copy) /* tmp page */ + mdbx_dpage_free(env, copy, 1); + if (unlikely(rc)) + mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + return rc; +} + +int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, + unsigned flags) { + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!key || !data)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_RESERVE | + MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT))) + return MDBX_EINVAL; + + if (unlikely(txn->mt_flags & (MDBX_RDONLY | MDBX_TXN_BLOCKED))) + return (txn->mt_flags & MDBX_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + + MDBX_cursor_couple cx; + rc = mdbx_cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + cx.outer.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &cx.outer; + + /* LY: support for update (explicit overwrite) */ + if (flags & MDBX_CURRENT) { + rc = mdbx_cursor_get(&cx.outer, key, NULL, MDBX_SET); + if (likely(rc == MDBX_SUCCESS) && + (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT)) { + /* LY: allows update (explicit overwrite) only for unique keys */ + MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top], + cx.outer.mc_ki[cx.outer.mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + mdbx_tassert(txn, XCURSOR_INITED(&cx.outer) && + cx.outer.mc_xcursor->mx_db.md_entries > 1); + rc = MDBX_EMULTIVAL; + } + } + } + + if (likely(rc == MDBX_SUCCESS)) + rc = mdbx_cursor_put(&cx.outer, key, data, flags); + txn->mt_cursors[dbi] = cx.outer.mc_next; + + return rc; +} + +/**** COPYING *****************************************************************/ + +#ifndef MDBX_WBUF +#define MDBX_WBUF ((size_t)1024 * 1024) +#endif +#define MDBX_EOF 0x10 /* mdbx_env_copyfd1() is done reading */ + +/* State needed for a double-buffering compacting copy. */ +typedef struct mdbx_copy { + MDBX_env *mc_env; + MDBX_txn *mc_txn; + mdbx_condmutex_t mc_condmutex; + uint8_t *mc_wbuf[2]; + uint8_t *mc_over[2]; + size_t mc_wlen[2]; + size_t mc_olen[2]; + mdbx_filehandle_t mc_fd; + volatile int mc_error; + pgno_t mc_next_pgno; + short mc_toggle; /* Buffer number in provider */ + short mc_new; /* (0-2 buffers to write) | (MDBX_EOF at end) */ + /* Error code. Never cleared if set. Both threads can set nonzero + * to fail the copy. Not mutex-protected, MDBX expects atomic int. */ +} mdbx_copy; + +/* Dedicated writer thread for compacting copy. */ +static THREAD_RESULT __cold THREAD_CALL mdbx_env_copythr(void *arg) { + mdbx_copy *my = arg; + uint8_t *ptr; + int toggle = 0; + int rc; + + mdbx_condmutex_lock(&my->mc_condmutex); + while (!my->mc_error) { + while (!my->mc_new) + mdbx_condmutex_wait(&my->mc_condmutex); + if (my->mc_new == 0 + MDBX_EOF) /* 0 buffers, just EOF */ + break; + size_t wsize = my->mc_wlen[toggle]; + ptr = my->mc_wbuf[toggle]; + again: + if (wsize > 0 && !my->mc_error) { + rc = mdbx_write(my->mc_fd, ptr, wsize); + if (rc != MDBX_SUCCESS) { + my->mc_error = rc; + break; + } + } + + /* If there's an overflow page tail, write it too */ + if (my->mc_olen[toggle]) { + wsize = my->mc_olen[toggle]; + ptr = my->mc_over[toggle]; + my->mc_olen[toggle] = 0; + goto again; + } + my->mc_wlen[toggle] = 0; + toggle ^= 1; + /* Return the empty buffer to provider */ + my->mc_new--; + mdbx_condmutex_signal(&my->mc_condmutex); + } + mdbx_condmutex_unlock(&my->mc_condmutex); + return (THREAD_RESULT)0; +} + +/* Give buffer and/or MDBX_EOF to writer thread, await unused buffer. + * + * [in] my control structure. + * [in] adjust (1 to hand off 1 buffer) | (MDBX_EOF when ending). */ +static int __cold mdbx_env_cthr_toggle(mdbx_copy *my, int adjust) { + mdbx_condmutex_lock(&my->mc_condmutex); + my->mc_new += (short)adjust; + mdbx_condmutex_signal(&my->mc_condmutex); + while (my->mc_new & 2) /* both buffers in use */ + mdbx_condmutex_wait(&my->mc_condmutex); + mdbx_condmutex_unlock(&my->mc_condmutex); + + my->mc_toggle ^= (adjust & 1); + /* Both threads reset mc_wlen, to be safe from threading errors */ + my->mc_wlen[my->mc_toggle] = 0; + return my->mc_error; +} + +/* Depth-first tree traversal for compacting copy. + * [in] my control structure. + * [in,out] pg database root. + * [in] flags includes F_DUPDATA if it is a sorted-duplicate sub-DB. */ +static int __cold mdbx_env_cwalk(mdbx_copy *my, pgno_t *pg, int flags) { + MDBX_cursor mc; + MDBX_page *mo, *mp, *leaf; + char *buf, *ptr; + int rc, toggle; + unsigned i; + + /* Empty DB, nothing to do */ + if (*pg == P_INVALID) + return MDBX_SUCCESS; + + memset(&mc, 0, sizeof(mc)); + mc.mc_snum = 1; + mc.mc_txn = my->mc_txn; + + rc = mdbx_page_get(&mc, *pg, &mc.mc_pg[0], NULL); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_page_search_root(&mc, NULL, MDBX_PS_FIRST); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + /* Make cursor pages writable */ + buf = ptr = mdbx_malloc(pgno2bytes(my->mc_env, mc.mc_snum)); + if (buf == NULL) + return MDBX_ENOMEM; + + for (i = 0; i < mc.mc_top; i++) { + mdbx_page_copy((MDBX_page *)ptr, mc.mc_pg[i], my->mc_env->me_psize); + mc.mc_pg[i] = (MDBX_page *)ptr; + ptr += my->mc_env->me_psize; + } + + /* This is writable space for a leaf page. Usually not needed. */ + leaf = (MDBX_page *)ptr; + + toggle = my->mc_toggle; + while (mc.mc_snum > 0) { + unsigned n; + mp = mc.mc_pg[mc.mc_top]; + n = page_numkeys(mp); + + if (IS_LEAF(mp)) { + if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { + for (i = 0; i < n; i++) { + MDBX_node *node = page_node(mp, i); + if (node_flags(node) & F_BIGDATA) { + MDBX_page *omp; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdbx_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + node = page_node(mp, i); + } + + const pgno_t pgno = node_largedata_pgno(node); + poke_pgno(node_data(node), my->mc_next_pgno); + rc = mdbx_page_get(&mc, pgno, &omp, NULL); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + if (my->mc_wlen[toggle] >= MDBX_WBUF) { + rc = mdbx_env_cthr_toggle(my, 1); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + toggle = my->mc_toggle; + } + mo = (MDBX_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + memcpy(mo, omp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno; + my->mc_next_pgno += omp->mp_pages; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (omp->mp_pages > 1) { + my->mc_olen[toggle] = pgno2bytes(my->mc_env, omp->mp_pages - 1); + my->mc_over[toggle] = (uint8_t *)omp + my->mc_env->me_psize; + rc = mdbx_env_cthr_toggle(my, 1); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + toggle = my->mc_toggle; + } + } else if (node_flags(node) & F_SUBDATA) { + if (node_ds(node) < sizeof(MDBX_db)) { + rc = MDBX_CORRUPTED; + goto done; + } + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdbx_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + node = page_node(mp, i); + } + + MDBX_db db; + memcpy(&db, node_data(node), sizeof(MDBX_db)); + my->mc_toggle = (short)toggle; + rc = mdbx_env_cwalk(my, &db.md_root, node_flags(node) & F_DUPDATA); + if (rc) + goto done; + toggle = my->mc_toggle; + memcpy(node_data(node), &db, sizeof(MDBX_db)); + } + } + } + } else { + mc.mc_ki[mc.mc_top]++; + if (mc.mc_ki[mc.mc_top] < n) { + again: + rc = mdbx_page_get(&mc, node_pgno(page_node(mp, mc.mc_ki[mc.mc_top])), + &mp, NULL); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + mc.mc_top++; + mc.mc_snum++; + mc.mc_ki[mc.mc_top] = 0; + if (IS_BRANCH(mp)) { + /* Whenever we advance to a sibling branch page, + * we must proceed all the way down to its first leaf. */ + mdbx_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); + goto again; + } else + mc.mc_pg[mc.mc_top] = mp; + continue; + } + } + if (my->mc_wlen[toggle] >= MDBX_WBUF) { + rc = mdbx_env_cthr_toggle(my, 1); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + toggle = my->mc_toggle; + } + mo = (MDBX_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + mdbx_page_copy(mo, mp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno++; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (mc.mc_top) { + /* Update parent if there is one */ + node_set_pgno(page_node(mc.mc_pg[mc.mc_top - 1], mc.mc_ki[mc.mc_top - 1]), + mo->mp_pgno); + mdbx_cursor_pop(&mc); + } else { + /* Otherwise we're done */ + *pg = mo->mp_pgno; + break; + } + } +done: + mdbx_free(buf); + return rc; +} + +static __cold void compact_fixup_meta(MDBX_env *env, MDBX_meta *meta) { + /* Calculate filesize taking in account shrink/growing thresholds */ + if (meta->mm_geo.next > meta->mm_geo.now) { + const pgno_t aligned = pgno_align2os_pgno( + env, + pgno_add(meta->mm_geo.next, + meta->mm_geo.grow - meta->mm_geo.next % meta->mm_geo.grow)); + meta->mm_geo.now = aligned; + } else if (meta->mm_geo.next < meta->mm_geo.now) { + meta->mm_geo.now = meta->mm_geo.next; + const pgno_t aligner = + meta->mm_geo.grow ? meta->mm_geo.grow : meta->mm_geo.shrink; + const pgno_t aligned = pgno_align2os_pgno( + env, meta->mm_geo.next + aligner - meta->mm_geo.next % aligner); + meta->mm_geo.now = aligned; + } + + if (meta->mm_geo.now < meta->mm_geo.lower) + meta->mm_geo.now = meta->mm_geo.lower; + if (meta->mm_geo.now > meta->mm_geo.upper) + meta->mm_geo.now = meta->mm_geo.upper; + + /* Update signature */ + assert(meta->mm_geo.now >= meta->mm_geo.next); + meta->mm_datasync_sign = mdbx_meta_sign(meta); +} + +/* Copy environment with compaction. */ +static int __cold mdbx_env_compact(MDBX_env *env, MDBX_txn *read_txn, + mdbx_filehandle_t fd, uint8_t *buffer, + const bool dest_is_pipe) { + const size_t meta_bytes = pgno2bytes(env, NUM_METAS); + uint8_t *const data_buffer = + buffer + roundup_powerof2(meta_bytes, env->me_os_psize); + MDBX_meta *const meta = mdbx_init_metas(env, buffer); + /* copy canary sequenses if present */ + if (read_txn->mt_canary.v) { + meta->mm_canary = read_txn->mt_canary; + meta->mm_canary.v = mdbx_meta_txnid_stable(env, meta); + } + + /* Set metapage 1 with current main DB */ + pgno_t new_root, root = read_txn->mt_dbs[MAIN_DBI].md_root; + if ((new_root = root) == P_INVALID) { + /* When the DB is empty, handle it specially to + * fix any breakage like page leaks from ITS#8174. */ + meta->mm_dbs[MAIN_DBI].md_flags = read_txn->mt_dbs[MAIN_DBI].md_flags; + compact_fixup_meta(env, meta); + if (dest_is_pipe) { + int rc = mdbx_write(fd, buffer, meta_bytes); + if (rc != MDBX_SUCCESS) + return rc; + } + } else { + /* Count free pages + GC pages. Subtract from last_pg + * to find the new last_pg, which also becomes the new root. */ + pgno_t freecount = 0; + MDBX_cursor mc; + MDBX_val key, data; + + int rc = mdbx_cursor_init(&mc, read_txn, FREE_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + while ((rc = mdbx_cursor_get(&mc, &key, &data, MDBX_NEXT)) == 0) + freecount += *(pgno_t *)data.iov_base; + if (unlikely(rc != MDBX_NOTFOUND)) + return rc; + + freecount += read_txn->mt_dbs[FREE_DBI].md_branch_pages + + read_txn->mt_dbs[FREE_DBI].md_leaf_pages + + read_txn->mt_dbs[FREE_DBI].md_overflow_pages; + + new_root = read_txn->mt_next_pgno - 1 - freecount; + meta->mm_geo.next = new_root + 1; + meta->mm_dbs[MAIN_DBI] = read_txn->mt_dbs[MAIN_DBI]; + meta->mm_dbs[MAIN_DBI].md_root = new_root; + + mdbx_copy ctx; + memset(&ctx, 0, sizeof(ctx)); + rc = mdbx_condmutex_init(&ctx.mc_condmutex); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + memset(data_buffer, 0, MDBX_WBUF * 2); + ctx.mc_wbuf[0] = data_buffer; + ctx.mc_wbuf[1] = data_buffer + MDBX_WBUF; + ctx.mc_next_pgno = NUM_METAS; + ctx.mc_env = env; + ctx.mc_fd = fd; + ctx.mc_txn = read_txn; + + mdbx_thread_t thread; + int thread_err = mdbx_thread_create(&thread, mdbx_env_copythr, &ctx); + if (likely(thread_err == MDBX_SUCCESS)) { + if (dest_is_pipe) { + compact_fixup_meta(env, meta); + rc = mdbx_write(fd, buffer, meta_bytes); + } + if (rc == MDBX_SUCCESS) + rc = mdbx_env_cwalk(&ctx, &root, 0); + mdbx_env_cthr_toggle(&ctx, 1 | MDBX_EOF); + thread_err = mdbx_thread_join(thread); + mdbx_condmutex_destroy(&ctx.mc_condmutex); + } + if (unlikely(thread_err != MDBX_SUCCESS)) + return thread_err; + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + if (unlikely(ctx.mc_error != MDBX_SUCCESS)) + return ctx.mc_error; + + if (dest_is_pipe) { + if (root != new_root) { + mdbx_error("post-compactification root %" PRIaPGNO + " NE expected %" PRIaPGNO + " (source DB corrupted or has a page leak(s))", + root, new_root); + return MDBX_CORRUPTED; /* page leak or corrupt DB */ + } + } else { + if (root > new_root) { + mdbx_error("post-compactification root %" PRIaPGNO + " GT expected %" PRIaPGNO " (source DB corrupted)", + root, new_root); + return MDBX_CORRUPTED; /* page leak or corrupt DB */ + } + if (root < new_root) { + mdbx_notice("post-compactification root %" PRIaPGNO + " LT expected %" PRIaPGNO " (page leak(s) in source DB)", + root, new_root); + /* fixup meta */ + meta->mm_dbs[MAIN_DBI].md_root = root; + meta->mm_geo.next = root + 1; + } + compact_fixup_meta(env, meta); + } + } + + /* Extend file if required */ + if (meta->mm_geo.now != meta->mm_geo.next) { + const size_t whole_size = pgno2bytes(env, meta->mm_geo.now); + if (!dest_is_pipe) + return mdbx_ftruncate(fd, whole_size); + + const size_t used_size = pgno2bytes(env, meta->mm_geo.next); + memset(data_buffer, 0, MDBX_WBUF); + for (size_t offset = used_size; offset < whole_size;) { + const size_t chunk = + (MDBX_WBUF < whole_size - offset) ? MDBX_WBUF : whole_size - offset; + /* copy to avoit EFAULT in case swapped-out */ + int rc = mdbx_write(fd, data_buffer, chunk); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + offset += chunk; + } + } + return MDBX_SUCCESS; +} + +/* Copy environment as-is. */ +static int __cold mdbx_env_copy_asis(MDBX_env *env, MDBX_txn *read_txn, + mdbx_filehandle_t fd, uint8_t *buffer, + const bool dest_is_pipe) { + /* We must start the actual read txn after blocking writers */ + int rc = mdbx_txn_end(read_txn, MDBX_END_RESET_TMP); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + /* Temporarily block writers until we snapshot the meta pages */ + rc = mdbx_txn_lock(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + rc = mdbx_txn_renew0(read_txn, MDBX_RDONLY); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_txn_unlock(env); + return rc; + } + + mdbx_jitter4testing(false); + const size_t meta_bytes = pgno2bytes(env, NUM_METAS); + /* Make a snapshot of meta-pages, + * but writing ones after the data was flushed */ + memcpy(buffer, env->me_map, meta_bytes); + MDBX_meta *const headcopy = /* LY: get pointer to the spanshot copy */ + (MDBX_meta *)(buffer + ((uint8_t *)mdbx_meta_head(env) - env->me_map)); + /* Update signature to steady */ + headcopy->mm_datasync_sign = mdbx_meta_sign(headcopy); + mdbx_txn_unlock(env); + + /* Copy the data */ + const size_t whole_size = pgno_align2os_bytes(env, read_txn->mt_end_pgno); + const size_t used_size = pgno2bytes(env, read_txn->mt_next_pgno); + mdbx_jitter4testing(false); + + if (dest_is_pipe) + rc = mdbx_write(fd, buffer, meta_bytes); + + uint8_t *const data_buffer = + buffer + roundup_powerof2(meta_bytes, env->me_os_psize); + for (size_t offset = meta_bytes; rc == MDBX_SUCCESS && offset < used_size;) { + if (dest_is_pipe) { +#if defined(__linux__) || defined(__gnu_linux__) + off_t in_offset = offset; + const intptr_t written = + sendfile(fd, env->me_fd, &in_offset, used_size - offset); + if (unlikely(written <= 0)) { + rc = written ? errno : MDBX_ENODATA; + break; + } + offset = in_offset; + continue; +#endif + } else { +#if __GLIBC_PREREQ(2, 27) && defined(_GNU_SOURCE) + off_t in_offset = offset, out_offset = offset; + ssize_t bytes_copied = copy_file_range( + env->me_fd, &in_offset, fd, &out_offset, used_size - offset, 0); + if (unlikely(bytes_copied <= 0)) { + rc = bytes_copied ? errno : MDBX_ENODATA; + break; + } + offset = in_offset; + continue; +#endif + } + + /* fallback to portable */ + const size_t chunk = + (MDBX_WBUF < used_size - offset) ? MDBX_WBUF : used_size - offset; + /* copy to avoit EFAULT in case swapped-out */ + memcpy(data_buffer, env->me_map + offset, chunk); + rc = mdbx_write(fd, data_buffer, chunk); + offset += chunk; + } + + /* Extend file if required */ + if (likely(rc == MDBX_SUCCESS) && whole_size != used_size) { + if (!dest_is_pipe) + rc = mdbx_ftruncate(fd, whole_size); + else { + memset(data_buffer, 0, MDBX_WBUF); + for (size_t offset = used_size; + rc == MDBX_SUCCESS && offset < whole_size;) { + const size_t chunk = + (MDBX_WBUF < whole_size - offset) ? MDBX_WBUF : whole_size - offset; + /* copy to avoit EFAULT in case swapped-out */ + rc = mdbx_write(fd, data_buffer, chunk); + offset += chunk; + } + } + } + + return rc; +} + +int __cold mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, + unsigned flags) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + const int dest_is_pipe = mdbx_is_pipe(fd); + if (MDBX_IS_ERROR(dest_is_pipe)) + return dest_is_pipe; + + if (!dest_is_pipe) { + int rc = mdbx_fseek(fd, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + + const size_t buffer_size = + pgno_align2os_bytes(env, NUM_METAS) + + roundup_powerof2(((flags & MDBX_CP_COMPACT) ? MDBX_WBUF * 2 : MDBX_WBUF), + env->me_os_psize); + + uint8_t *buffer = NULL; + int rc = mdbx_memalign_alloc(env->me_os_psize, buffer_size, (void **)&buffer); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + MDBX_txn *read_txn = NULL; + /* Do the lock/unlock of the reader mutex before starting the + * write txn. Otherwise other read txns could block writers. */ + rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &read_txn); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_memalign_free(buffer); + return rc; + } + + if (!dest_is_pipe) { + /* Firstly write a stub to meta-pages. + * Now we sure to incomplete copy will not be used. */ + memset(buffer, -1, pgno2bytes(env, NUM_METAS)); + rc = mdbx_write(fd, buffer, pgno2bytes(env, NUM_METAS)); + } + + if (likely(rc == MDBX_SUCCESS)) { + memset(buffer, 0, pgno2bytes(env, NUM_METAS)); + rc = (flags & MDBX_CP_COMPACT) + ? mdbx_env_compact(env, read_txn, fd, buffer, dest_is_pipe) + : mdbx_env_copy_asis(env, read_txn, fd, buffer, dest_is_pipe); + } + mdbx_txn_abort(read_txn); + + if (!dest_is_pipe) { + if (likely(rc == MDBX_SUCCESS)) + rc = mdbx_filesync(fd, MDBX_SYNC_DATA | MDBX_SYNC_SIZE | MDBX_SYNC_IODQ); + + /* Write actual meta */ + if (likely(rc == MDBX_SUCCESS)) + rc = mdbx_pwrite(fd, buffer, pgno2bytes(env, NUM_METAS), 0); + + if (likely(rc == MDBX_SUCCESS)) + rc = mdbx_filesync(fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + } + + mdbx_memalign_free(buffer); + return rc; +} + +int __cold mdbx_env_copy(MDBX_env *env, const char *dest_path, unsigned flags) { + if (unlikely(!env || !dest_path)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + char *dxb_pathname; + mdbx_filehandle_t newfd = INVALID_HANDLE_VALUE; + + if (env->me_flags & MDBX_NOSUBDIR) { + dxb_pathname = (char *)dest_path; + } else { + size_t len = strlen(dest_path); + len += sizeof(MDBX_DATANAME); + dxb_pathname = mdbx_malloc(len); + if (!dxb_pathname) + return MDBX_ENOMEM; + sprintf(dxb_pathname, "%s" MDBX_DATANAME, dest_path); + } + + /* The destination path must exist, but the destination file must not. + * We don't want the OS to cache the writes, since the source data is + * already in the OS cache. */ + int rc = mdbx_openfile(dxb_pathname, O_WRONLY | O_CREAT | O_EXCL, 0640, + &newfd, true); + if (rc == MDBX_SUCCESS) { + if (env->me_psize >= env->me_os_psize) { +#ifdef F_NOCACHE /* __APPLE__ */ + (void)fcntl(newfd, F_NOCACHE, 1); +#elif defined(O_DIRECT) && defined(F_GETFL) + /* Set O_DIRECT if the file system supports it */ + if ((rc = fcntl(newfd, F_GETFL)) != -1) + (void)fcntl(newfd, F_SETFL, rc | O_DIRECT); +#endif + } + rc = mdbx_env_copy2fd(env, newfd, flags); + } + + if (newfd != INVALID_HANDLE_VALUE) { + int err = mdbx_closefile(newfd); + if (rc == MDBX_SUCCESS && err != rc) + rc = err; + if (rc != MDBX_SUCCESS) + (void)mdbx_removefile(dxb_pathname); + } + + if (dxb_pathname != dest_path) + mdbx_free(dxb_pathname); + + return rc; +} + +/******************************************************************************/ + +int __cold mdbx_env_set_flags(MDBX_env *env, unsigned flags, int onoff) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely(flags & ~CHANGEABLE)) + return MDBX_EPERM; + + if (unlikely(env->me_flags & MDBX_RDONLY)) + return MDBX_EACCESS; + + if (unlikely(env->me_txn0->mt_owner == mdbx_thread_self())) + return MDBX_BUSY; + + int rc = mdbx_txn_lock(env, false); + if (unlikely(rc)) + return rc; + + if (onoff) + env->me_flags |= flags; + else + env->me_flags &= ~flags; + + mdbx_txn_unlock(env); + return MDBX_SUCCESS; +} + +int __cold mdbx_env_get_flags(MDBX_env *env, unsigned *arg) { + if (unlikely(!env || !arg)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + *arg = env->me_flags & (CHANGEABLE | CHANGELESS); + return MDBX_SUCCESS; +} + +int __cold mdbx_env_set_userctx(MDBX_env *env, void *ctx) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + env->me_userctx = ctx; + return MDBX_SUCCESS; +} + +void *__cold mdbx_env_get_userctx(MDBX_env *env) { + return env ? env->me_userctx : NULL; +} + +int __cold mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + +#if MDBX_DEBUG + env->me_assert_func = func; + return MDBX_SUCCESS; +#else + (void)func; + return MDBX_ENOSYS; +#endif +} + +int __cold mdbx_env_get_path(MDBX_env *env, const char **arg) { + if (unlikely(!env || !arg)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + *arg = env->me_path; + return MDBX_SUCCESS; +} + +int __cold mdbx_env_get_fd(MDBX_env *env, mdbx_filehandle_t *arg) { + if (unlikely(!env || !arg)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + *arg = env->me_fd; + return MDBX_SUCCESS; +} + +/* Common code for mdbx_dbi_stat() and mdbx_env_stat(). + * [in] env the environment to operate in. + * [in] db the MDBX_db record containing the stats to return. + * [out] arg the address of an MDBX_stat structure to receive the stats. + * Returns 0, this function always succeeds. */ +static void mdbx_stat0(const MDBX_env *env, const MDBX_db *db, MDBX_stat *dest, + size_t bytes) { + dest->ms_psize = env->me_psize; + dest->ms_depth = db->md_depth; + dest->ms_branch_pages = db->md_branch_pages; + dest->ms_leaf_pages = db->md_leaf_pages; + dest->ms_overflow_pages = db->md_overflow_pages; + dest->ms_entries = db->md_entries; + if (likely(bytes >= + offsetof(MDBX_stat, ms_mod_txnid) + sizeof(dest->ms_mod_txnid))) + dest->ms_mod_txnid = db->md_mod_txnid; +} + +int __cold mdbx_env_stat(MDBX_env *env, MDBX_stat *dest, size_t bytes) { + return mdbx_env_stat_ex(env, NULL, dest, bytes); +} + +int __cold mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, + MDBX_stat *dest, size_t bytes) { + if (unlikely((env == NULL && txn == NULL) || dest == NULL)) + return MDBX_EINVAL; + + if (txn) { + int err = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + if (env) { + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + if (txn && unlikely(txn->mt_env != env)) + return MDBX_EINVAL; + } + + const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); + if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid) + return MDBX_EINVAL; + + if (txn) { + mdbx_stat0(txn->mt_env, &txn->mt_dbs[MAIN_DBI], dest, bytes); + return MDBX_SUCCESS; + } + + while (1) { + const MDBX_meta *const recent_meta = mdbx_meta_head(env); + const txnid_t txnid = mdbx_meta_txnid_fluid(env, recent_meta); + mdbx_stat0(env, &recent_meta->mm_dbs[MAIN_DBI], dest, bytes); + mdbx_compiler_barrier(); + if (likely(txnid == mdbx_meta_txnid_fluid(env, recent_meta) && + recent_meta == mdbx_meta_head(env))) + return MDBX_SUCCESS; + } +} + +int __cold mdbx_env_info(MDBX_env *env, MDBX_envinfo *arg, size_t bytes) { + return mdbx_env_info_ex(env, NULL, arg, bytes); +} + +int __cold mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, + MDBX_envinfo *arg, size_t bytes) { + if (unlikely((env == NULL && txn == NULL) || arg == NULL)) + return MDBX_EINVAL; + + if (txn) { + int err = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + if (env) { + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + if (txn && unlikely(txn->mt_env != env)) + return MDBX_EINVAL; + } else { + env = txn->mt_env; + } + + const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); + if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid) + return MDBX_EINVAL; + + const MDBX_meta *const meta0 = METAPAGE(env, 0); + const MDBX_meta *const meta1 = METAPAGE(env, 1); + const MDBX_meta *const meta2 = METAPAGE(env, 2); + pgno_t unsynced_pages; + while (1) { + if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) + return MDBX_PANIC; + + const MDBX_meta *const recent_meta = mdbx_meta_head(env); + arg->mi_recent_txnid = mdbx_meta_txnid_fluid(env, recent_meta); + arg->mi_meta0_txnid = mdbx_meta_txnid_fluid(env, meta0); + arg->mi_meta0_sign = meta0->mm_datasync_sign; + arg->mi_meta1_txnid = mdbx_meta_txnid_fluid(env, meta1); + arg->mi_meta1_sign = meta1->mm_datasync_sign; + arg->mi_meta2_txnid = mdbx_meta_txnid_fluid(env, meta2); + arg->mi_meta2_sign = meta2->mm_datasync_sign; + + const MDBX_meta *txn_meta = recent_meta; + arg->mi_last_pgno = txn_meta->mm_geo.next - 1; + arg->mi_geo.current = pgno2bytes(env, txn_meta->mm_geo.now); + if (txn) { + arg->mi_last_pgno = txn->mt_next_pgno - 1; + arg->mi_geo.current = pgno2bytes(env, txn->mt_end_pgno); + + const txnid_t wanna_meta_txnid = (txn->mt_flags & MDBX_RDONLY) + ? txn->mt_txnid + : txn->mt_txnid - MDBX_TXNID_STEP; + txn_meta = (arg->mi_meta0_txnid == wanna_meta_txnid) ? meta0 : txn_meta; + txn_meta = (arg->mi_meta1_txnid == wanna_meta_txnid) ? meta1 : txn_meta; + txn_meta = (arg->mi_meta2_txnid == wanna_meta_txnid) ? meta2 : txn_meta; + } + arg->mi_geo.lower = pgno2bytes(env, txn_meta->mm_geo.lower); + arg->mi_geo.upper = pgno2bytes(env, txn_meta->mm_geo.upper); + arg->mi_geo.shrink = pgno2bytes(env, txn_meta->mm_geo.shrink); + arg->mi_geo.grow = pgno2bytes(env, txn_meta->mm_geo.grow); + unsynced_pages = *env->me_unsynced_pages + + (*env->me_meta_sync_txnid != (uint32_t)arg->mi_last_pgno); + + arg->mi_mapsize = env->me_dxb_mmap.limit; + mdbx_compiler_barrier(); + if (likely(arg->mi_meta0_txnid == mdbx_meta_txnid_fluid(env, meta0) && + arg->mi_meta0_sign == meta0->mm_datasync_sign && + arg->mi_meta1_txnid == mdbx_meta_txnid_fluid(env, meta1) && + arg->mi_meta1_sign == meta1->mm_datasync_sign && + arg->mi_meta2_txnid == mdbx_meta_txnid_fluid(env, meta2) && + arg->mi_meta2_sign == meta2->mm_datasync_sign && + recent_meta == mdbx_meta_head(env) && + arg->mi_recent_txnid == mdbx_meta_txnid_fluid(env, recent_meta))) + break; + } + + arg->mi_maxreaders = env->me_maxreaders; + arg->mi_numreaders = env->me_lck ? env->me_lck->mti_numreaders : INT32_MAX; + arg->mi_dxb_pagesize = env->me_psize; + arg->mi_sys_pagesize = env->me_os_psize; + + const MDBX_lockinfo *const lck = env->me_lck; + if (likely(bytes > size_before_bootid)) { + arg->mi_unsync_volume = pgno2bytes(env, unsynced_pages); + const uint64_t monotime_now = mdbx_osal_monotime(); + arg->mi_since_sync_seconds16dot16 = + mdbx_osal_monotime_to_16dot16(monotime_now - *env->me_sync_timestamp); + arg->mi_since_reader_check_seconds16dot16 = + lck ? mdbx_osal_monotime_to_16dot16(monotime_now - + lck->mti_reader_check_timestamp) + : 0; + arg->mi_autosync_threshold = pgno2bytes(env, *env->me_autosync_threshold); + arg->mi_autosync_period_seconds16dot16 = + mdbx_osal_monotime_to_16dot16(*env->me_autosync_period); + arg->mi_bootid[0] = lck ? lck->mti_bootid.x : 0; + arg->mi_bootid[1] = lck ? lck->mti_bootid.y : 0; + arg->mi_mode = lck ? lck->mti_envmode : env->me_flags; + } + + arg->mi_self_latter_reader_txnid = arg->mi_latter_reader_txnid = 0; + if (lck) { + arg->mi_self_latter_reader_txnid = arg->mi_latter_reader_txnid = + arg->mi_recent_txnid; + for (unsigned i = 0; i < arg->mi_numreaders; ++i) { + const uint32_t pid = lck->mti_readers[i].mr_pid; + if (pid) { + const txnid_t txnid = safe64_read(&lck->mti_readers[i].mr_txnid); + if (arg->mi_latter_reader_txnid > txnid) + arg->mi_latter_reader_txnid = txnid; + if (pid == env->me_pid && arg->mi_self_latter_reader_txnid > txnid) + arg->mi_self_latter_reader_txnid = txnid; + } + } + } + + return MDBX_SUCCESS; +} + +static MDBX_cmp_func *mdbx_default_keycmp(unsigned flags) { + return (flags & MDBX_REVERSEKEY) + ? mdbx_cmp_memnr + : (flags & MDBX_INTEGERKEY) ? mdbx_cmp_int_align2 : mdbx_cmp_memn; +} + +static MDBX_cmp_func *mdbx_default_datacmp(unsigned flags) { + return !(flags & MDBX_DUPSORT) + ? mdbx_cmp_memn + : ((flags & MDBX_INTEGERDUP) + ? mdbx_cmp_int_unaligned + : ((flags & MDBX_REVERSEDUP) ? mdbx_cmp_memnr + : mdbx_cmp_memn)); +} + +static int mdbx_dbi_bind(MDBX_txn *txn, const MDBX_dbi dbi, unsigned user_flags, + MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { + /* LY: so, accepting only three cases for the table's flags: + * 1) user_flags and both comparators are zero + * = assume that a by-default mode/flags is requested for reading; + * 2) user_flags exactly the same + * = assume that the target mode/flags are requested properly; + * 3) user_flags differs, but table is empty and MDBX_CREATE is provided + * = assume that a properly create request with custom flags; + */ + if ((user_flags ^ txn->mt_dbs[dbi].md_flags) & PERSISTENT_FLAGS) { + /* flags ara differs, check other conditions */ + if (!user_flags && (!keycmp || keycmp == txn->mt_dbxs[dbi].md_cmp) && + (!datacmp || datacmp == txn->mt_dbxs[dbi].md_dcmp)) { + /* no comparators were provided and flags are zero, + * seems that is case #1 above */ + user_flags = txn->mt_dbs[dbi].md_flags; + } else if ((user_flags & MDBX_CREATE) && txn->mt_dbs[dbi].md_entries == 0) { + if (txn->mt_flags & MDBX_RDONLY) + return /* FIXME: return extended info */ MDBX_EACCESS; + /* make sure flags changes get committed */ + txn->mt_dbs[dbi].md_flags = user_flags & PERSISTENT_FLAGS; + txn->mt_flags |= MDBX_TXN_DIRTY; + } else { + return /* FIXME: return extended info */ MDBX_INCOMPATIBLE; + } + } + + if (!txn->mt_dbxs[dbi].md_cmp || MDBX_DEBUG) { + if (!keycmp) + keycmp = mdbx_default_keycmp(user_flags); + mdbx_tassert(txn, !txn->mt_dbxs[dbi].md_cmp || + txn->mt_dbxs[dbi].md_cmp == keycmp); + txn->mt_dbxs[dbi].md_cmp = keycmp; + } + + if (!txn->mt_dbxs[dbi].md_dcmp || MDBX_DEBUG) { + if (!datacmp) + datacmp = mdbx_default_datacmp(user_flags); + mdbx_tassert(txn, !txn->mt_dbxs[dbi].md_dcmp || + txn->mt_dbxs[dbi].md_dcmp == datacmp); + txn->mt_dbxs[dbi].md_dcmp = datacmp; + } + + return MDBX_SUCCESS; +} + +int mdbx_dbi_open_ex(MDBX_txn *txn, const char *table_name, unsigned user_flags, + MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!dbi || (user_flags & ~VALID_FLAGS) != 0)) + return MDBX_EINVAL; + + switch (user_flags & + (MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT | MDBX_REVERSEDUP)) { + default: + return MDBX_EINVAL; + case MDBX_DUPSORT: + case MDBX_DUPSORT | MDBX_REVERSEDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: + case 0: + break; + } + + /* main table? */ + if (!table_name) { + *dbi = MAIN_DBI; + return mdbx_dbi_bind(txn, MAIN_DBI, user_flags, keycmp, datacmp); + } + + if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { + txn->mt_dbxs[MAIN_DBI].md_cmp = + mdbx_default_keycmp(txn->mt_dbs[MAIN_DBI].md_flags); + txn->mt_dbxs[MAIN_DBI].md_dcmp = + mdbx_default_datacmp(txn->mt_dbs[MAIN_DBI].md_flags); + } + + /* Is the DB already open? */ + size_t len = strlen(table_name); + MDBX_dbi scan, slot; + for (slot = scan = txn->mt_numdbs; --scan >= CORE_DBS;) { + if (!txn->mt_dbxs[scan].md_name.iov_len) { + /* Remember this free slot */ + slot = scan; + continue; + } + if (len == txn->mt_dbxs[scan].md_name.iov_len && + !strncmp(table_name, txn->mt_dbxs[scan].md_name.iov_base, len)) { + *dbi = scan; + return mdbx_dbi_bind(txn, scan, user_flags, keycmp, datacmp); + } + } + + /* Fail, if no free slot and max hit */ + MDBX_env *env = txn->mt_env; + if (unlikely(slot >= env->me_maxdbs)) + return MDBX_DBS_FULL; + + /* Cannot mix named table with some main-table flags */ + if (unlikely(txn->mt_dbs[MAIN_DBI].md_flags & + (MDBX_DUPSORT | MDBX_INTEGERKEY))) + return (user_flags & MDBX_CREATE) ? MDBX_INCOMPATIBLE : MDBX_NOTFOUND; + + /* Find the DB info */ + int exact = 0; + MDBX_val key, data; + key.iov_len = len; + key.iov_base = (void *)table_name; + MDBX_cursor mc; + rc = mdbx_cursor_init(&mc, txn, MAIN_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_cursor_set(&mc, &key, &data, MDBX_SET, &exact); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc != MDBX_NOTFOUND || !(user_flags & MDBX_CREATE)) + return rc; + } else { + /* make sure this is actually a table */ + MDBX_node *node = page_node(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); + if (unlikely((node_flags(node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA)) + return MDBX_INCOMPATIBLE; + if (unlikely(data.iov_len < sizeof(MDBX_db))) + return MDBX_CORRUPTED; + } + + if (rc != MDBX_SUCCESS && unlikely(txn->mt_flags & MDBX_RDONLY)) + return MDBX_EACCESS; + + /* Done here so we cannot fail after creating a new DB */ + char *namedup = mdbx_strdup(table_name); + if (unlikely(!namedup)) + return MDBX_ENOMEM; + + int err = mdbx_fastmutex_acquire(&env->me_dbi_lock); + if (unlikely(err != MDBX_SUCCESS)) { + mdbx_free(namedup); + return err; + } + + if (txn->mt_numdbs < env->me_numdbs) { + for (unsigned i = txn->mt_numdbs; i < env->me_numdbs; ++i) { + txn->mt_dbflags[i] = 0; + if (env->me_dbflags[i] & MDBX_VALID) { + txn->mt_dbs[i].md_flags = env->me_dbflags[i] & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = DB_VALID | DB_USRVALID | DB_STALE; + mdbx_tassert(txn, txn->mt_dbxs[i].md_cmp != NULL); + } + } + txn->mt_numdbs = env->me_numdbs; + } + + for (slot = scan = txn->mt_numdbs; --scan >= CORE_DBS;) { + if (!txn->mt_dbxs[scan].md_name.iov_len) { + /* Remember this free slot */ + slot = scan; + continue; + } + if (len == txn->mt_dbxs[scan].md_name.iov_len && + !strncmp(table_name, txn->mt_dbxs[scan].md_name.iov_base, len)) { + *dbi = scan; + rc = mdbx_dbi_bind(txn, scan, user_flags, keycmp, datacmp); + goto bailout; + } + } + + if (unlikely(slot >= env->me_maxdbs)) { + rc = MDBX_DBS_FULL; + goto bailout; + } + + unsigned dbflag = DB_FRESH | DB_VALID | DB_USRVALID; + MDBX_db db_dummy; + if (unlikely(rc)) { + /* MDBX_NOTFOUND and MDBX_CREATE: Create new DB */ + mdbx_tassert(txn, rc == MDBX_NOTFOUND); + memset(&db_dummy, 0, sizeof(db_dummy)); + db_dummy.md_root = P_INVALID; + db_dummy.md_flags = user_flags & PERSISTENT_FLAGS; + data.iov_len = sizeof(db_dummy); + data.iov_base = &db_dummy; + WITH_CURSOR_TRACKING( + mc, + rc = mdbx_cursor_put(&mc, &key, &data, F_SUBDATA | MDBX_NOOVERWRITE)); + + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + dbflag |= DB_DIRTY | DB_CREAT; + } + + /* Got info, register DBI in this txn */ + txn->mt_dbxs[slot].md_cmp = nullptr; + txn->mt_dbxs[slot].md_dcmp = nullptr; + txn->mt_dbs[slot] = *(MDBX_db *)data.iov_base; + env->me_dbflags[slot] = 0; + rc = mdbx_dbi_bind(txn, slot, user_flags, keycmp, datacmp); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_tassert(txn, (dbflag & DB_CREAT) == 0); + bailout: + mdbx_free(namedup); + } else { + txn->mt_dbflags[slot] = (uint8_t)dbflag; + txn->mt_dbxs[slot].md_name.iov_base = namedup; + txn->mt_dbxs[slot].md_name.iov_len = len; + txn->mt_numdbs += (slot == txn->mt_numdbs); + if ((dbflag & DB_CREAT) == 0) { + env->me_dbflags[slot] = txn->mt_dbs[slot].md_flags | MDBX_VALID; + mdbx_compiler_barrier(); + if (env->me_numdbs <= slot) + env->me_numdbs = slot + 1; + } else { + env->me_dbiseqs[slot] += 1; + } + txn->mt_dbiseqs[slot] = env->me_dbiseqs[slot]; + *dbi = slot; + } + + mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); + return rc; +} + +int mdbx_dbi_open(MDBX_txn *txn, const char *table_name, unsigned table_flags, + MDBX_dbi *dbi) { + return mdbx_dbi_open_ex(txn, table_name, table_flags, dbi, nullptr, nullptr); +} + +int __cold mdbx_dbi_stat(MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, + size_t bytes) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!dest)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_VALID))) + return MDBX_EINVAL; + + const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); + if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid) + return MDBX_EINVAL; + + if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED)) + return MDBX_BAD_TXN; + + if (unlikely(txn->mt_dbflags[dbi] & DB_STALE)) { + rc = mdbx_fetch_sdb(txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + mdbx_stat0(txn->mt_env, &txn->mt_dbs[dbi], dest, bytes); + return MDBX_SUCCESS; +} + +static int mdbx_dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) { + if (unlikely(dbi < CORE_DBS || dbi >= env->me_maxdbs)) + return MDBX_EINVAL; + + char *ptr = env->me_dbxs[dbi].md_name.iov_base; + /* If there was no name, this was already closed */ + if (unlikely(!ptr)) + return MDBX_BAD_DBI; + + env->me_dbflags[dbi] = 0; + env->me_dbxs[dbi].md_name.iov_len = 0; + mdbx_compiler_barrier(); + env->me_dbiseqs[dbi]++; + env->me_dbxs[dbi].md_name.iov_base = NULL; + mdbx_free(ptr); + return MDBX_SUCCESS; +} + +int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely(dbi < CORE_DBS || dbi >= env->me_maxdbs)) + return MDBX_EINVAL; + + int rc = mdbx_fastmutex_acquire(&env->me_dbi_lock); + if (likely(rc == MDBX_SUCCESS)) { + rc = mdbx_dbi_close_locked(env, dbi); + mdbx_ensure(env, mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); + } + return rc; +} + +int mdbx_dbi_flags_ex(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, + unsigned *state) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!flags || !state)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_VALID))) + return MDBX_EINVAL; + + *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; + *state = txn->mt_dbflags[dbi] & (DB_FRESH | DB_CREAT | DB_DIRTY | DB_STALE); + + return MDBX_SUCCESS; +} + +int mdbx_dbi_flags(MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags) { + unsigned state; + return mdbx_dbi_flags_ex(txn, dbi, flags, &state); +} + +/* Add all the DB's pages to the free list. + * [in] mc Cursor on the DB to free. + * [in] subs non-Zero to check for sub-DBs in this DB. + * Returns 0 on success, non-zero on failure. */ +static int mdbx_drop0(MDBX_cursor *mc, int subs) { + int rc = mdbx_page_search(mc, NULL, MDBX_PS_FIRST); + if (likely(rc == MDBX_SUCCESS)) { + MDBX_txn *txn = mc->mc_txn; + MDBX_cursor mx; + unsigned i; + + /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves. + * This also avoids any P_LEAF2 pages, which have no nodes. + * Also if the DB doesn't have sub-DBs and has no overflow + * pages, omit scanning leaves. */ + if ((mc->mc_flags & C_SUB) || (subs | mc->mc_db->md_overflow_pages) == 0) + mdbx_cursor_pop(mc); + + rc = mdbx_pnl_need(&txn->tw.retired_pages, + mc->mc_db->md_branch_pages + mc->mc_db->md_leaf_pages + + mc->mc_db->md_overflow_pages); + if (unlikely(rc)) + goto done; + + mdbx_cursor_copy(mc, &mx); + while (mc->mc_snum > 0) { + MDBX_page *mp = mc->mc_pg[mc->mc_top]; + unsigned n = page_numkeys(mp); + if (IS_LEAF(mp)) { + for (i = 0; i < n; i++) { + MDBX_node *node = page_node(mp, i); + if (node_flags(node) & F_BIGDATA) { + MDBX_page *omp; + rc = mdbx_page_get(mc, node_largedata_pgno(node), &omp, NULL); + if (unlikely(rc)) + goto done; + mdbx_cassert(mc, IS_OVERFLOW(omp)); + rc = mdbx_page_retire(mc, omp); + if (unlikely(rc)) + goto done; + if (!mc->mc_db->md_overflow_pages && !subs) + break; + } else if (subs && (node_flags(node) & F_SUBDATA)) { + rc = mdbx_xcursor_init1(mc, node); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + rc = mdbx_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (unlikely(rc)) + goto done; + } + } + if (!subs && !mc->mc_db->md_overflow_pages) + goto pop; + } else { + for (i = 0; i < n; i++) { + /* free it */ + rc = mdbx_retire_pgno(mc, node_pgno(page_node(mp, i))); + if (unlikely(rc)) + goto done; + } + } + if (!mc->mc_top) + break; + mdbx_cassert(mc, i <= UINT16_MAX); + mc->mc_ki[mc->mc_top] = (indx_t)i; + rc = mdbx_cursor_sibling(mc, 1); + if (rc) { + if (unlikely(rc != MDBX_NOTFOUND)) + goto done; + /* no more siblings, go back to beginning + * of previous level. */ + pop: + mdbx_cursor_pop(mc); + mc->mc_ki[0] = 0; + for (i = 1; i < mc->mc_snum; i++) { + mc->mc_ki[i] = 0; + mc->mc_pg[i] = mx.mc_pg[i]; + } + } + } + /* free it */ + rc = mdbx_retire_pgno(mc, mc->mc_db->md_root); + done: + if (unlikely(rc)) + txn->mt_flags |= MDBX_TXN_ERROR; + } else if (rc == MDBX_NOTFOUND) { + rc = MDBX_SUCCESS; + } + mc->mc_flags &= ~C_INITIALIZED; + return rc; +} + +int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, int del) { + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(1 < (unsigned)del)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + if (unlikely(TXN_DBI_CHANGED(txn, dbi))) + return MDBX_BAD_DBI; + + MDBX_cursor *mc; + rc = mdbx_cursor_open(txn, dbi, &mc); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) { + rc = MDBX_EINVAL; + goto bailout; + } + + if (unlikely(TXN_DBI_CHANGED(txn, dbi))) { + rc = MDBX_BAD_DBI; + goto bailout; + } + + rc = mdbx_drop0(mc, mc->mc_db->md_flags & MDBX_DUPSORT); + /* Invalidate the dropped DB's cursors */ + for (MDBX_cursor *m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) + m2->mc_flags &= ~(C_INITIALIZED | C_EOF); + if (unlikely(rc)) + goto bailout; + + /* Can't delete the main DB */ + if (del && dbi >= CORE_DBS) { + rc = mdbx_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA); + if (likely(rc == MDBX_SUCCESS)) { + txn->mt_dbflags[dbi] = DB_STALE; + MDBX_env *env = txn->mt_env; + rc = mdbx_fastmutex_acquire(&env->me_dbi_lock); + if (unlikely(rc != MDBX_SUCCESS)) { + txn->mt_flags |= MDBX_TXN_ERROR; + goto bailout; + } + mdbx_dbi_close_locked(env, dbi); + mdbx_ensure(env, + mdbx_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); + } else { + txn->mt_flags |= MDBX_TXN_ERROR; + } + } else { + /* reset the DB record, mark it dirty */ + txn->mt_dbflags[dbi] |= DB_DIRTY; + txn->mt_dbs[dbi].md_depth = 0; + txn->mt_dbs[dbi].md_branch_pages = 0; + txn->mt_dbs[dbi].md_leaf_pages = 0; + txn->mt_dbs[dbi].md_overflow_pages = 0; + txn->mt_dbs[dbi].md_entries = 0; + txn->mt_dbs[dbi].md_root = P_INVALID; + txn->mt_dbs[dbi].md_seq = 0; + txn->mt_flags |= MDBX_TXN_DIRTY; + } + +bailout: + mdbx_cursor_close(mc); + return rc; +} + +int mdbx_set_compare(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + txn->mt_dbxs[dbi].md_cmp = cmp; + return MDBX_SUCCESS; +} + +int mdbx_set_dupsort(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cmp_func *cmp) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + txn->mt_dbxs[dbi].md_dcmp = cmp; + return MDBX_SUCCESS; +} + +int __cold mdbx_reader_list(MDBX_env *env, MDBX_reader_list_func *func, + void *ctx) { + if (unlikely(!env || !func)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + int rc = MDBX_RESULT_TRUE; + int serial = 0; + if (likely(env->me_lck)) { + const unsigned snap_nreaders = env->me_lck->mti_numreaders; + for (unsigned i = 0; i < snap_nreaders; i++) { + const MDBX_reader *r = env->me_lck->mti_readers + i; + retry_reader:; + const uint32_t pid = r->mr_pid; + if (!pid) + continue; + txnid_t txnid = safe64_read(&r->mr_txnid); + const size_t tid = r->mr_tid; + const pgno_t pages_used = r->mr_snapshot_pages_used; + const uint64_t reader_pages_retired = r->mr_snapshot_pages_retired; + mdbx_compiler_barrier(); + if (unlikely(tid != r->mr_tid || + pages_used != r->mr_snapshot_pages_used || + reader_pages_retired != r->mr_snapshot_pages_retired || + txnid != safe64_read(&r->mr_txnid) || pid != r->mr_pid)) + goto retry_reader; + + mdbx_assert(env, txnid > 0); + if (txnid >= SAFE64_INVALID_THRESHOLD) + txnid = 0; + + size_t bytes_used = 0; + size_t bytes_retained = 0; + uint64_t lag = 0; + if (txnid) { + retry_header:; + const MDBX_meta *const recent_meta = mdbx_meta_head(env); + const uint64_t head_pages_retired = recent_meta->mm_pages_retired; + const txnid_t head_txnid = mdbx_meta_txnid_fluid(env, recent_meta); + mdbx_compiler_barrier(); + if (unlikely(recent_meta != mdbx_meta_head(env) || + head_pages_retired != recent_meta->mm_pages_retired) || + head_txnid != mdbx_meta_txnid_fluid(env, recent_meta)) + goto retry_header; + + lag = (head_txnid - txnid) / MDBX_TXNID_STEP; + bytes_used = pgno2bytes(env, pages_used); + bytes_retained = (head_pages_retired > reader_pages_retired) + ? pgno2bytes(env, (pgno_t)(head_pages_retired - + reader_pages_retired)) + : 0; + } + rc = func(ctx, ++serial, i, pid, (mdbx_tid_t)tid, txnid, lag, bytes_used, + bytes_retained); + if (unlikely(rc != MDBX_SUCCESS)) + break; + } + } + + return rc; +} + +/* Insert pid into list if not already present. + * return -1 if already present. */ +static int __cold mdbx_pid_insert(uint32_t *ids, uint32_t pid) { + /* binary search of pid in list */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while (n > 0) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = pid - ids[cursor]; + + if (val < 0) { + n = pivot; + } else if (val > 0) { + base = cursor; + n -= pivot + 1; + } else { + /* found, so it's a duplicate */ + return -1; + } + } + + if (val > 0) + ++cursor; + + ids[0]++; + for (n = ids[0]; n > cursor; n--) + ids[n] = ids[n - 1]; + ids[n] = pid; + return 0; +} + +int __cold mdbx_reader_check(MDBX_env *env, int *dead) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + if (dead) + *dead = 0; + return mdbx_reader_check0(env, false, dead); +} + +/* Return: + * MDBX_RESULT_TRUE - done and mutex recovered + * MDBX_SUCCESS - done + * Otherwise errcode. */ +int __cold mdbx_reader_check0(MDBX_env *env, int rdt_locked, int *dead) { + mdbx_assert(env, rdt_locked >= 0); + +#if MDBX_TXN_CHECKPID + if (unlikely(env->me_pid != mdbx_getpid())) { + env->me_flags |= MDBX_FATAL_ERROR; + return MDBX_PANIC; + } +#endif /* MDBX_TXN_CHECKPID */ + + MDBX_lockinfo *const lck = env->me_lck; + if (unlikely(lck == NULL)) { + /* exclusive mode */ + if (dead) + *dead = 0; + return MDBX_SUCCESS; + } + + lck->mti_reader_check_timestamp = mdbx_osal_monotime(); + const unsigned snap_nreaders = lck->mti_numreaders; + uint32_t pidsbuf_onstask[142]; + uint32_t *const pids = + (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask)) + ? pidsbuf_onstask + : mdbx_malloc((snap_nreaders + 1) * sizeof(uint32_t)); + if (unlikely(!pids)) + return MDBX_ENOMEM; + + pids[0] = 0; + + int rc = MDBX_SUCCESS, count = 0; + for (unsigned i = 0; i < snap_nreaders; i++) { + const uint32_t pid = lck->mti_readers[i].mr_pid; + if (pid == 0) + continue /* skip empty */; + if (pid == env->me_pid) + continue /* skip self */; + if (mdbx_pid_insert(pids, pid) != 0) + continue /* such pid already processed */; + + int err = mdbx_rpid_check(env, pid); + if (err == MDBX_RESULT_TRUE) + continue /* reader is live */; + + if (err != MDBX_SUCCESS) { + rc = err; + break /* mdbx_rpid_check() failed */; + } + + /* stale reader found */ + if (!rdt_locked) { + err = mdbx_rdt_lock(env); + if (MDBX_IS_ERROR(err)) { + rc = err; + break; + } + + rdt_locked = -1; + if (err == MDBX_RESULT_TRUE) { + /* mutex recovered, the mdbx_mutex_failed() checked all readers */ + rc = MDBX_RESULT_TRUE; + break; + } + + /* a other process may have clean and reused slot, recheck */ + if (lck->mti_readers[i].mr_pid != pid) + continue; + + err = mdbx_rpid_check(env, pid); + if (MDBX_IS_ERROR(err)) { + rc = err; + break; + } + + if (err != MDBX_SUCCESS) + continue /* the race with other process, slot reused */; + } + + /* clean it */ + for (unsigned j = i; j < snap_nreaders; j++) { + if (lck->mti_readers[j].mr_pid == pid) { + mdbx_debug("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, + (size_t)pid, lck->mti_readers[j].mr_txnid.inconsistent); + lck->mti_readers[j].mr_pid = 0; + lck->mti_readers_refresh_flag = true; + count++; + } + } + } + + if (rdt_locked < 0) + mdbx_rdt_unlock(env); + + if (pids != pidsbuf_onstask) + mdbx_free(pids); + + if (dead) + *dead = count; + return rc; +} + +int __cold mdbx_setup_debug(int loglevel, int flags, MDBX_debug_func *logger) { + const int rc = mdbx_runtime_flags | (mdbx_loglevel << 16); + +#if !MDBX_DEBUG + (void)loglevel; +#else + if (loglevel != -1) + mdbx_loglevel = (uint8_t)loglevel; +#endif + + if (flags != -1) { +#if !MDBX_DEBUG + flags &= MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN; +#else + flags &= MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER | + MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN; +#endif +#if defined(__linux__) || defined(__gnu_linux__) + if ((mdbx_runtime_flags ^ flags) & MDBX_DBG_DUMP) { + /* http://man7.org/linux/man-pages/man5/core.5.html */ + const unsigned long dump_bits = + 1 << 3 /* Dump file-backed shared mappings */ + | 1 << 6 /* Dump shared huge pages */ + | 1 << 8 /* Dump shared DAX pages */; + const int core_filter_fd = + open("/proc/self/coredump_filter", O_TRUNC | O_RDWR); + if (core_filter_fd != -1) { + char buf[32]; + intptr_t bytes = pread(core_filter_fd, buf, sizeof(buf), 0); + if (bytes > 0 && (size_t)bytes < sizeof(buf)) { + buf[bytes] = 0; + const unsigned long present_mask = strtoul(buf, NULL, 16); + const unsigned long wanna_mask = (flags & MDBX_DBG_DUMP) + ? present_mask | dump_bits + : present_mask & ~dump_bits; + if (wanna_mask != present_mask) { + bytes = snprintf(buf, sizeof(buf), "0x%lx\n", wanna_mask); + if (bytes > 0 && (size_t)bytes < sizeof(buf)) { + bytes = pwrite(core_filter_fd, buf, bytes, 0); + (void)bytes; + } + } + } + close(core_filter_fd); + } + } +#endif /* Linux */ + mdbx_runtime_flags = (uint8_t)flags; + } + + if (-1 != (intptr_t)logger) + mdbx_debug_logger = logger; + return rc; +} + +static txnid_t __cold mdbx_oomkick(MDBX_env *env, const txnid_t laggard) { + mdbx_debug("%s", "DB size maxed out"); + + int retry; + for (retry = 0; retry < INT_MAX; ++retry) { + txnid_t oldest = mdbx_reclaiming_detent(env); + mdbx_assert(env, oldest < env->me_txn0->mt_txnid); + mdbx_assert(env, oldest >= laggard); + mdbx_assert(env, oldest >= *env->me_oldest); + if (oldest == laggard || unlikely(env->me_lck == NULL /* exclusive mode */)) + return oldest; + + if (MDBX_IS_ERROR(mdbx_reader_check0(env, false, NULL))) + break; + + MDBX_reader *asleep = nullptr; + MDBX_lockinfo *const lck = env->me_lck; + uint64_t oldest_retired = UINT64_MAX; + const unsigned snap_nreaders = lck->mti_numreaders; + for (unsigned i = 0; i < snap_nreaders; ++i) { + retry: + if (lck->mti_readers[i].mr_pid) { + /* mdbx_jitter4testing(true); */ + const uint64_t snap_retired = + lck->mti_readers[i].mr_snapshot_pages_retired; + const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid); + mdbx_memory_barrier(); + if (unlikely(snap_retired != + lck->mti_readers[i].mr_snapshot_pages_retired || + snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid))) + goto retry; + if (oldest > snap_txnid && + laggard <= /* ignore pending updates */ snap_txnid) { + oldest = snap_txnid; + oldest_retired = snap_retired; + asleep = &lck->mti_readers[i]; + } + } + } + + if (laggard < oldest || !asleep) { + if (retry && env->me_oom_func) { + /* LY: notify end of oom-loop */ + const txnid_t gap = oldest - laggard; + env->me_oom_func(env, 0, 0, laggard, + (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, 0, + -retry); + } + mdbx_notice("oom-kick: update oldest %" PRIaTXN " -> %" PRIaTXN, + *env->me_oldest, oldest); + mdbx_assert(env, *env->me_oldest <= oldest); + return *env->me_oldest = oldest; + } + + if (!env->me_oom_func) + break; + + uint32_t pid = asleep->mr_pid; + size_t tid = asleep->mr_tid; + if (safe64_read(&asleep->mr_txnid) != laggard || pid <= 0) + continue; + + const MDBX_meta *head_meta = mdbx_meta_head(env); + const txnid_t gap = + (mdbx_meta_txnid_stable(env, head_meta) - laggard) / MDBX_TXNID_STEP; + const uint64_t head_retired = head_meta->mm_pages_retired; + const size_t space = + (oldest_retired > head_retired) + ? pgno2bytes(env, (pgno_t)(oldest_retired - head_retired)) + : 0; + int rc = env->me_oom_func(env, pid, (mdbx_tid_t)tid, laggard, + (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, + space, retry); + if (rc < 0) + break; + + if (rc > 0) { + if (rc == 1) { + safe64_reset_compare(&asleep->mr_txnid, laggard); + } else { + safe64_reset(&asleep->mr_txnid, true); + asleep->mr_tid = 0; + asleep->mr_pid = 0; + } + lck->mti_readers_refresh_flag = true; + mdbx_flush_noncoherent_cpu_writeback(); + } + } + + if (retry && env->me_oom_func) { + /* LY: notify end of oom-loop */ + env->me_oom_func(env, 0, 0, laggard, 0, 0, -retry); + } + return mdbx_find_oldest(env->me_txn); +} + +int __cold mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely(env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR))) + return MDBX_EACCESS; + + if (unlikely(!env->me_map)) + return MDBX_EPERM; + + *env->me_autosync_threshold = bytes2pgno(env, threshold + env->me_psize - 1); + if (threshold) { + int err = mdbx_env_sync_poll(env); + if (unlikely(MDBX_IS_ERROR(err))) + return err; + } + return MDBX_SUCCESS; +} + +int __cold mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + if (unlikely(env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR))) + return MDBX_EACCESS; + + if (unlikely(!env->me_map)) + return MDBX_EPERM; + + *env->me_autosync_period = mdbx_osal_16dot16_to_monotime(seconds_16dot16); + if (seconds_16dot16) { + int err = mdbx_env_sync_poll(env); + if (unlikely(MDBX_IS_ERROR(err))) + return err; + } + return MDBX_SUCCESS; +} + +int __cold mdbx_env_set_oomfunc(MDBX_env *env, MDBX_oom_func *oomfunc) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->me_signature != MDBX_ME_SIGNATURE)) + return MDBX_EBADSIGN; + + env->me_oom_func = oomfunc; + return MDBX_SUCCESS; +} + +MDBX_oom_func *__cold mdbx_env_get_oomfunc(MDBX_env *env) { + return likely(env && env->me_signature == MDBX_ME_SIGNATURE) + ? env->me_oom_func + : NULL; +} + +#ifdef __SANITIZE_THREAD__ +/* LY: avoid tsan-trap by me_txn, mm_last_pg and mt_next_pgno */ +__attribute__((__no_sanitize_thread__, __noinline__)) +#endif +int mdbx_txn_straggler(MDBX_txn *txn, int *percent) +{ + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc > 0) ? -rc : rc; + + MDBX_env *env = txn->mt_env; + if (unlikely((txn->mt_flags & MDBX_RDONLY) == 0)) { + if (percent) + *percent = + (int)((txn->mt_next_pgno * UINT64_C(100) + txn->mt_end_pgno / 2) / + txn->mt_end_pgno); + return 0; + } + + txnid_t recent; + MDBX_meta *meta; + do { + meta = mdbx_meta_head(env); + recent = mdbx_meta_txnid_fluid(env, meta); + if (percent) { + const pgno_t maxpg = meta->mm_geo.now; + *percent = (int)((meta->mm_geo.next * UINT64_C(100) + maxpg / 2) / maxpg); + } + } while (unlikely(recent != mdbx_meta_txnid_fluid(env, meta))); + + txnid_t lag = (recent - txn->mt_txnid) / MDBX_TXNID_STEP; + return (lag > INT_MAX) ? INT_MAX : (int)lag; +} + +typedef struct mdbx_walk_ctx { + void *mw_user; + MDBX_pgvisitor_func *mw_visitor; + MDBX_cursor mw_cursor; +} mdbx_walk_ctx_t; + +/* Depth-first tree traversal. */ +static int __cold mdbx_env_walk(mdbx_walk_ctx_t *ctx, const char *dbi, + pgno_t pgno, int deep) { + if (unlikely(pgno == P_INVALID)) + return MDBX_SUCCESS; /* empty db */ + + MDBX_page *mp; + int rc = mdbx_page_get(&ctx->mw_cursor, pgno, &mp, NULL); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + rc = mdbx_page_check(ctx->mw_cursor.mc_txn->mt_env, mp, false); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + const int nkeys = page_numkeys(mp); + size_t header_size = IS_LEAF2(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->mp_lower; + size_t unused_size = page_room(mp); + size_t payload_size = 0; + size_t align_bytes = 0; + MDBX_page_type_t type; + + /* LY: Don't use mask here, e.g bitwise + * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP). + * Pages should not me marked dirty/loose or otherwise. */ + switch (mp->mp_flags) { + case P_BRANCH: + type = MDBX_page_branch; + if (unlikely(nkeys < 2)) + return MDBX_CORRUPTED; + break; + case P_LEAF: + type = MDBX_page_leaf; + break; + case P_LEAF | P_LEAF2: + type = MDBX_page_dupfixed_leaf; + break; + default: + return MDBX_CORRUPTED; + } + + for (int i = 0; i < nkeys; + align_bytes += ((payload_size + align_bytes) & 1), i++) { + if (type == MDBX_page_dupfixed_leaf) { + /* LEAF2 pages have no mp_ptrs[] or node headers */ + payload_size += mp->mp_leaf2_ksize; + continue; + } + + MDBX_node *node = page_node(mp, i); + payload_size += NODESIZE + node_ks(node); + + if (type == MDBX_page_branch) { + assert(i > 0 || node_ks(node) == 0); + continue; + } + + assert(type == MDBX_page_leaf); + switch (node_flags(node)) { + case 0 /* usual node */: { + payload_size += node_ds(node); + } break; + + case F_BIGDATA /* long data on the large/overflow page */: { + payload_size += sizeof(pgno_t); + + const pgno_t large_pgno = node_largedata_pgno(node); + MDBX_page *op; + rc = mdbx_page_get(&ctx->mw_cursor, large_pgno, &op, NULL); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_page_check(ctx->mw_cursor.mc_txn->mt_env, op, false); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + /* LY: Don't use mask here, e.g bitwise + * (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP). + * Pages should not me marked dirty/loose or otherwise. */ + if (unlikely(P_OVERFLOW != op->mp_flags)) + return MDBX_CORRUPTED; + + const size_t over_header = PAGEHDRSZ; + const size_t over_payload = node_ds(node); + const size_t over_unused = + pgno2bytes(ctx->mw_cursor.mc_txn->mt_env, op->mp_pages) - + over_payload - over_header; + + rc = ctx->mw_visitor( + large_pgno, op->mp_pages, ctx->mw_user, deep, dbi, + pgno2bytes(ctx->mw_cursor.mc_txn->mt_env, op->mp_pages), + MDBX_page_large, 1, over_payload, over_header, over_unused); + } break; + + case F_SUBDATA /* sub-db */: { + const size_t namelen = node_ks(node); + if (unlikely(namelen == 0 || node_ds(node) < sizeof(MDBX_db))) + return MDBX_CORRUPTED; + payload_size += node_ds(node); + } break; + + case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: { + if (unlikely(node_ds(node) != sizeof(MDBX_db))) + return MDBX_CORRUPTED; + payload_size += sizeof(MDBX_db); + } break; + + case F_DUPDATA /* short sub-page */: { + if (unlikely(node_ds(node) <= PAGEHDRSZ)) + return MDBX_CORRUPTED; + + MDBX_page *sp = node_data(node); + const int nsubkeys = page_numkeys(sp); + size_t subheader_size = + IS_LEAF2(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->mp_lower; + size_t subunused_size = page_room(sp); + size_t subpayload_size = 0; + size_t subalign_bytes = 0; + MDBX_page_type_t subtype; + + switch (sp->mp_flags & ~P_DIRTY /* ignore for sub-pages */) { + case P_LEAF | P_SUBP: + subtype = MDBX_subpage_leaf; + break; + case P_LEAF | P_LEAF2 | P_SUBP: + subtype = MDBX_subpage_dupfixed_leaf; + break; + default: + return MDBX_CORRUPTED; + } + + for (int j = 0; j < nsubkeys; + subalign_bytes += ((subpayload_size + subalign_bytes) & 1), j++) { + + if (subtype == MDBX_subpage_dupfixed_leaf) { + /* LEAF2 pages have no mp_ptrs[] or node headers */ + subpayload_size += sp->mp_leaf2_ksize; + } else { + assert(subtype == MDBX_subpage_leaf); + MDBX_node *subnode = page_node(sp, j); + subpayload_size += NODESIZE + node_ks(subnode) + node_ds(subnode); + if (unlikely(node_flags(subnode) != 0)) + return MDBX_CORRUPTED; + } + } + + rc = ctx->mw_visitor(pgno, 0, ctx->mw_user, deep + 1, dbi, node_ds(node), + subtype, nsubkeys, subpayload_size, subheader_size, + subunused_size + subalign_bytes); + header_size += subheader_size; + unused_size += subunused_size; + payload_size += subpayload_size; + align_bytes += subalign_bytes; + } break; + + default: + return MDBX_CORRUPTED; + } + + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + + rc = ctx->mw_visitor(mp->mp_pgno, 1, ctx->mw_user, deep, dbi, + ctx->mw_cursor.mc_txn->mt_env->me_psize, type, nkeys, + payload_size, header_size, unused_size + align_bytes); + + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; + + for (int i = 0; i < nkeys; i++) { + if (type == MDBX_page_dupfixed_leaf) + continue; + + MDBX_node *node = page_node(mp, i); + if (type == MDBX_page_branch) { + rc = mdbx_env_walk(ctx, dbi, node_pgno(node), deep + 1); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc != MDBX_RESULT_TRUE) + return rc; + break; + } + continue; + } + + assert(type == MDBX_page_leaf); + MDBX_db db; + switch (node_flags(node)) { + default: + continue; + + case F_SUBDATA /* sub-db */: { + const size_t namelen = node_ks(node); + if (unlikely(namelen == 0 || node_ds(node) != sizeof(MDBX_db))) + return MDBX_CORRUPTED; + + char namebuf_onstask[142]; + char *const name = (namelen < sizeof(namebuf_onstask)) + ? namebuf_onstask + : mdbx_malloc(namelen + 1); + if (name) { + memcpy(name, node_key(node), namelen); + name[namelen] = 0; + memcpy(&db, node_data(node), sizeof(db)); + rc = mdbx_env_walk(ctx, name, db.md_root, deep + 1); + if (name != namebuf_onstask) + mdbx_free(name); + } else { + rc = MDBX_ENOMEM; + } + } break; + + case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: + if (unlikely(node_ds(node) != sizeof(MDBX_db))) + return MDBX_CORRUPTED; + + memcpy(&db, node_data(node), sizeof(db)); + rc = mdbx_env_walk(ctx, dbi, db.md_root, deep + 1); + break; + } + + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + + return MDBX_SUCCESS; +} + +int __cold mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor, + void *user) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + mdbx_walk_ctx_t ctx; + memset(&ctx, 0, sizeof(ctx)); + ctx.mw_cursor.mc_snum = 1; + ctx.mw_cursor.mc_txn = txn; + ctx.mw_user = user; + ctx.mw_visitor = visitor; + + rc = visitor(0, NUM_METAS, user, 0, MDBX_PGWALK_META, + pgno2bytes(txn->mt_env, NUM_METAS), MDBX_page_meta, NUM_METAS, + sizeof(MDBX_meta) * NUM_METAS, PAGEHDRSZ * NUM_METAS, + (txn->mt_env->me_psize - sizeof(MDBX_meta) - PAGEHDRSZ) * + NUM_METAS); + if (!MDBX_IS_ERROR(rc)) + rc = mdbx_env_walk(&ctx, MDBX_PGWALK_GC, txn->mt_dbs[FREE_DBI].md_root, 0); + if (!MDBX_IS_ERROR(rc)) + rc = + mdbx_env_walk(&ctx, MDBX_PGWALK_MAIN, txn->mt_dbs[MAIN_DBI].md_root, 0); + if (!MDBX_IS_ERROR(rc)) + rc = visitor(P_INVALID, 0, user, INT_MIN, NULL, 0, MDBX_page_void, 0, 0, 0, + 0); + return rc; +} + +int mdbx_canary_put(MDBX_txn *txn, const mdbx_canary *canary) { + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (likely(canary)) { + if (txn->mt_canary.x == canary->x && txn->mt_canary.y == canary->y && + txn->mt_canary.z == canary->z) + return MDBX_SUCCESS; + txn->mt_canary.x = canary->x; + txn->mt_canary.y = canary->y; + txn->mt_canary.z = canary->z; + } + txn->mt_canary.v = txn->mt_txnid; + + if ((txn->mt_flags & MDBX_TXN_DIRTY) == 0) { + txn->mt_flags |= MDBX_TXN_DIRTY; + *txn->mt_env->me_unsynced_pages += 1; + } + return MDBX_SUCCESS; +} + +int mdbx_canary_get(MDBX_txn *txn, mdbx_canary *canary) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(canary == NULL)) + return MDBX_EINVAL; + + *canary = txn->mt_canary; + return MDBX_SUCCESS; +} + +int mdbx_cursor_on_first(MDBX_cursor *mc) { + if (unlikely(mc == NULL)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return MDBX_RESULT_FALSE; + + for (unsigned i = 0; i < mc->mc_snum; ++i) { + if (mc->mc_ki[i]) + return MDBX_RESULT_FALSE; + } + + return MDBX_RESULT_TRUE; +} + +int mdbx_cursor_on_last(MDBX_cursor *mc) { + if (unlikely(mc == NULL)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return MDBX_RESULT_FALSE; + + for (unsigned i = 0; i < mc->mc_snum; ++i) { + unsigned nkeys = page_numkeys(mc->mc_pg[i]); + if (mc->mc_ki[i] < nkeys - 1) + return MDBX_RESULT_FALSE; + } + + return MDBX_RESULT_TRUE; +} + +int mdbx_cursor_eof(MDBX_cursor *mc) { + if (unlikely(mc == NULL)) + return MDBX_EINVAL; + + if (unlikely(mc->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + if ((mc->mc_flags & C_INITIALIZED) == 0) + return MDBX_RESULT_TRUE; + + if (mc->mc_snum == 0) + return MDBX_RESULT_TRUE; + + if ((mc->mc_flags & C_EOF) && + mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top])) + return MDBX_RESULT_TRUE; + + return MDBX_RESULT_FALSE; +} + +//------------------------------------------------------------------------------ + +struct diff_result { + ptrdiff_t diff; + int level; + int root_nkeys; +}; + +/* calculates: r = x - y */ +__hot static int cursor_diff(const MDBX_cursor *const __restrict x, + const MDBX_cursor *const __restrict y, + struct diff_result *const __restrict r) { + r->diff = 0; + r->level = 0; + r->root_nkeys = 0; + + if (unlikely(y->mc_signature != MDBX_MC_SIGNATURE || + x->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + int rc = check_txn(x->mc_txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(x->mc_txn != y->mc_txn)) + return MDBX_BAD_TXN; + + if (unlikely(y->mc_dbi != x->mc_dbi)) + return MDBX_EINVAL; + + if (unlikely(!(y->mc_flags & x->mc_flags & C_INITIALIZED))) + return MDBX_ENODATA; + + while (likely(r->level < y->mc_snum && r->level < x->mc_snum)) { + if (unlikely(y->mc_pg[r->level] != x->mc_pg[r->level])) + return MDBX_PROBLEM; + + int nkeys = page_numkeys(y->mc_pg[r->level]); + assert(nkeys > 0); + if (r->level == 0) + r->root_nkeys = nkeys; + + const int limit_ki = nkeys - 1; + const int x_ki = x->mc_ki[r->level]; + const int y_ki = y->mc_ki[r->level]; + r->diff = ((x_ki < limit_ki) ? x_ki : limit_ki) - + ((y_ki < limit_ki) ? y_ki : limit_ki); + if (r->diff == 0) { + r->level += 1; + continue; + } + + while (unlikely(r->diff == 1) && + likely(r->level + 1 < y->mc_snum && r->level + 1 < x->mc_snum)) { + r->level += 1; + /* DB'PAGEs: 0------------------>MAX + * + * CURSORs: y < x + * STACK[i ]: | + * STACK[+1]: ...y++N|0++x... + */ + nkeys = page_numkeys(y->mc_pg[r->level]); + r->diff = (nkeys - y->mc_ki[r->level]) + x->mc_ki[r->level]; + assert(r->diff > 0); + } + + while (unlikely(r->diff == -1) && + likely(r->level + 1 < y->mc_snum && r->level + 1 < x->mc_snum)) { + r->level += 1; + /* DB'PAGEs: 0------------------>MAX + * + * CURSORs: x < y + * STACK[i ]: | + * STACK[+1]: ...x--N|0--y... + */ + nkeys = page_numkeys(x->mc_pg[r->level]); + r->diff = -(nkeys - x->mc_ki[r->level]) - y->mc_ki[r->level]; + assert(r->diff < 0); + } + + return MDBX_SUCCESS; + } + + r->diff = CMP2INT(x->mc_flags & C_EOF, y->mc_flags & C_EOF); + return MDBX_SUCCESS; +} + +__hot static ptrdiff_t estimate(const MDBX_db *db, + struct diff_result *const __restrict dr) { + /* root: branch-page => scale = leaf-factor * branch-factor^(N-1) + * level-1: branch-page(s) => scale = leaf-factor * branch-factor^2 + * level-2: branch-page(s) => scale = leaf-factor * branch-factor + * level-N: branch-page(s) => scale = leaf-factor + * leaf-level: leaf-page(s) => scale = 1 + */ + ptrdiff_t btree_power = db->md_depth - 2 - dr->level; + if (btree_power < 0) + return dr->diff; + + ptrdiff_t estimated = + (ptrdiff_t)db->md_entries * dr->diff / (ptrdiff_t)db->md_leaf_pages; + if (btree_power == 0) + return estimated; + + if (db->md_depth < 4) { + assert(dr->level == 0 && btree_power == 1); + return (ptrdiff_t)db->md_entries * dr->diff / (ptrdiff_t)dr->root_nkeys; + } + + /* average_branchpage_fillfactor = total(branch_entries) / branch_pages + total(branch_entries) = leaf_pages + branch_pages - 1 (root page) */ + const size_t log2_fixedpoint = sizeof(size_t) - 1; + const size_t half = UINT64_C(1) << (log2_fixedpoint - 1); + const size_t factor = + ((db->md_leaf_pages + db->md_branch_pages - 1) << log2_fixedpoint) / + db->md_branch_pages; + while (1) { + switch ((size_t)btree_power) { + default: { + const size_t square = (factor * factor + half) >> log2_fixedpoint; + const size_t quad = (square * square + half) >> log2_fixedpoint; + do { + estimated = estimated * quad + half; + estimated >>= log2_fixedpoint; + btree_power -= 4; + } while (btree_power >= 4); + continue; + } + case 3: + estimated = estimated * factor + half; + estimated >>= log2_fixedpoint; + __fallthrough /* fall through */; + case 2: + estimated = estimated * factor + half; + estimated >>= log2_fixedpoint; + __fallthrough /* fall through */; + case 1: + estimated = estimated * factor + half; + estimated >>= log2_fixedpoint; + __fallthrough /* fall through */; + case 0: + if (unlikely(estimated > (ptrdiff_t)db->md_entries)) + return (ptrdiff_t)db->md_entries; + if (unlikely(estimated < -(ptrdiff_t)db->md_entries)) + return -(ptrdiff_t)db->md_entries; + return estimated; + } + } +} + +__hot int mdbx_estimate_distance(const MDBX_cursor *first, + const MDBX_cursor *last, + ptrdiff_t *distance_items) { + if (unlikely(first == NULL || last == NULL || distance_items == NULL)) + return MDBX_EINVAL; + + *distance_items = 0; + struct diff_result dr; + int rc = cursor_diff(last, first, &dr); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(dr.diff == 0) && + F_ISSET(first->mc_db->md_flags & first->mc_db->md_flags, + MDBX_DUPSORT | C_INITIALIZED)) { + first = &first->mc_xcursor->mx_cursor; + last = &last->mc_xcursor->mx_cursor; + rc = cursor_diff(first, last, &dr); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + + if (likely(dr.diff != 0)) + *distance_items = estimate(first->mc_db, &dr); + + return MDBX_SUCCESS; +} + +int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, + MDBX_cursor_op move_op, ptrdiff_t *distance_items) { + if (unlikely(cursor == NULL || distance_items == NULL || + move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE)) + return MDBX_EINVAL; + + if (unlikely(cursor->mc_signature != MDBX_MC_SIGNATURE)) + return MDBX_EBADSIGN; + + int rc = check_txn(cursor->mc_txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (!(cursor->mc_flags & C_INITIALIZED)) + return MDBX_ENODATA; + + MDBX_cursor_couple next; + mdbx_cursor_copy(cursor, &next.outer); + next.outer.mc_xcursor = NULL; + if (cursor->mc_db->md_flags & MDBX_DUPSORT) { + next.outer.mc_xcursor = &next.inner; + rc = mdbx_xcursor_init0(&next.outer); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + MDBX_xcursor *mx = &container_of(cursor, MDBX_cursor_couple, outer)->inner; + mdbx_cursor_copy(&mx->mx_cursor, &next.inner.mx_cursor); + } + + MDBX_val stub = {0, 0}; + if (data == NULL) { + const unsigned mask = + 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY; + if (unlikely(mask & (1 << move_op))) + return MDBX_EINVAL; + data = &stub; + } + + if (key == NULL) { + const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | + 1 << MDBX_SET_KEY | 1 << MDBX_SET | + 1 << MDBX_SET_RANGE; + if (unlikely(mask & (1 << move_op))) + return MDBX_EINVAL; + key = &stub; + } + + rc = mdbx_cursor_get(&next.outer, key, data, move_op); + if (unlikely(rc != MDBX_SUCCESS && + (rc != MDBX_NOTFOUND || !(next.outer.mc_flags & C_INITIALIZED)))) + return rc; + + return mdbx_estimate_distance(cursor, &next.outer, distance_items); +} + +static int mdbx_is_samedata(const MDBX_val *a, const MDBX_val *b) { + return a->iov_len == b->iov_len && + memcmp(a->iov_base, b->iov_base, a->iov_len) == 0; +} + +int mdbx_estimate_range(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *begin_key, + MDBX_val *begin_data, MDBX_val *end_key, + MDBX_val *end_data, ptrdiff_t *size_items) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!size_items)) + return MDBX_EINVAL; + + if (unlikely(begin_data && (begin_key == NULL || begin_key == MDBX_EPSILON))) + return MDBX_EINVAL; + + if (unlikely(end_data && (end_key == NULL || end_key == MDBX_EPSILON))) + return MDBX_EINVAL; + + if (unlikely(begin_key == MDBX_EPSILON && end_key == MDBX_EPSILON)) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + MDBX_cursor_couple begin; + /* LY: first, initialize cursor to refresh a DB in case it have DB_STALE */ + rc = mdbx_cursor_init(&begin.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(begin.outer.mc_db->md_entries == 0)) { + *size_items = 0; + return MDBX_SUCCESS; + } + + if (!begin_key) { + if (unlikely(!end_key)) { + /* LY: FIRST..LAST case */ + *size_items = (ptrdiff_t)begin.outer.mc_db->md_entries; + return MDBX_SUCCESS; + } + MDBX_val stub = {0, 0}; + rc = mdbx_cursor_first(&begin.outer, &stub, &stub); + if (unlikely(end_key == MDBX_EPSILON)) { + /* LY: FIRST..+epsilon case */ + return (rc == MDBX_SUCCESS) + ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) + : rc; + } + } else { + if (unlikely(begin_key == MDBX_EPSILON)) { + if (end_key == NULL) { + /* LY: -epsilon..LAST case */ + MDBX_val stub = {0, 0}; + rc = mdbx_cursor_last(&begin.outer, &stub, &stub); + return (rc == MDBX_SUCCESS) + ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) + : rc; + } + /* LY: -epsilon..value case */ + assert(end_key != MDBX_EPSILON); + begin_key = end_key; + } else if (unlikely(end_key == MDBX_EPSILON)) { + /* LY: value..+epsilon case */ + assert(begin_key != MDBX_EPSILON); + end_key = begin_key; + } + if (end_key && !begin_data && !end_data && + (begin_key == end_key || mdbx_is_samedata(begin_key, end_key))) { + /* LY: single key case */ + int exact = 0; + rc = mdbx_cursor_set(&begin.outer, begin_key, NULL, MDBX_SET, &exact); + if (unlikely(rc != MDBX_SUCCESS)) { + *size_items = 0; + return (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc; + } + *size_items = 1; + if (begin.outer.mc_xcursor != NULL) { + MDBX_node *node = page_node(begin.outer.mc_pg[begin.outer.mc_top], + begin.outer.mc_ki[begin.outer.mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + /* LY: return the number of duplicates for given key */ + mdbx_tassert(txn, + begin.outer.mc_xcursor == &begin.inner && + (begin.inner.mx_cursor.mc_flags & C_INITIALIZED)); + *size_items = + (sizeof(*size_items) >= sizeof(begin.inner.mx_db.md_entries) || + begin.inner.mx_db.md_entries <= PTRDIFF_MAX) + ? (size_t)begin.inner.mx_db.md_entries + : PTRDIFF_MAX; + } + } + return MDBX_SUCCESS; + } else { + rc = mdbx_cursor_set(&begin.outer, begin_key, begin_data, + begin_data ? MDBX_GET_BOTH_RANGE : MDBX_SET_RANGE, + NULL); + } + } + + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc != MDBX_NOTFOUND || !(begin.outer.mc_flags & C_INITIALIZED)) + return rc; + } + + MDBX_cursor_couple end; + rc = mdbx_cursor_init(&end.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + if (!end_key) { + MDBX_val stub = {0, 0}; + rc = mdbx_cursor_last(&end.outer, &stub, &stub); + } else { + rc = mdbx_cursor_set(&end.outer, end_key, end_data, + end_data ? MDBX_GET_BOTH_RANGE : MDBX_SET_RANGE, NULL); + } + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc != MDBX_NOTFOUND || !(end.outer.mc_flags & C_INITIALIZED)) + return rc; + } + + rc = mdbx_estimate_distance(&begin.outer, &end.outer, size_items); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + assert(*size_items >= -(ptrdiff_t)begin.outer.mc_db->md_entries && + *size_items <= (ptrdiff_t)begin.outer.mc_db->md_entries); + +#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \ + * results for an inverted ranges. */ + + /* Commit 8ddfd1f34ad7cf7a3c4aa75d2e248ca7e639ed63 + Change-Id: If59eccf7311123ab6384c4b93f9b1fed5a0a10d1 */ + + if (*size_items < 0) { + /* LY: inverted range case */ + *size_items += (ptrdiff_t)begin.outer.mc_db->md_entries; + } else if (*size_items == 0 && begin_key && end_key) { + int cmp = begin.outer.mc_dbx->md_cmp(&origin_begin_key, &origin_end_key); + if (cmp == 0 && (begin.inner.mx_cursor.mc_flags & C_INITIALIZED) && + begin_data && end_data) + cmp = begin.outer.mc_dbx->md_dcmp(&origin_begin_data, &origin_end_data); + if (cmp > 0) { + /* LY: inverted range case with empty scope */ + *size_items = (ptrdiff_t)begin.outer.mc_db->md_entries; + } + } + assert(*size_items >= 0 && + *size_items <= (ptrdiff_t)begin.outer.mc_db->md_entries); +#endif + + return MDBX_SUCCESS; +} + +//------------------------------------------------------------------------------ + +/* ПозволÑет обновить или удалить ÑущеÑтвующую запиÑÑŒ Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸ÐµÐ¼ + * в old_data предыдущего Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð´Ð°Ð½Ð½Ñ‹Ñ…. При Ñтом еÑли new_data равен + * нулю, то выполнÑетÑÑ ÑƒÐ´Ð°Ð»ÐµÐ½Ð¸Ðµ, иначе обновление/вÑтавка. + * + * Текущее значение может находитьÑÑ Ð² уже измененной (грÑзной) Ñтранице. + * Ð’ Ñтом Ñлучае Ñтраница будет перезапиÑана при обновлении, а Ñамо Ñтарое + * значение утрачено. ПоÑтому иÑходно в old_data должен быть передан + * дополнительный буфер Ð´Ð»Ñ ÐºÐ¾Ð¿Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ñтарого значениÑ. + * ЕÑли переданный буфер Ñлишком мал, то Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ Ð²ÐµÑ€Ð½ÐµÑ‚ -1, уÑтановив + * old_data->iov_len в ÑоответÑтвующее значение. + * + * Ð”Ð»Ñ Ð½Ðµ-уникальных ключей также возможен второй Ñценарий иÑпользованиÑ, + * когда поÑредÑтвом old_data из запиÑей Ñ Ð¾Ð´Ð¸Ð½Ð°ÐºÐ¾Ð²Ñ‹Ð¼ ключом Ð´Ð»Ñ + * удалениÑ/Ð¾Ð±Ð½Ð¾Ð²Ð»ÐµÐ½Ð¸Ñ Ð²Ñ‹Ð±Ð¸Ñ€Ð°ÐµÑ‚ÑÑ ÐºÐ¾Ð½ÐºÑ€ÐµÑ‚Ð½Ð°Ñ. Ð”Ð»Ñ Ð²Ñ‹Ð±Ð¾Ñ€Ð° Ñтого ÑÑ†ÐµÐ½Ð°Ñ€Ð¸Ñ + * во flags Ñледует одновременно указать MDBX_CURRENT и MDBX_NOOVERWRITE. + * Именно Ñта ÐºÐ¾Ð¼Ð±Ð¸Ð½Ð°Ñ†Ð¸Ñ Ð²Ñ‹Ð±Ñ€Ð°Ð½Ð°, так как она лишена ÑмыÑла, и Ñтим позволÑет + * идентифицировать Ð·Ð°Ð¿Ñ€Ð¾Ñ Ñ‚Ð°ÐºÐ¾Ð³Ð¾ ÑценариÑ. + * + * Ð¤ÑƒÐ½ÐºÑ†Ð¸Ñ Ð¼Ð¾Ð¶ÐµÑ‚ быть замещена ÑоответÑтвующими операциÑми Ñ ÐºÑƒÑ€Ñорами + * поÑле двух доработок (TODO): + * - внешнÑÑ Ð°Ð»Ð»Ð¾ÐºÐ°Ñ†Ð¸Ñ ÐºÑƒÑ€Ñоров, в том чиÑле на Ñтеке (без malloc). + * - Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ ÑтатуÑа Ñтраницы по адреÑу (знать о P_DIRTY). + */ +int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *new_data, + MDBX_val *old_data, unsigned flags) { + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!key || !old_data || old_data == new_data)) + return MDBX_EINVAL; + + if (unlikely(old_data->iov_base == NULL && old_data->iov_len)) + return MDBX_EINVAL; + + if (unlikely(new_data == NULL && !(flags & MDBX_CURRENT))) + return MDBX_EINVAL; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_RESERVE | + MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT))) + return MDBX_EINVAL; + + MDBX_cursor_couple cx; + rc = mdbx_cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + cx.outer.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &cx.outer; + + MDBX_val present_key = *key; + if (F_ISSET(flags, MDBX_CURRENT | MDBX_NOOVERWRITE)) { + /* в old_data значение Ð´Ð»Ñ Ð²Ñ‹Ð±Ð¾Ñ€Ð° конкретного дубликата */ + if (unlikely(!(txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT))) { + rc = MDBX_EINVAL; + goto bailout; + } + + /* убираем лишний бит, он был признаком запрошенного режима */ + flags -= MDBX_NOOVERWRITE; + + rc = mdbx_cursor_get(&cx.outer, &present_key, old_data, MDBX_GET_BOTH); + if (rc != MDBX_SUCCESS) + goto bailout; + + if (new_data) { + /* обновление конкретного дубликата */ + if (mdbx_is_samedata(old_data, new_data)) + /* еÑли данные Ñовпадают, то ничего делать не надо */ + goto bailout; + } + } else { + /* в old_data буфер Ð´Ð»Ñ ÑÐ¾Ñ…Ñ€Ð°Ð½ÐµÐ½Ð¸Ñ Ð¿Ñ€ÐµÐ´Ñ‹Ð´ÑƒÑ‰ÐµÐ³Ð¾ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ */ + if (unlikely(new_data && old_data->iov_base == new_data->iov_base)) + return MDBX_EINVAL; + MDBX_val present_data; + rc = mdbx_cursor_get(&cx.outer, &present_key, &present_data, MDBX_SET_KEY); + if (unlikely(rc != MDBX_SUCCESS)) { + old_data->iov_base = NULL; + old_data->iov_len = 0; + if (rc != MDBX_NOTFOUND || (flags & MDBX_CURRENT)) + goto bailout; + } else if (flags & MDBX_NOOVERWRITE) { + rc = MDBX_KEYEXIST; + *old_data = present_data; + goto bailout; + } else { + MDBX_page *page = cx.outer.mc_pg[cx.outer.mc_top]; + if (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) { + if (flags & MDBX_CURRENT) { + /* Ð´Ð»Ñ Ð½Ðµ-уникальных ключей позволÑем update/delete только еÑли ключ + * один */ + MDBX_node *node = page_node(page, cx.outer.mc_ki[cx.outer.mc_top]); + if (F_ISSET(node_flags(node), F_DUPDATA)) { + mdbx_tassert(txn, XCURSOR_INITED(&cx.outer) && + cx.outer.mc_xcursor->mx_db.md_entries > 1); + if (cx.outer.mc_xcursor->mx_db.md_entries > 1) { + rc = MDBX_EMULTIVAL; + goto bailout; + } + } + /* еÑли данные Ñовпадают, то ничего делать не надо */ + if (new_data && mdbx_is_samedata(&present_data, new_data)) { + *old_data = *new_data; + goto bailout; + } + /* Ð’ оригинальной LMDB флажок MDBX_CURRENT здеÑÑŒ приведет + * к замене данных без учета MDBX_DUPSORT Ñортировки, + * но здеÑÑŒ Ñто в любом Ñлучае допуÑтимо, так как мы + * проверили что Ð´Ð»Ñ ÐºÐ»ÑŽÑ‡Ð° еÑÑ‚ÑŒ только одно значение. */ + } else if ((flags & MDBX_NODUPDATA) && + mdbx_is_samedata(&present_data, new_data)) { + /* еÑли данные Ñовпадают и уÑтановлен MDBX_NODUPDATA */ + rc = MDBX_KEYEXIST; + goto bailout; + } + } else { + /* еÑли данные Ñовпадают, то ничего делать не надо */ + if (new_data && mdbx_is_samedata(&present_data, new_data)) { + *old_data = *new_data; + goto bailout; + } + flags |= MDBX_CURRENT; + } + + if (IS_DIRTY(page)) { + if (unlikely(old_data->iov_len < present_data.iov_len)) { + old_data->iov_base = NULL; + old_data->iov_len = present_data.iov_len; + rc = MDBX_RESULT_TRUE; + goto bailout; + } + memcpy(old_data->iov_base, present_data.iov_base, present_data.iov_len); + old_data->iov_len = present_data.iov_len; + } else { + *old_data = present_data; + } + } + } + + if (likely(new_data)) + rc = mdbx_cursor_put(&cx.outer, key, new_data, flags); + else + rc = mdbx_cursor_del(&cx.outer, 0); + +bailout: + txn->mt_cursors[dbi] = cx.outer.mc_next; + return rc; +} + +/* Ð¤ÑƒÐ½ÐºÑ†Ð¸Ñ Ñообщает находитÑÑ Ð»Ð¸ указанный Ð°Ð´Ñ€ÐµÑ Ð² "грÑзной" Ñтранице у + * заданной пишущей транзакции. Ð’ конечном Ñчете Ñто позволÑет избавитьÑÑ Ð¾Ñ‚ + * лишнего ÐºÐ¾Ð¿Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð´Ð°Ð½Ð½Ñ‹Ñ… из ÐЕ-грÑзных Ñтраниц. + * + * "ГрÑзные" Ñтраницы - Ñто те, которые уже были изменены в ходе пишущей + * транзакции. СоответÑтвенно, какие-либо дальнейшие Ð¸Ð·Ð¼ÐµÐ½ÐµÐ½Ð¸Ñ Ð¼Ð¾Ð³ÑƒÑ‚ привеÑти + * к перезапиÑи таких Ñтраниц. ПоÑтому вÑе функции, выполнÑющие изменениÑ, в + * качеÑтве аргументов ÐЕ должны получать указатели на данные в таких + * Ñтраницах. Ð’ Ñвою очередь "ÐЕ грÑзные" Ñтраницы перед модификацией будут + * Ñкопированы. + * + * Другими Ñловами, данные из "грÑзных" Ñтраниц должны быть либо Ñкопированы + * перед передачей в качеÑтве аргументов Ð´Ð»Ñ Ð´Ð°Ð»ÑŒÐ½ÐµÐ¹ÑˆÐ¸Ñ… модификаций, либо + * отвергнуты на Ñтадии проверки корректноÑти аргументов. + * + * Таким образом, Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ Ð¿Ð¾Ð·Ð²Ð¾Ð»Ñет как избавитьÑÑ Ð¾Ñ‚ лишнего копированиÑ, + * так и выполнить более полную проверку аргументов. + * + * Ð’ÐЖÐО: Передаваемый указатель должен указывать на начало данных. Только + * так гарантируетÑÑ Ñ‡Ñ‚Ð¾ актуальный заголовок Ñтраницы будет физичеÑки + * раÑположен в той-же Ñтранице памÑти, в том чиÑле Ð´Ð»Ñ Ð¼Ð½Ð¾Ð³Ð¾Ñтраничных + * P_OVERFLOW Ñтраниц Ñ Ð´Ð»Ð¸Ð½Ð½Ñ‹Ð¼Ð¸ данными. */ +int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (txn->mt_flags & MDBX_RDONLY) + return MDBX_RESULT_FALSE; + + const MDBX_env *env = txn->mt_env; + const ptrdiff_t offset = (uint8_t *)ptr - env->me_map; + if (offset >= 0) { + const pgno_t pgno = bytes2pgno(env, offset); + if (likely(pgno < txn->mt_next_pgno)) { + const MDBX_page *page = pgno2page(env, pgno); + if (unlikely(page->mp_pgno != pgno)) { + /* The ptr pointed into middle of a large page, + * not to the beginning of a data. */ + return MDBX_EINVAL; + } + if (unlikely(page->mp_flags & (P_DIRTY | P_LOOSE | P_KEEP))) + return MDBX_RESULT_TRUE; + if (likely(txn->tw.spill_pages == nullptr)) + return MDBX_RESULT_FALSE; + return mdbx_pnl_exist(txn->tw.spill_pages, pgno << 1) ? MDBX_RESULT_TRUE + : MDBX_RESULT_FALSE; + } + if ((size_t)offset < env->me_dxb_mmap.limit) { + /* Указатель адреÑует что-то в пределах mmap, но за границей + * раÑпределенных Ñтраниц. Такое может ÑлучитÑÑ ÐµÑли mdbx_is_dirty() + * вызывает поÑле операции, в ходе которой Ð³Ñ€Ð°Ð·Ð½Ð°Ñ Ñтраница попала + * в loose и затем была возвращена в нераÑпределенное проÑтранÑтво. */ + return MDBX_RESULT_TRUE; + } + } + + /* Страница вне иÑпользуемого mmap-диапазона, Ñ‚.е. либо в функцию был + * передан некорректный адреÑ, либо Ð°Ð´Ñ€ÐµÑ Ð² теневой Ñтранице, ÐºÐ¾Ñ‚Ð¾Ñ€Ð°Ñ Ð±Ñ‹Ð»Ð° + * выделена поÑредÑтвом malloc(). + * + * Ð”Ð»Ñ WRITE_MAP режима Ñ‚Ð°ÐºÐ°Ñ Ñтраница однозначно "не грÑзнаÑ", + * а Ð´Ð»Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¾Ð² без WRITE_MAP Ñледует проÑматривать ÑпиÑки dirty + * и spilled Ñтраниц у каких-либо транзакций (в том чиÑле дочерних). + * + * ПоÑтому Ð´Ð»Ñ WRITE_MAP возвращаем false, а Ð´Ð»Ñ Ð¾Ñтальных режимов + * вÑегда true. Ð¢Ð°ÐºÐ°Ñ Ð»Ð¾Ð³Ð¸ÐºÐ° имеет Ñ€Ñд преимущеÑтв: + * - не тратим Ð²Ñ€ÐµÐ¼Ñ Ð½Ð° проÑмотр ÑпиÑков; + * - результат вÑегда безопаÑен (может быть ложно-положительным, + * но не ложно-отрицательным); + * - результат не завиÑит от вложенноÑти транзакций и от отноÑительного + * Ð¿Ð¾Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð¿ÐµÑ€ÐµÐ´Ð°Ð½Ð½Ð¾Ð¹ транзакции в Ñтой рекурÑии. */ + return (env->me_flags & MDBX_WRITEMAP) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; +} + +int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, + uint64_t increment) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + if (unlikely(TXN_DBI_CHANGED(txn, dbi))) + return MDBX_BAD_DBI; + + if (unlikely(txn->mt_dbflags[dbi] & DB_STALE)) { + rc = mdbx_fetch_sdb(txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + + MDBX_db *dbs = &txn->mt_dbs[dbi]; + if (likely(result)) + *result = dbs->md_seq; + + if (likely(increment > 0)) { + if (unlikely(txn->mt_flags & MDBX_RDONLY)) + return MDBX_EACCESS; + + uint64_t new = dbs->md_seq + increment; + if (unlikely(new < increment)) + return MDBX_RESULT_TRUE; + + mdbx_tassert(txn, new > dbs->md_seq); + dbs->md_seq = new; + txn->mt_flags |= MDBX_TXN_DIRTY; + txn->mt_dbflags[dbi] |= DB_DIRTY; + } + + return MDBX_SUCCESS; +} + +/*----------------------------------------------------------------------------*/ + +__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_syspagesize(); + else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || + pagesize > (intptr_t)MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return (MDBX_EINVAL > 0) ? -MDBX_EINVAL : MDBX_EINVAL; + + return mdbx_maxkey(mdbx_nodemax(pagesize)); +} + +__cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_syspagesize(); + else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || + pagesize > (intptr_t)MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return (MDBX_EINVAL > 0) ? -MDBX_EINVAL : MDBX_EINVAL; + + return MIN_PAGENO * pagesize; +} + +__cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_syspagesize(); + else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || + pagesize > (intptr_t)MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return (MDBX_EINVAL > 0) ? -MDBX_EINVAL : MDBX_EINVAL; + + const uint64_t limit = MAX_PAGENO * (uint64_t)pagesize; + return (limit < (intptr_t)MAX_MAPSIZE) ? (intptr_t)limit + : (intptr_t)MAX_MAPSIZE; +} + +__cold intptr_t mdbx_limits_txnsize_max(intptr_t pagesize) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_syspagesize(); + else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || + pagesize > (intptr_t)MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return (MDBX_EINVAL > 0) ? -MDBX_EINVAL : MDBX_EINVAL; + + return pagesize * (MDBX_DPL_TXNFULL - 1); +} + +/*** Attribute support functions for Nexenta **********************************/ +#ifdef MDBX_NEXENTA_ATTRS + +static __inline int mdbx_attr_peek(MDBX_val *data, mdbx_attr_t *attrptr) { + if (unlikely(data->iov_len < sizeof(mdbx_attr_t))) + return MDBX_INCOMPATIBLE; + + if (likely(attrptr != NULL)) + *attrptr = *(mdbx_attr_t *)data->iov_base; + data->iov_len -= sizeof(mdbx_attr_t); + data->iov_base = + likely(data->iov_len > 0) ? ((mdbx_attr_t *)data->iov_base) + 1 : NULL; + + return MDBX_SUCCESS; +} + +static __inline int mdbx_attr_poke(MDBX_val *reserved, MDBX_val *data, + mdbx_attr_t attr, unsigned flags) { + mdbx_attr_t *space = reserved->iov_base; + if (flags & MDBX_RESERVE) { + if (likely(data != NULL)) { + data->iov_base = data->iov_len ? space + 1 : NULL; + } + } else { + *space = attr; + if (likely(data != NULL)) { + memcpy(space + 1, data->iov_base, data->iov_len); + } + } + + return MDBX_SUCCESS; +} + +int mdbx_cursor_get_attr(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, + mdbx_attr_t *attrptr, MDBX_cursor_op op) { + int rc = mdbx_cursor_get(mc, key, data, op); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + return mdbx_attr_peek(data, attrptr); +} + +int mdbx_get_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, + uint64_t *attrptr) { + int rc = mdbx_get(txn, dbi, key, data); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + return mdbx_attr_peek(data, attrptr); +} + +int mdbx_put_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, + mdbx_attr_t attr, unsigned flags) { + MDBX_val reserve; + reserve.iov_base = NULL; + reserve.iov_len = (data ? data->iov_len : 0) + sizeof(mdbx_attr_t); + + int rc = mdbx_put(txn, dbi, key, &reserve, flags | MDBX_RESERVE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + return mdbx_attr_poke(&reserve, data, attr, flags); +} + +int mdbx_cursor_put_attr(MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, + mdbx_attr_t attr, unsigned flags) { + MDBX_val reserve; + reserve.iov_base = NULL; + reserve.iov_len = (data ? data->iov_len : 0) + sizeof(mdbx_attr_t); + + int rc = mdbx_cursor_put(cursor, key, &reserve, flags | MDBX_RESERVE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + return mdbx_attr_poke(&reserve, data, attr, flags); +} + +int mdbx_set_attr(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, + mdbx_attr_t attr) { + if (unlikely(!key || !txn)) + return MDBX_EINVAL; + + if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE)) + return MDBX_VERSION_MISMATCH; + + if (unlikely(!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))) + return MDBX_EINVAL; + + if (unlikely(txn->mt_flags & (MDBX_RDONLY | MDBX_TXN_BLOCKED))) + return (txn->mt_flags & MDBX_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + + MDBX_cursor_couple cx; + MDBX_val old_data; + int rc = mdbx_cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = mdbx_cursor_set(&cx.outer, key, &old_data, MDBX_SET, NULL); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_NOTFOUND && data) { + cx.outer.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &cx.outer; + rc = mdbx_cursor_put_attr(&cx.outer, key, data, attr, 0); + txn->mt_cursors[dbi] = cx.outer.mc_next; + } + return rc; + } + + mdbx_attr_t old_attr = 0; + rc = mdbx_attr_peek(&old_data, &old_attr); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (old_attr == attr && (!data || (data->iov_len == old_data.iov_len && + memcmp(data->iov_base, old_data.iov_base, + old_data.iov_len) == 0))) + return MDBX_SUCCESS; + + cx.outer.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &cx.outer; + rc = mdbx_cursor_put_attr(&cx.outer, key, data ? data : &old_data, attr, + MDBX_CURRENT); + txn->mt_cursors[dbi] = cx.outer.mc_next; + return rc; +} +#endif /* MDBX_NEXENTA_ATTRS */ + +/******************************************************************************/ +/* *INDENT-OFF* */ +/* clang-format off */ + +__dll_export +#ifdef __attribute_used__ + __attribute_used__ +#elif defined(__GNUC__) || __has_attribute(__used__) + __attribute__((__used__)) +#endif +#ifdef __attribute_externally_visible__ + __attribute_externally_visible__ +#elif (defined(__GNUC__) && !defined(__clang__)) || \ + __has_attribute(__externally_visible__) + __attribute__((__externally_visible__)) +#endif + const mdbx_build_info mdbx_build = { +#ifdef MDBX_BUILD_TIMESTAMP + MDBX_BUILD_TIMESTAMP +#else + __DATE__ " " __TIME__ +#endif /* MDBX_BUILD_TIMESTAMP */ + + , +#ifdef MDBX_BUILD_TARGET + MDBX_BUILD_TARGET +#else + #if defined(__ANDROID__) + "Android" + #elif defined(__linux__) || defined(__gnu_linux__) + "Linux" + #elif defined(EMSCRIPTEN) || defined(__EMSCRIPTEN__) + "webassembly" + #elif defined(__CYGWIN__) + "CYGWIN" + #elif defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) \ + || defined(__WINDOWS__) + "Windows" + #elif defined(__APPLE__) + #if (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) \ + || (defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR) + "iOS" + #else + "MacOS" + #endif + #elif defined(__FreeBSD__) + "FreeBSD" + #elif defined(__DragonFly__) + "DragonFlyBSD" + #elif defined(__NetBSD__) || defined(__NETBSD__) + "NetBSD" + #elif defined(__OpenBSD__) + "OpenBSD" + #elif defined(__bsdi__) + "UnixBSDI" + #elif defined(__MACH__) + "MACH" + #elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) + "HPUX" + #elif defined(_AIX) + "AIX" + #elif defined(__sun) && defined(__SVR4) + "Solaris" + #elif defined(__BSD__) || defined(BSD) + "UnixBSD" + #elif defined(__unix__) || defined(UNIX) || defined(__unix) \ + || defined(__UNIX) || defined(__UNIX__) + "UNIX" + #elif defined(_POSIX_VERSION) + "POSIX" STRINGIFY(_POSIX_VERSION) + #else + "UnknownOS" + #endif /* Target OS */ + + "-" + + #if defined(__amd64__) + "AMD64" + #elif defined(__ia32__) + "IA32" + #elif defined(__e2k__) || defined(__elbrus__) + "Elbrus" + #elif defined(__alpha__) || defined(__alpha) || defined(_M_ALPHA) + "Alpha" + #elif defined(__aarch64__) || defined(_M_ARM64) + "ARM64" + #elif defined(__arm__) || defined(__thumb__) || defined(__TARGET_ARCH_ARM) \ + || defined(__TARGET_ARCH_THUMB) || defined(_ARM) || defined(_M_ARM) \ + || defined(_M_ARMT) || defined(__arm) + "ARM" + #elif defined(__mips64) || defined(__mips64__) || (defined(__mips) && (__mips >= 64)) + "MIPS64" + #elif if defined(__mips__) || defined(__mips) || defined(_R4000) || defined(__MIPS__) + "MIPS" + #elif defined(__hppa64__) || defined(__HPPA64__) || defined(__hppa64) + "PARISC64" + #elif defined(__hppa__) || defined(__HPPA__) || defined(__hppa) + "PARISC" + #elif defined(__ia64__) || defined(__ia64) || defined(_IA64) \ + || defined(__IA64__) || defined(_M_IA64) || defined(__itanium__) + "Itanium" + #elif defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) \ + || defined(__powerpc64) || defined(_ARCH_PPC64) + "PowerPC64" + #elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) \ + || defined(__ppc) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__POWERPC__) + "PowerPC" + #elif defined(__sparc64__) || defined(__sparc64) + "SPARC64" + #elif defined(__sparc__) || defined(__sparc) + "SPARC" + #elif defined(__s390__) || defined(__s390) || defined(__zarch__) || defined(__zarch) + "S390" + #else + "UnknownARCH" + #endif +#endif /* MDBX_BUILD_TARGET */ + +#ifdef MDBX_BUILD_CONFIG +# if defined(_MSC_VER) +# pragma message("Configuration-depended MDBX_BUILD_CONFIG: " MDBX_BUILD_CONFIG) +# endif + "-" MDBX_BUILD_CONFIG +#endif /* MDBX_BUILD_CONFIG */ + , + "MDBX_DEBUG=" STRINGIFY(MDBX_DEBUG) +#ifdef MDBX_LOGLEVEL_BUILD + " MDBX_LOGLEVEL_BUILD=" STRINGIFY(MDBX_LOGLEVEL_BUILD) +#endif /* MDBX_LOGLEVEL_BUILD */ + " MDBX_WORDBITS=" STRINGIFY(MDBX_WORDBITS) + " BYTE_ORDER=" +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + "LITTLE_ENDIAN" +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + "BIG_ENDIAN" +#else + #error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + " MDBX_TXN_CHECKPID=" MDBX_TXN_CHECKPID_CONFIG + " MDBX_TXN_CHECKOWNER=" MDBX_TXN_CHECKOWNER_CONFIG + " MDBX_64BIT_ATOMIC=" MDBX_64BIT_ATOMIC_CONFIG + " MDBX_64BIT_CAS=" MDBX_64BIT_CAS_CONFIG + " MDBX_TRUST_RTC=" MDBX_TRUST_RTC_CONFIG +#ifdef __SANITIZE_ADDRESS__ + " SANITIZE_ADDRESS=YES" +#endif /* __SANITIZE_ADDRESS__ */ +#ifdef MDBX_USE_VALGRIND + " MDBX_USE_VALGRIND=YES" +#endif /* MDBX_USE_VALGRIND */ +#ifdef _GNU_SOURCE + " _GNU_SOURCE=YES" +#else + " _GNU_SOURCE=NO" +#endif /* _GNU_SOURCE */ +#ifdef __APPLE__ + " MDBX_OSX_SPEED_INSTEADOF_DURABILITY=" STRINGIFY(MDBX_OSX_SPEED_INSTEADOF_DURABILITY) +#endif /* MacOS */ +#if defined(_WIN32) || defined(_WIN64) + " MDBX_AVOID_CRT=" STRINGIFY(MDBX_AVOID_CRT) + " MDBX_CONFIG_MANUAL_TLS_CALLBACK=" STRINGIFY(MDBX_CONFIG_MANUAL_TLS_CALLBACK) + " MDBX_BUILD_SHARED_LIBRARY=" STRINGIFY(MDBX_BUILD_SHARED_LIBRARY) + " WINVER=" STRINGIFY(WINVER) +#else /* Windows */ + " MDBX_USE_ROBUST=" MDBX_USE_ROBUST_CONFIG + " MDBX_USE_OFDLOCKS=" MDBX_USE_OFDLOCKS_CONFIG +#endif /* !Windows */ +#ifdef MDBX_OSAL_LOCK + " MDBX_OSAL_LOCK=" STRINGIFY(MDBX_OSAL_LOCK) +#endif + " MDBX_CACHELINE_SIZE=" STRINGIFY(MDBX_CACHELINE_SIZE) + " MDBX_CPU_WRITEBACK_IS_COHERENT=" STRINGIFY(MDBX_CPU_WRITEBACK_IS_COHERENT) + " MDBX_UNALIGNED_OK=" STRINGIFY(MDBX_UNALIGNED_OK) + " MDBX_PNL_ASCENDING=" STRINGIFY(MDBX_PNL_ASCENDING) + , +#ifdef MDBX_BUILD_COMPILER + MDBX_BUILD_COMPILER +#else + #ifdef __INTEL_COMPILER + "Intel C/C++ " STRINGIFY(__INTEL_COMPILER) + #elsif defined(__apple_build_version__) + "Apple clang " STRINGIFY(__apple_build_version__) + #elif defined(__ibmxl__) + "IBM clang C " STRINGIFY(__ibmxl_version__) "." STRINGIFY(__ibmxl_release__) + "." STRINGIFY(__ibmxl_modification__) "." STRINGIFY(__ibmxl_ptf_fix_level__) + #elif defined(__clang__) + "clang " STRINGIFY(__clang_version__) + #elif defined(__MINGW64__) + "MINGW-64 " STRINGIFY(__MINGW64_MAJOR_VERSION) "." STRINGIFY(__MINGW64_MINOR_VERSION) + #elif defined(__MINGW32__) + "MINGW-32 " STRINGIFY(__MINGW32_MAJOR_VERSION) "." STRINGIFY(__MINGW32_MINOR_VERSION) + #elif defined(__IBMC__) + "IBM C " STRINGIFY(__IBMC__) + #elif defined(__GNUC__) + "GNU C/C++ " + #ifdef __VERSION__ + __VERSION__ + #else + STRINGIFY(__GNUC__) "." STRINGIFY(__GNUC_MINOR__) "." STRINGIFY(__GNUC_PATCHLEVEL__) + #endif + #elif defined(_MSC_VER) + "MSVC " STRINGIFY(_MSC_FULL_VER) "-" STRINGIFY(_MSC_BUILD) + #else + "Unknown compiler" + #endif +#endif /* MDBX_BUILD_COMPILER */ + , +#ifdef MDBX_BUILD_FLAGS + MDBX_BUILD_FLAGS +#endif /* MDBX_BUILD_FLAGS */ +#ifdef MDBX_BUILD_FLAGS_CONFIG + MDBX_BUILD_FLAGS_CONFIG +#endif /* MDBX_BUILD_FLAGS_CONFIG */ +}; + +#ifdef __SANITIZE_ADDRESS__ +LIBMDBX_API __attribute__((__weak__)) const char *__asan_default_options() { + return "symbolize=1:allow_addr2line=1:" +#ifdef _DEBUG + "debug=1:" +#endif /* _DEBUG */ + "report_globals=1:" + "replace_str=1:replace_intrin=1:" + "malloc_context_size=9:" + "detect_leaks=1:" + "check_printf=1:" + "detect_deadlocks=1:" +#ifndef LTO_ENABLED + "check_initialization_order=1:" +#endif + "detect_stack_use_after_return=1:" + "intercept_tls_get_addr=1:" + "decorate_proc_maps=1:" + "abort_on_error=1"; +} +#endif /* __SANITIZE_ADDRESS__ */ + +/* *INDENT-ON* */ +/* clang-format on */ diff --git a/src/contrib/db/libmdbx/src/elements/defs.h b/src/contrib/db/libmdbx/src/elements/defs.h new file mode 100644 index 0000000..aba07f9 --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/defs.h @@ -0,0 +1,450 @@ +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once +/* *INDENT-OFF* */ +/* clang-format off */ + +#ifndef __GNUC_PREREQ +# if defined(__GNUC__) && defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(maj, min) \ + ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GNUC_PREREQ(maj, min) (0) +# endif +#endif /* __GNUC_PREREQ */ + +#ifndef __CLANG_PREREQ +# ifdef __clang__ +# define __CLANG_PREREQ(maj,min) \ + ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +# else +# define __CLANG_PREREQ(maj,min) (0) +# endif +#endif /* __CLANG_PREREQ */ + +#ifndef __GLIBC_PREREQ +# if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +# define __GLIBC_PREREQ(maj, min) \ + ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +# else +# define __GLIBC_PREREQ(maj, min) (0) +# endif +#endif /* __GLIBC_PREREQ */ + +#ifndef __has_attribute +# define __has_attribute(x) (0) +#endif + +#ifndef __has_feature +# define __has_feature(x) (0) +#endif + +#ifndef __has_extension +# define __has_extension(x) (0) +#endif + +#ifndef __has_builtin +# define __has_builtin(x) (0) +#endif + +#ifndef __has_warning +# define __has_warning(x) (0) +#endif + +#ifndef __has_include +# define __has_include(x) (0) +#endif + +#if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ 1 +#endif + +#if __has_feature(address_sanitizer) +# define __SANITIZE_ADDRESS__ 1 +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __extern_C +# ifdef __cplusplus +# define __extern_C extern "C" +# else +# define __extern_C +# endif +#endif /* __extern_C */ + +#ifndef __cplusplus +# ifndef bool +# define bool _Bool +# endif +# ifndef true +# define true (1) +# endif +# ifndef false +# define false (0) +# endif +#endif + +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) +# define nullptr NULL +#endif + +/*----------------------------------------------------------------------------*/ + +#ifndef __always_inline +# if defined(__GNUC__) || __has_attribute(__always_inline__) +# define __always_inline __inline __attribute__((__always_inline__)) +# elif defined(_MSC_VER) +# define __always_inline __forceinline +# else +# define __always_inline +# endif +#endif /* __always_inline */ + +#ifndef __noinline +# if defined(__GNUC__) || __has_attribute(__noinline__) +# define __noinline __attribute__((__noinline__)) +# elif defined(_MSC_VER) +# define __noinline __declspec(noinline) +# elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +# define __noinline inline +# elif !defined(__INTEL_COMPILER) +# define __noinline /* FIXME ? */ +# endif +#endif /* __noinline */ + +#ifndef __must_check_result +# if defined(__GNUC__) || __has_attribute(__warn_unused_result__) +# define __must_check_result __attribute__((__warn_unused_result__)) +# else +# define __must_check_result +# endif +#endif /* __must_check_result */ + +#ifndef __maybe_unused +# if defined(__GNUC__) || __has_attribute(__unused__) +# define __maybe_unused __attribute__((__unused__)) +# else +# define __maybe_unused +# endif +#endif /* __maybe_unused */ + +#ifndef __deprecated +# if defined(__GNUC__) || __has_attribute(__deprecated__) +# define __deprecated __attribute__((__deprecated__)) +# elif defined(_MSC_VER) +# define __deprecated __declspec(deprecated) +# else +# define __deprecated +# endif +#endif /* __deprecated */ + +#if !defined(__noop) && !defined(_MSC_VER) +# define __noop(...) do {} while(0) +#endif /* __noop */ + +#ifndef __fallthrough +# if __GNUC_PREREQ(7, 0) || __has_attribute(__fallthrough__) +# define __fallthrough __attribute__((__fallthrough__)) +# else +# define __fallthrough __noop() +# endif +#endif /* __fallthrough */ + +#ifndef __unreachable +# if __GNUC_PREREQ(4,5) +# define __unreachable() __builtin_unreachable() +# elif defined(_MSC_VER) +# define __unreachable() __assume(0) +# else +# define __unreachable() __noop() +# endif +#endif /* __unreachable */ + +#ifndef __prefetch +# if defined(__GNUC__) || defined(__clang__) +# define __prefetch(ptr) __builtin_prefetch(ptr) +# else +# define __prefetch(ptr) __noop(ptr) +# endif +#endif /* __prefetch */ + +#ifndef __noreturn +# if defined(__GNUC__) || __has_attribute(__noreturn__) +# define __noreturn __attribute__((__noreturn__)) +# elif defined(_MSC_VER) +# define __noreturn __declspec(noreturn) +# else +# define __noreturn +# endif +#endif /* __noreturn */ + +#ifndef __nothrow +# if defined(__cplusplus) +# if __cplusplus < 201703L +# define __nothrow throw() +# else +# define __nothrow noexcept(true) +# endif /* __cplusplus */ +# elif defined(__GNUC__) || __has_attribute(__nothrow__) +# define __nothrow __attribute__((__nothrow__)) +# elif defined(_MSC_VER) && defined(__cplusplus) +# define __nothrow __declspec(nothrow) +# else +# define __nothrow +# endif +#endif /* __nothrow */ + +#ifndef __pure_function + /* Many functions have no effects except the return value and their + * return value depends only on the parameters and/or global variables. + * Such a function can be subject to common subexpression elimination + * and loop optimization just as an arithmetic operator would be. + * These functions should be declared with the attribute pure. */ +# if defined(__GNUC__) || __has_attribute(__pure__) +# define __pure_function __attribute__((__pure__)) +# else +# define __pure_function +# endif +#endif /* __pure_function */ + +#ifndef __const_function + /* Many functions do not examine any values except their arguments, + * and have no effects except the return value. Basically this is just + * slightly more strict class than the PURE attribute, since function + * is not allowed to read global memory. + * + * Note that a function that has pointer arguments and examines the + * data pointed to must not be declared const. Likewise, a function + * that calls a non-const function usually must not be const. + * It does not make sense for a const function to return void. */ +# if defined(__GNUC__) || __has_attribute(__const__) +# define __const_function __attribute__((__const__)) +# else +# define __const_function +# endif +#endif /* __const_function */ + +#ifndef __hidden +# if defined(__GNUC__) || __has_attribute(__visibility__) +# define __hidden __attribute__((__visibility__("hidden"))) +# else +# define __hidden +# endif +#endif /* __hidden */ + +#ifndef __optimize +# if defined(__OPTIMIZE__) +# if defined(__clang__) && !__has_attribute(__optimize__) +# define __optimize(ops) +# elif defined(__GNUC__) || __has_attribute(__optimize__) +# define __optimize(ops) __attribute__((__optimize__(ops))) +# else +# define __optimize(ops) +# endif +# else +# define __optimize(ops) +# endif +#endif /* __optimize */ + +#ifndef __hot +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __hot __attribute__((__hot__)) __optimize(3) +# elif defined(__clang__) && !__has_attribute(__hot_) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put frequently used functions in separate section */ +# define __hot __attribute__((__section__("text.hot"))) __optimize("O3") +# elif defined(__GNUC__) || __has_attribute(__hot__) +# define __hot __attribute__((__hot__)) __optimize("O3") +# else +# define __hot __optimize("O3") +# endif +# else +# define __hot +# endif +#endif /* __hot */ + +#ifndef __cold +# if defined(__OPTIMIZE__) +# if defined(__e2k__) +# define __cold __attribute__((__cold__)) __optimize(1) +# elif defined(__clang__) && !__has_attribute(cold) \ + && __has_attribute(__section__) && (defined(__linux__) || defined(__gnu_linux__)) + /* just put infrequently used functions in separate section */ +# define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") +# elif defined(__GNUC__) || __has_attribute(cold) +# define __cold __attribute__((__cold__)) __optimize("Os") +# else +# define __cold __optimize("Os") +# endif +# else +# define __cold +# endif +#endif /* __cold */ + +#ifndef __flatten +# if defined(__OPTIMIZE__) && (defined(__GNUC__) || __has_attribute(__flatten__)) +# define __flatten __attribute__((__flatten__)) +# else +# define __flatten +# endif +#endif /* __flatten */ + +#ifndef likely +# if (defined(__GNUC__) || defined(__clang__)) && !defined(__COVERITY__) +# define likely(cond) __builtin_expect(!!(cond), 1) +# else +# define likely(x) (x) +# endif +#endif /* likely */ + +#ifndef unlikely +# if (defined(__GNUC__) || defined(__clang__)) && !defined(__COVERITY__) +# define unlikely(cond) __builtin_expect(!!(cond), 0) +# else +# define unlikely(x) (x) +# endif +#endif /* unlikely */ + +/* Workaround for Coverity Scan */ +#if defined(__COVERITY__) && __GNUC_PREREQ(7, 0) && !defined(__cplusplus) +typedef float _Float32; +typedef double _Float32x; +typedef double _Float64; +typedef long double _Float64x; +typedef float _Float128 __attribute__((__mode__(__TF__))); +typedef __complex__ float __cfloat128 __attribute__ ((__mode__ (__TC__))); +typedef _Complex float __cfloat128 __attribute__ ((__mode__ (__TC__))); +#endif /* Workaround for Coverity Scan */ + +#ifndef __printf_args +# if defined(__GNUC__) || __has_attribute(__format__) +# define __printf_args(format_index, first_arg) \ + __attribute__((__format__(printf, format_index, first_arg))) +# else +# define __printf_args(format_index, first_arg) +# endif +#endif /* __printf_args */ + +#ifndef __anonymous_struct_extension__ +# if defined(__GNUC__) +# define __anonymous_struct_extension__ __extension__ +# else +# define __anonymous_struct_extension__ +# endif +#endif /* __anonymous_struct_extension__ */ + +#ifndef __Wpedantic_format_voidptr + static __inline const void* __pure_function + __Wpedantic_format_voidptr(const void* ptr) {return ptr;} +# define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +/*----------------------------------------------------------------------------*/ + +#if defined(MDBX_USE_VALGRIND) +# include +# ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE + /* LY: available since Valgrind 3.10 */ +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# endif +#elif !defined(RUNNING_ON_VALGRIND) +# define VALGRIND_CREATE_MEMPOOL(h,r,z) +# define VALGRIND_DESTROY_MEMPOOL(h) +# define VALGRIND_MEMPOOL_TRIM(h,a,s) +# define VALGRIND_MEMPOOL_ALLOC(h,a,s) +# define VALGRIND_MEMPOOL_FREE(h,a) +# define VALGRIND_MEMPOOL_CHANGE(h,a,b,s) +# define VALGRIND_MAKE_MEM_NOACCESS(a,s) +# define VALGRIND_MAKE_MEM_DEFINED(a,s) +# define VALGRIND_MAKE_MEM_UNDEFINED(a,s) +# define VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_ENABLE_ADDR_ERROR_REPORTING_IN_RANGE(a,s) +# define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,s) (0) +# define VALGRIND_CHECK_MEM_IS_DEFINED(a,s) (0) +# define RUNNING_ON_VALGRIND (0) +#endif /* MDBX_USE_VALGRIND */ + +#ifdef __SANITIZE_ADDRESS__ +# include +#elif !defined(ASAN_POISON_MEMORY_REGION) +# define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +# define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif /* __SANITIZE_ADDRESS__ */ + +/*----------------------------------------------------------------------------*/ + +#ifndef ARRAY_LENGTH +# ifdef __cplusplus + template + char (&__ArraySizeHelper(T (&array)[N]))[N]; +# define ARRAY_LENGTH(array) (sizeof(::__ArraySizeHelper(array))) +# else +# define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) +# endif +#endif /* ARRAY_LENGTH */ + +#ifndef ARRAY_END +# define ARRAY_END(array) (&array[ARRAY_LENGTH(array)]) +#endif /* ARRAY_END */ + +#ifndef STRINGIFY +# define STRINGIFY_HELPER(x) #x +# define STRINGIFY(x) STRINGIFY_HELPER(x) +#endif /* STRINGIFY */ + +#define CONCAT(a,b) a##b +#define XCONCAT(a,b) CONCAT(a,b) + +#ifndef offsetof +# define offsetof(type, member) __builtin_offsetof(type, member) +#endif /* offsetof */ + +#ifndef container_of +# define container_of(ptr, type, member) \ + ((type *)((char *)(ptr) - offsetof(type, member))) +#endif /* container_of */ + +#define MDBX_TETRAD(a, b, c, d) \ + ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) + +#define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) + +#define FIXME "FIXME: " __FILE__ ", " STRINGIFY(__LINE__) + +#ifndef STATIC_ASSERT_MSG +# if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || __has_feature(c_static_assert) +# define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) +# elif defined(static_assert) +# define STATIC_ASSERT_MSG(expr, msg) static_assert(expr, msg) +# elif defined(_MSC_VER) +# include +# define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) +# else +# define STATIC_ASSERT_MSG(expr, msg) switch (0) {case 0:case (expr):;} +# endif +#endif /* STATIC_ASSERT */ + +#ifndef STATIC_ASSERT +# define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) +#endif + +/* *INDENT-ON* */ +/* clang-format on */ diff --git a/src/contrib/db/libmdbx/src/elements/internals.h b/src/contrib/db/libmdbx/src/elements/internals.h new file mode 100644 index 0000000..6aba374 --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/internals.h @@ -0,0 +1,1367 @@ +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . */ + +#pragma once +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* *INDENT-OFF* */ +/* clang-format off */ + +/* In case the MDBX_DEBUG is undefined set it corresponding to NDEBUG */ +#ifndef MDBX_DEBUG +# ifdef NDEBUG +# define MDBX_DEBUG 0 +# else +# define MDBX_DEBUG 1 +# endif +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if MDBX_DEBUG +# undef NDEBUG +#endif + +#define MDBX_OSX_WANNA_DURABILITY 0 /* using fcntl(F_FULLFSYNC) with 5-10 times slowdown */ +#define MDBX_OSX_WANNA_SPEED 1 /* using fsync() with chance of data lost on power failure */ +#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +# define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif + +#ifdef MDBX_ALLOY +/* Amalgamated build */ +# define MDBX_INTERNAL_FUNC static +# define MDBX_INTERNAL_VAR static +#else +/* Non-amalgamated build */ +# define MDBX_INTERNAL_FUNC +# define MDBX_INTERNAL_VAR extern +#endif /* MDBX_ALLOY */ + +#ifndef MDBX_DISABLE_GNU_SOURCE +#define MDBX_DISABLE_GNU_SOURCE 0 +#endif +#if MDBX_DISABLE_GNU_SOURCE +#undef _GNU_SOURCE +#elif defined(__linux__) || defined(__gnu_linux__) +#define _GNU_SOURCE +#endif + +/*----------------------------------------------------------------------------*/ + +/* Should be defined before any includes */ +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#ifdef _MSC_VER +# if _MSC_VER < 1400 +# error "Microsoft Visual C++ 8.0 (Visual Studio 2005) or later version is required" +# endif +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#if _MSC_VER > 1800 +# pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#if _MSC_VER > 1913 +# pragma warning(disable : 5045) /* Compiler will insert Spectre mitigation... */ +#endif +#pragma warning(disable : 4710) /* 'xyz': function not inlined */ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic inline expansion */ +#pragma warning(disable : 4201) /* nonstandard extension used : nameless struct / union */ +#pragma warning(disable : 4702) /* unreachable code */ +#pragma warning(disable : 4706) /* assignment within conditional expression */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to alignment specifier */ +#pragma warning(disable : 4310) /* cast truncates constant value */ +#pragma warning(disable : 4820) /* bytes padding added after data member for aligment */ +#pragma warning(disable : 4548) /* expression before comma has no effect; expected expression with side - effect */ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be unaligned */ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized array in struct/union */ +#endif /* _MSC_VER (warnings) */ + +#include "../../mdbx.h" +#include "defs.h" + +#if defined(__GNUC__) && !__GNUC_PREREQ(4,2) + /* Actualy libmdbx was not tested with compilers older than GCC from RHEL6. + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3,8) + /* Actualy libmdbx was not tested with CLANG older than 3.8. + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +# warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2,12) + /* Actualy libmdbx was not tested with something older than glibc 2.12 (from RHEL6). + * But you could remove this #error and try to continue at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +# warning "libmdbx required at least GLIBC 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +# warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + +#if __has_warning("-Wconstant-logical-operand") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Wconstant-logical-operand" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wconstant-logical-operand" +# else +# pragma warning disable "constant-logical-operand" +# endif +#endif /* -Wconstant-logical-operand */ + +#if defined(__LCC__) && (__LCC__ <= 121) + /* bug #2798 */ +# pragma diag_suppress alignment_reduction_ignored +#elif defined(__ICC) +# pragma warning(disable: 3453 1366) +#elif __has_warning("-Walignment-reduction-ignored") +# if defined(__clang__) +# pragma clang diagnostic ignored "-Walignment-reduction-ignored" +# elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Walignment-reduction-ignored" +# else +# pragma warning disable "alignment-reduction-ignored" +# endif +#endif /* -Walignment-reduction-ignored */ + +#include "osal.h" + +/* *INDENT-ON* */ +/* clang-format on */ + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + +#ifndef MDBX_64BIT_ATOMIC +#if MDBX_WORDBITS >= 64 +#define MDBX_64BIT_ATOMIC 1 +#else +#define MDBX_64BIT_ATOMIC 0 +#endif +#define MDBX_64BIT_ATOMIC_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_ATOMIC) +#else +#define MDBX_64BIT_ATOMIC_CONFIG STRINGIFY(MDBX_64BIT_ATOMIC) +#endif /* MDBX_64BIT_ATOMIC */ + +#ifndef MDBX_64BIT_CAS +#if defined(ATOMIC_LLONG_LOCK_FREE) +#if ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__GCC_ATOMIC_LLONG_LOCK_FREE) +#if __GCC_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(__CLANG_ATOMIC_LLONG_LOCK_FREE) +#if __CLANG_ATOMIC_LLONG_LOCK_FREE > 1 +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS 0 +#endif +#elif defined(_MSC_VER) || defined(__APPLE__) +#define MDBX_64BIT_CAS 1 +#else +#define MDBX_64BIT_CAS MDBX_64BIT_ATOMIC +#endif +#define MDBX_64BIT_CAS_CONFIG "AUTO=" STRINGIFY(MDBX_64BIT_CAS) +#else +#define MDBX_64BIT_CAS_CONFIG STRINGIFY(MDBX_64BIT_CAS) +#endif /* MDBX_64BIT_CAS */ + +/* Some platforms define the EOWNERDEAD error code even though they + * don't support Robust Mutexes. Compile with -DMDBX_USE_ROBUST=0. */ +#ifndef MDBX_USE_ROBUST +/* Howard Chu: Android currently lacks Robust Mutex support */ +#if defined(EOWNERDEAD) && !defined(__ANDROID__) && !defined(__APPLE__) && \ + (!defined(__GLIBC__) || \ + __GLIBC_PREREQ( \ + 2, \ + 10) /* LY: glibc before 2.10 has a troubles with Robust Mutex too. */ \ + || _POSIX_C_SOURCE >= 200809L) +#define MDBX_USE_ROBUST 1 +#else +#define MDBX_USE_ROBUST 0 +#endif +#define MDBX_USE_ROBUST_CONFIG "AUTO=" STRINGIFY(MDBX_USE_ROBUST) +#else +#define MDBX_USE_ROBUST_CONFIG STRINGIFY(MDBX_USE_ROBUST) +#endif /* MDBX_USE_ROBUST */ + +#ifndef MDBX_USE_OFDLOCKS +#if defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK) && \ + !defined(MDBX_SAFE4QEMU) +#define MDBX_USE_OFDLOCKS 1 +#else +#define MDBX_USE_OFDLOCKS 0 +#endif +#define MDBX_USE_OFDLOCKS_CONFIG "AUTO=" STRINGIFY(MDBX_USE_OFDLOCKS) +#else +#define MDBX_USE_OFDLOCKS_CONFIG STRINGIFY(MDBX_USE_OFDLOCKS) +#endif /* MDBX_USE_OFDLOCKS */ + +/* Controls checking PID against reuse DB environment after the fork() */ +#ifndef MDBX_TXN_CHECKPID +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) +/* PID check could be ommited: + * - on Linux when madvise(MADV_DONTFORK) is available. i.e. after the fork() + * mapped pages will not be available for child process. + * - in Windows where fork() not available. */ +#define MDBX_TXN_CHECKPID 0 +#else +#define MDBX_TXN_CHECKPID 1 +#endif +#define MDBX_TXN_CHECKPID_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKPID) +#else +#define MDBX_TXN_CHECKPID_CONFIG STRINGIFY(MDBX_TXN_CHECKPID) +#endif /* MDBX_TXN_CHECKPID */ + +/* Controls checking transaction owner thread against misuse transactions from + * other threads. */ +#ifndef MDBX_TXN_CHECKOWNER +#define MDBX_TXN_CHECKOWNER 1 +#define MDBX_TXN_CHECKOWNER_CONFIG "AUTO=" STRINGIFY(MDBX_TXN_CHECKOWNER) +#else +#define MDBX_TXN_CHECKOWNER_CONFIG STRINGIFY(MDBX_TXN_CHECKOWNER) +#endif /* MDBX_TXN_CHECKOWNER */ + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(MDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +/* Does a system have battery-backed Real-Time Clock or just a fake. */ +#ifndef MDBX_TRUST_RTC +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ + defined(__OpenBSD__) +#define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ +#else +#define MDBX_TRUST_RTC 1 +#endif +#define MDBX_TRUST_RTC_CONFIG "AUTO=" STRINGIFY(MDBX_TRUST_RTC) +#else +#define MDBX_TRUST_RTC_CONFIG STRINGIFY(MDBX_TRUST_RTC) +#endif /* MDBX_TRUST_RTC */ + +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ + +/* The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. */ +#define MDBX_MINKEYS 2 + +/* A stamp that identifies a file as an MDBX file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. */ +#define MDBX_MAGIC UINT64_C(/* 56-bit prime */ 0x59659DBDEF4C11) + +/* The version number for a database's datafile format. */ +#define MDBX_DATA_VERSION 2 +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 3 + +/* handle for the DB used to track free pages. */ +#define FREE_DBI 0 +/* handle for the default DB. */ +#define MAIN_DBI 1 +/* Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 +#define MAX_DBI (INT16_MAX - CORE_DBS) +#if MAX_DBI != MDBX_MAX_DBI +#error "Opps, MAX_DBI != MDBX_MAX_DBI" +#endif + +/* Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 3 + +/* A page number in the database. + * + * MDBX uses 32 bit for page numbers. This limits database + * size up to 2^44 bytes, in case of 4K pages. */ +typedef uint32_t pgno_t; +#define PRIaPGNO PRIu32 +#define MAX_PAGENO UINT32_C(0x7FFFffff) +#define MIN_PAGENO NUM_METAS + +/* A transaction ID. */ +typedef uint64_t txnid_t; +#define PRIaTXN PRIi64 +#define MIN_TXNID UINT64_C(1) +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define MDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef MDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define MDBX_TXNID_STEP 1u +#else +#define MDBX_TXNID_STEP 2u +#endif +#endif /* MDBX_TXNID_STEP */ + +/* Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. */ +typedef uint16_t indx_t; + +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ +/* Core structures for database and shared memory (i.e. format definition) */ +#pragma pack(push, 1) + +typedef union mdbx_safe64 { + volatile uint64_t inconsistent; +#if MDBX_64BIT_ATOMIC + volatile uint64_t atomic; +#endif /* MDBX_64BIT_ATOMIC */ + struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + volatile uint32_t low; + volatile uint32_t high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + volatile uint32_t high; + volatile uint32_t low; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + }; +} mdbx_safe64_t; + +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) + +/* Information about a single database in the environment. */ +typedef struct MDBX_db { + uint16_t md_flags; /* see mdbx_dbi_open */ + uint16_t md_depth; /* depth of this tree */ + uint32_t md_xsize; /* also ksize for LEAF2 pages */ + pgno_t md_root; /* the root page of this tree */ + pgno_t md_branch_pages; /* number of internal pages */ + pgno_t md_leaf_pages; /* number of leaf pages */ + pgno_t md_overflow_pages; /* number of overflow pages */ + uint64_t md_seq; /* table sequence counter */ + uint64_t md_entries; /* number of data items */ + uint64_t md_mod_txnid; /* txnid of last commited modification */ +} MDBX_db; + +/* database size-related parameters */ +typedef struct mdbx_geo_t { + uint16_t grow; /* datafile growth step in pages */ + uint16_t shrink; /* datafile shrink threshold in pages */ + pgno_t lower; /* minimal size of datafile in pages */ + pgno_t upper; /* maximal size of datafile in pages */ + pgno_t now; /* current size of datafile in pages */ + pgno_t next; /* first unused page in the datafile, + * but actually the file may be shorter. */ +} mdbx_geo_t; + +/* Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ +typedef struct MDBX_meta { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ + uint64_t mm_magic_and_version; + + /* txnid that committed this page, the first of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_a; + + uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ + uint8_t mm_validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + uint8_t mm_extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + mdbx_geo_t mm_geo; /* database size-related parameters */ + + MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ + /* The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_xsize +/* Any persistent environment flags, see mdbx_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + mdbx_canary mm_canary; + +#define MDBX_DATASIGN_NONE 0u +#define MDBX_DATASIGN_WEAK 1u +#define SIGN_IS_WEAK(sign) ((sign) == MDBX_DATASIGN_WEAK) +#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) +#define META_IS_WEAK(meta) SIGN_IS_WEAK((meta)->mm_datasync_sign) +#define META_IS_STEADY(meta) SIGN_IS_STEADY((meta)->mm_datasync_sign) + volatile uint64_t mm_datasync_sign; + + /* txnid that committed this page, the second of a two-phase-update pair */ + mdbx_safe64_t mm_txnid_b; + + /* Number of non-meta pages which were put in GC after COW. May be 0 in case + * DB was previously handled by libmdbx without corresponding feature. + * This value in couple with mr_snapshot_pages_retired allows fast estimation + * of "how much reader is restraining GC recycling". */ + uint64_t mm_pages_retired; +} MDBX_meta; + +/* Common header for all page types. The page type depends on mp_flags. + * + * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with + * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages + * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * + * P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of F_BIGDATA nodes. + * + * P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * + * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a GC record. */ +typedef struct MDBX_page { + union { + struct MDBX_page *mp_next; /* for in-memory list of freed pages */ + uint64_t mp_validator; /* checksum of page content or a txnid during + * which the page has been updated */ + }; + uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ +#define P_BRANCH 0x01 /* branch page */ +#define P_LEAF 0x02 /* leaf page */ +#define P_OVERFLOW 0x04 /* overflow page */ +#define P_META 0x08 /* meta page */ +#define P_DIRTY 0x10 /* dirty page, also set for P_SUBP pages */ +#define P_LEAF2 0x20 /* for MDBX_DUPFIXED records */ +#define P_SUBP 0x40 /* for MDBX_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /* page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /* leave this page alone during spill */ + uint16_t mp_flags; + union { + struct { + indx_t mp_lower; /* lower bound of free space */ + indx_t mp_upper; /* upper bound of free space */ + }; + uint32_t mp_pages; /* number of overflow pages */ + }; + pgno_t mp_pgno; /* page number */ + + /* dynamic size */ + indx_t mp_ptrs[/* C99 */]; +} MDBX_page; + +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)offsetof(MDBX_page, mp_ptrs)) + +#pragma pack(pop) + +/* Reader Lock Table + * + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent + * read transactions started by the same thread need no further locking to + * proceed. + * + * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. + * No reader table is used if the database is on a read-only filesystem. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. */ + +/* The actual reader record, with cacheline padding. */ +typedef struct MDBX_reader { + /* Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. */ + mdbx_safe64_t /* txnid_t */ mr_txnid; + + /* The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * + * NOTE: We currently don't check for stale records. + * We simply re-init the table when we know that we're the only process + * opening the lock file. */ + + /* The thread ID of the thread owning this txn. */ +#if MDBX_WORDBITS >= 64 + volatile uint64_t mr_tid; +#else + volatile uint32_t mr_tid; + volatile uint32_t mr_aba_curer; /* CSN to resolve ABA_problems on 32-bit arch, + unused for now */ +#endif + /* The process ID of the process owning this reader txn. */ + volatile uint32_t mr_pid; + + /* The number of pages used in the reader's MVCC snapshot, + * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ + volatile pgno_t mr_snapshot_pages_used; + /* Number of retired pages at the time this reader starts transaction. So, + * at any time the difference mm_pages_retired - mr_snapshot_pages_retired + * will give the number of pages which this reader restraining from reuse. */ + volatile uint64_t mr_snapshot_pages_retired; +} MDBX_reader; + +/* The header for the reader table (a memory-mapped lock file). */ +typedef struct MDBX_lockinfo { + /* Stamp identifying this as an MDBX file. + * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ + uint64_t mti_magic_and_version; + + /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ + uint32_t mti_os_and_format; + + /* Flags which environment was opened. */ + volatile uint32_t mti_envmode; + + /* Threshold of un-synced-with-disk pages for auto-sync feature, + * zero means no-threshold, i.e. auto-sync is disabled. */ + volatile pgno_t mti_autosync_threshold; + + /* Low 32-bit of txnid with which meta-pages was synced, + * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ + volatile uint32_t mti_meta_sync_txnid; + + /* Period for timed auto-sync feature, i.e. at the every steady checkpoint + * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * The time value is represented in a suitable system-dependent form, for + * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). + * Zero means timed auto-sync is disabled. */ + volatile uint64_t mti_autosync_period; + + /* Marker to distinguish uniqueness of DB/CLK.*/ + volatile uint64_t mti_bait_uniqueness; + + /* The analogue /proc/sys/kernel/random/boot_id or similar to determine + * whether the system was rebooted after the last use of the database files. + * If there was no reboot, but there is no need to rollback to the last + * steady sync point. Zeros mean that no relevant information is available + * from the system. */ + volatile bin128_t mti_bootid; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ +#ifdef MDBX_OSAL_LOCK + /* Mutex protecting write-txn. */ + MDBX_OSAL_LOCK mti_wmutex; +#endif + + volatile txnid_t mti_oldest_reader; + + /* Timestamp of the last steady sync. Value is represented in a suitable + * system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) or + * clock_gettime(CLOCK_MONOTONIC). */ + volatile uint64_t mti_sync_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + volatile pgno_t mti_unsynced_pages; + + /* Number of page which was discarded last time by madvise(MADV_FREE). */ + volatile pgno_t mti_discarded_tail; + + /* Timestamp of the last readers check. */ + volatile uint64_t mti_reader_check_timestamp; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + +#ifdef MDBX_OSAL_LOCK + /* Mutex protecting readers registration access to this table. */ + MDBX_OSAL_LOCK mti_rmutex; +#endif + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + volatile unsigned mti_numreaders; + volatile unsigned mti_readers_refresh_flag; + + alignas(MDBX_CACHELINE_SIZE) /* cacheline ---------------------------------*/ + MDBX_reader mti_readers[/* C99 */]; +} MDBX_lockinfo; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_OSAL_LOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ + (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ + (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_DEVEL ((MDBX_MAGIC << 8) + 255) + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#ifndef MDBX_ASSUME_MALLOC_OVERHEAD +#define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) +#endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ + +/* The maximum size of a database page. + * + * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. + * + * MDBX will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. */ +#define MAX_PAGESIZE MDBX_MAX_PAGESIZE +#define MIN_PAGESIZE MDBX_MIN_PAGESIZE + +#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7ff80000) +#endif +#define MAX_MAPSIZE64 (MAX_PAGENO * (uint64_t)MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define MDBX_READERS_LIMIT \ + ((65536 - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader)) +#else +#define MDBX_READERS_LIMIT 1024 +#define MAX_MAPSIZE MAX_MAPSIZE32 +#endif /* MDBX_WORDBITS */ + +/*----------------------------------------------------------------------------*/ +/* Two kind lists of pages (aka PNL) */ + +/* An PNL is an Page Number List, a sorted array of IDs. The first element of + * the array is a counter for how many actual page-numbers are in the list. + * PNLs are sorted in descending order, this allow cut off a page with lowest + * pgno (at the tail) just truncating the list */ +#define MDBX_PNL_ASCENDING 0 +typedef pgno_t *MDBX_PNL; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ +typedef txnid_t *MDBX_TXL; + +/* An Dirty-Page list item is an pgno/pointer pair. */ +typedef union MDBX_DP { + struct { + pgno_t pgno; + MDBX_page *ptr; + }; + struct { + unsigned sorted; + unsigned length; + }; +} MDBX_DP; + +/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. + * The first element's length member is a count of how many actual + * elements are in the array. */ +typedef MDBX_DP *MDBX_DPL; + +/* PNL sizes - likely should be even bigger */ +#define MDBX_PNL_GRANULATE 1024 +#define MDBX_PNL_INITIAL \ + (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_PNL_MAX \ + ((1u << 24) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#define MDBX_DPL_TXNFULL (MDBX_PNL_MAX / 4) + +#define MDBX_TXL_GRANULATE 32 +#define MDBX_TXL_INITIAL \ + (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_TXL_MAX \ + ((1u << 17) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_SIZE(pl) ((pl)[0]) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_SIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_SIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_SIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_SIZE(pl) == 0) + +/*----------------------------------------------------------------------------*/ +/* Internal structures */ + +/* Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. */ +typedef struct MDBX_dbx { + MDBX_val md_name; /* name of the database */ + MDBX_cmp_func *md_cmp; /* function for comparing keys */ + MDBX_cmp_func *md_dcmp; /* function for comparing data items */ +} MDBX_dbx; + +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { +#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) + size_t mt_signature; + MDBX_txn *mt_parent; /* parent of a nested txn */ + /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ + MDBX_txn *mt_child; + mdbx_geo_t mt_geo; + /* next unallocated page */ +#define mt_next_pgno mt_geo.next + /* corresponding to the current size of datafile */ +#define mt_end_pgno mt_geo.now + + /* Transaction Flags */ + /* mdbx_txn_begin() flags */ +#define MDBX_TXN_BEGIN_FLAGS \ + (MDBX_NOMETASYNC | MDBX_NOSYNC | MDBX_MAPASYNC | MDBX_RDONLY | MDBX_TRYTXN) + /* internal txn flags */ +#define MDBX_TXN_FINISHED 0x01 /* txn is finished or never began */ +#define MDBX_TXN_ERROR 0x02 /* txn is unusable after an error */ +#define MDBX_TXN_DIRTY 0x04 /* must write, even if dirty list is empty */ +#define MDBX_TXN_SPILLS 0x08 /* txn or a parent has spilled pages */ +#define MDBX_TXN_HAS_CHILD 0x10 /* txn has an MDBX_txn.mt_child */ + /* most operations on the txn are currently illegal */ +#define MDBX_TXN_BLOCKED \ + (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD) + unsigned mt_flags; + /* The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. */ + txnid_t mt_txnid; + MDBX_env *mt_env; /* the DB environment */ + /* Array of records for each DB known in the environment. */ + MDBX_dbx *mt_dbxs; + /* Array of MDBX_db records for each known DB */ + MDBX_db *mt_dbs; + /* Array of sequence numbers for each DB handle */ + unsigned *mt_dbiseqs; + + /* Transaction DB Flags */ +#define DB_DIRTY MDBX_TBL_DIRTY /* DB was written in this txn */ +#define DB_STALE MDBX_TBL_STALE /* Named-DB record is older than txnID */ +#define DB_FRESH MDBX_TBL_FRESH /* Named-DB handle opened in this txn */ +#define DB_CREAT MDBX_TBL_CREAT /* Named-DB handle created in this txn */ +#define DB_VALID 0x10 /* DB handle is valid, see also MDBX_VALID */ +#define DB_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ +#define DB_DUPDATA 0x40 /* DB is MDBX_DUPSORT data */ +#define DB_AUDITED 0x80 /* Internal flag for accounting during audit */ + /* In write txns, array of cursors for each DB */ + MDBX_cursor **mt_cursors; + /* Array of flags for each DB */ + uint8_t *mt_dbflags; + /* Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. */ + MDBX_dbi mt_numdbs; + size_t mt_owner; /* thread ID that owns this transaction */ + mdbx_canary mt_canary; + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or NULL. */ + MDBX_reader *reader; + } to; + struct { + pgno_t *reclaimed_pglist; /* Reclaimed GC pages */ + txnid_t last_reclaimed; /* ID of last used record */ + pgno_t loose_refund_wl /* FIXME: describe */; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into mt_parent after freeing hidden mt_parent pages. */ + unsigned dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + MDBX_DPL dirtylist; + /* The list of reclaimed txns from GC */ + MDBX_TXL lifo_reclaimed; + /* The list of pages that became unused during this transaction. */ + MDBX_PNL retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `mp_next`. */ + MDBX_page *loose_pages; + /* Number of loose pages (tw.loose_pages) */ + unsigned loose_count; + /* Number of retired to parent pages (tw.retired2parent_pages) */ + unsigned retired2parent_count; + /* The list of parent's txn dirty pages that retired (became unused) + * in this transaction, linked through `mp_next`. */ + MDBX_page *retired2parent_pages; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + MDBX_PNL spill_pages; + } tw; + }; +}; + +/* Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. */ +#if MDBX_WORDBITS >= 64 +#define CURSOR_STACK 28 +#else +#define CURSOR_STACK 20 +#endif + +struct MDBX_xcursor; + +/* Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. MDBX_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a P_SUBP page can be stale. + * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ +struct MDBX_cursor { +#define MDBX_MC_SIGNATURE UINT32_C(0xFE05D5B1) +#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) +#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) + uint32_t mc_signature; + /* The database handle this cursor operates on */ + MDBX_dbi mc_dbi; + /* Next cursor on this DB in this txn */ + MDBX_cursor *mc_next; + /* Backup of the original cursor if this cursor is a shadow */ + MDBX_cursor *mc_backup; + /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ + struct MDBX_xcursor *mc_xcursor; + /* The transaction that owns this cursor */ + MDBX_txn *mc_txn; + /* The database record for this cursor */ + MDBX_db *mc_db; + /* The database auxiliary record for this cursor */ + MDBX_dbx *mc_dbx; + /* The mt_dbflag for this database */ + uint8_t *mc_dbflag; + uint16_t mc_snum; /* number of pushed pages */ + uint16_t mc_top; /* index of top page, normally mc_snum-1 */ + /* Cursor state flags. */ +#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ +#define C_EOF 0x02 /* No more data */ +#define C_SUB 0x04 /* Cursor is a sub-cursor */ +#define C_DEL 0x08 /* last op was a cursor_del */ +#define C_UNTRACK 0x10 /* Un-track cursor when closing */ +#define C_RECLAIMING 0x20 /* GC lookup is prohibited */ +#define C_GCFREEZE 0x40 /* reclaimed_pglist must not be updated */ + unsigned mc_flags; /* see mdbx_cursor */ + MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ +}; + +/* Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. */ +typedef struct MDBX_xcursor { + /* A sub-cursor for traversing the Dup DB */ + MDBX_cursor mx_cursor; + /* The database record for this Dup DB */ + MDBX_db mx_db; + /* The auxiliary DB record for this Dup DB */ + MDBX_dbx mx_dbx; + /* The mt_dbflag for this Dup DB */ + uint8_t mx_dbflag; +} MDBX_xcursor; + +typedef struct MDBX_cursor_couple { + MDBX_cursor outer; + MDBX_xcursor inner; +} MDBX_cursor_couple; + +/* The database environment. */ +struct MDBX_env { +#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) + size_t me_signature; + mdbx_mmap_t me_dxb_mmap; /* The main data file */ +#define me_map me_dxb_mmap.dxb +#define me_fd me_dxb_mmap.fd + mdbx_mmap_t me_lck_mmap; /* The lock file */ +#define me_lfd me_lck_mmap.fd +#define me_lck me_lck_mmap.lck + + /* Failed to update the meta page. Probably an I/O error. */ +#define MDBX_FATAL_ERROR UINT32_C(0x80000000) + /* Additional flag for mdbx_sync_locked() */ +#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) + /* Some fields are initialized. */ +#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) + /* me_txkey is set */ +#define MDBX_ENV_TXKEY UINT32_C(0x10000000) + uint32_t me_flags; /* see mdbx_env */ + unsigned me_psize; /* DB page size, inited from me_os_psize */ + unsigned me_psize2log; /* log2 of DB page size */ + unsigned me_os_psize; /* OS page size, from mdbx_syspagesize() */ + unsigned me_maxreaders; /* size of the reader table */ + mdbx_fastmutex_t me_dbi_lock; + MDBX_dbi me_numdbs; /* number of DBs opened */ + MDBX_dbi me_maxdbs; /* size of the DB table */ + uint32_t me_pid; /* process ID of this env */ + mdbx_thread_key_t me_txkey; /* thread-key for readers */ + char *me_path; /* path to the DB files */ + void *me_pbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *me_txn; /* current write transaction */ + MDBX_txn *me_txn0; /* prealloc'd write transaction */ +#ifdef MDBX_OSAL_LOCK + MDBX_OSAL_LOCK *me_wmutex; /* write-txn mutex */ +#endif + MDBX_dbx *me_dbxs; /* array of static DB info */ + uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ + unsigned *me_dbiseqs; /* array of dbi sequence numbers */ + volatile txnid_t *me_oldest; /* ID of oldest reader last time we looked */ + MDBX_page *me_dpages; /* list of malloc'd blocks for re-use */ + /* PNL of pages that became unused in a write txn */ + MDBX_PNL me_retired_pages; + /* MDBX_DP of pages written during a write txn. Length MDBX_DPL_TXNFULL. */ + MDBX_DPL me_dirtylist; + /* Number of freelist items that can fit in a single overflow page */ + unsigned me_maxgc_ov1page; + /* Max size of a node on a page */ + unsigned me_nodemax; + unsigned me_maxkey_limit; /* max size of a key */ + uint32_t me_live_reader; /* have liveness lock in reader table */ + void *me_userctx; /* User-settable context */ + volatile uint64_t *me_sync_timestamp; + volatile uint64_t *me_autosync_period; + volatile pgno_t *me_unsynced_pages; + volatile pgno_t *me_autosync_threshold; + volatile pgno_t *me_discarded_tail; + volatile uint32_t *me_meta_sync_txnid; + MDBX_oom_func *me_oom_func; /* Callback for kicking laggard readers */ + struct { +#ifdef MDBX_OSAL_LOCK + MDBX_OSAL_LOCK wmutex; +#endif + txnid_t oldest; + uint64_t sync_timestamp; + uint64_t autosync_period; + pgno_t autosync_pending; + pgno_t autosync_threshold; + pgno_t discarded_tail; + uint32_t meta_sync_txnid; + } me_lckless_stub; +#if MDBX_DEBUG + MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ +#endif +#ifdef MDBX_USE_VALGRIND + int me_valgrind_handle; +#endif +#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) + pgno_t me_poison_edge; +#endif + MDBX_env *me_lcklist_next; + + /* struct me_dbgeo used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } me_dbgeo; + +#if defined(_WIN32) || defined(_WIN64) + MDBX_srwlock me_remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION me_windowsbug_lock; +#else + mdbx_fastmutex_t me_remap_guard; +#endif +}; + +/*----------------------------------------------------------------------------*/ +/* Debug and Logging stuff */ + +#define MDBX_RUNTIME_FLAGS_INIT \ + ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT + +#ifndef mdbx_runtime_flags /* avoid override from tools */ +MDBX_INTERNAL_VAR uint8_t mdbx_runtime_flags; +#endif +#ifndef mdbx_runtime_flags /* avoid override from tools */ +MDBX_INTERNAL_VAR uint8_t mdbx_loglevel; +#endif +MDBX_INTERNAL_VAR MDBX_debug_func *mdbx_debug_logger; + +MDBX_INTERNAL_FUNC void mdbx_debug_log(int type, const char *function, int line, + const char *fmt, ...) + __printf_args(4, 5); + +MDBX_INTERNAL_FUNC void mdbx_panic(const char *fmt, ...) __printf_args(1, 2); + +#if MDBX_DEBUG + +#define mdbx_assert_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_ASSERT) + +#define mdbx_audit_enabled() unlikely(mdbx_runtime_flags &MDBX_DBG_AUDIT) + +#ifdef MDBX_LOGLEVEL_BUILD +#define mdbx_log_enabled(msg) \ + (msg <= MDBX_LOGLEVEL_BUILD && unlikely(msg <= mdbx_loglevel)) +#else +#define mdbx_log_enabled(msg) unlikely(msg <= mdbx_loglevel) +#endif /* MDBX_LOGLEVEL_BUILD */ + +#else /* MDBX_DEBUG */ + +#define mdbx_audit_enabled() (0) + +#if !defined(NDEBUG) || defined(MDBX_FORCE_ASSERT) +#define mdbx_assert_enabled() (1) +#else +#define mdbx_assert_enabled() (0) +#endif /* NDEBUG */ + +#ifdef MDBX_LOGLEVEL_BUILD +#define mdbx_log_enabled(msg) (msg <= MDBX_LOGLEVEL_BUILD) +#else +#define mdbx_log_enabled(msg) (0) +#endif /* MDBX_LOGLEVEL_BUILD */ + +#endif /* MDBX_DEBUG */ + +MDBX_INTERNAL_FUNC void mdbx_assert_fail(const MDBX_env *env, const char *msg, + const char *func, int line); + +#define mdbx_debug_extra(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_extra_print(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_EXTRA)) \ + mdbx_debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_trace(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_TRACE)) \ + mdbx_debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_debug_print(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_DEBUG)) \ + mdbx_debug_log(MDBX_LOG_DEBUG, NULL, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define mdbx_verbose(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_VERBOSE)) \ + mdbx_debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_notice(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_NOTICE)) \ + mdbx_debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_warning(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_WARN)) \ + mdbx_debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_error(fmt, ...) \ + do { \ + if (mdbx_log_enabled(MDBX_LOG_ERROR)) \ + mdbx_debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", \ + __VA_ARGS__); \ + } while (0) + +#define mdbx_fatal(fmt, ...) \ + mdbx_debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#define mdbx_ensure_msg(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + mdbx_assert_fail(env, msg, __func__, __LINE__); \ + } while (0) + +#define mdbx_ensure(env, expr) mdbx_ensure_msg(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define mdbx_assert(env, expr) \ + do { \ + if (mdbx_assert_enabled()) \ + mdbx_ensure(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define mdbx_cassert(mc, expr) mdbx_assert((mc)->mc_txn->mt_env, expr) + +/* assert(3) variant in transaction context */ +#define mdbx_tassert(txn, expr) mdbx_assert((txn)->mt_env, expr) + +#ifndef MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#undef assert +#define assert(expr) mdbx_assert(NULL, expr) +#endif + +/*----------------------------------------------------------------------------*/ +/* Internal prototypes */ + +MDBX_INTERNAL_FUNC int mdbx_reader_check0(MDBX_env *env, int rlocked, + int *dead); +MDBX_INTERNAL_FUNC int mdbx_rthc_alloc(mdbx_thread_key_t *key, + MDBX_reader *begin, MDBX_reader *end); +MDBX_INTERNAL_FUNC void mdbx_rthc_remove(const mdbx_thread_key_t key); + +MDBX_INTERNAL_FUNC void mdbx_rthc_global_init(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_global_dtor(void); +MDBX_INTERNAL_FUNC void mdbx_rthc_thread_dtor(void *ptr); + +#define MDBX_IS_ERROR(rc) \ + ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/* Internal error codes, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_ERRCODE + 10) + +/* Debuging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) + +/* Key size which fits in a DKBUF. */ +#define DKBUF_MAXKEYSIZE 511 /* FIXME */ + +#if MDBX_DEBUG +#define DKBUF char _kbuf[DKBUF_MAXKEYSIZE * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAXKEYSIZE * 2 + 1) +#define DVAL(x) \ + mdbx_dump_val(x, _kbuf + DKBUF_MAXKEYSIZE * 2 + 1, DKBUF_MAXKEYSIZE * 2 + 1) +#else +#define DKBUF ((void)(0)) +#define DKEY(x) ("-") +#define DVAL(x) ("-") +#endif + +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + +/* Default size of memory map. + * This is certainly too small for any actual applications. Apps should + * always set the size explicitly using mdbx_env_set_mapsize(). */ +#define DEFAULT_MAPSIZE MEGABYTE + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +/* Test if a page is a leaf page */ +#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) +/* Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) +/* Test if a page is a branch page */ +#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) +/* Test if a page is an overflow page */ +#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) +/* Test if a page is a sub page */ +#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +/* Test if a page is dirty */ +#define IS_DIRTY(p) (((p)->mp_flags & P_DIRTY) != 0) + +#define PAGETYPE(p) ((p)->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW)) + +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. + * We guarantee 2-byte alignment for 'MDBX_node's. + * + * mn_lo and mn_hi are used for data size on leaf nodes, and for child + * pgno on branch nodes. On 64 bit platforms, mn_flags is also used + * for pgno. (Branch nodes have no flags). Lo and hi are in host byte + * order in case some accesses can be optimized to 32-bit word access. + * + * Leaf node flags describe node contents. F_BIGDATA says the node's + * data part is the page number of an overflow page with actual data. + * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just F_SUBDATA). */ +typedef struct MDBX_node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + struct { + uint16_t mn_lo, mn_hi; /* part of data size or pgno */ + }; + uint32_t mn_dsize; + uint32_t mn_pgno32; + }; + uint8_t mn_flags; /* see mdbx_node flags */ + uint8_t mn_extra; + uint16_t mn_ksize; /* key size */ +#else + uint16_t mn_ksize; /* key size */ + uint8_t mn_extra; + uint8_t mn_flags; /* see mdbx_node flags */ + union { + uint32_t mn_pgno32; + uint32_t mn_dsize; + struct { + uint16_t mn_hi, mn_lo; /* part of data size or pgno */ + }; + }; +#endif + + /* mdbx_node Flags */ +#define F_BIGDATA 0x01 /* data put on overflow page */ +#define F_SUBDATA 0x02 /* data is a sub-database */ +#define F_DUPDATA 0x04 /* data has duplicates */ + + /* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) + uint8_t mn_data[/* C99 */]; /* key and data are appended here */ +} MDBX_node; + +#define MDBX_VALID 0x8000 /* DB handle is valid, for me_dbflags */ +#define PERSISTENT_FLAGS (0xffff & ~(MDBX_VALID)) +/* mdbx_dbi_open() flags */ +#define VALID_FLAGS \ + (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ + MDBX_INTEGERDUP | MDBX_REVERSEDUP | MDBX_CREATE) + +/* max number of pages to commit in one writev() call */ +#define MDBX_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDBX_COMMIT_PAGES /* sysconf(_SC_IOV_MAX) */ +#undef MDBX_COMMIT_PAGES +#define MDBX_COMMIT_PAGES IOV_MAX +#endif + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#if 1 +/* LY: fast enough on most systems */ +#define CMP2INT(a, b) (((b) > (a)) ? -1 : (a) > (b)) +#else +#define CMP2INT(a, b) (((a) > (b)) - ((b) > (a))) +#endif + +/* Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDBX_NOSPILL 0x8000 + +static __maybe_unused __inline pgno_t pgno_add(pgno_t base, pgno_t augend) { + assert(base <= MAX_PAGENO); + return (augend < MAX_PAGENO - base) ? base + augend : MAX_PAGENO; +} + +static __maybe_unused __inline pgno_t pgno_sub(pgno_t base, pgno_t subtrahend) { + assert(base >= MIN_PAGENO); + return (subtrahend < base - MIN_PAGENO) ? base - subtrahend : MIN_PAGENO; +} + +static __maybe_unused __inline void mdbx_jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (MDBX_DBG_JITTER & mdbx_runtime_flags) + mdbx_osal_jitter(tiny); +#else + (void)tiny; +#endif +} diff --git a/src/contrib/db/libmdbx/src/elements/lck-posix.c b/src/contrib/db/libmdbx/src/elements/lck-posix.c new file mode 100644 index 0000000..147778d --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/lck-posix.c @@ -0,0 +1,551 @@ +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "internals.h" + +/*----------------------------------------------------------------------------*/ +/* global constructor/destructor */ + +#if defined(__linux__) || defined(__gnu_linux__) +#include +#ifndef MDBX_ALLOY +uint32_t mdbx_linux_kernel_version; +#endif /* MDBX_ALLOY */ +#endif /* Linux */ + +static __cold __attribute__((__constructor__)) void +mdbx_global_constructor(void) { +#if defined(__linux__) || defined(__gnu_linux__) + struct utsname buffer; + if (uname(&buffer) == 0) { + int i = 0; + char *p = buffer.release; + while (*p && i < 4) { + if (*p >= '0' && *p <= '9') { + long number = strtol(p, &p, 10); + if (number > 0) { + if (number > 255) + number = 255; + mdbx_linux_kernel_version += number << (24 - i * 8); + } + ++i; + } else { + ++p; + } + } + } +#endif /* Linux */ + + mdbx_rthc_global_init(); +} + +static __cold __attribute__((__destructor__)) void +mdbx_global_destructor(void) { + mdbx_rthc_global_dtor(); +} + +/*----------------------------------------------------------------------------*/ +/* lck */ + +/* ОпиÑание реализации блокировок Ð´Ð»Ñ POSIX & Linux: + * + * lck-файл отображаетÑÑ Ð² памÑÑ‚ÑŒ, в нём организуетÑÑ Ñ‚Ð°Ð±Ð»Ð¸Ñ†Ð° читателей и + * размещаютÑÑ ÑовмеÑтно иÑпользуемые posix-мьютекÑÑ‹ (futex). ПоÑредÑтвом + * Ñтих мьютекÑов (Ñм struct MDBX_lockinfo) реализуютÑÑ: + * - Блокировка таблицы читателей Ð´Ð»Ñ Ñ€ÐµÐ³Ð¸Ñтрации, + * Ñ‚.е. функции mdbx_rdt_lock() и mdbx_rdt_unlock(). + * - Блокировка БД Ð´Ð»Ñ Ð¿Ð¸ÑˆÑƒÑ‰Ð¸Ñ… транзакций, + * Ñ‚.е. функции mdbx_txn_lock() и mdbx_txn_unlock(). + * + * ОÑтальной функционал реализуетÑÑ Ð¾Ñ‚Ð´ÐµÐ»ÑŒÐ½Ð¾ поÑредÑтвом файловых блокировок: + * - Первоначальный захват БД в режиме exclusive/shared и поÑледующий перевод + * в операционный режим, функции mdbx_lck_seize() и mdbx_lck_downgrade(). + * - Проверка приÑутÑтвие процеÑÑов-читателей, + * Ñ‚.е. функции mdbx_rpid_set(), mdbx_rpid_clear() и mdbx_rpid_check(). + * + * Ð”Ð»Ñ Ð±Ð»Ð¾ÐºÐ¸Ñ€Ð¾Ð²ÐºÐ¸ файлов иÑпользуетÑÑ fcntl(F_SETLK), так как: + * - lockf() оперирует только ÑкÑклюзивной блокировкой и требует + * Ð¾Ñ‚ÐºÑ€Ñ‹Ñ‚Ð¸Ñ Ñ„Ð°Ð¹Ð»Ð° в RW-режиме. + * - flock() не гарантирует атомарноÑти при Ñмене блокировок + * и оперирует только вÑем файлом целиком. + * - Ð”Ð»Ñ ÐºÐ¾Ð½Ñ‚Ñ€Ð¾Ð»Ñ Ð¿Ñ€Ð¾Ñ†ÐµÑÑов-читателей иÑпользуютÑÑ Ð¾Ð´Ð½Ð¾Ð±Ð°Ð¹Ñ‚Ð¾Ð²Ñ‹Ðµ + * range-блокировки lck-файла поÑредÑтвом fcntl(F_SETLK). При Ñтом + * в качеÑтве позиции иÑпользуетÑÑ pid процеÑÑа-читателÑ. + * - Ð”Ð»Ñ Ð¿ÐµÑ€Ð²Ð¾Ð½Ð°Ñ‡Ð°Ð»ÑŒÐ½Ð¾Ð³Ð¾ захвата и shared/exclusive выполнÑетÑÑ Ð±Ð»Ð¾ÐºÐ¸Ñ€Ð¾Ð²ÐºÐ° + * оÑновного файла БД и при уÑпехе lck-файла. + * + * ---------------------------------------------------------------------------- + * УДЕРЖИВÐЕМЫЕ БЛОКИРОВКИ Ð’ ЗÐВИСИМОСТИ ОТ РЕЖИМРИ СОСТОЯÐИЯ + * + * ЭкÑклюзивный режим без lck-файла: + * = заблокирован веÑÑŒ dxb-файл поÑредÑтвом F_RDLCK или F_WRLCK, + * в завиÑимоÑти от MDBX_RDONLY. + * + * Ðе-операционный режим на Ð²Ñ€ÐµÐ¼Ñ Ð¿ÐµÑ€Ðµ-инициализации и разрушении lck-файла: + * = F_WRLCK блокировка первого байта lck-файла, другие процеÑÑÑ‹ ждут её + * ÑнÑÑ‚Ð¸Ñ Ð¿Ñ€Ð¸ получении F_RDLCK через F_SETLKW. + * - блокировки dxb-файла могут менÑÑ‚ÑŒÑÑ Ð´Ð¾ ÑнÑтие ÑкÑклюзивной блокировки + * lck-файла: + * + Ð´Ð»Ñ ÐЕ-ÑкÑклюзивного режима блокировка pid-байта в dxb-файле + * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY. + * + Ð´Ð»Ñ Ð­ÐšÐ¡ÐšÐ›Ð®Ð—Ð˜Ð’ÐОГО режима блокировка pid-байта вÑего dxb-файла + * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY. + * + * ОПЕРÐЦИОÐÐЫЙ режим Ñ lck-файлом: + * = F_RDLCK блокировка первого байта lck-файла, другие процеÑÑÑ‹ не могут + * получить F_WRLCK и таким образом видÑÑ‚ что БД иÑпользуетÑÑ. + * + F_WRLCK блокировка pid-байта в clk-файле поÑле первой транзакции чтениÑ. + * + Ð´Ð»Ñ ÐЕ-ÑкÑклюзивного режима блокировка pid-байта в dxb-файле + * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY. + * + Ð´Ð»Ñ Ð­ÐšÐ¡ÐšÐ›Ð®Ð—Ð˜Ð’ÐОГО режима блокировка pid-байта вÑего dxb-файла + * поÑредÑтвом F_RDLCK или F_WRLCK, в завиÑимоÑти от MDBX_RDONLY. + */ + +#if MDBX_USE_OFDLOCKS +static int op_setlk, op_setlkw, op_getlk; +static void __cold choice_fcntl() { + assert(!op_setlk && !op_setlkw && !op_getlk); + if ((mdbx_runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 +#if defined(__linux__) || defined(__gnu_linux__) + && mdbx_linux_kernel_version > + 0x030f0000 /* OFD locks are available since 3.15, but engages here + only for 3.16 and larer kernels (LTS) for reliability reasons */ +#endif /* linux */ + ) { + op_setlk = F_OFD_SETLK; + op_setlkw = F_OFD_SETLKW; + op_getlk = F_OFD_GETLK; + return; + } + op_setlk = F_SETLK; + op_setlkw = F_SETLKW; + op_getlk = F_GETLK; +} +#else +#define op_setlk F_SETLK +#define op_setlkw F_SETLKW +#define op_getlk F_GETLK +#endif /* MDBX_USE_OFDLOCKS */ + +#ifndef OFF_T_MAX +#define OFF_T_MAX \ + ((sizeof(off_t) > 4 ? INT64_MAX : INT32_MAX) & ~(size_t)0xffff) +#endif + +static int lck_op(mdbx_filehandle_t fd, int cmd, int lck, off_t offset, + off_t len) { + mdbx_jitter4testing(true); + for (;;) { + struct flock lock_op; + memset(&lock_op, 0, sizeof(lock_op)); + lock_op.l_type = lck; + lock_op.l_whence = SEEK_SET; + lock_op.l_start = offset; + lock_op.l_len = len; + int rc = fcntl(fd, cmd, &lock_op); + mdbx_jitter4testing(true); + if (rc != -1) { + if (cmd == op_getlk) { + /* Checks reader by pid. Returns: + * MDBX_RESULT_TRUE - if pid is live (unable to acquire lock) + * MDBX_RESULT_FALSE - if pid is dead (lock acquired). */ + return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE + : MDBX_RESULT_TRUE; + } + return MDBX_SUCCESS; + } + rc = errno; + if (rc != EINTR || cmd == op_setlkw) { + mdbx_assert(nullptr, MDBX_IS_ERROR(rc)); + return rc; + } + } +} + +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) { + assert(env->me_lfd != INVALID_HANDLE_VALUE); + assert(env->me_pid > 0); + if (unlikely(mdbx_getpid() != env->me_pid)) + return MDBX_PANIC; + return lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1); +} + +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) { + assert(env->me_lfd != INVALID_HANDLE_VALUE); + assert(env->me_pid > 0); + return lck_op(env->me_lfd, op_setlk, F_UNLCK, env->me_pid, 1); +} + +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) { + assert(env->me_lfd != INVALID_HANDLE_VALUE); + assert(pid > 0); + return lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1); +} + +/*---------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int __cold mdbx_lck_seize(MDBX_env *env) { + assert(env->me_fd != INVALID_HANDLE_VALUE); + if (unlikely(mdbx_getpid() != env->me_pid)) + return MDBX_PANIC; +#if MDBX_USE_OFDLOCKS + if (unlikely(op_setlk == 0)) + choice_fcntl(); +#endif /* MDBX_USE_OFDLOCKS */ + + int rc; + if (env->me_lfd == INVALID_HANDLE_VALUE) { + /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ + rc = + lck_op(env->me_fd, op_setlk, + (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); + if (rc != MDBX_SUCCESS) { + mdbx_error("%s(%s) failed: errcode %u", __func__, "without-lck", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; + } + + /* Firstly try to get exclusive locking. */ + rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1); + if (rc == MDBX_SUCCESS) { + continue_dxb_exclusive: + rc = + lck_op(env->me_fd, op_setlk, + (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); + if (rc == MDBX_SUCCESS) + return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; + + /* the cause may be a collision with POSIX's file-lock recovery. */ + if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || + rc == EDEADLK)) { + mdbx_error("%s(%s) failed: errcode %u", __func__, "dxb-exclusive", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Fallback to lck-shared */ + rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1); + if (rc != MDBX_SUCCESS) { + mdbx_error("%s(%s) failed: errcode %u", __func__, "fallback-shared", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + /* Done: return with shared locking. */ + return MDBX_RESULT_FALSE; + } + + /* Wait for lck-shared now. */ + /* Here may be await during transient processes, for instance until another + * competing process doesn't call lck_downgrade(). */ + rc = lck_op(env->me_lfd, op_setlkw, F_RDLCK, 0, 1); + if (rc != MDBX_SUCCESS) { + mdbx_error("%s(%s) failed: errcode %u", __func__, "try-shared", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Lock against another process operating in without-lck or exclusive mode. */ + rc = + lck_op(env->me_fd, op_setlk, + (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->me_pid, 1); + if (rc != MDBX_SUCCESS) { + mdbx_error("%s(%s) failed: errcode %u", __func__, + "lock-against-without-lck", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* got shared, retry exclusive */ + rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1); + if (rc == MDBX_SUCCESS) + goto continue_dxb_exclusive; + + if (rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || + rc == EDEADLK) + return MDBX_RESULT_FALSE /* Done: exclusive is unavailable, + but shared locks are alive. */ + ; + + mdbx_error("%s(%s) failed: errcode %u", __func__, "try-exclusive", rc); + mdbx_assert(env, MDBX_IS_ERROR(rc)); + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) { + assert(env->me_lfd != INVALID_HANDLE_VALUE); + if (unlikely(mdbx_getpid() != env->me_pid)) + return MDBX_PANIC; + + int rc = MDBX_SUCCESS; + if ((env->me_flags & MDBX_EXCLUSIVE) == 0) { + rc = lck_op(env->me_fd, op_setlk, F_UNLCK, 0, env->me_pid); + if (rc == MDBX_SUCCESS) + rc = lck_op(env->me_fd, op_setlk, F_UNLCK, env->me_pid + 1, + OFF_T_MAX - env->me_pid - 1); + } + if (rc == MDBX_SUCCESS) + rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1); + if (unlikely(rc != 0)) { + mdbx_error("%s(%s) failed: errcode %u", __func__, "lck", rc); + assert(MDBX_IS_ERROR(rc)); + } + return rc; +} + +MDBX_INTERNAL_FUNC int __cold mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor) { + if (unlikely(mdbx_getpid() != env->me_pid)) + return MDBX_PANIC; + + int rc = MDBX_SUCCESS; + if (env->me_lfd != INVALID_HANDLE_VALUE && !inprocess_neighbor && + env->me_lck && + /* try get exclusive access */ + lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 && + lck_op(env->me_fd, op_setlk, + (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX)) { + mdbx_verbose("%s: got exclusive, drown mutexes", __func__); + rc = pthread_mutex_destroy(&env->me_lck->mti_rmutex); + if (rc == 0) + rc = pthread_mutex_destroy(&env->me_lck->mti_wmutex); + mdbx_assert(env, rc == 0); + if (rc == 0) { + memset(env->me_lck, 0x81, sizeof(MDBX_lockinfo)); + msync(env->me_lck, env->me_os_psize, MS_ASYNC); + } + mdbx_jitter4testing(false); + } + + /* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored + * after file was closed. + * + * 2) File locks would be released (by kernel) while the file-descriptors will + * be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel, + * locks should be released here explicitly with properly order. */ + + /* close dxb and restore lock */ + if (env->me_fd != INVALID_HANDLE_VALUE) { + if (unlikely(close(env->me_fd) != 0) && rc == MDBX_SUCCESS) + rc = errno; + env->me_fd = INVALID_HANDLE_VALUE; + if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { + /* restore file-lock */ + rc = lck_op( + inprocess_neighbor->me_fd, F_SETLKW, + (inprocess_neighbor->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, + (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) + ? 0 + : inprocess_neighbor->me_pid, + (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1); + } + } + + /* close clk and restore locks */ + if (env->me_lfd != INVALID_HANDLE_VALUE) { + if (unlikely(close(env->me_lfd) != 0) && rc == MDBX_SUCCESS) + rc = errno; + env->me_lfd = INVALID_HANDLE_VALUE; + if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { + /* restore file-locks */ + rc = lck_op(inprocess_neighbor->me_lfd, F_SETLKW, F_RDLCK, 0, 1); + if (rc == MDBX_SUCCESS && inprocess_neighbor->me_live_reader) + rc = mdbx_rpid_set(inprocess_neighbor); + } + } + + if (inprocess_neighbor && rc != MDBX_SUCCESS) + inprocess_neighbor->me_flags |= MDBX_FATAL_ERROR; + return rc; +} + +/*---------------------------------------------------------------------------*/ + +static int mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex, + const int rc); + +MDBX_INTERNAL_FUNC int __cold mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag) { + if (inprocess_neighbor) + return MDBX_SUCCESS /* currently don't need any initialization + if LCK already opened/used inside current process */ + ; + + /* FIXME: Unfortunately, there is no other reliable way but to long testing + * on each platform. On the other hand, behavior like FreeBSD is incorrect + * and we can expect it to be rare. Moreover, even on FreeBSD without + * additional in-process initialization, the probability of an problem + * occurring is vanishingly small, and the symptom is a return of EINVAL + * while locking a mutex. In other words, in the worst case, the problem + * results in an EINVAL error at the start of the transaction, but NOT data + * loss, nor database corruption, nor other fatal troubles. Thus, the code + * below I am inclined to think the workaround for erroneous platforms (like + * FreeBSD), rather than a defect of libmdbx. */ +#if defined(__FreeBSD__) + /* seems that shared mutexes on FreeBSD required in-process initialization */ + (void)global_uniqueness_flag; +#else + /* shared mutexes on many other platforms (including Darwin and Linux's + * futexes) doesn't need any addition in-process initialization */ + if (global_uniqueness_flag != MDBX_RESULT_TRUE) + return MDBX_SUCCESS; +#endif + + pthread_mutexattr_t ma; + int rc = pthread_mutexattr_init(&ma); + if (rc) + return rc; + + rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED); + if (rc) + goto bailout; + +#if MDBX_USE_ROBUST +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) && \ + !defined(pthread_mutex_consistent) && _POSIX_C_SOURCE < 200809L + rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); +#else + rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); +#endif + if (rc) + goto bailout; +#endif /* MDBX_USE_ROBUST */ + +#if _POSIX_C_SOURCE >= 199506L && !defined(MDBX_SAFE4QEMU) + rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT); + if (rc == ENOTSUP) + rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE); + if (rc) + goto bailout; +#endif /* PTHREAD_PRIO_INHERIT */ + + rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK); + if (rc) + goto bailout; + + rc = pthread_mutex_init(&env->me_lck->mti_rmutex, &ma); + if (rc) + goto bailout; + rc = pthread_mutex_init(&env->me_lck->mti_wmutex, &ma); + +bailout: + pthread_mutexattr_destroy(&ma); + return rc; +} + +static int mdbx_robust_lock(MDBX_env *env, pthread_mutex_t *mutex) { + mdbx_jitter4testing(true); + int rc = pthread_mutex_lock(mutex); + if (unlikely(rc != 0)) + rc = mdbx_mutex_failed(env, mutex, rc); + return rc; +} + +static int mdbx_robust_trylock(MDBX_env *env, pthread_mutex_t *mutex) { + mdbx_jitter4testing(true); + int rc = pthread_mutex_trylock(mutex); + if (unlikely(rc != 0 && rc != EBUSY)) + rc = mdbx_mutex_failed(env, mutex, rc); + return (rc != EBUSY) ? rc : MDBX_BUSY; +} + +static int mdbx_robust_unlock(MDBX_env *env, pthread_mutex_t *mutex) { + int rc = pthread_mutex_unlock(mutex); + mdbx_jitter4testing(true); + if (unlikely(rc != 0)) + env->me_flags |= MDBX_FATAL_ERROR; + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) { + mdbx_trace("%s", ">>"); + int rc = mdbx_robust_lock(env, &env->me_lck->mti_rmutex); + mdbx_trace("<< rc %d", rc); + return rc; +} + +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) { + mdbx_trace("%s", ">>"); + int rc = mdbx_robust_unlock(env, &env->me_lck->mti_rmutex); + mdbx_trace("<< rc %d", rc); + if (unlikely(MDBX_IS_ERROR(rc))) + mdbx_panic("%s() failed: errcode %d\n", __func__, rc); +} + +int mdbx_txn_lock(MDBX_env *env, bool dontwait) { + mdbx_trace("%s", ">>"); + int rc = dontwait ? mdbx_robust_trylock(env, env->me_wmutex) + : mdbx_robust_lock(env, env->me_wmutex); + mdbx_trace("<< rc %d", rc); + return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS; +} + +void mdbx_txn_unlock(MDBX_env *env) { + mdbx_trace("%s", ">>"); + int rc = mdbx_robust_unlock(env, env->me_wmutex); + mdbx_trace("<< rc %d", rc); + if (unlikely(MDBX_IS_ERROR(rc))) + mdbx_panic("%s() failed: errcode %d\n", __func__, rc); +} + +static int __cold mdbx_mutex_failed(MDBX_env *env, pthread_mutex_t *mutex, + const int err) { + int rc = err; +#if MDBX_USE_ROBUST + if (err == EOWNERDEAD) { + /* We own the mutex. Clean up after dead previous owner. */ + + int rlocked = (env->me_lck && mutex == &env->me_lck->mti_rmutex); + rc = MDBX_SUCCESS; + if (!rlocked) { + if (unlikely(env->me_txn)) { + /* env is hosed if the dead thread was ours */ + env->me_flags |= MDBX_FATAL_ERROR; + env->me_txn = NULL; + rc = MDBX_PANIC; + } + } + mdbx_notice("%cmutex owner died, %s", (rlocked ? 'r' : 'w'), + (rc ? "this process' env is hosed" : "recovering")); + + int check_rc = mdbx_reader_check0(env, rlocked, NULL); + check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc; + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) && \ + !defined(pthread_mutex_consistent) && _POSIX_C_SOURCE < 200809L + int mreco_rc = pthread_mutex_consistent_np(mutex); +#else + int mreco_rc = pthread_mutex_consistent(mutex); +#endif + check_rc = (mreco_rc == 0) ? check_rc : mreco_rc; + + if (unlikely(mreco_rc)) + mdbx_error("mutex recovery failed, %s", mdbx_strerror(mreco_rc)); + + rc = (rc == MDBX_SUCCESS) ? check_rc : rc; + if (MDBX_IS_ERROR(rc)) + pthread_mutex_unlock(mutex); + return rc; + } +#else + (void)mutex; +#endif /* MDBX_USE_ROBUST */ + + mdbx_error("mutex (un)lock failed, %s", mdbx_strerror(err)); + if (rc != EDEADLK) + env->me_flags |= MDBX_FATAL_ERROR; + return rc; +} diff --git a/src/contrib/db/libmdbx/src/elements/lck-windows.c b/src/contrib/db/libmdbx/src/elements/lck-windows.c new file mode 100644 index 0000000..5d74bb8 --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/lck-windows.c @@ -0,0 +1,777 @@ +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "internals.h" + +/* PREAMBLE FOR WINDOWS: + * + * We are not concerned for performance here. + * If you are running Windows a performance could NOT be the goal. + * Otherwise please use Linux. + * + * Regards, + * LY + */ + +static void mdbx_winnt_import(void); + +#if MDBX_BUILD_SHARED_LIBRARY +#if MDBX_AVOID_CRT && defined(NDEBUG) +/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks. + * + * Define dll's entry point only for Release build when NDEBUG is defined and + * MDBX_AVOID_CRT=ON. if the entry point isn't defined then MSVC's will + * automatically use DllMainCRTStartup() from CRT library, which also + * automatically call DllMain() from our mdbx.dll */ +#pragma comment(linker, "/ENTRY:DllMain") +#endif /* MDBX_AVOID_CRT */ + +BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved) +#else +#if !MDBX_CONFIG_MANUAL_TLS_CALLBACK +static +#endif /* !MDBX_CONFIG_MANUAL_TLS_CALLBACK */ + void NTAPI + mdbx_dll_handler(PVOID module, DWORD reason, PVOID reserved) +#endif /* MDBX_BUILD_SHARED_LIBRARY */ +{ + (void)reserved; + switch (reason) { + case DLL_PROCESS_ATTACH: + mdbx_winnt_import(); + mdbx_rthc_global_init(); + break; + case DLL_PROCESS_DETACH: + mdbx_rthc_global_dtor(); + break; + + case DLL_THREAD_ATTACH: + break; + case DLL_THREAD_DETACH: + mdbx_rthc_thread_dtor(module); + break; + } +#if MDBX_BUILD_SHARED_LIBRARY + return TRUE; +#endif +} + +#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(_MSC_VER) +# pragma const_seg(push) +# pragma data_seg(push) + +# ifdef _WIN64 + /* kick a linker to create the TLS directory if not already done */ +# pragma comment(linker, "/INCLUDE:_tls_used") + /* Force some symbol references. */ +# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor") + /* specific const-segment for WIN64 */ +# pragma const_seg(".CRT$XLB") + const +# else + /* kick a linker to create the TLS directory if not already done */ +# pragma comment(linker, "/INCLUDE:__tls_used") + /* Force some symbol references. */ +# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor") + /* specific data-segment for WIN32 */ +# pragma data_seg(".CRT$XLB") +# endif + + __declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_dll_handler; +# pragma data_seg(pop) +# pragma const_seg(pop) + +#elif defined(__GNUC__) +# ifdef _WIN64 + const +# endif + PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_dll_handler; +#else +# error FIXME +#endif +/* *INDENT-ON* */ +/* clang-format on */ +#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_CONFIG_MANUAL_TLS_CALLBACK */ + +/*----------------------------------------------------------------------------*/ + +#define LCK_SHARED 0 +#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK +#define LCK_WAITFOR 0 +#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY + +static __inline BOOL flock(mdbx_filehandle_t fd, DWORD flags, uint64_t offset, + size_t bytes) { + OVERLAPPED ov; + ov.hEvent = 0; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); + return LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov); +} + +static __inline BOOL funlock(mdbx_filehandle_t fd, uint64_t offset, + size_t bytes) { + return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, + HIGH_DWORD(bytes)); +} + +/*----------------------------------------------------------------------------*/ +/* global `write` lock for write-txt processing, + * exclusive locking both meta-pages) */ + +#define LCK_MAXLEN (1u + (size_t)(MAXSSIZE_T)) +#define LCK_META_OFFSET 0 +#define LCK_META_LEN 0x10000u +#define LCK_BODY_OFFSET LCK_META_LEN +#define LCK_BODY_LEN (LCK_MAXLEN - LCK_BODY_OFFSET) +#define LCK_META LCK_META_OFFSET, LCK_META_LEN +#define LCK_BODY LCK_BODY_OFFSET, LCK_BODY_LEN +#define LCK_WHOLE 0, LCK_MAXLEN + +int mdbx_txn_lock(MDBX_env *env, bool dontwait) { + if (dontwait) { + if (!TryEnterCriticalSection(&env->me_windowsbug_lock)) + return MDBX_BUSY; + } else { + EnterCriticalSection(&env->me_windowsbug_lock); + } + + if ((env->me_flags & MDBX_EXCLUSIVE) || + flock(env->me_fd, + dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) + : (LCK_EXCLUSIVE | LCK_WAITFOR), + LCK_BODY)) + return MDBX_SUCCESS; + int rc = GetLastError(); + LeaveCriticalSection(&env->me_windowsbug_lock); + return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY; +} + +void mdbx_txn_unlock(MDBX_env *env) { + int rc = + (env->me_flags & MDBX_EXCLUSIVE) ? TRUE : funlock(env->me_fd, LCK_BODY); + LeaveCriticalSection(&env->me_windowsbug_lock); + if (!rc) + mdbx_panic("%s failed: errcode %u", __func__, GetLastError()); +} + +/*----------------------------------------------------------------------------*/ +/* global `read` lock for readers registration, + * exclusive locking `mti_numreaders` (second) cacheline */ + +#define LCK_LO_OFFSET 0 +#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders) +#define LCK_UP_OFFSET LCK_LO_LEN +#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET) +#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN +#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN + +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env) { + mdbx_srwlock_AcquireShared(&env->me_remap_guard); + if (env->me_lfd == INVALID_HANDLE_VALUE) + return MDBX_SUCCESS; /* readonly database in readonly filesystem */ + + /* transite from S-? (used) to S-E (locked), e.g. exclusive lock upper-part */ + if ((env->me_flags & MDBX_EXCLUSIVE) || + flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) + return MDBX_SUCCESS; + + int rc = GetLastError(); + mdbx_srwlock_ReleaseShared(&env->me_remap_guard); + return rc; +} + +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env) { + if (env->me_lfd != INVALID_HANDLE_VALUE) { + /* transite from S-E (locked) to S-? (used), e.g. unlock upper-part */ + if ((env->me_flags & MDBX_EXCLUSIVE) == 0 && + !funlock(env->me_lfd, LCK_UPPER)) + mdbx_panic("%s failed: errcode %u", __func__, GetLastError()); + } + mdbx_srwlock_ReleaseShared(&env->me_remap_guard); +} + +static int suspend_and_append(mdbx_handle_array_t **array, + const DWORD ThreadId) { + const unsigned limit = (*array)->limit; + if ((*array)->count == limit) { + void *ptr = mdbx_realloc( + (limit > ARRAY_LENGTH((*array)->handles)) + ? *array + : /* don't free initial array on the stack */ NULL, + sizeof(mdbx_handle_array_t) + + sizeof(HANDLE) * (limit * 2 - ARRAY_LENGTH((*array)->handles))); + if (!ptr) + return MDBX_ENOMEM; + if (limit == ARRAY_LENGTH((*array)->handles)) + memcpy(ptr, *array, sizeof(mdbx_handle_array_t)); + *array = (mdbx_handle_array_t *)ptr; + (*array)->limit = limit * 2; + } + + HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, + FALSE, ThreadId); + if (hThread == NULL) + return GetLastError(); + + if (SuspendThread(hThread) == -1) { + int err = GetLastError(); + DWORD ExitCode; + if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || + !GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE) + err = MDBX_SUCCESS; + CloseHandle(hThread); + return err; + } + + (*array)->handles[(*array)->count++] = hThread; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) { + const size_t CurrentTid = GetCurrentThreadId(); + int rc; + if (env->me_lck) { + /* Scan LCK for threads of the current process */ + const MDBX_reader *const begin = env->me_lck->mti_readers; + const MDBX_reader *const end = begin + env->me_lck->mti_numreaders; + const size_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0; + for (const MDBX_reader *reader = begin; reader < end; ++reader) { + if (reader->mr_pid != env->me_pid || !reader->mr_tid) { + skip_lck: + continue; + } + if (reader->mr_tid == CurrentTid || reader->mr_tid == WriteTxnOwner) + goto skip_lck; + if (env->me_flags & MDBX_NOTLS) { + /* Skip duplicates in no-tls mode */ + for (const MDBX_reader *scan = reader; --scan >= begin;) + if (scan->mr_tid == reader->mr_tid) + goto skip_lck; + } + + rc = suspend_and_append(array, (mdbx_tid_t)reader->mr_tid); + if (rc != MDBX_SUCCESS) { + bailout_lck: + (void)mdbx_resume_threads_after_remap(*array); + return rc; + } + } + if (WriteTxnOwner && WriteTxnOwner != CurrentTid) { + rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner); + if (rc != MDBX_SUCCESS) + goto bailout_lck; + } + } else { + /* Without LCK (i.e. read-only mode). + * Walk thougth a snapshot of all running threads */ + mdbx_assert(env, + env->me_txn0 == NULL || (env->me_flags & MDBX_EXCLUSIVE) != 0); + const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (hSnapshot == INVALID_HANDLE_VALUE) + return GetLastError(); + + THREADENTRY32 entry; + entry.dwSize = sizeof(THREADENTRY32); + + if (!Thread32First(hSnapshot, &entry)) { + rc = GetLastError(); + bailout_toolhelp: + CloseHandle(hSnapshot); + (void)mdbx_resume_threads_after_remap(*array); + return rc; + } + + do { + if (entry.th32OwnerProcessID != env->me_pid || + entry.th32ThreadID == CurrentTid) + continue; + + rc = suspend_and_append(array, entry.th32ThreadID); + if (rc != MDBX_SUCCESS) + goto bailout_toolhelp; + + } while (Thread32Next(hSnapshot, &entry)); + + rc = GetLastError(); + if (rc != ERROR_NO_MORE_FILES) + goto bailout_toolhelp; + CloseHandle(hSnapshot); + } + + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array) { + int rc = MDBX_SUCCESS; + for (unsigned i = 0; i < array->count; ++i) { + const HANDLE hThread = array->handles[i]; + if (ResumeThread(hThread) == -1) { + const int err = GetLastError(); + DWORD ExitCode; + if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && + GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE) + rc = err; + } + CloseHandle(hThread); + } + return rc; +} + +/*----------------------------------------------------------------------------*/ +/* global `initial` lock for lockfile initialization, + * exclusive/shared locking first cacheline */ + +/* Briefly descritpion of locking schema/algorithm: + * - Windows does not support upgrading or downgrading for file locking. + * - Therefore upgrading/downgrading is emulated by shared and exclusive + * locking of upper and lower halves. + * - In other words, we have FSM with possible 9 states, + * i.e. free/shared/exclusive x free/shared/exclusive == 9. + * Only 6 states of FSM are used, which 2 of ones are transitive. + * + * The mdbx_lck_seize() moves the locking-FSM from the initial free/unlocked + * state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible, + * or to the "used" (and returns MDBX_RESULT_FALSE). + * + * The mdbx_lck_downgrade() moves the locking-FSM from "exclusive write" + * state to the "used" (i.e. shared) state. + * + * States: + * ?-? = free, i.e. unlocked + * S-? = used, i.e. shared lock + * E-? = exclusive-read, i.e. operational exclusive + * ?-S + * ?-E = middle (transitive state) + * S-S + * S-E = locked (transitive state) + * E-S + * E-E = exclusive-write, i.e. exclusive due (re)initialization + */ + +static void lck_unlock(MDBX_env *env) { + int rc; + + if (env->me_lfd != INVALID_HANDLE_VALUE) { + /* double `unlock` for robustly remove overlapped shared/exclusive locks */ + while (funlock(env->me_lfd, LCK_LOWER)) + ; + rc = GetLastError(); + assert(rc == ERROR_NOT_LOCKED); + (void)rc; + SetLastError(ERROR_SUCCESS); + + while (funlock(env->me_lfd, LCK_UPPER)) + ; + rc = GetLastError(); + assert(rc == ERROR_NOT_LOCKED); + (void)rc; + SetLastError(ERROR_SUCCESS); + } + + if (env->me_fd != INVALID_HANDLE_VALUE) { + /* explicitly unlock to avoid latency for other processes (windows kernel + * releases such locks via deferred queues) */ + while (funlock(env->me_fd, LCK_BODY)) + ; + rc = GetLastError(); + assert(rc == ERROR_NOT_LOCKED); + (void)rc; + SetLastError(ERROR_SUCCESS); + + while (funlock(env->me_fd, LCK_META)) + ; + rc = GetLastError(); + assert(rc == ERROR_NOT_LOCKED); + (void)rc; + SetLastError(ERROR_SUCCESS); + + while (funlock(env->me_fd, LCK_WHOLE)) + ; + rc = GetLastError(); + assert(rc == ERROR_NOT_LOCKED); + (void)rc; + SetLastError(ERROR_SUCCESS); + } +} + +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag) { + (void)env; + (void)inprocess_neighbor; + (void)global_uniqueness_flag; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor) { + (void)inprocess_neighbor; + + /* LY: should unmap before releasing the locks to avoid race condition and + * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */ + if (env->me_map) + mdbx_munmap(&env->me_dxb_mmap); + if (env->me_lck) + mdbx_munmap(&env->me_lck_mmap); + + lck_unlock(env); + return MDBX_SUCCESS; +} + +/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE) + * or as 'used' (S-? and returns MDBX_RESULT_FALSE). + * Oherwise returns an error. */ +static int internal_seize_lck(HANDLE lfd) { + int rc; + assert(lfd != INVALID_HANDLE_VALUE); + + /* 1) now on ?-? (free), get ?-E (middle) */ + mdbx_jitter4testing(false); + if (!flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER)) { + rc = GetLastError() /* 2) something went wrong, give up */; + mdbx_error("%s(%s) failed: errcode %u", __func__, + "?-?(free) >> ?-E(middle)", rc); + return rc; + } + + /* 3) now on ?-E (middle), try E-E (exclusive-write) */ + mdbx_jitter4testing(false); + if (flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER)) + return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */; + + /* 5) still on ?-E (middle) */ + rc = GetLastError(); + mdbx_jitter4testing(false); + if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) { + /* 6) something went wrong, give up */ + if (!funlock(lfd, LCK_UPPER)) + mdbx_panic("%s(%s) failed: errcode %u", __func__, + "?-E(middle) >> ?-?(free)", GetLastError()); + return rc; + } + + /* 7) still on ?-E (middle), try S-E (locked) */ + mdbx_jitter4testing(false); + rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER) ? MDBX_RESULT_FALSE + : GetLastError(); + + mdbx_jitter4testing(false); + if (rc != MDBX_RESULT_FALSE) + mdbx_error("%s(%s) failed: errcode %u", __func__, + "?-E(middle) >> S-E(locked)", rc); + + /* 8) now on S-E (locked) or still on ?-E (middle), + * transite to S-? (used) or ?-? (free) */ + if (!funlock(lfd, LCK_UPPER)) + mdbx_panic("%s(%s) failed: errcode %u", __func__, + "X-E(locked/middle) >> X-?(used/free)", GetLastError()); + + /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */ + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env) { + int rc; + + assert(env->me_fd != INVALID_HANDLE_VALUE); + if (env->me_flags & MDBX_EXCLUSIVE) + return MDBX_RESULT_TRUE /* nope since files were must be opened + non-shareable */ + ; + + if (env->me_lfd == INVALID_HANDLE_VALUE) { + /* LY: without-lck mode (e.g. on read-only filesystem) */ + mdbx_jitter4testing(false); + if (!flock(env->me_fd, LCK_SHARED | LCK_DONTWAIT, LCK_WHOLE)) { + rc = GetLastError(); + mdbx_error("%s(%s) failed: errcode %u", __func__, "without-lck", rc); + return rc; + } + return MDBX_RESULT_FALSE; + } + + rc = internal_seize_lck(env->me_lfd); + mdbx_jitter4testing(false); + if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) { + /* Check that another process don't operates in without-lck mode. + * Doing such check by exclusive locking the body-part of db. Should be + * noted: + * - we need an exclusive lock for do so; + * - we can't lock meta-pages, otherwise other process could get an error + * while opening db in valid (non-conflict) mode. */ + if (!flock(env->me_fd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_BODY)) { + rc = GetLastError(); + mdbx_error("%s(%s) failed: errcode %u", __func__, + "lock-against-without-lck", rc); + mdbx_jitter4testing(false); + lck_unlock(env); + } else { + mdbx_jitter4testing(false); + if (!funlock(env->me_fd, LCK_BODY)) + mdbx_panic("%s(%s) failed: errcode %u", __func__, + "unlock-against-without-lck", GetLastError()); + } + } + + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env) { + /* Transite from exclusive state (E-?) to used (S-?) */ + assert(env->me_fd != INVALID_HANDLE_VALUE); + assert(env->me_lfd != INVALID_HANDLE_VALUE); + +#if 1 + if (env->me_flags & MDBX_EXCLUSIVE) + return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ + ; +#else + /* 1) must be at E-E (exclusive-write) */ + if (env->me_flags & MDBX_EXCLUSIVE) { + /* transite from E-E to E_? (exclusive-read) */ + if (!funlock(env->me_lfd, LCK_UPPER)) + mdbx_panic("%s(%s) failed: errcode %u", __func__, + "E-E(exclusive-write) >> E-?(exclusive-read)", GetLastError()); + return MDBX_SUCCESS /* 2) now at E-? (exclusive-read), done */; + } +#endif + + /* 3) now at E-E (exclusive-write), transite to ?_E (middle) */ + if (!funlock(env->me_lfd, LCK_LOWER)) + mdbx_panic("%s(%s) failed: errcode %u", __func__, + "E-E(exclusive-write) >> ?-E(middle)", GetLastError()); + + /* 4) now at ?-E (middle), transite to S-E (locked) */ + if (!flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER)) { + int rc = GetLastError() /* 5) something went wrong, give up */; + mdbx_error("%s(%s) failed: errcode %u", __func__, + "?-E(middle) >> S-E(locked)", rc); + return rc; + } + + /* 6) got S-E (locked), continue transition to S-? (used) */ + if (!funlock(env->me_lfd, LCK_UPPER)) + mdbx_panic("%s(%s) failed: errcode %u", __func__, + "S-E(locked) >> S-?(used)", GetLastError()); + + return MDBX_SUCCESS /* 7) now at S-? (used), done */; +} + +/*----------------------------------------------------------------------------*/ +/* reader checking (by pid) */ + +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env) { + (void)env; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env) { + (void)env; + return MDBX_SUCCESS; +} + +/* Checks reader by pid. + * + * Returns: + * MDBX_RESULT_TRUE, if pid is live (unable to acquire lock) + * MDBX_RESULT_FALSE, if pid is dead (lock acquired) + * or otherwise the errcode. */ +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid) { + (void)env; + HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid); + int rc; + if (likely(hProcess)) { + rc = WaitForSingleObject(hProcess, 0); + if (unlikely(rc == WAIT_FAILED)) + rc = GetLastError(); + CloseHandle(hProcess); + } else { + rc = GetLastError(); + } + + switch (rc) { + case ERROR_INVALID_PARAMETER: + /* pid seems invalid */ + return MDBX_RESULT_FALSE; + case WAIT_OBJECT_0: + /* process just exited */ + return MDBX_RESULT_FALSE; + case WAIT_TIMEOUT: + /* pid running */ + return MDBX_RESULT_TRUE; + default: + /* failure */ + return rc; + } +} + +//---------------------------------------------------------------------------- +// Stub for slim read-write lock +// Copyright (C) 1995-2002 Brad Wilson + +static void WINAPI stub_srwlock_Init(MDBX_srwlock *srwl) { + srwl->readerCount = srwl->writerCount = 0; +} + +static void WINAPI stub_srwlock_AcquireShared(MDBX_srwlock *srwl) { + while (true) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + + // If there's a writer already, spin without unnecessarily + // interlocking the CPUs + if (srwl->writerCount != 0) { + YieldProcessor(); + continue; + } + + // Add to the readers list + _InterlockedIncrement(&srwl->readerCount); + + // Check for writers again (we may have been pre-empted). If + // there are no writers writing or waiting, then we're done. + if (srwl->writerCount == 0) + break; + + // Remove from the readers list, spin, try again + _InterlockedDecrement(&srwl->readerCount); + YieldProcessor(); + } +} + +static void WINAPI stub_srwlock_ReleaseShared(MDBX_srwlock *srwl) { + assert(srwl->readerCount > 0); + _InterlockedDecrement(&srwl->readerCount); +} + +static void WINAPI stub_srwlock_AcquireExclusive(MDBX_srwlock *srwl) { + while (true) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + + // If there's a writer already, spin without unnecessarily + // interlocking the CPUs + if (srwl->writerCount != 0) { + YieldProcessor(); + continue; + } + + // See if we can become the writer (expensive, because it inter- + // locks the CPUs, so writing should be an infrequent process) + if (_InterlockedExchange(&srwl->writerCount, 1) == 0) + break; + } + + // Now we're the writer, but there may be outstanding readers. + // Spin until there aren't any more; new readers will wait now + // that we're the writer. + while (srwl->readerCount != 0) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + YieldProcessor(); + } +} + +static void WINAPI stub_srwlock_ReleaseExclusive(MDBX_srwlock *srwl) { + assert(srwl->writerCount == 1 && srwl->readerCount >= 0); + srwl->writerCount = 0; +} + +MDBX_srwlock_function mdbx_srwlock_Init, mdbx_srwlock_AcquireShared, + mdbx_srwlock_ReleaseShared, mdbx_srwlock_AcquireExclusive, + mdbx_srwlock_ReleaseExclusive; + +/*----------------------------------------------------------------------------*/ + +#if 0 /* LY: unused for now */ +static DWORD WINAPI stub_DiscardVirtualMemory(PVOID VirtualAddress, + SIZE_T Size) { + return VirtualAlloc(VirtualAddress, Size, MEM_RESET, PAGE_NOACCESS) + ? ERROR_SUCCESS + : GetLastError(); +} +#endif /* unused for now */ + +static uint64_t WINAPI stub_GetTickCount64(void) { + LARGE_INTEGER Counter, Frequency; + return (QueryPerformanceFrequency(&Frequency) && + QueryPerformanceCounter(&Counter)) + ? Counter.QuadPart * 1000ul / Frequency.QuadPart + : 0; +} + +/*----------------------------------------------------------------------------*/ +#ifndef MDBX_ALLOY +MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx; +MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW; +MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; +MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle; +MDBX_NtFsControlFile mdbx_NtFsControlFile; +MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; +MDBX_GetTickCount64 mdbx_GetTickCount64; +#if 0 /* LY: unused for now */ +MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; +MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; +#endif /* unused for now */ +#endif /* MDBX_ALLOY */ + +static void mdbx_winnt_import(void) { + const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll"); + const MDBX_srwlock_function init = + (MDBX_srwlock_function)GetProcAddress(hKernel32dll, "InitializeSRWLock"); + if (init != NULL) { + mdbx_srwlock_Init = init; + mdbx_srwlock_AcquireShared = (MDBX_srwlock_function)GetProcAddress( + hKernel32dll, "AcquireSRWLockShared"); + mdbx_srwlock_ReleaseShared = (MDBX_srwlock_function)GetProcAddress( + hKernel32dll, "ReleaseSRWLockShared"); + mdbx_srwlock_AcquireExclusive = (MDBX_srwlock_function)GetProcAddress( + hKernel32dll, "AcquireSRWLockExclusive"); + mdbx_srwlock_ReleaseExclusive = (MDBX_srwlock_function)GetProcAddress( + hKernel32dll, "ReleaseSRWLockExclusive"); + } else { + mdbx_srwlock_Init = stub_srwlock_Init; + mdbx_srwlock_AcquireShared = stub_srwlock_AcquireShared; + mdbx_srwlock_ReleaseShared = stub_srwlock_ReleaseShared; + mdbx_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive; + mdbx_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive; + } + +#define GET_KERNEL32_PROC(ENTRY) \ + mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(hKernel32dll, #ENTRY) + GET_KERNEL32_PROC(GetFileInformationByHandleEx); + GET_KERNEL32_PROC(GetVolumeInformationByHandleW); + GET_KERNEL32_PROC(GetFinalPathNameByHandleW); + GET_KERNEL32_PROC(SetFileInformationByHandle); + GET_KERNEL32_PROC(PrefetchVirtualMemory); + GET_KERNEL32_PROC(GetTickCount64); + if (!mdbx_GetTickCount64) + mdbx_GetTickCount64 = stub_GetTickCount64; +#if 0 /* LY: unused for now */ + GET_KERNEL32_PROC(DiscardVirtualMemory); + if (!mdbx_DiscardVirtualMemory) + mdbx_DiscardVirtualMemory = stub_DiscardVirtualMemory; + GET_KERNEL32_PROC(OfferVirtualMemory); + GET_KERNEL32_PROC(ReclaimVirtualMemory); +#endif /* unused for now */ +#undef GET_KERNEL32_PROC + + const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll"); + mdbx_NtFsControlFile = + (MDBX_NtFsControlFile)GetProcAddress(hNtdll, "NtFsControlFile"); +} diff --git a/src/contrib/db/libmdbx/src/elements/ntdll.def b/src/contrib/db/libmdbx/src/elements/ntdll.def new file mode 100644 index 0000000..e3a6e33 --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/ntdll.def @@ -0,0 +1,1244 @@ +LIBRARY ntdll +EXPORTS +CsrAllocateCaptureBuffer +CsrAllocateMessagePointer +CsrCaptureMessageBuffer +CsrCaptureMessageMultiUnicodeStringsInPlace +CsrCaptureMessageString +CsrCaptureTimeout +CsrClientCallServer +CsrClientConnectToServer +CsrFreeCaptureBuffer +CsrGetProcessId +CsrIdentifyAlertableThread +CsrSetPriorityClass +DbgBreakPoint +DbgPrint +DbgPrintEx +DbgPrintReturnControlC +DbgPrompt +DbgQueryDebugFilterState +DbgSetDebugFilterState +DbgUiConnectToDbg +DbgUiContinue +DbgUiConvertStateChangeStructure +DbgUiDebugActiveProcess +DbgUiGetThreadDebugObject +DbgUiIssueRemoteBreakin +DbgUiRemoteBreakin +DbgUiSetThreadDebugObject +DbgUiStopDebugging +DbgUiWaitStateChange +DbgUserBreakPoint +KiRaiseUserExceptionDispatcher +KiUserApcDispatcher +KiUserCallbackDispatcher +KiUserExceptionDispatcher +LdrAccessResource +LdrAddRefDll +LdrDisableThreadCalloutsForDll +LdrEnumResources +LdrEnumerateLoadedModules +LdrFindEntryForAddress +LdrFindResourceDirectory_U +LdrFindResourceEx_U +LdrFindResource_U +LdrFlushAlternateResourceModules +LdrGetDllHandle +LdrGetDllHandleEx +LdrGetProcedureAddress +LdrInitShimEngineDynamic +LdrInitializeThunk +LdrLoadAlternateResourceModule +LdrLoadDll +LdrLockLoaderLock +LdrProcessRelocationBlock +LdrQueryImageFileExecutionOptions +LdrQueryProcessModuleInformation +LdrSetAppCompatDllRedirectionCallback +LdrSetDllManifestProber +LdrShutdownProcess +LdrShutdownThread +LdrUnloadAlternateResourceModule +LdrUnloadDll +LdrUnlockLoaderLock +LdrVerifyImageMatchesChecksum +NlsAnsiCodePage +NlsMbCodePageTag +NlsMbOemCodePageTag +NtAcceptConnectPort +NtAccessCheck +NtAccessCheckAndAuditAlarm +NtAccessCheckByType +NtAccessCheckByTypeAndAuditAlarm +NtAccessCheckByTypeResultList +NtAccessCheckByTypeResultListAndAuditAlarm +NtAccessCheckByTypeResultListAndAuditAlarmByHandle +NtAddAtom +NtAddBootEntry +NtAdjustGroupsToken +NtAdjustPrivilegesToken +NtAlertResumeThread +NtAlertThread +NtAllocateLocallyUniqueId +NtAllocateUserPhysicalPages +NtAllocateUuids +NtAllocateVirtualMemory +NtAreMappedFilesTheSame +NtAssignProcessToJobObject +NtCallbackReturn +NtCancelIoFile +NtCancelTimer +NtClearEvent +NtClose +NtCloseObjectAuditAlarm +NtCompactKeys +NtCompareTokens +NtCompleteConnectPort +NtCompressKey +NtConnectPort +NtContinue +NtCreateDebugObject +NtCreateDirectoryObject +NtCreateEvent +NtCreateEventPair +NtCreateFile +NtCreateIoCompletion +NtCreateJobObject +NtCreateJobSet +NtCreateKey +NtCreateKeyedEvent +NtCreateMailslotFile +NtCreateMutant +NtCreateNamedPipeFile +NtCreatePagingFile +NtCreatePort +NtCreateProcess +NtCreateProcessEx +NtCreateProfile +NtCreateSection +NtCreateSemaphore +NtCreateSymbolicLinkObject +NtCreateThread +NtCreateTimer +NtCreateToken +NtCreateWaitablePort +NtDebugActiveProcess +NtDebugContinue +NtDelayExecution +NtDeleteAtom +NtDeleteBootEntry +NtDeleteFile +NtDeleteKey +NtDeleteObjectAuditAlarm +NtDeleteValueKey +NtDeviceIoControlFile +NtDisplayString +NtDuplicateObject +NtDuplicateToken +NtEnumerateBootEntries +NtEnumerateKey +NtEnumerateSystemEnvironmentValuesEx +NtEnumerateValueKey +NtExtendSection +NtFilterToken +NtFindAtom +NtFlushBuffersFile +NtFlushInstructionCache +NtFlushKey +NtFlushVirtualMemory +NtFlushWriteBuffer +NtFreeUserPhysicalPages +NtFreeVirtualMemory +NtFsControlFile +NtGetContextThread +NtGetDevicePowerState +NtGetWriteWatch +NtImpersonateAnonymousToken +NtImpersonateClientOfPort +NtImpersonateThread +NtInitializeRegistry +NtInitiatePowerAction +NtIsProcessInJob +NtIsSystemResumeAutomatic +NtListenPort +NtLoadDriver +NtLoadKey +NtLoadKey2 +NtLockFile +NtLockProductActivationKeys +NtLockRegistryKey +NtLockVirtualMemory +NtMakePermanentObject +NtMakeTemporaryObject +NtMapUserPhysicalPages +NtMapUserPhysicalPagesScatter +NtMapViewOfSection +NtModifyBootEntry +NtNotifyChangeDirectoryFile +NtNotifyChangeKey +NtNotifyChangeMultipleKeys +NtOpenDirectoryObject +NtOpenEvent +NtOpenEventPair +NtOpenFile +NtOpenIoCompletion +NtOpenJobObject +NtOpenKey +NtOpenKeyedEvent +NtOpenMutant +NtOpenObjectAuditAlarm +NtOpenProcess +NtOpenProcessToken +NtOpenProcessTokenEx +NtOpenSection +NtOpenSemaphore +NtOpenSymbolicLinkObject +NtOpenThread +NtOpenThreadToken +NtOpenThreadTokenEx +NtOpenTimer +NtPlugPlayControl +NtPowerInformation +NtPrivilegeCheck +NtPrivilegeObjectAuditAlarm +NtPrivilegedServiceAuditAlarm +NtProtectVirtualMemory +NtPulseEvent +NtQueryAttributesFile +NtQueryBootEntryOrder +NtQueryBootOptions +NtQueryDebugFilterState +NtQueryDefaultLocale +NtQueryDefaultUILanguage +NtQueryDirectoryFile +NtQueryDirectoryObject +NtQueryEaFile +NtQueryEvent +NtQueryFullAttributesFile +NtQueryInformationAtom +NtQueryInformationFile +NtQueryInformationJobObject +NtQueryInformationPort +NtQueryInformationProcess +NtQueryInformationThread +NtQueryInformationToken +NtQueryInstallUILanguage +NtQueryIntervalProfile +NtQueryIoCompletion +NtQueryKey +NtQueryMultipleValueKey +NtQueryMutant +NtQueryObject +NtQueryOpenSubKeys +NtQueryPerformanceCounter +NtQueryPortInformationProcess +NtQueryQuotaInformationFile +NtQuerySection +NtQuerySecurityObject +NtQuerySemaphore +NtQuerySymbolicLinkObject +NtQuerySystemEnvironmentValue +NtQuerySystemEnvironmentValueEx +NtQuerySystemInformation +NtQuerySystemTime +NtQueryTimer +NtQueryTimerResolution +NtQueryValueKey +NtQueryVirtualMemory +NtQueryVolumeInformationFile +NtQueueApcThread +NtRaiseException +NtRaiseHardError +NtReadFile +NtReadFileScatter +NtReadRequestData +NtReadVirtualMemory +NtRegisterThreadTerminatePort +NtReleaseKeyedEvent +NtReleaseMutant +NtReleaseSemaphore +NtRemoveIoCompletion +NtRemoveProcessDebug +NtRenameKey +NtReplaceKey +NtReplyPort +NtReplyWaitReceivePort +NtReplyWaitReceivePortEx +NtReplyWaitReplyPort +NtRequestPort +NtRequestWaitReplyPort +NtResetEvent +NtResetWriteWatch +NtRestoreKey +NtResumeProcess +NtResumeThread +NtSaveKey +NtSaveKeyEx +NtSaveMergedKeys +NtSecureConnectPort +NtSetBootEntryOrder +NtSetBootOptions +NtSetContextThread +NtSetDebugFilterState +NtSetDefaultHardErrorPort +NtSetDefaultLocale +NtSetDefaultUILanguage +NtSetEaFile +NtSetEvent +NtSetEventBoostPriority +NtSetHighEventPair +NtSetHighWaitLowEventPair +NtSetInformationDebugObject +NtSetInformationFile +NtSetInformationJobObject +NtSetInformationKey +NtSetInformationObject +NtSetInformationProcess +NtSetInformationThread +NtSetInformationToken +NtSetIntervalProfile +NtSetIoCompletion +NtSetLdtEntries +NtSetLowEventPair +NtSetLowWaitHighEventPair +NtSetQuotaInformationFile +NtSetSecurityObject +NtSetSystemEnvironmentValue +NtSetSystemEnvironmentValueEx +NtSetSystemInformation +NtSetSystemPowerState +NtSetSystemTime +NtSetThreadExecutionState +NtSetTimer +NtSetTimerResolution +NtSetUuidSeed +NtSetValueKey +NtSetVolumeInformationFile +NtShutdownSystem +NtSignalAndWaitForSingleObject +NtStartProfile +NtStopProfile +NtSuspendProcess +NtSuspendThread +NtSystemDebugControl +NtTerminateJobObject +NtTerminateProcess +NtTerminateThread +NtTestAlert +NtTraceEvent +NtTranslateFilePath +NtUnloadDriver +NtUnloadKey +NtUnloadKeyEx +NtUnlockFile +NtUnlockVirtualMemory +NtUnmapViewOfSection +NtVdmControl +NtWaitForDebugEvent +NtWaitForKeyedEvent +NtWaitForMultipleObjects +NtWaitForSingleObject +NtWaitHighEventPair +NtWaitLowEventPair +NtWriteFile +NtWriteFileGather +NtWriteRequestData +NtWriteVirtualMemory +NtYieldExecution +PfxFindPrefix +PfxInitialize +PfxInsertPrefix +PfxRemovePrefix +RtlAbortRXact +RtlAbsoluteToSelfRelativeSD +RtlAcquirePebLock +RtlAcquireResourceExclusive +RtlAcquireResourceShared +RtlActivateActivationContext +RtlActivateActivationContextEx +RtlActivateActivationContextUnsafeFast +RtlAddAccessAllowedAce +RtlAddAccessAllowedAceEx +RtlAddAccessAllowedObjectAce +RtlAddAccessDeniedAce +RtlAddAccessDeniedAceEx +RtlAddAccessDeniedObjectAce +RtlAddAce +RtlAddActionToRXact +RtlAddAtomToAtomTable +RtlAddAttributeActionToRXact +RtlAddAuditAccessAce +RtlAddAuditAccessAceEx +RtlAddAuditAccessObjectAce +RtlAddCompoundAce +RtlAddRefActivationContext +RtlAddRefMemoryStream +RtlAddVectoredExceptionHandler +RtlAddressInSectionTable +RtlAdjustPrivilege +RtlAllocateAndInitializeSid +RtlAllocateHandle +RtlAllocateHeap +RtlAnsiCharToUnicodeChar +RtlAnsiStringToUnicodeSize +RtlAnsiStringToUnicodeString +RtlAppendAsciizToString +RtlAppendPathElement +RtlAppendStringToString +RtlAppendUnicodeStringToString +RtlAppendUnicodeToString +RtlApplicationVerifierStop +RtlApplyRXact +RtlApplyRXactNoFlush +RtlAreAllAccessesGranted +RtlAreAnyAccessesGranted +RtlAreBitsClear +RtlAreBitsSet +RtlAssert +RtlCancelTimer +RtlCaptureContext +RtlCaptureStackBackTrace +RtlCharToInteger +RtlCheckForOrphanedCriticalSections +RtlCheckRegistryKey +RtlClearAllBits +RtlClearBits +RtlCloneMemoryStream +RtlCommitMemoryStream +RtlCompactHeap +RtlCompareMemory +RtlCompareMemoryUlong +RtlCompareString +RtlCompareUnicodeString +RtlCompressBuffer +RtlComputeCrc32 +RtlComputeImportTableHash +RtlComputePrivatizedDllName_U +RtlConsoleMultiByteToUnicodeN +RtlConvertExclusiveToShared +RtlConvertSharedToExclusive +RtlConvertSidToUnicodeString +RtlConvertToAutoInheritSecurityObject +RtlCopyLuid +RtlCopyLuidAndAttributesArray +RtlCopyMemoryStreamTo +RtlCopyOutOfProcessMemoryStreamTo +RtlCopySecurityDescriptor +RtlCopySid +RtlCopySidAndAttributesArray +RtlCopyString +RtlCopyUnicodeString +RtlCreateAcl +RtlCreateActivationContext +RtlCreateAndSetSD +RtlCreateAtomTable +RtlCreateBootStatusDataFile +RtlCreateEnvironment +RtlCreateHeap +RtlCreateProcessParameters +RtlCreateQueryDebugBuffer +RtlCreateRegistryKey +RtlCreateSecurityDescriptor +RtlCreateServiceSid +RtlCreateSystemVolumeInformationFolder +RtlCreateTagHeap +RtlCreateTimer +RtlCreateTimerQueue +RtlCreateUnicodeString +RtlCreateUnicodeStringFromAsciiz +RtlCreateUserProcess +RtlCreateUserSecurityObject +RtlCreateUserThread +RtlCustomCPToUnicodeN +RtlCutoverTimeToSystemTime +RtlDeNormalizeProcessParams +RtlDeactivateActivationContext +RtlDeactivateActivationContextUnsafeFast +RtlDebugPrintTimes +RtlDecodePointer +RtlDecodeSystemPointer +RtlDecompressBuffer +RtlDecompressFragment +RtlDefaultNpAcl +RtlDelete +RtlDeleteAce +RtlDeleteAtomFromAtomTable +RtlDeleteCriticalSection +RtlDeleteElementGenericTable +RtlDeleteElementGenericTableAvl +RtlDeleteNoSplay +RtlDeleteRegistryValue +RtlDeleteResource +RtlDeleteSecurityObject +RtlDeleteTimer +RtlDeleteTimerQueue +RtlDeleteTimerQueueEx +RtlDeregisterWait +RtlDeregisterWaitEx +RtlDestroyAtomTable +RtlDestroyEnvironment +RtlDestroyHandleTable +RtlDestroyHeap +RtlDestroyProcessParameters +RtlDestroyQueryDebugBuffer +RtlDetermineDosPathNameType_U +RtlDllShutdownInProgress +RtlDnsHostNameToComputerName +RtlDoesFileExists_U +RtlDosApplyFileIsolationRedirection_Ustr +RtlDosPathNameToNtPathName_U +RtlDosSearchPath_U +RtlDosSearchPath_Ustr +RtlDowncaseUnicodeChar +RtlDowncaseUnicodeString +RtlDumpResource +RtlDuplicateUnicodeString +RtlEmptyAtomTable +RtlEnableEarlyCriticalSectionEventCreation +RtlEncodePointer +RtlEncodeSystemPointer +RtlEnterCriticalSection +RtlEnumProcessHeaps +RtlEnumerateGenericTable +RtlEnumerateGenericTableAvl +RtlEnumerateGenericTableLikeADirectory +RtlEnumerateGenericTableWithoutSplaying +RtlEnumerateGenericTableWithoutSplayingAvl +RtlEqualComputerName +RtlEqualDomainName +RtlEqualLuid +RtlEqualPrefixSid +RtlEqualSid +RtlEqualString +RtlEqualUnicodeString +RtlEraseUnicodeString +RtlExitUserThread +RtlExpandEnvironmentStrings_U +RtlFillMemory +RtlFinalReleaseOutOfProcessMemoryStream +RtlFindActivationContextSectionGuid +RtlFindActivationContextSectionString +RtlFindCharInUnicodeString +RtlFindClearBits +RtlFindClearBitsAndSet +RtlFindClearRuns +RtlFindLastBackwardRunClear +RtlFindLeastSignificantBit +RtlFindLongestRunClear +RtlFindMessage +RtlFindMostSignificantBit +RtlFindNextForwardRunClear +RtlFindSetBits +RtlFindSetBitsAndClear +RtlFirstEntrySList +RtlFirstFreeAce +RtlFlushSecureMemoryCache +RtlFormatCurrentUserKeyPath +RtlFormatMessage +RtlFreeAnsiString +RtlFreeHandle +RtlFreeHeap +RtlFreeOemString +RtlFreeSid +RtlFreeThreadActivationContextStack +RtlFreeUnicodeString +RtlGUIDFromString +RtlGenerate8dot3Name +RtlGetAce +RtlGetActiveActivationContext +RtlGetCallersAddress +RtlGetCompressionWorkSpaceSize +RtlGetControlSecurityDescriptor +RtlGetCurrentDirectory_U +RtlGetCurrentPeb +RtlGetDaclSecurityDescriptor +RtlGetElementGenericTable +RtlGetElementGenericTableAvl +RtlGetFrame +RtlGetFullPathName_U +RtlGetGroupSecurityDescriptor +RtlGetLastNtStatus +RtlGetLastWin32Error +RtlGetLengthWithoutLastFullDosOrNtPathElement +RtlGetLengthWithoutTrailingPathSeperators +RtlGetLongestNtPathLength +RtlGetNativeSystemInformation +RtlGetNtGlobalFlags +RtlGetNtProductType +RtlGetNtVersionNumbers +RtlGetOwnerSecurityDescriptor +RtlGetProcessHeaps +RtlGetSaclSecurityDescriptor +RtlGetSecurityDescriptorRMControl +RtlGetSetBootStatusData +RtlGetUnloadEventTrace +RtlGetUserInfoHeap +RtlGetVersion +RtlHashUnicodeString +RtlIdentifierAuthoritySid +RtlImageDirectoryEntryToData +RtlImageNtHeader +RtlImageRvaToSection +RtlImageRvaToVa +RtlImpersonateSelf +RtlInitAnsiString +RtlInitCodePageTable +RtlInitMemoryStream +RtlInitNlsTables +RtlInitOutOfProcessMemoryStream +RtlInitString +RtlInitUnicodeString +RtlInitUnicodeStringEx +RtlInitializeAtomPackage +RtlInitializeBitMap +RtlInitializeContext +RtlInitializeCriticalSection +RtlInitializeCriticalSectionAndSpinCount +RtlInitializeGenericTable +RtlInitializeGenericTableAvl +RtlInitializeHandleTable +RtlInitializeRXact +RtlInitializeResource +RtlInitializeSListHead +RtlInitializeSid +RtlInsertElementGenericTable +RtlInsertElementGenericTableAvl +RtlInt64ToUnicodeString +RtlIntegerToChar +RtlIntegerToUnicodeString +RtlInterlockedFlushSList +RtlInterlockedPopEntrySList +RtlInterlockedPushEntrySList +RtlInterlockedPushListSList +RtlIpv4AddressToStringA +RtlIpv4AddressToStringExA +RtlIpv4AddressToStringExW +RtlIpv4AddressToStringW +RtlIpv4StringToAddressA +RtlIpv4StringToAddressExA +RtlIpv4StringToAddressExW +RtlIpv4StringToAddressW +RtlIpv6AddressToStringA +RtlIpv6AddressToStringExA +RtlIpv6AddressToStringExW +RtlIpv6AddressToStringW +RtlIpv6StringToAddressA +RtlIpv6StringToAddressExA +RtlIpv6StringToAddressExW +RtlIpv6StringToAddressW +RtlIsActivationContextActive +RtlIsDosDeviceName_U +RtlIsGenericTableEmpty +RtlIsGenericTableEmptyAvl +RtlIsNameLegalDOS8Dot3 +RtlIsTextUnicode +RtlIsThreadWithinLoaderCallout +RtlIsValidHandle +RtlIsValidIndexHandle +RtlLargeIntegerToChar +RtlLeaveCriticalSection +RtlLengthRequiredSid +RtlLengthSecurityDescriptor +RtlLengthSid +RtlLocalTimeToSystemTime +RtlLockBootStatusData +RtlLockHeap +RtlLockMemoryStreamRegion +RtlLogStackBackTrace +RtlLookupAtomInAtomTable +RtlLookupElementGenericTable +RtlLookupElementGenericTableAvl +RtlMakeSelfRelativeSD +RtlMapGenericMask +RtlMapSecurityErrorToNtStatus +RtlMoveMemory +RtlMultiAppendUnicodeStringBuffer +RtlMultiByteToUnicodeN +RtlMultiByteToUnicodeSize +RtlNewInstanceSecurityObject +RtlNewSecurityGrantedAccess +RtlNewSecurityObject +RtlNewSecurityObjectEx +RtlNewSecurityObjectWithMultipleInheritance +RtlNormalizeProcessParams +RtlNtPathNameToDosPathName +RtlNtStatusToDosError +RtlNtStatusToDosErrorNoTeb +RtlNumberGenericTableElements +RtlNumberGenericTableElementsAvl +RtlNumberOfClearBits +RtlNumberOfSetBits +RtlOemStringToUnicodeSize +RtlOemStringToUnicodeString +RtlOemToUnicodeN +RtlOpenCurrentUser +RtlPcToFileHeader +RtlPinAtomInAtomTable +RtlPopFrame +RtlPrefixString +RtlPrefixUnicodeString +RtlProtectHeap +RtlPushFrame +RtlQueryAtomInAtomTable +RtlQueryDepthSList +RtlQueryEnvironmentVariable_U +RtlQueryHeapInformation +RtlQueryInformationAcl +RtlQueryInformationActivationContext +RtlQueryInformationActiveActivationContext +RtlQueryInterfaceMemoryStream +RtlQueryProcessBackTraceInformation +RtlQueryProcessDebugInformation +RtlQueryProcessHeapInformation +RtlQueryProcessLockInformation +RtlQueryRegistryValues +RtlQuerySecurityObject +RtlQueryTagHeap +RtlQueryTimeZoneInformation +RtlQueueApcWow64Thread +RtlQueueWorkItem +RtlRaiseException +RtlRaiseStatus +RtlRandom +RtlRandomEx +RtlReAllocateHeap +RtlReadMemoryStream +RtlReadOutOfProcessMemoryStream +RtlRealPredecessor +RtlRealSuccessor +RtlRegisterSecureMemoryCacheCallback +RtlRegisterWait +RtlReleaseActivationContext +RtlReleaseMemoryStream +RtlReleasePebLock +RtlReleaseResource +RtlRemoteCall +RtlRemoveVectoredExceptionHandler +RtlResetRtlTranslations +RtlRestoreLastWin32Error +RtlRevertMemoryStream +RtlRunDecodeUnicodeString +RtlRunEncodeUnicodeString +RtlSecondsSince1970ToTime +RtlSecondsSince1980ToTime +RtlSeekMemoryStream +RtlSelfRelativeToAbsoluteSD +RtlSelfRelativeToAbsoluteSD2 +RtlSetAllBits +RtlSetAttributesSecurityDescriptor +RtlSetBits +RtlSetControlSecurityDescriptor +RtlSetCriticalSectionSpinCount +RtlSetCurrentDirectory_U +RtlSetCurrentEnvironment +RtlSetDaclSecurityDescriptor +RtlSetEnvironmentVariable +RtlSetGroupSecurityDescriptor +RtlSetHeapInformation +RtlSetInformationAcl +RtlSetIoCompletionCallback +RtlSetLastWin32Error +RtlSetLastWin32ErrorAndNtStatusFromNtStatus +RtlSetMemoryStreamSize +RtlSetOwnerSecurityDescriptor +RtlSetProcessIsCritical +RtlSetSaclSecurityDescriptor +RtlSetSecurityDescriptorRMControl +RtlSetSecurityObject +RtlSetSecurityObjectEx +RtlSetThreadIsCritical +RtlSetThreadPoolStartFunc +RtlSetTimeZoneInformation +RtlSetTimer +RtlSetUserFlagsHeap +RtlSetUserValueHeap +RtlSizeHeap +RtlSplay +RtlStartRXact +RtlStatMemoryStream +RtlStringFromGUID +RtlSubAuthorityCountSid +RtlSubAuthoritySid +RtlSubtreePredecessor +RtlSubtreeSuccessor +RtlSystemTimeToLocalTime +RtlTimeFieldsToTime +RtlTimeToElapsedTimeFields +RtlTimeToSecondsSince1970 +RtlTimeToSecondsSince1980 +RtlTimeToTimeFields +RtlTraceDatabaseAdd +RtlTraceDatabaseCreate +RtlTraceDatabaseDestroy +RtlTraceDatabaseEnumerate +RtlTraceDatabaseFind +RtlTraceDatabaseLock +RtlTraceDatabaseUnlock +RtlTraceDatabaseValidate +RtlTryEnterCriticalSection +RtlUnhandledExceptionFilter +RtlUnhandledExceptionFilter2 +RtlUnicodeStringToAnsiSize +RtlUnicodeStringToAnsiString +RtlUnicodeStringToCountedOemString +RtlUnicodeStringToInteger +RtlUnicodeStringToOemSize +RtlUnicodeStringToOemString +RtlUnicodeToCustomCPN +RtlUnicodeToMultiByteN +RtlUnicodeToMultiByteSize +RtlUnicodeToOemN +RtlUniform +RtlUnlockBootStatusData +RtlUnlockHeap +RtlUnlockMemoryStreamRegion +RtlUnwind +RtlUpcaseUnicodeChar +RtlUpcaseUnicodeString +RtlUpcaseUnicodeStringToAnsiString +RtlUpcaseUnicodeStringToCountedOemString +RtlUpcaseUnicodeStringToOemString +RtlUpcaseUnicodeToCustomCPN +RtlUpcaseUnicodeToMultiByteN +RtlUpcaseUnicodeToOemN +RtlUpdateTimer +RtlUpperChar +RtlUpperString +RtlValidAcl +RtlValidRelativeSecurityDescriptor +RtlValidSecurityDescriptor +RtlValidSid +RtlValidateHeap +RtlValidateProcessHeaps +RtlValidateUnicodeString +RtlVerifyVersionInfo +RtlWalkFrameChain +RtlWalkHeap +RtlWriteMemoryStream +RtlWriteRegistryValue +RtlZeroHeap +RtlZeroMemory +RtlZombifyActivationContext +RtlpApplyLengthFunction +RtlpEnsureBufferSize +RtlpNotOwnerCriticalSection +RtlpNtCreateKey +RtlpNtEnumerateSubKey +RtlpNtMakeTemporaryKey +RtlpNtOpenKey +RtlpNtQueryValueKey +RtlpNtSetValueKey +RtlpUnWaitCriticalSection +RtlpWaitForCriticalSection +RtlxAnsiStringToUnicodeSize +RtlxOemStringToUnicodeSize +RtlxUnicodeStringToAnsiSize +RtlxUnicodeStringToOemSize +VerSetConditionMask +ZwAcceptConnectPort +ZwAccessCheck +ZwAccessCheckAndAuditAlarm +ZwAccessCheckByType +ZwAccessCheckByTypeAndAuditAlarm +ZwAccessCheckByTypeResultList +ZwAccessCheckByTypeResultListAndAuditAlarm +ZwAccessCheckByTypeResultListAndAuditAlarmByHandle +ZwAddAtom +ZwAddBootEntry +ZwAdjustGroupsToken +ZwAdjustPrivilegesToken +ZwAlertResumeThread +ZwAlertThread +ZwAllocateLocallyUniqueId +ZwAllocateUserPhysicalPages +ZwAllocateUuids +ZwAllocateVirtualMemory +ZwAreMappedFilesTheSame +ZwAssignProcessToJobObject +ZwCallbackReturn +ZwCancelIoFile +ZwCancelTimer +ZwClearEvent +ZwClose +ZwCloseObjectAuditAlarm +ZwCompactKeys +ZwCompareTokens +ZwCompleteConnectPort +ZwCompressKey +ZwConnectPort +ZwContinue +ZwCreateDebugObject +ZwCreateDirectoryObject +ZwCreateEvent +ZwCreateEventPair +ZwCreateFile +ZwCreateIoCompletion +ZwCreateJobObject +ZwCreateJobSet +ZwCreateKey +ZwCreateKeyedEvent +ZwCreateMailslotFile +ZwCreateMutant +ZwCreateNamedPipeFile +ZwCreatePagingFile +ZwCreatePort +ZwCreateProcess +ZwCreateProcessEx +ZwCreateProfile +ZwCreateSection +ZwCreateSemaphore +ZwCreateSymbolicLinkObject +ZwCreateThread +ZwCreateTimer +ZwCreateToken +ZwCreateWaitablePort +ZwDebugActiveProcess +ZwDebugContinue +ZwDelayExecution +ZwDeleteAtom +ZwDeleteBootEntry +ZwDeleteFile +ZwDeleteKey +ZwDeleteObjectAuditAlarm +ZwDeleteValueKey +ZwDeviceIoControlFile +ZwDisplayString +ZwDuplicateObject +ZwDuplicateToken +ZwEnumerateBootEntries +ZwEnumerateKey +ZwEnumerateSystemEnvironmentValuesEx +ZwEnumerateValueKey +ZwExtendSection +ZwFilterToken +ZwFindAtom +ZwFlushBuffersFile +ZwFlushInstructionCache +ZwFlushKey +ZwFlushVirtualMemory +ZwFlushWriteBuffer +ZwFreeUserPhysicalPages +ZwFreeVirtualMemory +ZwFsControlFile +ZwGetContextThread +ZwGetDevicePowerState +ZwGetWriteWatch +ZwImpersonateAnonymousToken +ZwImpersonateClientOfPort +ZwImpersonateThread +ZwInitializeRegistry +ZwInitiatePowerAction +ZwIsProcessInJob +ZwIsSystemResumeAutomatic +ZwListenPort +ZwLoadDriver +ZwLoadKey +ZwLoadKey2 +ZwLockFile +ZwLockProductActivationKeys +ZwLockRegistryKey +ZwLockVirtualMemory +ZwMakePermanentObject +ZwMakeTemporaryObject +ZwMapUserPhysicalPages +ZwMapUserPhysicalPagesScatter +ZwMapViewOfSection +ZwModifyBootEntry +ZwNotifyChangeDirectoryFile +ZwNotifyChangeKey +ZwNotifyChangeMultipleKeys +ZwOpenDirectoryObject +ZwOpenEvent +ZwOpenEventPair +ZwOpenFile +ZwOpenIoCompletion +ZwOpenJobObject +ZwOpenKey +ZwOpenKeyedEvent +ZwOpenMutant +ZwOpenObjectAuditAlarm +ZwOpenProcess +ZwOpenProcessToken +ZwOpenProcessTokenEx +ZwOpenSection +ZwOpenSemaphore +ZwOpenSymbolicLinkObject +ZwOpenThread +ZwOpenThreadToken +ZwOpenThreadTokenEx +ZwOpenTimer +ZwPlugPlayControl +ZwPowerInformation +ZwPrivilegeCheck +ZwPrivilegeObjectAuditAlarm +ZwPrivilegedServiceAuditAlarm +ZwProtectVirtualMemory +ZwPulseEvent +ZwQueryAttributesFile +ZwQueryBootEntryOrder +ZwQueryBootOptions +ZwQueryDebugFilterState +ZwQueryDefaultLocale +ZwQueryDefaultUILanguage +ZwQueryDirectoryFile +ZwQueryDirectoryObject +ZwQueryEaFile +ZwQueryEvent +ZwQueryFullAttributesFile +ZwQueryInformationAtom +ZwQueryInformationFile +ZwQueryInformationJobObject +ZwQueryInformationPort +ZwQueryInformationProcess +ZwQueryInformationThread +ZwQueryInformationToken +ZwQueryInstallUILanguage +ZwQueryIntervalProfile +ZwQueryIoCompletion +ZwQueryKey +ZwQueryMultipleValueKey +ZwQueryMutant +ZwQueryObject +ZwQueryOpenSubKeys +ZwQueryPerformanceCounter +ZwQueryPortInformationProcess +ZwQueryQuotaInformationFile +ZwQuerySection +ZwQuerySecurityObject +ZwQuerySemaphore +ZwQuerySymbolicLinkObject +ZwQuerySystemEnvironmentValue +ZwQuerySystemEnvironmentValueEx +ZwQuerySystemInformation +ZwQuerySystemTime +ZwQueryTimer +ZwQueryTimerResolution +ZwQueryValueKey +ZwQueryVirtualMemory +ZwQueryVolumeInformationFile +ZwQueueApcThread +ZwRaiseException +ZwRaiseHardError +ZwReadFile +ZwReadFileScatter +ZwReadRequestData +ZwReadVirtualMemory +ZwRegisterThreadTerminatePort +ZwReleaseKeyedEvent +ZwReleaseMutant +ZwReleaseSemaphore +ZwRemoveIoCompletion +ZwRemoveProcessDebug +ZwRenameKey +ZwReplaceKey +ZwReplyPort +ZwReplyWaitReceivePort +ZwReplyWaitReceivePortEx +ZwReplyWaitReplyPort +ZwRequestPort +ZwRequestWaitReplyPort +ZwResetEvent +ZwResetWriteWatch +ZwRestoreKey +ZwResumeProcess +ZwResumeThread +ZwSaveKey +ZwSaveKeyEx +ZwSaveMergedKeys +ZwSecureConnectPort +ZwSetBootEntryOrder +ZwSetBootOptions +ZwSetContextThread +ZwSetDebugFilterState +ZwSetDefaultHardErrorPort +ZwSetDefaultLocale +ZwSetDefaultUILanguage +ZwSetEaFile +ZwSetEvent +ZwSetEventBoostPriority +ZwSetHighEventPair +ZwSetHighWaitLowEventPair +ZwSetInformationDebugObject +ZwSetInformationFile +ZwSetInformationJobObject +ZwSetInformationKey +ZwSetInformationObject +ZwSetInformationProcess +ZwSetInformationThread +ZwSetInformationToken +ZwSetIntervalProfile +ZwSetIoCompletion +ZwSetLdtEntries +ZwSetLowEventPair +ZwSetLowWaitHighEventPair +ZwSetQuotaInformationFile +ZwSetSecurityObject +ZwSetSystemEnvironmentValue +ZwSetSystemEnvironmentValueEx +ZwSetSystemInformation +ZwSetSystemPowerState +ZwSetSystemTime +ZwSetThreadExecutionState +ZwSetTimer +ZwSetTimerResolution +ZwSetUuidSeed +ZwSetValueKey +ZwSetVolumeInformationFile +ZwShutdownSystem +ZwSignalAndWaitForSingleObject +ZwStartProfile +ZwStopProfile +ZwSuspendProcess +ZwSuspendThread +ZwSystemDebugControl +ZwTerminateJobObject +ZwTerminateProcess +ZwTerminateThread +ZwTestAlert +ZwTraceEvent +ZwTranslateFilePath +ZwUnloadDriver +ZwUnloadKey +ZwUnloadKeyEx +ZwUnlockFile +ZwUnlockVirtualMemory +ZwUnmapViewOfSection +ZwVdmControl +ZwWaitForDebugEvent +ZwWaitForKeyedEvent +ZwWaitForMultipleObjects +ZwWaitForSingleObject +ZwWaitHighEventPair +ZwWaitLowEventPair +ZwWriteFile +ZwWriteFileGather +ZwWriteRequestData +ZwWriteVirtualMemory +ZwYieldExecution +__isascii +__iscsym +__iscsymf +__toascii +_alldiv +_alldvrm +_allmul +_allrem +_allshl +_allshr +_atoi64 +_aulldiv +_aulldvrm +_aullrem +_aullshr +_fltused +_i64toa +_i64tow +_itoa +_itow +_lfind +_ltoa +_ltow +_memccpy +_memicmp +_snprintf +_snwprintf +_splitpath +_strcmpi +_stricmp +_strlwr +_strnicmp +_strupr +_ui64toa +_ui64tow +_ultoa +_ultow +_vsnprintf +_vsnwprintf +_wcsicmp +_wcslwr +_wcsnicmp +_wcsupr +_wtoi +_wtoi64 +_wtol +abs +atan +atoi +atol +bsearch +ceil +cos +fabs +floor +isalnum +isalpha +iscntrl +isdigit +isgraph +islower +isprint +ispunct +isspace +isupper +iswalpha +iswctype +iswdigit +iswlower +iswspace +iswxdigit +isxdigit +labs +log +mbstowcs +memchr +memcmp +memcpy +memmove +memset +pow +qsort +sin +sprintf +sqrt +sscanf +strcat +strchr +strcmp +strcpy +strcspn +strlen +strncat +strncmp +strncpy +strpbrk +strrchr +strspn +strstr +strtol +strtoul +swprintf +tan +tolower +toupper +towlower +towupper +vDbgPrintEx +vDbgPrintExWithPrefix +vsprintf +wcscat +wcschr +wcscmp +wcscpy +wcscspn +wcslen +wcsncat +wcsncmp +wcsncpy +wcspbrk +wcsrchr +wcsspn +wcsstr +wcstol +wcstombs +wcstoul diff --git a/src/contrib/db/libmdbx/src/elements/osal.c b/src/contrib/db/libmdbx/src/elements/osal.c new file mode 100644 index 0000000..9f13d51 --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/osal.c @@ -0,0 +1,1899 @@ +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "internals.h" + +#if defined(_WIN32) || defined(_WIN64) + +#include + +static int waitstatus2errcode(DWORD result) { + switch (result) { + case WAIT_OBJECT_0: + return MDBX_SUCCESS; + case WAIT_FAILED: + return GetLastError(); + case WAIT_ABANDONED: + return ERROR_ABANDONED_WAIT_0; + case WAIT_IO_COMPLETION: + return ERROR_USER_APC; + case WAIT_TIMEOUT: + return ERROR_TIMEOUT; + default: + return ERROR_UNHANDLED_ERROR; + } +} + +/* Map a result from an NTAPI call to WIN32 error code. */ +static int ntstatus2errcode(NTSTATUS status) { + DWORD dummy; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Internal = status; + return GetOverlappedResult(NULL, &ov, &dummy, FALSE) ? MDBX_SUCCESS + : GetLastError(); +} + +/* We use native NT APIs to setup the memory map, so that we can + * let the DB file grow incrementally instead of always preallocating + * the full size. These APIs are defined in and + * but those headers are meant for driver-level development and + * conflict with the regular user-level headers, so we explicitly + * declare them here. Using these APIs also means we must link to + * ntdll.dll, which is not linked by default in user code. */ + +extern NTSTATUS NTAPI NtCreateSection( + OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess, + IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes, + IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection, + IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle); + +typedef struct _SECTION_BASIC_INFORMATION { + ULONG Unknown; + ULONG SectionAttributes; + LARGE_INTEGER SectionSize; +} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION; + +typedef enum _SECTION_INFORMATION_CLASS { + SectionBasicInformation, + SectionImageInformation, + SectionRelocationInformation, // name:wow64:whNtQuerySection_SectionRelocationInformation + MaxSectionInfoClass +} SECTION_INFORMATION_CLASS; + +extern NTSTATUS NTAPI NtQuerySection( + IN HANDLE SectionHandle, IN SECTION_INFORMATION_CLASS InformationClass, + OUT PVOID InformationBuffer, IN ULONG InformationBufferSize, + OUT PULONG ResultLength OPTIONAL); + +extern NTSTATUS NTAPI NtExtendSection(IN HANDLE SectionHandle, + IN PLARGE_INTEGER NewSectionSize); + +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; + +extern NTSTATUS NTAPI NtMapViewOfSection( + IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, + IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize, + IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize, + IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType, + IN ULONG Win32Protect); + +extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, + IN OPTIONAL PVOID BaseAddress); + +extern NTSTATUS NTAPI NtClose(HANDLE Handle); + +extern NTSTATUS NTAPI NtAllocateVirtualMemory( + IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits, + IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect); + +extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, + IN PVOID *BaseAddress, + IN OUT PSIZE_T RegionSize, + IN ULONG FreeType); + +#ifndef WOF_CURRENT_VERSION +typedef struct _WOF_EXTERNAL_INFO { + DWORD Version; + DWORD Provider; +} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO; +#endif /* WOF_CURRENT_VERSION */ + +#ifndef WIM_PROVIDER_CURRENT_VERSION +#define WIM_PROVIDER_HASH_SIZE 20 + +typedef struct _WIM_PROVIDER_EXTERNAL_INFO { + DWORD Version; + DWORD Flags; + LARGE_INTEGER DataSourceId; + BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE]; +} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO; +#endif /* WIM_PROVIDER_CURRENT_VERSION */ + +#ifndef FILE_PROVIDER_CURRENT_VERSION +typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 { + ULONG Version; + ULONG Algorithm; + ULONG Flags; +} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1; +#endif /* FILE_PROVIDER_CURRENT_VERSION */ + +#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED +#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL) +#endif +#ifndef STATUS_INVALID_DEVICE_REQUEST +#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L) +#endif + +#ifndef FILE_DEVICE_FILE_SYSTEM +#define FILE_DEVICE_FILE_SYSTEM 0x00000009 +#endif + +#ifndef FSCTL_GET_EXTERNAL_BACKING +#define FSCTL_GET_EXTERNAL_BACKING \ + CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS) +#endif + +#endif /* _WIN32 || _WIN64 */ + +/*----------------------------------------------------------------------------*/ + +#if _POSIX_C_SOURCE > 200212 && \ + /* workaround for avoid musl libc wrong prototype */ ( \ + defined(__GLIBC__) || defined(__GNU_LIBRARY__)) +/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */ +__extern_C void __assert_fail(const char *assertion, const char *file, + unsigned line, const char *function) +#ifdef __THROW + __THROW +#else + __nothrow +#endif /* __THROW */ + __noreturn; + +#elif defined(__APPLE__) || defined(__MACH__) +__extern_C void __assert_rtn(const char *function, const char *file, int line, + const char *assertion) /* __nothrow */ +#ifdef __dead2 + __dead2 +#else + __noreturn +#endif /* __dead2 */ +#ifdef __disable_tail_calls + __disable_tail_calls +#endif /* __disable_tail_calls */ + ; + +#define __assert_fail(assertion, file, line, function) \ + __assert_rtn(function, file, line, assertion) +#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__NETBSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) +__extern_C void __assert(const char *function, const char *file, int line, + const char *assertion) /* __nothrow */ +#ifdef __dead2 + __dead2 +#else + __noreturn +#endif /* __dead2 */ +#ifdef __disable_tail_calls + __disable_tail_calls +#endif /* __disable_tail_calls */ + ; +#define __assert_fail(assertion, file, line, function) \ + __assert(function, file, line, assertion) + +#endif /* __assert_fail */ + +MDBX_INTERNAL_FUNC void __cold mdbx_assert_fail(const MDBX_env *env, + const char *msg, + const char *func, int line) { +#if MDBX_DEBUG + if (env && env->me_assert_func) { + env->me_assert_func(env, msg, func, line); + return; + } +#else + (void)env; +#endif /* MDBX_DEBUG */ + + if (mdbx_debug_logger) + mdbx_debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg); + else { +#if defined(_WIN32) || defined(_WIN64) + char *message = nullptr; + const int num = mdbx_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", + msg, func ? func : "unknown", line); + if (num < 1 || !message) + message = ""; + OutputDebugStringA(message); + if (IsDebuggerPresent()) + DebugBreak(); +#else + __assert_fail(msg, "mdbx", line, func); +#endif + } + +#if defined(_WIN32) || defined(_WIN64) + FatalExit(ERROR_UNHANDLED_ERROR); +#else + abort(); +#endif +} + +MDBX_INTERNAL_FUNC __cold void mdbx_panic(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + + char *message = nullptr; + const int num = mdbx_vasprintf(&message, fmt, ap); + va_end(ap); + const char *const const_message = + (num < 1 || !message) ? "" + : message; + +#if defined(_WIN32) || defined(_WIN64) + OutputDebugStringA("\r\nMDBX-PANIC: "); + OutputDebugStringA(const_message); + if (IsDebuggerPresent()) + DebugBreak(); + FatalExit(ERROR_UNHANDLED_ERROR); +#else + __assert_fail(const_message, "mdbx", 0, "panic"); + abort(); +#endif +} + +/*----------------------------------------------------------------------------*/ + +#ifndef mdbx_vasprintf +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, + va_list ap) { + va_list ones; + va_copy(ones, ap); + int needed = vsnprintf(nullptr, 0, fmt, ap); + + if (unlikely(needed < 0 || needed >= INT_MAX)) { + *strp = nullptr; + va_end(ones); + return needed; + } + + *strp = mdbx_malloc(needed + 1); + if (unlikely(*strp == nullptr)) { + va_end(ones); +#if defined(_WIN32) || defined(_WIN64) + SetLastError(MDBX_ENOMEM); +#else + errno = MDBX_ENOMEM; +#endif + return -1; + } + + int actual = vsnprintf(*strp, needed + 1, fmt, ones); + va_end(ones); + + assert(actual == needed); + if (unlikely(actual < 0)) { + mdbx_free(*strp); + *strp = nullptr; + } + return actual; +} +#endif /* mdbx_vasprintf */ + +#ifndef mdbx_asprintf +MDBX_INTERNAL_FUNC int mdbx_asprintf(char **strp, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + int rc = mdbx_vasprintf(strp, fmt, ap); + va_end(ap); + return rc; +} +#endif /* mdbx_asprintf */ + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result) { +#if defined(_WIN32) || defined(_WIN64) + (void)alignment; + *result = VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */; +#elif defined(_ISOC11_SOURCE) + *result = aligned_alloc(alignment, bytes); + return *result ? MDBX_SUCCESS : errno; +#elif _POSIX_VERSION >= 200112L + *result = nullptr; + return posix_memalign(result, alignment, bytes); +#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L + *result = memalign(alignment, bytes); + return *result ? MDBX_SUCCESS : errno; +#else +#error FIXME +#endif +} +#endif /* mdbx_memalign_alloc */ + +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr) { +#if defined(_WIN32) || defined(_WIN64) + VirtualFree(ptr, 0, MEM_RELEASE); +#else + mdbx_free(ptr); +#endif +} +#endif /* mdbx_memalign_free */ + +#ifndef mdbx_strdup +char *mdbx_strdup(const char *str) { + if (!str) + return NULL; + size_t bytes = strlen(str) + 1; + char *dup = mdbx_malloc(bytes); + if (dup) + memcpy(dup, str, bytes); + return dup; +} +#endif /* mdbx_strdup */ + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int mdbx_condmutex_init(mdbx_condmutex_t *condmutex) { +#if defined(_WIN32) || defined(_WIN64) + int rc = MDBX_SUCCESS; + condmutex->event = NULL; + condmutex->mutex = CreateMutex(NULL, FALSE, NULL); + if (!condmutex->mutex) + return GetLastError(); + + condmutex->event = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!condmutex->event) { + rc = GetLastError(); + (void)CloseHandle(condmutex->mutex); + condmutex->mutex = NULL; + } + return rc; +#else + memset(condmutex, 0, sizeof(mdbx_condmutex_t)); + int rc = pthread_mutex_init(&condmutex->mutex, NULL); + if (rc == 0) { + rc = pthread_cond_init(&condmutex->cond, NULL); + if (rc != 0) + (void)pthread_mutex_destroy(&condmutex->mutex); + } + return rc; +#endif +} + +static bool is_allzeros(const void *ptr, size_t bytes) { + const uint8_t *u8 = ptr; + for (size_t i = 0; i < bytes; ++i) + if (u8[i] != 0) + return false; + return true; +} + +MDBX_INTERNAL_FUNC int mdbx_condmutex_destroy(mdbx_condmutex_t *condmutex) { + int rc = MDBX_EINVAL; +#if defined(_WIN32) || defined(_WIN64) + if (condmutex->event) { + rc = CloseHandle(condmutex->event) ? MDBX_SUCCESS : GetLastError(); + if (rc == MDBX_SUCCESS) + condmutex->event = NULL; + } + if (condmutex->mutex) { + rc = CloseHandle(condmutex->mutex) ? MDBX_SUCCESS : GetLastError(); + if (rc == MDBX_SUCCESS) + condmutex->mutex = NULL; + } +#else + if (!is_allzeros(&condmutex->cond, sizeof(condmutex->cond))) { + rc = pthread_cond_destroy(&condmutex->cond); + if (rc == 0) + memset(&condmutex->cond, 0, sizeof(condmutex->cond)); + } + if (!is_allzeros(&condmutex->mutex, sizeof(condmutex->mutex))) { + rc = pthread_mutex_destroy(&condmutex->mutex); + if (rc == 0) + memset(&condmutex->mutex, 0, sizeof(condmutex->mutex)); + } +#endif + return rc; +} + +MDBX_INTERNAL_FUNC int mdbx_condmutex_lock(mdbx_condmutex_t *condmutex) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = WaitForSingleObject(condmutex->mutex, INFINITE); + return waitstatus2errcode(code); +#else + return pthread_mutex_lock(&condmutex->mutex); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_condmutex_unlock(mdbx_condmutex_t *condmutex) { +#if defined(_WIN32) || defined(_WIN64) + return ReleaseMutex(condmutex->mutex) ? MDBX_SUCCESS : GetLastError(); +#else + return pthread_mutex_unlock(&condmutex->mutex); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_condmutex_signal(mdbx_condmutex_t *condmutex) { +#if defined(_WIN32) || defined(_WIN64) + return SetEvent(condmutex->event) ? MDBX_SUCCESS : GetLastError(); +#else + return pthread_cond_signal(&condmutex->cond); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_condmutex_wait(mdbx_condmutex_t *condmutex) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = + SignalObjectAndWait(condmutex->mutex, condmutex->event, INFINITE, FALSE); + if (code == WAIT_OBJECT_0) + code = WaitForSingleObject(condmutex->mutex, INFINITE); + return waitstatus2errcode(code); +#else + return pthread_cond_wait(&condmutex->cond, &condmutex->mutex); +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + InitializeCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_init(fastmutex, NULL); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + DeleteCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_destroy(fastmutex); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + EnterCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_lock(fastmutex); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + LeaveCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_unlock(fastmutex); +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname) { +#if defined(_WIN32) || defined(_WIN64) + return DeleteFileA(pathname) ? MDBX_SUCCESS : GetLastError(); +#else + return unlink(pathname) ? errno : MDBX_SUCCESS; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_openfile(const char *pathname, int flags, + mode_t mode, mdbx_filehandle_t *fd, + bool exclusive) { + *fd = INVALID_HANDLE_VALUE; +#if defined(_WIN32) || defined(_WIN64) + (void)mode; + size_t wlen = mbstowcs(nullptr, pathname, INT_MAX); + if (wlen < 1 || wlen > /* MAX_PATH */ INT16_MAX) + return ERROR_INVALID_NAME; + wchar_t *const pathnameW = _alloca((wlen + 1) * sizeof(wchar_t)); + if (wlen != mbstowcs(pathnameW, pathname, wlen + 1)) + return ERROR_INVALID_NAME; + + DWORD DesiredAccess, ShareMode; + DWORD FlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + switch (flags & (O_RDONLY | O_WRONLY | O_RDWR)) { + default: + return ERROR_INVALID_PARAMETER; + case O_RDONLY: + DesiredAccess = GENERIC_READ; + ShareMode = + exclusive ? FILE_SHARE_READ : (FILE_SHARE_READ | FILE_SHARE_WRITE); + break; + case O_WRONLY: /* assume for MDBX_env_copy() and friends output */ + DesiredAccess = GENERIC_WRITE; + ShareMode = 0; + FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH; + break; + case O_RDWR: + DesiredAccess = GENERIC_READ | GENERIC_WRITE; + ShareMode = exclusive ? 0 : (FILE_SHARE_READ | FILE_SHARE_WRITE); + break; + } + + DWORD CreationDisposition; + switch (flags & (O_EXCL | O_CREAT)) { + default: + return ERROR_INVALID_PARAMETER; + case 0: + CreationDisposition = OPEN_EXISTING; + break; + case O_EXCL | O_CREAT: + CreationDisposition = CREATE_NEW; + FlagsAndAttributes |= FILE_ATTRIBUTE_NOT_CONTENT_INDEXED; + break; + case O_CREAT: + CreationDisposition = OPEN_ALWAYS; + FlagsAndAttributes |= FILE_ATTRIBUTE_NOT_CONTENT_INDEXED; + break; + } + + *fd = CreateFileW(pathnameW, DesiredAccess, ShareMode, NULL, + CreationDisposition, FlagsAndAttributes, NULL); + + if (*fd == INVALID_HANDLE_VALUE) + return GetLastError(); + if ((flags & O_CREAT) && GetLastError() != ERROR_ALREADY_EXISTS) { + /* set FILE_ATTRIBUTE_NOT_CONTENT_INDEXED for new file */ + DWORD FileAttributes = GetFileAttributesA(pathname); + if (FileAttributes == INVALID_FILE_ATTRIBUTES || + !SetFileAttributesA(pathname, FileAttributes | + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED)) { + int rc = GetLastError(); + CloseHandle(*fd); + *fd = INVALID_HANDLE_VALUE; + return rc; + } + } +#else + (void)exclusive; +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif /* O_CLOEXEC */ + *fd = open(pathname, flags, mode); + if (*fd < 0) + return errno; + +#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC) + int fd_flags = fcntl(*fd, F_GETFD); + if (fd_flags != -1) + (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC); +#endif /* FD_CLOEXEC && !O_CLOEXEC */ + + if ((flags & (O_RDONLY | O_WRONLY | O_RDWR)) == O_WRONLY) { + /* assume for MDBX_env_copy() and friends output */ +#if defined(O_DIRECT) + int fd_flags = fcntl(*fd, F_GETFD); + if (fd_flags != -1) + (void)fcntl(*fd, F_SETFL, fd_flags | O_DIRECT); +#endif /* O_DIRECT */ +#if defined(F_NOCACHE) + (void)fcntl(*fd, F_NOCACHE, 1); +#endif /* F_NOCACHE */ + } +#endif + + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd) { +#if defined(_WIN32) || defined(_WIN64) + return CloseHandle(fd) ? MDBX_SUCCESS : GetLastError(); +#else + return (close(fd) == 0) ? MDBX_SUCCESS : errno; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, + uint64_t offset) { + if (bytes > MAX_WRITE) + return MDBX_EINVAL; +#if defined(_WIN32) || defined(_WIN64) + OVERLAPPED ov; + ov.hEvent = 0; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); + + DWORD read = 0; + if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) { + int rc = GetLastError(); + return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc; + } +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + intptr_t read = pread(fd, buf, bytes, offset); + if (read < 0) { + int rc = errno; + return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc; + } +#endif + return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA; +} + +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t bytes, uint64_t offset) { + while (true) { +#if defined(_WIN32) || defined(_WIN64) + OVERLAPPED ov; + ov.hEvent = 0; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); + + DWORD written; + if (unlikely(!WriteFile( + fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, + &written, &ov))) + return GetLastError(); + if (likely(bytes == written)) + return MDBX_SUCCESS; +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + const intptr_t written = + pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset); + if (likely(bytes == (size_t)written)) + return MDBX_SUCCESS; + if (written < 0) { + const int rc = errno; + if (rc != EINTR) + return rc; + continue; + } +#endif + bytes -= written; + offset += written; + buf = (char *)buf + written; + } +} + +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t bytes) { + while (true) { +#if defined(_WIN32) || defined(_WIN64) + DWORD written; + if (unlikely(!WriteFile( + fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, + &written, nullptr))) + return GetLastError(); + if (likely(bytes == written)) + return MDBX_SUCCESS; +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + const intptr_t written = + write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE); + if (likely(bytes == (size_t)written)) + return MDBX_SUCCESS; + if (written < 0) { + const int rc = errno; + if (rc != EINTR) + return rc; + continue; + } +#endif + bytes -= written; + buf = (char *)buf + written; + } +} + +int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, int iovcnt, + uint64_t offset, size_t expected_written) { +#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) + size_t written = 0; + for (int i = 0; i < iovcnt; ++i) { + int rc = mdbx_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + written += iov[i].iov_len; + offset += iov[i].iov_len; + } + return (expected_written == written) ? MDBX_SUCCESS + : MDBX_EIO /* ERROR_WRITE_FAULT */; +#else + int rc; + intptr_t written; + do { + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + written = pwritev(fd, iov, iovcnt, offset); + if (likely(expected_written == (size_t)written)) + return MDBX_SUCCESS; + rc = errno; + } while (rc == EINTR); + return (written < 0) ? rc : MDBX_EIO /* Use which error code? */; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_filesync(mdbx_filehandle_t fd, + enum mdbx_syncmode_bits mode_bits) { +#if defined(_WIN32) || defined(_WIN64) + return ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) == 0 || + FlushFileBuffers(fd)) + ? MDBX_SUCCESS + : GetLastError(); +#else + +#if defined(__APPLE__) && \ + MDBX_OSX_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY + if (mode_bits & MDBX_SYNC_IODQ) + return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno; +#endif /* MacOS */ +#if defined(__linux__) || defined(__gnu_linux__) + if (mode_bits == MDBX_SYNC_SIZE && mdbx_linux_kernel_version >= 0x03060000) + return MDBX_SUCCESS; +#endif /* Linux */ + int rc; + do { +#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0 + /* LY: This code is always safe and without appreciable performance + * degradation, even on a kernel with fdatasync's bug. + * + * For more info about of a corresponding fdatasync() bug + * see http://www.spinics.net/lists/linux-ext4/msg33714.html */ + if ((mode_bits & MDBX_SYNC_SIZE) == 0) { + if (fdatasync(fd) == 0) + return MDBX_SUCCESS; + } else +#else + (void)mode_bits; +#endif + if (fsync(fd) == 0) + return MDBX_SUCCESS; + rc = errno; + } while (rc == EINTR); + return rc; +#endif +} + +int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length) { +#if defined(_WIN32) || defined(_WIN64) + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(fd, &info)) + return GetLastError(); + *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; +#else + struct stat st; + + STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + if (fstat(fd, &st)) + return errno; + + *length = st.st_size; +#endif + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd) { +#if defined(_WIN32) || defined(_WIN64) + switch (GetFileType(fd)) { + case FILE_TYPE_DISK: + return MDBX_RESULT_FALSE; + case FILE_TYPE_CHAR: + case FILE_TYPE_PIPE: + return MDBX_RESULT_TRUE; + default: + return GetLastError(); + } +#else + struct stat info; + if (fstat(fd, &info)) + return errno; + switch (info.st_mode & S_IFMT) { + case S_IFBLK: + case S_IFREG: + return MDBX_RESULT_FALSE; + case S_IFCHR: + case S_IFIFO: + case S_IFSOCK: + return MDBX_RESULT_TRUE; + case S_IFDIR: + case S_IFLNK: + default: + return MDBX_INCOMPATIBLE; + } +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length) { +#if defined(_WIN32) || defined(_WIN64) + if (mdbx_SetFileInformationByHandle) { + FILE_END_OF_FILE_INFO EndOfFileInfo; + EndOfFileInfo.EndOfFile.QuadPart = length; + return mdbx_SetFileInformationByHandle(fd, FileEndOfFileInfo, + &EndOfFileInfo, + sizeof(FILE_END_OF_FILE_INFO)) + ? MDBX_SUCCESS + : GetLastError(); + } else { + LARGE_INTEGER li; + li.QuadPart = length; + return (SetFilePointerEx(fd, li, NULL, FILE_BEGIN) && SetEndOfFile(fd)) + ? MDBX_SUCCESS + : GetLastError(); + } +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos) { +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER li; + li.QuadPart = pos; + return SetFilePointerEx(fd, li, NULL, FILE_BEGIN) ? MDBX_SUCCESS + : GetLastError(); +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS; +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg) { +#if defined(_WIN32) || defined(_WIN64) + *thread = CreateThread(NULL, 0, start_routine, arg, 0, NULL); + return *thread ? MDBX_SUCCESS : GetLastError(); +#else + return pthread_create(thread, NULL, start_routine, arg); +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = WaitForSingleObject(thread, INFINITE); + return waitstatus2errcode(code); +#else + void *unused_retval = &unused_retval; + return pthread_join(thread, &unused_retval); +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, int async) { + uint8_t *ptr = (uint8_t *)map->address + offset; +#if defined(_WIN32) || defined(_WIN64) + if (FlushViewOfFile(ptr, length) && (async || FlushFileBuffers(map->fd))) + return MDBX_SUCCESS; + return GetLastError(); +#else +#if defined(__linux__) || defined(__gnu_linux__) + if (async && mdbx_linux_kernel_version > 0x02061300) + /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op, + since the kernel properly tracks dirty pages and flushes them to storage + as necessary. */ + return MDBX_SUCCESS; +#endif /* Linux */ + const int mode = async ? MS_ASYNC : MS_SYNC; + int rc = (msync(ptr, length, mode) == 0) ? MDBX_SUCCESS : errno; +#if defined(__APPLE__) && \ + MDBX_OSX_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY + if (rc == MDBX_SUCCESS && mode == MS_SYNC) + rc = likely(fcntl(map->fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno; +#endif /* MacOS */ + return rc; +#endif +} + +MDBX_INTERNAL_FUNC int mdbx_check4nonlocal(mdbx_filehandle_t handle, + int flags) { +#if defined(_WIN32) || defined(_WIN64) + if (GetFileType(handle) != FILE_TYPE_DISK) + return ERROR_FILE_OFFLINE; + + if (mdbx_GetFileInformationByHandleEx) { + FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo; + if (mdbx_GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, + &RemoteProtocolInfo, + sizeof(RemoteProtocolInfo))) { + + if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && + !(flags & MDBX_RDONLY)) + return ERROR_FILE_OFFLINE; + if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && + !(flags & MDBX_EXCLUSIVE)) + return ERROR_REMOTE_STORAGE_MEDIA_ERROR; + } + } + + if (mdbx_NtFsControlFile) { + NTSTATUS rc; + struct { + WOF_EXTERNAL_INFO wof_info; + union { + WIM_PROVIDER_EXTERNAL_INFO wim_info; + FILE_PROVIDER_EXTERNAL_INFO_V1 file_info; + }; + size_t reserved_for_microsoft_madness[42]; + } GetExternalBacking_OutputBuffer; + IO_STATUS_BLOCK StatusBlock; + rc = mdbx_NtFsControlFile(handle, NULL, NULL, NULL, &StatusBlock, + FSCTL_GET_EXTERNAL_BACKING, NULL, 0, + &GetExternalBacking_OutputBuffer, + sizeof(GetExternalBacking_OutputBuffer)); + if (NT_SUCCESS(rc)) { + if (!(flags & MDBX_EXCLUSIVE)) + return ERROR_REMOTE_STORAGE_MEDIA_ERROR; + } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && + rc != STATUS_INVALID_DEVICE_REQUEST) + return ntstatus2errcode(rc); + } + + if (mdbx_GetVolumeInformationByHandleW && mdbx_GetFinalPathNameByHandleW) { + WCHAR *PathBuffer = mdbx_malloc(sizeof(WCHAR) * INT16_MAX); + if (!PathBuffer) + return MDBX_ENOMEM; + + int rc = MDBX_SUCCESS; + DWORD VolumeSerialNumber, FileSystemFlags; + if (!mdbx_GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, + &VolumeSerialNumber, NULL, + &FileSystemFlags, NULL, 0)) { + rc = GetLastError(); + goto bailout; + } + + if ((flags & MDBX_RDONLY) == 0) { + if (FileSystemFlags & + (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | + FILE_VOLUME_IS_COMPRESSED)) { + rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; + goto bailout; + } + } + + if (!mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, + FILE_NAME_NORMALIZED | + VOLUME_NAME_NT)) { + rc = GetLastError(); + goto bailout; + } + + if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) { + if (!(flags & MDBX_EXCLUSIVE)) { + rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; + goto bailout; + } + } else if (mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, + FILE_NAME_NORMALIZED | + VOLUME_NAME_DOS)) { + UINT DriveType = GetDriveTypeW(PathBuffer); + if (DriveType == DRIVE_NO_ROOT_DIR && + _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 && + _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) { + PathBuffer[7] = 0; + DriveType = GetDriveTypeW(PathBuffer + 4); + } + switch (DriveType) { + case DRIVE_CDROM: + if (flags & MDBX_RDONLY) + break; + // fall through + case DRIVE_UNKNOWN: + case DRIVE_NO_ROOT_DIR: + case DRIVE_REMOTE: + default: + if (!(flags & MDBX_EXCLUSIVE)) + rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; + // fall through + case DRIVE_REMOVABLE: + case DRIVE_FIXED: + case DRIVE_RAMDISK: + break; + } + } + bailout: + mdbx_free(PathBuffer); + return rc; + } +#else + (void)handle; + /* TODO: check for NFS handle ? */ + (void)flags; +#endif + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t size, const size_t limit, + const bool truncate) { + assert(size <= limit); + map->limit = 0; + map->current = 0; + map->address = nullptr; +#if defined(_WIN32) || defined(_WIN64) + map->section = NULL; + map->filesize = 0; +#endif /* Windows */ + + int err = mdbx_check4nonlocal(map->fd, flags); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + if ((flags & MDBX_RDONLY) == 0 && truncate) { + err = mdbx_ftruncate(map->fd, size); + if (err != MDBX_SUCCESS) + return err; +#if defined(_WIN32) || defined(_WIN64) + map->filesize = size; +#else + map->current = size; +#endif + } else { + uint64_t filesize; + err = mdbx_filesize(map->fd, &filesize); + if (err != MDBX_SUCCESS) + return err; +#if defined(_WIN32) || defined(_WIN64) + map->filesize = filesize; +#else + map->current = (filesize > limit) ? limit : (size_t)filesize; +#endif + } + +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER SectionSize; + SectionSize.QuadPart = size; + err = NtCreateSection( + &map->section, + /* DesiredAccess */ + (flags & MDBX_WRITEMAP) + ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | + SECTION_MAP_WRITE + : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, + /* ObjectAttributes */ NULL, /* MaximumSize (InitialSize) */ &SectionSize, + /* SectionPageProtection */ + (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, + /* AllocationAttributes */ SEC_RESERVE, map->fd); + if (!NT_SUCCESS(err)) + return ntstatus2errcode(err); + + SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 : limit; + err = NtMapViewOfSection( + map->section, GetCurrentProcess(), &map->address, + /* ZeroBits */ 0, + /* CommitSize */ 0, + /* SectionOffset */ NULL, &ViewSize, + /* InheritDisposition */ ViewUnmap, + /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, + /* Win32Protect */ + (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); + if (!NT_SUCCESS(err)) { + NtClose(map->section); + map->section = 0; + map->address = nullptr; + return ntstatus2errcode(err); + } + assert(map->address != MAP_FAILED); + + map->current = (size_t)SectionSize.QuadPart; + map->limit = ViewSize; + +#else + + map->address = mmap( + NULL, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, + MAP_SHARED, map->fd, 0); + + if (unlikely(map->address == MAP_FAILED)) { + map->limit = 0; + map->current = 0; + map->address = nullptr; + return errno; + } + map->limit = limit; + +#ifdef MADV_DONTFORK + if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0)) + return errno; +#endif +#ifdef MADV_NOHUGEPAGE + (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE); +#endif + +#endif + + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map) { +#if defined(_WIN32) || defined(_WIN64) + if (map->section) + NtClose(map->section); + NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->address); + if (!NT_SUCCESS(rc)) + ntstatus2errcode(rc); +#else + if (unlikely(munmap(map->address, map->limit))) + return errno; +#endif + + map->limit = 0; + map->current = 0; + map->address = nullptr; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t size, + size_t limit) { + assert(size <= limit); +#if defined(_WIN32) || defined(_WIN64) + assert(size != map->current || limit != map->limit || size < map->filesize); + + NTSTATUS status; + LARGE_INTEGER SectionSize; + int err, rc = MDBX_SUCCESS; + + if (!(flags & MDBX_RDONLY) && limit == map->limit && size > map->current) { + /* growth rw-section */ + SectionSize.QuadPart = size; + status = NtExtendSection(map->section, &SectionSize); + if (NT_SUCCESS(status)) { + map->current = size; + if (map->filesize < size) + map->filesize = size; + } + return ntstatus2errcode(status); + } + + if (limit > map->limit) { + /* check ability of address space for growth before umnap */ + PVOID BaseAddress = (PBYTE)map->address + map->limit; + SIZE_T RegionSize = limit - map->limit; + status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, + &RegionSize, MEM_RESERVE, PAGE_NOACCESS); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + + status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, + MEM_RELEASE); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + } + + /* Windows unable: + * - shrink a mapped file; + * - change size of mapped view; + * - extend read-only mapping; + * Therefore we should unmap/map entire section. */ + status = NtUnmapViewOfSection(GetCurrentProcess(), map->address); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + status = NtClose(map->section); + map->section = NULL; + PVOID ReservedAddress = NULL; + SIZE_T ReservedSize = limit; + + if (!NT_SUCCESS(status)) { + bailout_ntstatus: + err = ntstatus2errcode(status); + bailout: + map->address = NULL; + map->current = map->limit = 0; + if (ReservedAddress) + (void)NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, + &ReservedSize, MEM_RELEASE); + return err; + } + + /* resizing of the file may take a while, + * therefore we reserve address space to avoid occupy it by other threads */ + ReservedAddress = map->address; + status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, + &ReservedSize, MEM_RESERVE, PAGE_NOACCESS); + if (!NT_SUCCESS(status)) { + ReservedAddress = NULL; + if (status != /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) + goto bailout_ntstatus /* no way to recovery */; + + /* assume we can change base address if mapping size changed or prev address + * couldn't be used */ + map->address = NULL; + } + +retry_file_and_section: + err = mdbx_filesize(map->fd, &map->filesize); + if (err != MDBX_SUCCESS) + goto bailout; + + if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) { + err = mdbx_ftruncate(map->fd, size); + if (err == MDBX_SUCCESS) + map->filesize = size; + /* ignore error, because Windows unable shrink file + * that already mapped (by another process) */ + } + + SectionSize.QuadPart = size; + status = NtCreateSection( + &map->section, + /* DesiredAccess */ + (flags & MDBX_WRITEMAP) + ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | + SECTION_MAP_WRITE + : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, + /* ObjectAttributes */ NULL, + /* MaximumSize (InitialSize) */ &SectionSize, + /* SectionPageProtection */ + (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, + /* AllocationAttributes */ SEC_RESERVE, map->fd); + + if (!NT_SUCCESS(status)) + goto bailout_ntstatus; + + if (ReservedAddress) { + /* release reserved address space */ + status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, + &ReservedSize, MEM_RELEASE); + ReservedAddress = NULL; + if (!NT_SUCCESS(status)) + goto bailout_ntstatus; + } + +retry_mapview:; + SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit; + status = NtMapViewOfSection( + map->section, GetCurrentProcess(), &map->address, + /* ZeroBits */ 0, + /* CommitSize */ 0, + /* SectionOffset */ NULL, &ViewSize, + /* InheritDisposition */ ViewUnmap, + /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, + /* Win32Protect */ + (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); + + if (!NT_SUCCESS(status)) { + if (status == /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && + map->address) { + /* try remap at another base address */ + map->address = NULL; + goto retry_mapview; + } + NtClose(map->section); + map->section = NULL; + + if (map->address && (size != map->current || limit != map->limit)) { + /* try remap with previously size and limit, + * but will return MDBX_RESULT_TRUE on success */ + rc = MDBX_RESULT_TRUE; + size = map->current; + limit = map->limit; + goto retry_file_and_section; + } + + /* no way to recovery */ + goto bailout_ntstatus; + } + assert(map->address != MAP_FAILED); + + map->current = (size_t)SectionSize.QuadPart; + map->limit = ViewSize; +#else + + uint64_t filesize; + int rc = mdbx_filesize(map->fd, &filesize); + if (rc != MDBX_SUCCESS) + return rc; + + if (flags & MDBX_RDONLY) { + map->current = (filesize > limit) ? limit : (size_t)filesize; + if (map->current != size) + rc = MDBX_RESULT_TRUE; + } else if (filesize != size) { + rc = mdbx_ftruncate(map->fd, size); + if (rc != MDBX_SUCCESS) + return rc; + map->current = size; + } + + if (limit != map->limit) { +#if defined(_GNU_SOURCE) && (defined(__linux__) || defined(__gnu_linux__)) + void *ptr = mremap(map->address, map->limit, limit, + /* LY: in case changing the mapping size calling code + must guarantees the absence of competing threads, + and a willingness to another base address */ + MREMAP_MAYMOVE); + if (ptr == MAP_FAILED) { + rc = errno; + return (rc == EAGAIN || rc == ENOMEM) ? MDBX_RESULT_TRUE : rc; + } + map->address = ptr; + map->limit = limit; + +#ifdef MADV_DONTFORK + if (unlikely(madvise(map->address, map->limit, MADV_DONTFORK) != 0)) + return errno; +#endif + +#ifdef MADV_NOHUGEPAGE + (void)madvise(map->address, map->limit, MADV_NOHUGEPAGE); +#endif + +#else + rc = MDBX_RESULT_TRUE; +#endif /* _GNU_SOURCE && __linux__ */ + } +#endif + return rc; +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL_FUNC __cold void mdbx_osal_jitter(bool tiny) { + for (;;) { +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \ + defined(__x86_64__) + const unsigned salt = 277u * (unsigned)__rdtsc(); +#else + const unsigned salt = rand(); +#endif + + const unsigned coin = salt % (tiny ? 29u : 43u); + if (coin < 43 / 3) + break; +#if defined(_WIN32) || defined(_WIN64) + SwitchToThread(); + if (coin > 43 * 2 / 3) + Sleep(1); +#else + sched_yield(); + if (coin > 43 * 2 / 3) + usleep(coin); +#endif + } +} + +#if defined(_WIN32) || defined(_WIN64) +#elif defined(__APPLE__) || defined(__MACH__) +#include +#elif defined(__linux__) || defined(__gnu_linux__) +static __cold clockid_t choice_monoclock(void) { + struct timespec probe; +#if defined(CLOCK_BOOTTIME) + if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0) + return CLOCK_BOOTTIME; +#elif defined(CLOCK_MONOTONIC_RAW) + if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0) + return CLOCK_MONOTONIC_RAW; +#elif defined(CLOCK_MONOTONIC_COARSE) + if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0) + return CLOCK_MONOTONIC_COARSE; +#endif + return CLOCK_MONOTONIC; +} +#endif + +/*----------------------------------------------------------------------------*/ + +#if defined(_WIN32) || defined(_WIN64) +static LARGE_INTEGER performance_frequency; +#elif defined(__APPLE__) || defined(__MACH__) +static uint64_t ratio_16dot16_to_monotine; +#endif + +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16) { +#if defined(_WIN32) || defined(_WIN64) + if (unlikely(performance_frequency.QuadPart == 0)) + QueryPerformanceFrequency(&performance_frequency); + const uint64_t ratio = performance_frequency.QuadPart; +#elif defined(__APPLE__) || defined(__MACH__) + if (unlikely(ratio_16dot16_to_monotine == 0)) { + mach_timebase_info_data_t ti; + mach_timebase_info(&ti); + ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer; + } + const uint64_t ratio = ratio_16dot16_to_monotine; +#else + const uint64_t ratio = UINT64_C(1000000000); +#endif + return (ratio * seconds_16dot16 + 32768) >> 16; +} + +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime) { + static uint64_t limit; + if (unlikely(monotime > limit)) { + if (limit != 0) + return UINT32_MAX; + limit = mdbx_osal_16dot16_to_monotime(UINT32_MAX - 1); + if (monotime > limit) + return UINT32_MAX; + } +#if defined(_WIN32) || defined(_WIN64) + return (uint32_t)((monotime << 16) / performance_frequency.QuadPart); +#elif defined(__APPLE__) || defined(__MACH__) + return (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine); +#else + return (uint32_t)(monotime * 128 / 1953125); +#endif +} + +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void) { +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER counter; + counter.QuadPart = 0; + QueryPerformanceCounter(&counter); + return counter.QuadPart; +#elif defined(__APPLE__) || defined(__MACH__) + return mach_absolute_time(); +#else + +#if defined(__linux__) || defined(__gnu_linux__) + static clockid_t posix_clockid = -1; + if (unlikely(posix_clockid < 0)) + posix_clockid = choice_monoclock(); +#elif defined(CLOCK_MONOTONIC) +#define posix_clockid CLOCK_MONOTONIC +#else +#define posix_clockid CLOCK_REALTIME +#endif + + struct timespec ts; + if (unlikely(clock_gettime(posix_clockid, &ts) != 0)) { + ts.tv_nsec = 0; + ts.tv_sec = 0; + } + return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec; +#endif +} + +/*----------------------------------------------------------------------------*/ + +static void bootid_shake(bin128_t *p) { + /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */ + const uint32_t e = p->a - (p->b << 23 | p->b >> 9); + p->a = p->b ^ (p->c << 16 | p->c >> 16); + p->b = p->c + (p->d << 11 | p->d >> 21); + p->c = p->d + e; + p->d = e + p->a; +} + +static void bootid_collect(bin128_t *p, const void *s, size_t n) { + p->y += UINT64_C(64526882297375213); + bootid_shake(p); + for (size_t i = 0; i < n; ++i) { + bootid_shake(p); + p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i]; + bootid_shake(p); + p->y += 14621231; + } + bootid_shake(p); + + /* minor non-linear tomfoolery */ + const unsigned z = p->x % 61; + p->y = p->y << z | p->y >> (64 - z); + bootid_shake(p); + bootid_shake(p); + const unsigned q = p->x % 59; + p->y = p->y << q | p->y >> (64 - q); + bootid_shake(p); + bootid_shake(p); + bootid_shake(p); +} + +#if defined(_WIN32) || defined(_WIN64) + +static uint64_t windows_systemtime_ms() { + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul; +} + +static uint64_t windows_bootime(void) { + unsigned confirmed = 0; + uint64_t boottime = 0; + uint64_t up0 = mdbx_GetTickCount64(); + uint64_t st0 = windows_systemtime_ms(); + for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) { + YieldProcessor(); + const uint64_t up1 = mdbx_GetTickCount64(); + const uint64_t st1 = windows_systemtime_ms(); + if (st1 > fuse && st1 == st0 && up1 == up0) { + uint64_t diff = st1 - up1; + if (boottime == diff) { + if (++confirmed > 4) + return boottime; + } else { + confirmed = 0; + boottime = diff; + } + fuse = st1; + Sleep(1); + } + st0 = st1; + up0 = up1; + } + return 0; +} + +static LSTATUS mdbx_RegGetValue(HKEY hkey, LPCWSTR lpSubKey, LPCWSTR lpValue, + DWORD dwFlags, LPDWORD pdwType, PVOID pvData, + LPDWORD pcbData) { + LSTATUS rc = + RegGetValueW(hkey, lpSubKey, lpValue, dwFlags, pdwType, pvData, pcbData); + if (rc != ERROR_FILE_NOT_FOUND) + return rc; + + rc = RegGetValueW(hkey, lpSubKey, lpValue, + dwFlags | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, pdwType, + pvData, pcbData); + if (rc != ERROR_FILE_NOT_FOUND) + return rc; + return RegGetValueW(hkey, lpSubKey, lpValue, + dwFlags | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, pdwType, + pvData, pcbData); +} +#endif + +static __cold __maybe_unused bool bootid_parse_uuid(bin128_t *s, const void *p, + const size_t n) { + if (n > 31) { + unsigned bits = 0; + for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ { + uint8_t c = ((const uint8_t *)p)[i]; + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'a' && c <= 'f') + c -= 'a' - 10; + else if (c >= 'A' && c <= 'F') + c -= 'A' - 10; + else + continue; + assert(c <= 15); + c ^= s->y >> 60; + s->y = s->y << 4 | s->x >> 60; + s->x = s->x << 4 | c; + bits += 4; + } + if (bits > 42 * 3) + /* UUID parsed successfully */ + return true; + } + + if (n > 15) /* is enough handle it as a binary? */ { + if (n == sizeof(bin128_t)) { + bin128_t aligned; + memcpy(&aligned, p, sizeof(bin128_t)); + s->x += aligned.x; + s->y += aligned.y; + } else + bootid_collect(s, p, n); + return true; + } + + if (n) + bootid_collect(s, p, n); + return false; +} + +__cold MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void) { + bin128_t bin = {{0, 0}}; + bool got_machineid = false, got_bootime = false, got_bootseq = false; + +#if defined(__linux__) || defined(__gnu_linux__) + { + const int fd = + open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_NOFOLLOW); + if (fd != -1) { + struct statvfs fs; + char buf[42]; + const ssize_t len = + (fstatvfs(fd, &fs) == 0 && fs.f_fsid == /* procfs */ 0x00009FA0) + ? read(fd, buf, sizeof(buf)) + : -1; + close(fd); + if (len > 0 && bootid_parse_uuid(&bin, buf, len)) + return bin; + } + } +#endif /* Linux */ + +#if defined(__APPLE__) || defined(__MACH__) + { + char buf[42]; + size_t len = sizeof(buf); + if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && + bootid_parse_uuid(&bin, buf, len)) + return bin; + +#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \ + __MAC_OS_X_VERSION_MIN_REQUIRED > 1050 + uuid_t uuid; + struct timespec wait = {0, 1000000000u / 42}; + if (!gethostuuid(uuid, &wait) && + bootid_parse_uuid(&bin, uuid, sizeof(uuid))) + got_machineid = true; +#endif /* > 10.5 */ + + struct timeval boottime; + len = sizeof(boottime); + if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && + len == sizeof(boottime) && boottime.tv_sec) + got_bootime = true; + } +#endif /* Apple/Darwin */ + +#if defined(_WIN32) || defined(_WIN64) + { + union buf { + DWORD BootId; + DWORD BaseTime; + SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo; + struct { + LARGE_INTEGER BootTime; + LARGE_INTEGER CurrentTime; + LARGE_INTEGER TimeZoneBias; + ULONG TimeZoneId; + ULONG Reserved; + ULONGLONG BootTimeBias; + ULONGLONG SleepTimeBias; + } SysTimeOfDayInfoHacked; + wchar_t MachineGuid[42]; + char DigitalProductId[248]; + } buf; + + static const wchar_t HKLM_MicrosoftCryptography[] = + L"SOFTWARE\\Microsoft\\Cryptography"; + DWORD len = sizeof(buf); + /* Windows is madness and must die */ + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, + L"MachineGuid", RRF_RT_ANY, NULL, &buf.MachineGuid, + &len) == ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) + got_machineid = bootid_parse_uuid(&bin, &buf.MachineGuid, len); + + if (!got_machineid) { + /* again, Windows is madness */ + static const wchar_t HKLM_WindowsNT[] = + L"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion"; + static const wchar_t HKLM_WindowsNT_DPK[] = + L"SOFTWARE\\Microsoft\\Windows " + L"NT\\CurrentVersion\\DefaultProductKey"; + static const wchar_t HKLM_WindowsNT_DPK2[] = + L"SOFTWARE\\Microsoft\\Windows " + L"NT\\CurrentVersion\\DefaultProductKey2"; + + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, + L"DigitalProductId", RRF_RT_ANY, NULL, + &buf.DigitalProductId, &len) == ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&bin, &buf.DigitalProductId, len); + got_machineid = true; + } + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, + L"DigitalProductId", RRF_RT_ANY, NULL, + &buf.DigitalProductId, &len) == ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&bin, &buf.DigitalProductId, len); + got_machineid = true; + } + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, + L"DigitalProductId", RRF_RT_ANY, NULL, + &buf.DigitalProductId, &len) == ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&bin, &buf.DigitalProductId, len); + got_machineid = true; + } + } + + static const wchar_t HKLM_PrefetcherParams[] = + L"SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory " + L"Management\\PrefetchParameters"; + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, L"BootId", + RRF_RT_DWORD, NULL, &buf.BootId, + &len) == ERROR_SUCCESS && + len > 1 && len < sizeof(buf)) { + bootid_collect(&bin, &buf.BootId, len); + got_bootseq = true; + } + + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, L"BaseTime", + RRF_RT_DWORD, NULL, &buf.BaseTime, + &len) == ERROR_SUCCESS && + len >= sizeof(buf.BaseTime) && buf.BaseTime) { + bootid_collect(&bin, &buf.BaseTime, len); + got_bootime = true; + } + + /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */ + NTSTATUS status = NtQuerySystemInformation( + 0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo, + sizeof(buf.SysTimeOfDayInfo), &len); + if (NT_SUCCESS(status) && + len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTime) + + sizeof(buf.SysTimeOfDayInfoHacked.BootTime) && + buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) { + bootid_collect(&bin, &buf.SysTimeOfDayInfoHacked.BootTime, + sizeof(buf.SysTimeOfDayInfoHacked.BootTime)); + got_bootime = true; + } + + if (!got_bootime) { + uint64_t boottime = windows_bootime(); + if (boottime) { + bootid_collect(&bin, &boottime, sizeof(boottime)); + got_bootime = true; + } + } + } +#endif /* Windows */ + +#if defined(CTL_HW) && defined(HW_UUID) + if (!got_machineid) { + static const int mib[] = {CTL_HW, HW_UUID}; + char buf[42]; + size_t len = sizeof(buf); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) +#endif + mib, + ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0) + got_machineid = bootid_parse_uuid(&bin, buf, len); + } +#endif /* CTL_HW && HW_UUID */ + +#if defined(CTL_KERN) && defined(KERN_HOSTUUID) + if (!got_machineid) { + static const int mib[] = {CTL_KERN, KERN_HOSTUUID}; + char buf[42]; + size_t len = sizeof(buf); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) +#endif + mib, + ARRAY_LENGTH(mib), &buf, &len, NULL, 0) == 0) + got_machineid = bootid_parse_uuid(&bin, buf, len); + } +#endif /* CTL_KERN && KERN_HOSTUUID */ + +#if defined(__NetBSD__) + if (!got_machineid) { + char buf[42]; + size_t len = sizeof(buf); + if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, NULL, 0) == 0) + got_machineid = bootid_parse_uuid(&bin, buf, len); + } +#endif /* __NetBSD__ */ + +#if _XOPEN_SOURCE_EXTENDED + if (!got_machineid) { + const int hostid = gethostid(); + if (hostid > 0) { + bootid_collect(&bin, &hostid, sizeof(hostid)); + got_machineid = true; + } + } +#endif /* _XOPEN_SOURCE_EXTENDED */ + + if (!got_machineid) { + lack: + bin.x = bin.y = 0; + return bin; + } + + /*--------------------------------------------------------------------------*/ + +#if defined(CTL_KERN) && defined(KERN_BOOTTIME) + if (!got_bootime) { + static const int mib[] = {CTL_KERN, KERN_BOOTTIME}; + struct timeval boottime; + size_t len = sizeof(boottime); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) +#endif + mib, + ARRAY_LENGTH(mib), &boottime, &len, NULL, 0) == 0 && + len == sizeof(boottime) && boottime.tv_sec) { + bootid_collect(&bin, &boottime, len); + got_bootime = true; + } + } +#endif /* CTL_KERN && KERN_BOOTTIME */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) + if (!got_bootime) { + kstat_ctl_t *kc = kstat_open(); + if (kc) { + kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc"); + if (kp && kstat_read(kc, kp, 0) != -1) { + kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time"); + if (kn) { + switch (kn->data_type) { + case KSTAT_DATA_INT32: + case KSTAT_DATA_UINT32: + bootid_collect(&bin, &kn->value, sizeof(int32_t)); + got_boottime = true; + case KSTAT_DATA_INT64: + case KSTAT_DATA_UINT64: + bootid_collect(&bin, &kn->value, sizeof(int64_t)); + got_boottime = true; + } + } + } + kstat_close(kc); + } + } +#endif /* SunOS / Solaris */ + +#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME) + if (!got_bootime) { + setutxent(); + const struct utmpx id = {.ut_type = BOOT_TIME}; + const struct utmpx *entry = getutxid(&id); + if (entry) { + bootid_collect(&bin, entry, sizeof(*entry)); + got_bootime = true; + while (unlikely((entry = getutxid(&id)) != nullptr)) { + /* have multiple reboot records, assuming we can distinguish next + * bootsession even if RTC is wrong or absent */ + bootid_collect(&bin, entry, sizeof(*entry)); + got_bootseq = true; + } + } + endutxent(); + } +#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */ + + if (!got_bootseq) { + if (!got_bootime || !MDBX_TRUST_RTC) + goto lack; + +#if defined(_WIN32) || defined(_WIN64) + FILETIME now; + GetSystemTimeAsFileTime(&now); + if (0x1CCCCCC > now.dwHighDateTime) +#else + struct timespec mono, real; + if (clock_gettime(CLOCK_MONOTONIC, &mono) || + clock_gettime(CLOCK_REALTIME, &real) || + /* wrong time, RTC is mad or absent */ + 1555555555l > real.tv_sec || + /* seems no adjustment by RTC/NTP, i.e. a fake time */ + real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || + (real.tv_sec - mono.tv_sec) % 900u == 0) +#endif + goto lack; + } + + return bin; +} diff --git a/src/contrib/db/libmdbx/src/elements/osal.h b/src/contrib/db/libmdbx/src/elements/osal.h new file mode 100644 index 0000000..c1988ad --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/osal.h @@ -0,0 +1,959 @@ +/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ + +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +#if defined(_WIN32) || defined(_WIN64) +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ + !defined(MDBX_TOOLS) +#define _NO_CRT_STDIO_INLINE +#endif +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* C99 includes */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/* C11 stdalign.h */ +#if __has_include() +#include +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#define alignas(N) _Alignas(N) +#elif defined(_MSC_VER) +#define alignas(N) __declspec(align(N)) +#elif __has_attribute(__aligned__) || defined(__GNUC__) +#define alignas(N) __attribute__((__aligned__(N))) +#else +#error "FIXME: Required _alignas() or equivalent." +#endif + +/*----------------------------------------------------------------------------*/ +/* Systems includes */ + +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__BSD__) || defined(__NETBSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) +#include +#include +#include +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include +#elif defined(__OpenBSD__) || defined(__NetBSD__) +#include +#else +#define SYSCTL_LEGACY_NONCONST_MIB +#endif +#include +#else +#include +#ifndef _POSIX_C_SOURCE +#ifdef _POSIX_SOURCE +#define _POSIX_C_SOURCE 1 +#else +#define _POSIX_C_SOURCE 0 +#endif +#endif +#endif /* !xBSD */ + +#if defined(__FreeBSD__) || defined(__OpenBSD__) || __has_include() +#include +#endif + +#if defined(__APPLE__) || defined(__MACH__) || __has_include() +#include +#endif /* MacOS */ + +#if defined(__MACH__) +#include +#include +#include +#include +#undef P_DIRTY +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +#include +#include +#include +#endif /* Linux */ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 0 +#endif + +#ifndef _XOPEN_SOURCE_EXTENDED +#define _XOPEN_SOURCE_EXTENDED 0 +#else +#include +#endif /* _XOPEN_SOURCE_EXTENDED */ + +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) +#include +#endif /* SunOS/Solaris */ + +#if defined(_WIN32) || defined(_WIN64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#include +#include +#define HAVE_SYS_STAT_H +#define HAVE_SYS_TYPES_H +typedef HANDLE mdbx_thread_t; +typedef unsigned mdbx_thread_key_t; +#define MDBX_OSAL_SECTION HANDLE +#define MAP_FAILED NULL +#define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) +#define THREAD_CALL WINAPI +#define THREAD_RESULT DWORD +typedef struct { + HANDLE mutex; + HANDLE event; +} mdbx_condmutex_t; +typedef CRITICAL_SECTION mdbx_fastmutex_t; + +#if MDBX_AVOID_CRT +#ifndef mdbx_malloc +static inline void *mdbx_malloc(size_t bytes) { + return LocalAlloc(LMEM_FIXED, bytes); +} +#endif /* mdbx_malloc */ + +#ifndef mdbx_calloc +static inline void *mdbx_calloc(size_t nelem, size_t size) { + return LocalAlloc(LMEM_FIXED | LMEM_ZEROINIT, nelem * size); +} +#endif /* mdbx_calloc */ + +#ifndef mdbx_realloc +static inline void *mdbx_realloc(void *ptr, size_t bytes) { + return LocalReAlloc(ptr, bytes, LMEM_MOVEABLE); +} +#endif /* mdbx_realloc */ + +#ifndef mdbx_free +#define mdbx_free LocalFree +#endif /* mdbx_free */ +#else +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup _strdup +#endif /* MDBX_AVOID_CRT */ + +#ifndef snprintf +#define snprintf _snprintf /* ntdll */ +#endif + +#ifndef vsnprintf +#define vsnprintf _vsnprintf /* ntdll */ +#endif + +#else /*----------------------------------------------------------------------*/ + +#include +#include +#include +#include +#include +#include +#include +#include +typedef pthread_t mdbx_thread_t; +typedef pthread_key_t mdbx_thread_key_t; +#define INVALID_HANDLE_VALUE (-1) +#define THREAD_CALL +#define THREAD_RESULT void * +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; +} mdbx_condmutex_t; +typedef pthread_mutex_t mdbx_fastmutex_t; +#define mdbx_malloc malloc +#define mdbx_calloc calloc +#define mdbx_realloc realloc +#define mdbx_free free +#define mdbx_strdup strdup +#endif /* Platform */ + +#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) +/* malloc_usable_size() already provided */ +#elif defined(__APPLE__) +#define malloc_usable_size(ptr) malloc_size(ptr) +#elif defined(_MSC_VER) && !MDBX_AVOID_CRT +#define malloc_usable_size(ptr) _msize(ptr) +#endif /* malloc_usable_size */ + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(HAVE_SYS_STAT_H) || __has_include() +#include +#endif +#if defined(HAVE_SYS_TYPES_H) || __has_include() +#include +#endif +#if defined(HAVE_SYS_FILE_H) || __has_include() +#include +#endif +/* *INDENT-ON* */ +/* clang-format on */ + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if !defined(MADV_DODUMP) && defined(MADV_CORE) +#define MADV_DODUMP MADV_CORE +#endif /* MADV_CORE -> MADV_DODUMP */ + +#if !defined(MADV_DONTDUMP) && defined(MADV_NOCORE) +#define MADV_DONTDUMP MADV_NOCORE +#endif /* MADV_NOCORE -> MADV_DONTDUMP */ + +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ + defined(i486) || defined(__i486) || defined(__i486__) || \ + defined(i586) | defined(__i586) || defined(__i586__) || defined(i686) || \ + defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ + defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#ifndef __ia32__ +/* LY: define neutral __ia32__ for x86 and x86-64 archs */ +#define __ia32__ 1 +#endif /* __ia32__ */ +#if !defined(__amd64__) && (defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64) || defined(_M_X64)) +/* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ +#define __amd64__ 1 +#endif /* __amd64__ */ +#endif /* all x86 */ + +#if !defined(MDBX_UNALIGNED_OK) +#if defined(_MSC_VER) +#define MDBX_UNALIGNED_OK 1 /* avoid MSVC misoptimization */ +#elif __CLANG_PREREQ(5, 0) || __GNUC_PREREQ(5, 0) +#define MDBX_UNALIGNED_OK 0 /* expecting optimization is well done */ +#elif (defined(__ia32__) || defined(__ARM_FEATURE_UNALIGNED)) && \ + !defined(__ALIGNED__) +#define MDBX_UNALIGNED_OK 1 +#else +#define MDBX_UNALIGNED_OK 0 +#endif +#endif /* MDBX_UNALIGNED_OK */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error \ + "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +/*----------------------------------------------------------------------------*/ +/* Compiler's includes for builtins/intrinsics */ + +#if defined(_MSC_VER) || defined(__INTEL_COMPILER) +#include +#elif __GNUC_PREREQ(4, 4) || defined(__clang__) +#if defined(__ia32__) || defined(__e2k__) +#include +#endif /* __ia32__ */ +#if defined(__ia32__) +#include +#endif /* __ia32__ */ +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) +#include +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) +#include +#elif defined(__IBMC__) && defined(__powerpc) +#include +#elif defined(_AIX) +#include +#include +#elif (defined(__osf__) && defined(__DECC)) || defined(__alpha) +#include +#include +#elif defined(__MWERKS__) +/* CodeWarrior - troubles ? */ +#pragma gcc_extensions +#elif defined(__SNC__) +/* Sony PS3 - troubles ? */ +#elif defined(__hppa__) || defined(__hppa) +#include +#else +#error Unsupported C compiler, please use GNU C 4.4 or newer +#endif /* Compiler */ + +/*----------------------------------------------------------------------------*/ +/* Byteorder */ + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) + +/* *INDENT-OFF* */ +/* clang-format off */ +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) || \ + defined(HAVE_ENDIAN_H) || __has_include() +#include +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ + defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#include +#elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() +#include +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ + (__has_include() && __has_include()) +#include +#include +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__NETBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() +#include +#endif /* OS */ +/* *INDENT-ON* */ +/* clang-format on */ + +#if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN +#define __BYTE_ORDER__ __BYTE_ORDER +#elif defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN) +#define __ORDER_LITTLE_ENDIAN__ _LITTLE_ENDIAN +#define __ORDER_BIG_ENDIAN__ _BIG_ENDIAN +#define __BYTE_ORDER__ _BYTE_ORDER +#else +#define __ORDER_LITTLE_ENDIAN__ 1234 +#define __ORDER_BIG_ENDIAN__ 4321 + +#if defined(__LITTLE_ENDIAN__) || \ + (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ + defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ + defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ + defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ + defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ + defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) +#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ + +#elif defined(__BIG_ENDIAN__) || \ + (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ + defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ + defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ + defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ + defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ + +#else +#error __BYTE_ORDER__ should be defined. +#endif /* Arch */ + +#endif +#endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ + +/*----------------------------------------------------------------------------*/ +/* Memory/Compiler barriers, cache coherence */ + +static __maybe_unused __inline void mdbx_compiler_barrier(void) { +#if defined(__clang__) || defined(__GNUC__) + __asm__ __volatile__("" ::: "memory"); +#elif defined(_MSC_VER) + _ReadWriteBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ + __memory_barrier(); + if (type > MDBX_BARRIER_COMPILER) +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __compiler_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __fence(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __maybe_unused __inline void mdbx_memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__i386__) || defined(__x86_64__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and invalidation */ + +#ifndef MDBX_CPU_WRITEBACK_IS_COHERENT +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ + defined(__hppa__) +#define MDBX_CPU_WRITEBACK_IS_COHERENT 1 +#else +#define MDBX_CPU_WRITEBACK_IS_COHERENT 0 +#endif +#endif /* MDBX_CPU_WRITEBACK_IS_COHERENT */ + +#ifndef MDBX_CACHELINE_SIZE +#if defined(SYSTEM_CACHE_ALIGNMENT_SIZE) +#define MDBX_CACHELINE_SIZE SYSTEM_CACHE_ALIGNMENT_SIZE +#elif defined(__ia64__) || defined(__ia64) || defined(_M_IA64) +#define MDBX_CACHELINE_SIZE 128 +#else +#define MDBX_CACHELINE_SIZE 64 +#endif +#endif /* MDBX_CACHELINE_SIZE */ + +#if MDBX_CPU_WRITEBACK_IS_COHERENT +#define mdbx_flush_noncoherent_cpu_writeback() mdbx_compiler_barrier() +#else +#define mdbx_flush_noncoherent_cpu_writeback() mdbx_memory_barrier() +#endif + +#if __has_include() +#include +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS should have explicit cache control */ +#include +#endif + +#ifndef MDBX_CPU_CACHE_MMAP_NONCOHERENT +#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ + defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ + defined(__MWERKS__) || defined(__sgi) +/* MIPS has cache coherency issues. */ +#define MDBX_CPU_CACHE_MMAP_NONCOHERENT 1 +#else +/* LY: assume no relevant mmap/dcache issues. */ +#define MDBX_CPU_CACHE_MMAP_NONCOHERENT 0 +#endif +#endif /* ndef MDBX_CPU_CACHE_MMAP_NONCOHERENT */ + +static __maybe_unused __inline void +mdbx_invalidate_mmap_noncoherent_cache(void *addr, size_t nbytes) { +#if MDBX_CPU_CACHE_MMAP_NONCOHERENT +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush(addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#else /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */ + (void)addr; + (void)nbytes; +#endif /* MDBX_CPU_CACHE_MMAP_NONCOHERENT */ +} + +/*----------------------------------------------------------------------------*/ +/* libc compatibility stuff */ + +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ + (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#define mdbx_asprintf asprintf +#define mdbx_vasprintf vasprintf +#else +MDBX_INTERNAL_FUNC __printf_args(2, 3) int __maybe_unused + mdbx_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL_FUNC int mdbx_vasprintf(char **strp, const char *fmt, va_list ap); +#endif + +/*----------------------------------------------------------------------------*/ +/* OS abstraction layer stuff */ + +/* max bytes to write in one call */ +#if defined(_WIN32) || defined(_WIN64) +#define MAX_WRITE UINT32_C(0x01000000) +#else +#define MAX_WRITE UINT32_C(0x3fff0000) +#endif + +#if defined(__linux__) || defined(__gnu_linux__) +MDBX_INTERNAL_VAR uint32_t mdbx_linux_kernel_version; +#endif /* Linux */ + +/* Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. */ +static __maybe_unused __inline size_t mdbx_syspagesize(void) { +#if defined(_WIN32) || defined(_WIN64) + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +#else + return sysconf(_SC_PAGE_SIZE); +#endif +} + +#ifndef mdbx_strdup +LIBMDBX_API char *mdbx_strdup(const char *str); +#endif + +static __maybe_unused __inline int mdbx_get_errno(void) { +#if defined(_WIN32) || defined(_WIN64) + DWORD rc = GetLastError(); +#else + int rc = errno; +#endif + return rc; +} + +#ifndef mdbx_memalign_alloc +MDBX_INTERNAL_FUNC int mdbx_memalign_alloc(size_t alignment, size_t bytes, + void **result); +#endif +#ifndef mdbx_memalign_free +MDBX_INTERNAL_FUNC void mdbx_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL_FUNC int mdbx_condmutex_init(mdbx_condmutex_t *condmutex); +MDBX_INTERNAL_FUNC int mdbx_condmutex_lock(mdbx_condmutex_t *condmutex); +MDBX_INTERNAL_FUNC int mdbx_condmutex_unlock(mdbx_condmutex_t *condmutex); +MDBX_INTERNAL_FUNC int mdbx_condmutex_signal(mdbx_condmutex_t *condmutex); +MDBX_INTERNAL_FUNC int mdbx_condmutex_wait(mdbx_condmutex_t *condmutex); +MDBX_INTERNAL_FUNC int mdbx_condmutex_destroy(mdbx_condmutex_t *condmutex); + +MDBX_INTERNAL_FUNC int mdbx_fastmutex_init(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_acquire(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_release(mdbx_fastmutex_t *fastmutex); +MDBX_INTERNAL_FUNC int mdbx_fastmutex_destroy(mdbx_fastmutex_t *fastmutex); + +MDBX_INTERNAL_FUNC int mdbx_pwritev(mdbx_filehandle_t fd, struct iovec *iov, + int iovcnt, uint64_t offset, + size_t expected_written); +MDBX_INTERNAL_FUNC int mdbx_pread(mdbx_filehandle_t fd, void *buf, size_t count, + uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_pwrite(mdbx_filehandle_t fd, const void *buf, + size_t count, uint64_t offset); +MDBX_INTERNAL_FUNC int mdbx_write(mdbx_filehandle_t fd, const void *buf, + size_t count); + +MDBX_INTERNAL_FUNC int +mdbx_thread_create(mdbx_thread_t *thread, + THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL_FUNC int mdbx_thread_join(mdbx_thread_t thread); + +enum mdbx_syncmode_bits { + MDBX_SYNC_DATA = 1, + MDBX_SYNC_SIZE = 2, + MDBX_SYNC_IODQ = 4 +}; + +MDBX_INTERNAL_FUNC int mdbx_filesync(mdbx_filehandle_t fd, + enum mdbx_syncmode_bits mode_bits); +MDBX_INTERNAL_FUNC int mdbx_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL_FUNC int mdbx_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL_FUNC int mdbx_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL_FUNC int mdbx_openfile(const char *pathname, int flags, + mode_t mode, mdbx_filehandle_t *fd, + bool exclusive); +MDBX_INTERNAL_FUNC int mdbx_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL_FUNC int mdbx_removefile(const char *pathname); +MDBX_INTERNAL_FUNC int mdbx_is_pipe(mdbx_filehandle_t fd); + +typedef struct mdbx_mmap_param { + union { + void *address; + uint8_t *dxb; + struct MDBX_lockinfo *lck; + }; + mdbx_filehandle_t fd; + size_t limit; /* mapping length, but NOT a size of file nor DB */ + size_t current; /* mapped region size, i.e. the size of file and DB */ +#if defined(_WIN32) || defined(_WIN64) + uint64_t filesize /* in-process cache of a file size. */; +#endif +#ifdef MDBX_OSAL_SECTION + MDBX_OSAL_SECTION section; +#endif +} mdbx_mmap_t; + +MDBX_INTERNAL_FUNC int mdbx_mmap(const int flags, mdbx_mmap_t *map, + const size_t must, const size_t limit, + const bool truncate); +MDBX_INTERNAL_FUNC int mdbx_munmap(mdbx_mmap_t *map); +MDBX_INTERNAL_FUNC int mdbx_mresize(int flags, mdbx_mmap_t *map, size_t current, + size_t wanna); +#if defined(_WIN32) || defined(_WIN64) +typedef struct { + unsigned limit, count; + HANDLE handles[31]; +} mdbx_handle_array_t; +MDBX_INTERNAL_FUNC int +mdbx_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL_FUNC int +mdbx_resume_threads_after_remap(mdbx_handle_array_t *array); +#endif /* Windows */ +MDBX_INTERNAL_FUNC int mdbx_msync(mdbx_mmap_t *map, size_t offset, + size_t length, int async); +MDBX_INTERNAL_FUNC int mdbx_check4nonlocal(mdbx_filehandle_t handle, int flags); + +static __maybe_unused __inline uint32_t mdbx_getpid(void) { + STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentProcessId(); +#else + return getpid(); +#endif +} + +static __maybe_unused __inline size_t mdbx_thread_self(void) { + STATIC_ASSERT(sizeof(mdbx_tid_t) <= sizeof(size_t)); +#if defined(_WIN32) || defined(_WIN64) + return GetCurrentThreadId(); +#else + return (size_t)pthread_self(); +#endif +} + +MDBX_INTERNAL_FUNC void __maybe_unused mdbx_osal_jitter(bool tiny); +MDBX_INTERNAL_FUNC uint64_t mdbx_osal_monotime(void); +MDBX_INTERNAL_FUNC uint64_t +mdbx_osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL_FUNC uint32_t mdbx_osal_monotime_to_16dot16(uint64_t monotime); + +typedef union bin128 { + __anonymous_struct_extension__ struct { uint64_t x, y; }; + __anonymous_struct_extension__ struct { uint32_t a, b, c, d; }; +} bin128_t; + +MDBX_INTERNAL_FUNC bin128_t mdbx_osal_bootid(void); +/*----------------------------------------------------------------------------*/ +/* lck stuff */ + +#if defined(_WIN32) || defined(_WIN64) +#undef MDBX_OSAL_LOCK +#define MDBX_OSAL_LOCK_SIGN UINT32_C(0xF10C) +#else +#define MDBX_OSAL_LOCK pthread_mutex_t +#define MDBX_OSAL_LOCK_SIGN UINT32_C(0x8017) +#endif /* MDBX_OSAL_LOCK */ + +/// \brief Initialization of synchronization primitives linked with MDBX_env +/// instance both in LCK-file and within the current process. +/// \param +/// global_uniqueness_flag = true - denotes that there are no other processes +/// working with DB and LCK-file. Thus the function MUST initialize +/// shared synchronization objects in memory-mapped LCK-file. +/// global_uniqueness_flag = false - denotes that at least one process is +/// already working with DB and LCK-file, including the case when DB +/// has already been opened in the current process. Thus the function +/// MUST NOT initialize shared synchronization objects in memory-mapped +/// LCK-file that are already in use. +/// \return Error code or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_init(MDBX_env *env, + MDBX_env *inprocess_neighbor, + int global_uniqueness_flag); + +/// \brief Disconnects from shared interprocess objects and destructs +/// synchronization objects linked with MDBX_env instance +/// within the current process. +/// \param +/// inprocess_neighbor = NULL - if the current process does not have other +/// instances of MDBX_env linked with the DB being closed. +/// Thus the function MUST check for other processes working with DB or +/// LCK-file, and keep or destroy shared synchronization objects in +/// memory-mapped LCK-file depending on the result. +/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env +/// (anyone of there is several) working with DB or LCK-file within the +/// current process. Thus the function MUST NOT try to acquire exclusive +/// lock and/or try to destruct shared synchronization objects linked with +/// DB or LCK-file. Moreover, the implementation MUST ensure correct work +/// of other instances of MDBX_env within the current process, e.g. +/// restore POSIX-fcntl locks after the closing of file descriptors. +/// \return Error code (MDBX_PANIC) or zero on success. +MDBX_INTERNAL_FUNC int mdbx_lck_destroy(MDBX_env *env, + MDBX_env *inprocess_neighbor); + +/// \brief Connects to shared interprocess locking objects and tries to acquire +/// the maximum lock level (shared if exclusive is not available) +/// Depending on implementation or/and platform (Windows) this function may +/// acquire the non-OS super-level lock (e.g. for shared synchronization +/// objects initialization), which will be downgraded to OS-exclusive or +/// shared via explicit calling of mdbx_lck_downgrade(). +/// \return +/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus +/// the current process is the first and only after the last use of DB. +/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus +/// DB has already been opened and now is used by other processes. +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_lck_seize(MDBX_env *env); + +/// \brief Downgrades the level of initially acquired lock to +/// operational level specified by agrument. The reson for such downgrade: +/// - unblocking of other processes that are waiting for access, i.e. +/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes +/// should be made aware that access is unavailable rather than +/// wait for it. +/// - freeing locks that interfere file operation (expecially for Windows) +/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. +/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive +/// operational lock. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_lck_downgrade(MDBX_env *env); + +/// \brief Locks LCK-file or/and table of readers for (de)registering. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rdt_lock(MDBX_env *env); + +/// \brief Unlocks LCK-file or/and table of readers after (de)registering. +MDBX_INTERNAL_FUNC void mdbx_rdt_unlock(MDBX_env *env); + +/// \brief Acquires lock for DB change (on writing transaction start) +/// Reading transactions will not be blocked. +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +/// \return Error code or zero on success +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dontwait); + +/// \brief Releases lock once DB changes is made (after writing transaction +/// has finished). +/// Declared as LIBMDBX_API because it is used in mdbx_chk. +LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); + +/// \brief Sets alive-flag of reader presence (indicative lock) for PID of +/// the current process. The function does no more than needed for +/// the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_set(MDBX_env *env); + +/// \brief Resets alive-flag of reader presence (indicative lock) +/// for PID of the current process. The function does no more than needed +/// for the correct working of mdbx_rpid_check() in other processes. +/// \return Error code or zero on success +MDBX_INTERNAL_FUNC int mdbx_rpid_clear(MDBX_env *env); + +/// \brief Checks for reading process status with the given pid with help of +/// alive-flag of presence (indicative lock) or using another way. +/// \return +/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive +/// and working with DB (indicative lock is present). +/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent +/// or not working with DB (indicative lock is not present). +/// Otherwise (not 0 and not -1) - error code. +MDBX_INTERNAL_FUNC int mdbx_rpid_check(MDBX_env *env, uint32_t pid); + +#if defined(_WIN32) || defined(_WIN64) +typedef union MDBX_srwlock { + struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} MDBX_srwlock; + +typedef void(WINAPI *MDBX_srwlock_function)(MDBX_srwlock *); +MDBX_INTERNAL_VAR MDBX_srwlock_function mdbx_srwlock_Init, + mdbx_srwlock_AcquireShared, mdbx_srwlock_ReleaseShared, + mdbx_srwlock_AcquireExclusive, mdbx_srwlock_ReleaseExclusive; + +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx + mdbx_GetFileInformationByHandleEx; + +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, + _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, + _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, + _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); +MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW + mdbx_GetVolumeInformationByHandleW; + +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, + _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, + _In_ DWORD dwFlags); +MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; + +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( + _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); +MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle + mdbx_SetFileInformationByHandle; + +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( + IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); +MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; + +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); +MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ + +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( + HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); +MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; + +#if 0 /* LY: unused for now */ +#if !defined(_WIN32_WINNT_WIN81) || _WIN32_WINNT < _WIN32_WINNT_WIN81 +typedef enum OFFER_PRIORITY { + VmOfferPriorityVeryLow = 1, + VmOfferPriorityLow, + VmOfferPriorityBelowNormal, + VmOfferPriorityNormal +} OFFER_PRIORITY; +#endif /* Windows 8.1 */ + +typedef DWORD(WINAPI *MDBX_DiscardVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_DiscardVirtualMemory mdbx_DiscardVirtualMemory; + +typedef DWORD(WINAPI *MDBX_ReclaimVirtualMemory)(PVOID VirtualAddress, + SIZE_T Size); +MDBX_INTERNAL_VAR MDBX_ReclaimVirtualMemory mdbx_ReclaimVirtualMemory; + +typedef DWORD(WINAPI *MDBX_OfferVirtualMemory( + PVOID VirtualAddress, + SIZE_T Size, + OFFER_PRIORITY Priority +); +MDBX_INTERNAL_VAR MDBX_OfferVirtualMemory mdbx_OfferVirtualMemory; +#endif /* unused for now */ + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ +/* Atomics */ + +#if !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ + !(defined(__GNUC__) || defined(__clang__))) +#include +#elif defined(__GNUC__) || defined(__clang__) +/* LY: nothing required */ +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include +#else +#error FIXME atomic-ops +#endif + +/*----------------------------------------------------------------------------*/ + +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ + +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ + +#ifdef _MSC_VER +#pragma warning(pop) +#endif diff --git a/src/contrib/db/libmdbx/src/elements/version.c.in b/src/contrib/db/libmdbx/src/elements/version.c.in new file mode 100644 index 0000000..2854bd5 --- /dev/null +++ b/src/contrib/db/libmdbx/src/elements/version.c.in @@ -0,0 +1,46 @@ +/* This is CMake-template for libmdbx's version.c + ******************************************************************************/ + +#include "internals.h" + +#if MDBX_VERSION_MAJOR != ${MDBX_VERSION_MAJOR} || \ + MDBX_VERSION_MINOR != ${MDBX_VERSION_MINOR} +#error "API version mismatch! Had `git fetch --tags` done?" +#endif + +static const char sourcery[] = STRINGIFY(MDBX_BUILD_SOURCERY); + +__dll_export +#ifdef __attribute_used__ + __attribute_used__ +#elif defined(__GNUC__) || __has_attribute(__used__) + __attribute__((__used__)) +#endif +#ifdef __attribute_externally_visible__ + __attribute_externally_visible__ +#elif (defined(__GNUC__) && !defined(__clang__)) || \ + __has_attribute(__externally_visible__) + __attribute__((__externally_visible__)) +#endif + const mdbx_version_info mdbx_version = { + ${MDBX_VERSION_MAJOR}, + ${MDBX_VERSION_MINOR}, + ${MDBX_VERSION_RELEASE}, + ${MDBX_VERSION_REVISION}, + {"@MDBX_GIT_TIMESTAMP@", "@MDBX_GIT_TREE@", "@MDBX_GIT_COMMIT@", + "@MDBX_GIT_DESCRIBE@"}, + sourcery}; + +__dll_export +#ifdef __attribute_used__ + __attribute_used__ +#elif defined(__GNUC__) || __has_attribute(__used__) + __attribute__((__used__)) +#endif +#ifdef __attribute_externally_visible__ + __attribute_externally_visible__ +#elif (defined(__GNUC__) && !defined(__clang__)) || \ + __has_attribute(__externally_visible__) + __attribute__((__externally_visible__)) +#endif + const char *const mdbx_sourcery_anchor = sourcery; diff --git a/src/contrib/db/libmdbx/src/man1/mdbx_chk.1 b/src/contrib/db/libmdbx/src/man1/mdbx_chk.1 new file mode 100644 index 0000000..84b10a1 --- /dev/null +++ b/src/contrib/db/libmdbx/src/man1/mdbx_chk.1 @@ -0,0 +1,87 @@ +.\" Copyright 2015-2019 Leonid Yuriev . +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.TH MDBX_CHK 1 "2019-09-10" "MDBX 0.x" +.SH NAME +mdbx_chk \- MDBX checking tool +.SH SYNOPSIS +.B mdbx_chk +[\c +.BR \-V ] +[\c +.BR \-v [ v [ v ]]] +[\c +.BR \-n ] +[\c +.BR \-q ] +[\c +.BR \-w ] +[\c +.BR \-d ] +[\c +.BI \-s \ subdb\fR] +[\c +.BR \-c ] +[\c +.BR \-i ] +.BR \ envpath +.SH DESCRIPTION +The +.B mdbx_chk +utility intended to check an MDBX database file. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-v +Produce verbose output, including summarize space and page usage statistics. +If \fB\-vv\fP is given, be more verbose, show summarized B-tree info +and space allocation. +If \fB\-vvv\fP is given, be more verbose, include summarized statistics +of leaf B-tree pages. +If \fB\-vvvv\fP is given, be even more verbose, show info of each page +during B-tree traversal and basic info of each GC record. +If \fB\-vvvvv\fP is given, turn maximal verbosity, display the full list +of page IDs in the GC records and size of each key-value pair of database(s). +.TP +.BR \-n +Open MDBX environment(s) which do not use subdirectories. +.TP +.BR \-q +Be quiet; do not output anything even if an error was detected. +.TP +.BR \-w +Open environment in read-write mode and lock for writing while checking. +This could be impossible if environment already used by another process(s) +in an incompatible read-write mode. This allow rollback to last steady commit +(in case environment was not closed properly) and then check transaction IDs +of meta-pages. Otherwise, without \fB\-w\fP option environment will be +opened in read-only mode. +.TP +.BR \-d +Disable page-by-page traversal of B-tree. In this case, without B-tree +traversal, it is unable to check for lost-unused pages nor for double-used +pages. +.TP +.BR \-s \ subdb +Verify and show info only for a specific subdatabase. +.TP +.BR \-c +Force using cooperative mode while opening environment, i.e. don't try to open +in exclusive/monopolistic mode. Only exclusive/monopolistic mode allow complete +check, including full check of all meta-pages and actual size of database file. +.TP +.BR \-i +Ignore wrong order errors, which will likely false-positive if custom +comparator(s) was used. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. Errors result in a non-zero exit status +and a diagnostic message being written to standard error +if no quiet mode was requested. +.SH "SEE ALSO" +.BR mdbx_stat (1), +.BR mdbx_copy (1), +.BR mdbx_dump (1), +.BR mdbx_load (1) +.SH AUTHOR +Leonid Yuriev diff --git a/src/contrib/db/libmdbx/src/man1/mdbx_copy.1 b/src/contrib/db/libmdbx/src/man1/mdbx_copy.1 new file mode 100644 index 0000000..08717bc --- /dev/null +++ b/src/contrib/db/libmdbx/src/man1/mdbx_copy.1 @@ -0,0 +1,60 @@ +.\" Copyright 2015-2019 Leonid Yuriev . +.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copyright 2015,2016 Peter-Service R&D LLC . +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.TH MDBX_COPY 1 "2019-09-10" "MDBX 0.x" +.SH NAME +mdbx_copy \- MDBX environment copy tool +.SH SYNOPSIS +.B mdbx_copy +[\c +.BR \-V ] +[\c +.BR \-c ] +[\c +.BR \-n ] +.B srcpath +[\c +.BR dstpath ] +.SH DESCRIPTION +The +.B mdbx_copy +utility copies an MDBX environment. The environment can +be copied regardless of whether it is currently in use. +No lockfile is created, since it gets recreated at need. + +If +.I dstpath +is specified it must be the path of an empty directory +for storing the backup. Otherwise, the backup will be +written to stdout. + +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-c +Compact while copying. Only current data pages will be copied; freed +or unused pages will be omitted from the copy. This option will +slow down the backup process as it is more CPU-intensive. +Currently it fails if the environment has suffered a page leak. +.TP +.BR \-n +Open MDBX environment(s) which do not use subdirectories. + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH CAVEATS +This utility can trigger significant file size growth if run +in parallel with write transactions, because pages which they +free during copying cannot be reused until the copy is done. +.SH "SEE ALSO" +.BR mdbx_dump (1), +.BR mdbx_chk (1), +.BR mdbx_stat (1), +.BR mdbx_load (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/src/contrib/db/libmdbx/src/man1/mdbx_dump.1 b/src/contrib/db/libmdbx/src/man1/mdbx_dump.1 new file mode 100644 index 0000000..0e5ecf6 --- /dev/null +++ b/src/contrib/db/libmdbx/src/man1/mdbx_dump.1 @@ -0,0 +1,80 @@ +.\" Copyright 2015-2019 Leonid Yuriev . +.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copyright 2015,2016 Peter-Service R&D LLC . +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.TH MDBX_DUMP 1 "2019-09-10" "MDBX 0.x" +.SH NAME +mdbx_dump \- MDBX environment export tool +.SH SYNOPSIS +.B mdbx_dump +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-l ] +[\c +.BR \-n ] +[\c +.BR \-p ] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.BR \ envpath +.SH DESCRIPTION +The +.B mdbx_dump +utility reads a database and writes its contents to the +standard output using a portable flat-text format +understood by the +.BR mdbx_load (1) +utility. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Write to the specified file instead of to the standard output. +.TP +.BR \-l +List the databases stored in the environment. Just the +names will be listed, no data will be output. +.TP +.BR \-n +Dump an MDBX database which does not use subdirectories. +.TP +.BR \-p +If characters in either the key or data items are printing characters (as +defined by isprint(3)), output them directly. This option permits users to +use standard text editors and tools to modify the contents of databases. + +Note: different systems may have different notions about what characters +are considered printing characters, and databases dumped in this manner may +be less portable to external systems. +.TP +.BR \-a +Dump all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Dump a specific subdatabase. If no database is specified, only the main database is dumped. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +Dumping and reloading databases that use user-defined comparison functions +will result in new databases that use the default comparison functions. +\fBIn this case it is quite likely that the reloaded database will be +damaged beyond repair permitting neither record storage nor retrieval.\fP + +The only available workaround is to modify the source for the +.BR mdbx_load (1) +utility to load the database using the correct comparison functions. +.SH "SEE ALSO" +.BR mdbx_load (1), +.BR mdbx_copy (1), +.BR mdbx_chk (1), +.BR mdbx_stat (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/src/contrib/db/libmdbx/src/man1/mdbx_load.1 b/src/contrib/db/libmdbx/src/man1/mdbx_load.1 new file mode 100644 index 0000000..01b03ef --- /dev/null +++ b/src/contrib/db/libmdbx/src/man1/mdbx_load.1 @@ -0,0 +1,89 @@ +.\" Copyright 2015-2019 Leonid Yuriev . +.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copyright 2015,2016 Peter-Service R&D LLC . +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.TH MDBX_LOAD 1 "2019-09-10" "MDBX 0.x" +.SH NAME +mdbx_load \- MDBX environment import tool +.SH SYNOPSIS +.B mdbx_load +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-n ] +[\c +.BI \-s \ subdb\fR] +[\c +.BR \-N ] +[\c +.BR \-T ] +.BR \ envpath +.SH DESCRIPTION +The +.B mdbx_load +utility reads from the standard input and loads it into the +MDBX environment +.BR envpath . + +The input to +.B mdbx_load +must be in the output format specified by the +.BR mdbx_dump (1) +utility or as specified by the +.B -T +option below. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-a +Append all records in the order they appear in the input. The input is assumed to already be +in correctly sorted order and no sorting or checking for redundant values will be performed. +This option must be used to reload data that was produced by running +.B mdbx_dump +on a database that uses custom compare functions. +.TP +.BR \-f \ file +Read from the specified file instead of from the standard input. +.TP +.BR \-n +Load an MDBX database which does not use subdirectories. +.TP +.BR \-s \ subdb +Load a specific subdatabase. If no database is specified, data is loaded into the main database. +.TP +.BR \-N +Don't overwrite existing records when loading into an already existing database; just skip them. +.TP +.BR \-T +Load data from simple text files. The input must be paired lines of text, where the first +line of the pair is the key item, and the second line of the pair is its corresponding +data item. + +A simple escape mechanism, where newline and backslash (\\) characters are special, is +applied to the text input. Newline characters are interpreted as record separators. +Backslash characters in the text will be interpreted in one of two ways: If the backslash +character precedes another backslash character, the pair will be interpreted as a literal +backslash. If the backslash character precedes any other character, the two characters +following the backslash will be interpreted as a hexadecimal specification of a single +character; for example, \\0a is a newline character in the ASCII character set. + +For this reason, any backslash or newline characters that naturally occur in the text +input must be escaped to avoid misinterpretation by +.BR mdbx_load . + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +.SH "SEE ALSO" +.BR mdbx_dump (1), +.BR mdbx_chk (1), +.BR mdbx_stat (1), +.BR mdbx_copy (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/src/contrib/db/libmdbx/src/man1/mdbx_stat.1 b/src/contrib/db/libmdbx/src/man1/mdbx_stat.1 new file mode 100644 index 0000000..424e76c --- /dev/null +++ b/src/contrib/db/libmdbx/src/man1/mdbx_stat.1 @@ -0,0 +1,69 @@ +.\" Copyright 2015-2019 Leonid Yuriev . +.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copyright 2015,2016 Peter-Service R&D LLC . +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.TH MDBX_STAT 1 "2019-09-10" "MDBX 0.x" +.SH NAME +mdbx_stat \- MDBX environment status tool +.SH SYNOPSIS +.B mdbx_stat +[\c +.BR \-V ] +[\c +.BR \-e ] +[\c +.BR \-f [ f [ f ]]] +[\c +.BR \-n ] +[\c +.BR \-r [ r ]] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.BR \ envpath +.SH DESCRIPTION +The +.B mdbx_stat +utility displays the status of an MDBX environment. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-e +Display information about the database environment. +.TP +.BR \-f +Display information about the environment freelist. +If \fB\-ff\fP is given, summarize each freelist entry. +If \fB\-fff\fP is given, display the full list of page IDs in the freelist. +.TP +.BR \-n +Display the status of an MDBX database which does not use subdirectories. +.TP +.BR \-r +Display information about the environment reader table. +Shows the process ID, thread ID, and transaction ID for each active +reader slot. The process ID and transaction ID are in decimal, the +thread ID is in hexadecimal. The transaction ID is displayed as "-" +if the reader does not currently have a read transaction open. +If \fB\-rr\fP is given, check for stale entries in the reader +table and clear them. The reader table will be printed again +after the check is performed. +.TP +.BR \-a +Display the status of all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Display the status of a specific subdatabase. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH "SEE ALSO" +.BR mdbx_chk (1), +.BR mdbx_copy (1), +.BR mdbx_dump (1), +.BR mdbx_load (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/src/contrib/db/libmdbx/src/tools/CMakeLists.txt b/src/contrib/db/libmdbx/src/tools/CMakeLists.txt new file mode 100644 index 0000000..01ecd7d --- /dev/null +++ b/src/contrib/db/libmdbx/src/tools/CMakeLists.txt @@ -0,0 +1,42 @@ +set(MDBX_TOOLS mdbx_chk mdbx_copy mdbx_dump mdbx_load mdbx_stat) + +# use, i.e. don't skip the full RPATH for the build tree +set(CMAKE_SKIP_BUILD_RPATH FALSE) + +# when building, don't use the install RPATH already (but later on when installing) +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) + +# add the automatically determined parts of the RPATH +# which point to directories outside the build tree to the install RPATH +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + +# the RPATH to be used when installing, but only if it's not a system directory +list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir) +if(isSystemDir EQUAL -1) + if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + set(CMAKE_INSTALL_RPATH "@executable_path/../lib") + else() + set(CMAKE_INSTALL_RPATH "\$ORIGIN/../lib") + endif() +endif() + +foreach(TOOL ${MDBX_TOOLS}) + if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + add_executable(${TOOL} ${TOOL}.c wingetopt.c wingetopt.h) + else() + add_executable(${TOOL} ${TOOL}.c) + endif() + + target_link_libraries(${TOOL} mdbx ${CMAKE_THREAD_LIBS_INIT}) + set_target_properties(${TOOL} PROPERTIES + C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON + INTERPROCEDURAL_OPTIMIZATION $) + + install(TARGETS ${TOOL} DESTINATION ${CMAKE_INSTALL_PREFIX}/bin COMPONENT mdbx) + install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../man1/${TOOL}.1 DESTINATION ${CMAKE_INSTALL_PREFIX}/man/man1 COMPONENT mdbx) +endforeach() + +if(LIB_MATH) + target_link_libraries(mdbx_chk ${LIB_MATH}) + target_link_libraries(mdbx_stat ${LIB_MATH}) +endif() diff --git a/src/contrib/db/libmdbx/src/tools/mdbx_chk.c b/src/contrib/db/libmdbx/src/tools/mdbx_chk.c new file mode 100644 index 0000000..cd61e44 --- /dev/null +++ b/src/contrib/db/libmdbx/src/tools/mdbx_chk.c @@ -0,0 +1,1430 @@ +/* mdbx_chk.c - memory-mapped database check tool */ + +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#include "../elements/internals.h" + +typedef struct flagbit { + int bit; + char *name; +} flagbit; + +flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"}, + {MDBX_INTEGERKEY, "integerkey"}, + {MDBX_REVERSEKEY, "reversekey"}, + {MDBX_DUPFIXED, "dupfixed"}, + {MDBX_REVERSEDUP, "reversedup"}, + {MDBX_INTEGERDUP, "integerdup"}, + {0, NULL}}; + +#if defined(_WIN32) || defined(_WIN64) +#include "wingetopt.h" + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +#define EXIT_INTERRUPTED (EXIT_FAILURE + 4) +#define EXIT_FAILURE_SYS (EXIT_FAILURE + 3) +#define EXIT_FAILURE_MDB (EXIT_FAILURE + 2) +#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1) +#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE + +typedef struct { + const char *name; + struct { + uint64_t branch, large_count, large_volume, leaf; + uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed; + uint64_t total, empty, other; + } pages; + uint64_t payload_bytes; + uint64_t lost_bytes; +} walk_dbi_t; + +struct { + short *pagemap; + uint64_t total_payload_bytes; + uint64_t pgcount; + walk_dbi_t dbi[MAX_DBI + CORE_DBS + /* account pseudo-entry for meta */ 1]; +} walk; + +#define dbi_free walk.dbi[FREE_DBI] +#define dbi_main walk.dbi[MAIN_DBI] +#define dbi_meta walk.dbi[CORE_DBS] + +uint64_t total_unused_bytes; +int envflags = MDBX_RDONLY | MDBX_EXCLUSIVE; + +MDBX_env *env; +MDBX_txn *txn; +MDBX_envinfo envinfo; +MDBX_stat envstat; +size_t maxkeysize, userdb_count, skipped_subdb; +uint64_t reclaimable_pages, gc_pages, alloc_pages, unused_pages, backed_pages; +unsigned verbose; +bool ignore_wrong_order, quiet; +const char *only_subdb; + +struct problem { + struct problem *pr_next; + size_t count; + const char *caption; +}; + +struct problem *problems_list; +uint64_t total_problems; + +static void __printf_args(1, 2) print(const char *msg, ...) { + if (!quiet) { + va_list args; + + fflush(stderr); + va_start(args, msg); + vfprintf(stdout, msg, args); + va_end(args); + } +} + +static void __printf_args(1, 2) error(const char *msg, ...) { + total_problems++; + + if (!quiet) { + va_list args; + + fflush(NULL); + va_start(args, msg); + fputs(" ! ", stderr); + vfprintf(stderr, msg, args); + va_end(args); + fflush(NULL); + } +} + +static void pagemap_cleanup(void) { + for (size_t i = CORE_DBS + /* account pseudo-entry for meta */ 1; + i < ARRAY_LENGTH(walk.dbi); ++i) { + if (walk.dbi[i].name) { + mdbx_free((void *)walk.dbi[i].name); + walk.dbi[i].name = NULL; + } + } + + mdbx_free(walk.pagemap); + walk.pagemap = NULL; +} + +static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) { + static walk_dbi_t *last; + + if (dbi_name == MDBX_PGWALK_MAIN) + return &dbi_main; + if (dbi_name == MDBX_PGWALK_GC) + return &dbi_free; + if (dbi_name == MDBX_PGWALK_META) + return &dbi_meta; + + if (last && strcmp(last->name, dbi_name) == 0) + return last; + + walk_dbi_t *dbi = walk.dbi + CORE_DBS + /* account pseudo-entry for meta */ 1; + for (; dbi < ARRAY_END(walk.dbi) && dbi->name; ++dbi) { + if (strcmp(dbi->name, dbi_name) == 0) + return last = dbi; + } + + if (verbose > 0 && !silent) { + print(" - found '%s' area\n", dbi_name); + fflush(NULL); + } + + if (dbi == ARRAY_END(walk.dbi)) + return NULL; + + dbi->name = mdbx_strdup(dbi_name); + return last = dbi; +} + +static void __printf_args(4, 5) + + problem_add(const char *object, uint64_t entry_number, const char *msg, + const char *extra, ...) { + total_problems++; + + if (!quiet) { + int need_fflush = 0; + struct problem *p; + + for (p = problems_list; p; p = p->pr_next) + if (p->caption == msg) + break; + + if (!p) { + p = mdbx_calloc(1, sizeof(*p)); + p->caption = msg; + p->pr_next = problems_list; + problems_list = p; + need_fflush = 1; + } + + p->count++; + if (verbose > 1) { + print(" %s #%" PRIu64 ": %s", object, entry_number, msg); + if (extra) { + va_list args; + printf(" ("); + va_start(args, extra); + vfprintf(stdout, extra, args); + va_end(args); + printf(")"); + } + printf("\n"); + if (need_fflush) + fflush(NULL); + } + } +} + +static struct problem *problems_push(void) { + struct problem *p = problems_list; + problems_list = NULL; + return p; +} + +static size_t problems_pop(struct problem *list) { + size_t count = 0; + + if (problems_list) { + int i; + + print(" - problems: "); + for (i = 0; problems_list; ++i) { + struct problem *p = problems_list->pr_next; + count += problems_list->count; + print("%s%s (%" PRIuPTR ")", i ? ", " : "", problems_list->caption, + problems_list->count); + mdbx_free(problems_list); + problems_list = p; + } + print("\n"); + fflush(NULL); + } + + problems_list = list; + return count; +} + +static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, + void *const ctx, const int deep, + const char *const dbi_name_or_tag, const size_t page_size, + const MDBX_page_type_t pagetype, const size_t nentries, + const size_t payload_bytes, const size_t header_bytes, + const size_t unused_bytes) { + (void)ctx; + if (deep > 42) { + problem_add("deep", deep, "too large", nullptr); + return MDBX_CORRUPTED /* avoid infinite loop/recursion */; + } + + if (pagetype == MDBX_page_void) + return MDBX_SUCCESS; + + walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name_or_tag, false); + if (!dbi) + return MDBX_ENOMEM; + + const size_t page_bytes = payload_bytes + header_bytes + unused_bytes; + walk.pgcount += pgnumber; + + const char *pagetype_caption; + bool branch = false; + switch (pagetype) { + default: + problem_add("page", pgno, "unknown page-type", "type %u, deep %i", + (unsigned)pagetype, deep); + pagetype_caption = "unknown"; + dbi->pages.other += pgnumber; + break; + case MDBX_page_meta: + pagetype_caption = "meta"; + dbi->pages.other += pgnumber; + break; + case MDBX_page_large: + pagetype_caption = "large"; + dbi->pages.large_volume += pgnumber; + dbi->pages.large_count += 1; + break; + case MDBX_page_branch: + pagetype_caption = "branch"; + dbi->pages.branch += pgnumber; + branch = true; + break; + case MDBX_page_leaf: + pagetype_caption = "leaf"; + dbi->pages.leaf += pgnumber; + break; + case MDBX_page_dupfixed_leaf: + pagetype_caption = "leaf-dupfixed"; + dbi->pages.leaf_dupfixed += pgnumber; + break; + case MDBX_subpage_leaf: + pagetype_caption = "subleaf-dupsort"; + dbi->pages.subleaf_dupsort += 1; + break; + case MDBX_subpage_dupfixed_leaf: + pagetype_caption = "subleaf-dupfixed"; + dbi->pages.subleaf_dupfixed += 1; + break; + } + + if (pgnumber) { + if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi->name) == 0)) { + if (pgnumber == 1) + print(" %s-page %" PRIu64, pagetype_caption, pgno); + else + print(" %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber); + print(" of %s: header %" PRIiPTR ", payload %" PRIiPTR + ", unused %" PRIiPTR ", deep %i\n", + dbi->name, header_bytes, payload_bytes, unused_bytes, deep); + } + + bool already_used = false; + for (unsigned n = 0; n < pgnumber; ++n) { + uint64_t spanpgno = pgno + n; + if (spanpgno >= alloc_pages) + problem_add("page", spanpgno, "wrong page-no", + "%s-page: %" PRIu64 " > %" PRIu64 ", deep %i", + pagetype_caption, spanpgno, alloc_pages, deep); + else if (walk.pagemap[spanpgno]) { + walk_dbi_t *coll_dbi = &walk.dbi[walk.pagemap[spanpgno] - 1]; + problem_add("page", spanpgno, + (branch && coll_dbi == dbi) ? "loop" : "already used", + "%s-page: by %s, deep %i", pagetype_caption, coll_dbi->name, + deep); + already_used = true; + } else { + walk.pagemap[spanpgno] = (short)(dbi - walk.dbi + 1); + dbi->pages.total += 1; + } + } + + if (already_used) + return branch ? MDBX_RESULT_TRUE /* avoid infinite loop/recursion */ + : MDBX_SUCCESS; + } + + if (unused_bytes > page_size) + problem_add("page", pgno, "illegal unused-bytes", + "%s-page: %u < %" PRIuPTR " < %u", pagetype_caption, 0, + unused_bytes, envstat.ms_psize); + + if (header_bytes < (int)sizeof(long) || + (size_t)header_bytes >= envstat.ms_psize - sizeof(long)) + problem_add("page", pgno, "illegal header-length", + "%s-page: %" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR, + pagetype_caption, sizeof(long), header_bytes, + envstat.ms_psize - sizeof(long)); + if (payload_bytes < 1) { + if (nentries > 1) { + problem_add("page", pgno, "zero size-of-entry", + "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries", + pagetype_caption, payload_bytes, nentries); + /* if ((size_t)header_bytes + unused_bytes < page_size) { + // LY: hush a misuse error + page_bytes = page_size; + } */ + } else { + problem_add("page", pgno, "empty", + "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR + " entries, deep %i", + pagetype_caption, payload_bytes, nentries, deep); + dbi->pages.empty += 1; + } + } + + if (pgnumber) { + if (page_bytes != page_size) { + problem_add("page", pgno, "misused", + "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR + "h + %" PRIuPTR "p + %" PRIuPTR "u), deep %i", + pagetype_caption, page_size, page_bytes, header_bytes, + payload_bytes, unused_bytes, deep); + if (page_size > page_bytes) + dbi->lost_bytes += page_size - page_bytes; + } else { + dbi->payload_bytes += payload_bytes + header_bytes; + walk.total_payload_bytes += payload_bytes + header_bytes; + } + } + + return user_break ? MDBX_EINTR : MDBX_SUCCESS; +} + +typedef int(visitor)(const uint64_t record_number, const MDBX_val *key, + const MDBX_val *data); +static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler, + bool silent); + +static int handle_userdb(const uint64_t record_number, const MDBX_val *key, + const MDBX_val *data) { + (void)record_number; + (void)key; + (void)data; + return MDBX_SUCCESS; +} + +static int handle_freedb(const uint64_t record_number, const MDBX_val *key, + const MDBX_val *data) { + char *bad = ""; + pgno_t *iptr = data->iov_base; + + if (key->iov_len != sizeof(txnid_t)) + problem_add("entry", record_number, "wrong txn-id size", + "key-size %" PRIiPTR, key->iov_len); + else { + txnid_t txnid; + memcpy(&txnid, key->iov_base, sizeof(txnid)); + if (txnid < 1 || txnid > envinfo.mi_recent_txnid) + problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid); + else { + if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t)) + problem_add("entry", txnid, "wrong idl size", "%" PRIuPTR, + data->iov_len); + size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0; + if (number < 1 || number > MDBX_PNL_MAX) + problem_add("entry", txnid, "wrong idl length", "%" PRIuPTR, number); + else if ((number + 1) * sizeof(pgno_t) > data->iov_len) { + problem_add("entry", txnid, "trimmed idl", + "%" PRIuSIZE " > %" PRIuSIZE " (corruption)", + (number + 1) * sizeof(pgno_t), data->iov_len); + number = data->iov_len / sizeof(pgno_t) - 1; + } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >= + /* LY: allow gap upto one page. it is ok + * and better than shink-and-retry inside mdbx_update_gc() */ + envstat.ms_psize) + problem_add("entry", txnid, "extra idl space", + "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)", + (number + 1) * sizeof(pgno_t), data->iov_len); + + gc_pages += number; + if (envinfo.mi_latter_reader_txnid > txnid) + reclaimable_pages += number; + + pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : txn->mt_next_pgno; + pgno_t span = 1; + for (unsigned i = 0; i < number; ++i) { + const pgno_t pgno = iptr[i]; + if (pgno < NUM_METAS) + problem_add("entry", txnid, "wrong idl entry", + "pgno %" PRIaPGNO " < meta-pages %u", pgno, NUM_METAS); + else if (pgno >= backed_pages) + problem_add("entry", txnid, "wrong idl entry", + "pgno %" PRIaPGNO " > backed-pages %" PRIu64, pgno, + backed_pages); + else if (pgno >= alloc_pages) + problem_add("entry", txnid, "wrong idl entry", + "pgno %" PRIaPGNO " > alloc-pages %" PRIu64, pgno, + alloc_pages - 1); + else { + if (MDBX_PNL_DISORDERED(prev, pgno)) { + bad = " [bad sequence]"; + problem_add("entry", txnid, "bad sequence", + "%" PRIaPGNO " %c [%u].%" PRIaPGNO, prev, + (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'), + i, pgno); + } + if (walk.pagemap) { + int idx = walk.pagemap[pgno]; + if (idx == 0) + walk.pagemap[pgno] = -1; + else if (idx > 0) + problem_add("page", pgno, "already used", "by %s", + walk.dbi[idx - 1].name); + else + problem_add("page", pgno, "already listed in GC", nullptr); + } + } + prev = pgno; + while (i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) + : pgno_sub(pgno, span))) + ++span; + } + if (verbose > 3 && !only_subdb) { + print(" transaction %" PRIaTXN ", %" PRIuPTR + " pages, maxspan %" PRIaPGNO "%s\n", + txnid, number, span, bad); + if (verbose > 4) { + for (unsigned i = 0; i < number; i += span) { + const pgno_t pgno = iptr[i]; + for (span = 1; + i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) + : pgno_sub(pgno, span)); + ++span) + ; + if (span > 1) { + print(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pgno, span); + } else + print(" %9" PRIaPGNO "\n", pgno); + } + } + } + } + } + + return MDBX_SUCCESS; +} + +static int handle_maindb(const uint64_t record_number, const MDBX_val *key, + const MDBX_val *data) { + char *name; + int rc; + size_t i; + + name = key->iov_base; + for (i = 0; i < key->iov_len; ++i) { + if (name[i] < ' ') + return handle_userdb(record_number, key, data); + } + + name = mdbx_malloc(key->iov_len + 1); + memcpy(name, key->iov_base, key->iov_len); + name[key->iov_len] = '\0'; + userdb_count++; + + rc = process_db(~0u, name, handle_userdb, false); + mdbx_free(name); + if (rc != MDBX_INCOMPATIBLE) + return rc; + + return handle_userdb(record_number, key, data); +} + +static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler, + bool silent) { + MDBX_cursor *mc; + MDBX_stat ms; + MDBX_val key, data; + MDBX_val prev_key, prev_data; + unsigned flags; + int rc, i; + struct problem *saved_list; + uint64_t problems_count; + + uint64_t record_count = 0, dups = 0; + uint64_t key_bytes = 0, data_bytes = 0; + + if (dbi_handle == ~0u) { + rc = mdbx_dbi_open(txn, dbi_name, 0, &dbi_handle); + if (rc) { + if (!dbi_name || + rc != + MDBX_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ { + error("mdbx_open '%s' failed, error %d %s\n", + dbi_name ? dbi_name : "main", rc, mdbx_strerror(rc)); + } + return rc; + } + } + + if (dbi_handle >= CORE_DBS && dbi_name && only_subdb && + strcmp(only_subdb, dbi_name) != 0) { + if (verbose) { + print("Skip processing '%s'...\n", dbi_name); + fflush(NULL); + } + skipped_subdb++; + return MDBX_SUCCESS; + } + + if (!silent && verbose) { + print("Processing '%s'...\n", dbi_name ? dbi_name : "@MAIN"); + fflush(NULL); + } + + rc = mdbx_dbi_flags(txn, dbi_handle, &flags); + if (rc) { + error("mdbx_dbi_flags failed, error %d %s\n", rc, mdbx_strerror(rc)); + return rc; + } + + rc = mdbx_dbi_stat(txn, dbi_handle, &ms, sizeof(ms)); + if (rc) { + error("mdbx_dbi_stat failed, error %d %s\n", rc, mdbx_strerror(rc)); + return rc; + } + + if (!silent && verbose) { + print(" - dbi-id %d, flags:", dbi_handle); + if (!flags) + print(" none"); + else { + for (i = 0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + print(" %s", dbflags[i].name); + } + print(" (0x%02X)\n", flags); + if (verbose > 1) { + print(" - page size %u, entries %" PRIu64 "\n", ms.ms_psize, + ms.ms_entries); + print(" - b-tree depth %u, pages: branch %" PRIu64 ", leaf %" PRIu64 + ", overflow %" PRIu64 "\n", + ms.ms_depth, ms.ms_branch_pages, ms.ms_leaf_pages, + ms.ms_overflow_pages); + } + } + + walk_dbi_t *dbi = (dbi_handle < CORE_DBS) + ? &walk.dbi[dbi_handle] + : pagemap_lookup_dbi(dbi_name, true); + if (!dbi) { + error("too many DBIs or out of memory\n"); + return MDBX_ENOMEM; + } + const uint64_t subtotal_pages = + ms.ms_branch_pages + ms.ms_leaf_pages + ms.ms_overflow_pages; + if (subtotal_pages != dbi->pages.total) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "subtotal", + subtotal_pages, dbi->pages.total); + if (ms.ms_branch_pages != dbi->pages.branch) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "branch", + ms.ms_branch_pages, dbi->pages.branch); + const uint64_t allleaf_pages = dbi->pages.leaf + dbi->pages.leaf_dupfixed; + if (ms.ms_leaf_pages != allleaf_pages) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "all-leaf", + ms.ms_leaf_pages, allleaf_pages); + if (ms.ms_overflow_pages != dbi->pages.large_volume) + error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", + "large/overlow", ms.ms_overflow_pages, dbi->pages.large_volume); + + rc = mdbx_cursor_open(txn, dbi_handle, &mc); + if (rc) { + error("mdbx_cursor_open failed, error %d %s\n", rc, mdbx_strerror(rc)); + return rc; + } + + saved_list = problems_push(); + prev_key.iov_base = NULL; + prev_key.iov_len = 0; + prev_data.iov_base = NULL; + prev_data.iov_len = 0; + rc = mdbx_cursor_get(mc, &key, &data, MDBX_FIRST); + while (rc == MDBX_SUCCESS) { + if (user_break) { + print(" - interrupted by signal\n"); + fflush(NULL); + rc = MDBX_EINTR; + goto bailout; + } + + bool bad_key = false; + if (key.iov_len > maxkeysize) { + problem_add("entry", record_count, "key length exceeds max-key-size", + "%" PRIuPTR " > %" PRIuPTR, key.iov_len, maxkeysize); + bad_key = true; + } else if ((flags & MDBX_INTEGERKEY) && key.iov_len != sizeof(uint64_t) && + key.iov_len != sizeof(uint32_t)) { + problem_add("entry", record_count, "wrong key length", + "%" PRIuPTR " != 4or8", key.iov_len); + bad_key = true; + } + + bool bad_data = false; + if ((flags & MDBX_INTEGERDUP) && data.iov_len != sizeof(uint64_t) && + data.iov_len != sizeof(uint32_t)) { + problem_add("entry", record_count, "wrong data length", + "%" PRIuPTR " != 4or8", data.iov_len); + bad_data = true; + } + + if (prev_key.iov_base && !bad_data) { + if ((flags & MDBX_DUPFIXED) && prev_data.iov_len != data.iov_len) { + problem_add("entry", record_count, "different data length", + "%" PRIuPTR " != %" PRIuPTR, prev_data.iov_len, + data.iov_len); + bad_data = true; + } + + if (!bad_key) { + int cmp = mdbx_cmp(txn, dbi_handle, &prev_key, &key); + if (cmp == 0) { + ++dups; + if ((flags & MDBX_DUPSORT) == 0) { + problem_add("entry", record_count, "duplicated entries", NULL); + if (data.iov_len == prev_data.iov_len && + memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == 0) { + problem_add("entry", record_count, "complete duplicate", NULL); + } + } else if (!bad_data) { + cmp = mdbx_dcmp(txn, dbi_handle, &prev_data, &data); + if (cmp == 0) { + problem_add("entry", record_count, "complete duplicate", NULL); + } else if (cmp > 0 && !ignore_wrong_order) { + problem_add("entry", record_count, "wrong order of multi-values", + NULL); + } + } + } else if (cmp > 0 && !ignore_wrong_order) { + problem_add("entry", record_count, "wrong order of entries", NULL); + } + } + } else if (verbose) { + if (flags & MDBX_INTEGERKEY) + print(" - fixed key-size %" PRIuPTR "\n", key.iov_len); + if (flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) + print(" - fixed data-size %" PRIuPTR "\n", data.iov_len); + } + + if (handler) { + rc = handler(record_count, &key, &data); + if (MDBX_IS_ERROR(rc)) + goto bailout; + } + + record_count++; + key_bytes += key.iov_len; + data_bytes += data.iov_len; + + if (!bad_key) + prev_key = key; + if (!bad_data) + prev_data = data; + rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT); + } + if (rc != MDBX_NOTFOUND) + error("mdbx_cursor_get failed, error %d %s\n", rc, mdbx_strerror(rc)); + else + rc = 0; + + if (record_count != ms.ms_entries) + problem_add("entry", record_count, "differentent number of entries", + "%" PRIu64 " != %" PRIu64, record_count, ms.ms_entries); +bailout: + problems_count = problems_pop(saved_list); + if (!silent && verbose) { + print(" - summary: %" PRIu64 " records, %" PRIu64 " dups, %" PRIu64 + " key's bytes, %" PRIu64 " data's " + "bytes, %" PRIu64 " problems\n", + record_count, dups, key_bytes, data_bytes, problems_count); + fflush(NULL); + } + + mdbx_cursor_close(mc); + return (rc || problems_count) ? MDBX_RESULT_TRUE : MDBX_SUCCESS; +} + +static void usage(char *prog) { + fprintf(stderr, + "usage: %s dbpath [-V] [-v] [-n] [-q] [-w] [-c] [-d] [-s subdb]\n" + " -V\t\tshow version\n" + " -v\t\tmore verbose, could be used multiple times\n" + " -n\t\tNOSUBDIR mode for open\n" + " -q\t\tbe quiet\n" + " -w\t\tlock DB for writing while checking\n" + " -d\t\tdisable page-by-page traversal of B-tree\n" + " -s subdb\tprocess a specific subdatabase only\n" + " -c\t\tforce cooperative mode (don't try exclusive)\n" + " -i\t\tignore wrong order errors (for custom comparators case)\n", + prog); + exit(EXIT_INTERRUPTED); +} + +const char *meta_synctype(uint64_t sign) { + switch (sign) { + case MDBX_DATASIGN_NONE: + return "no-sync/legacy"; + case MDBX_DATASIGN_WEAK: + return "weak"; + default: + return "steady"; + } +} + +static __inline bool meta_ot(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b, + uint64_t sign_b, const bool roolback2steady) { + if (txn_a == txn_b) + return SIGN_IS_STEADY(sign_b); + + if (roolback2steady && SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b)) + return SIGN_IS_STEADY(sign_b); + + return txn_a < txn_b; +} + +static __inline bool meta_eq(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b, + uint64_t sign_b) { + if (txn_a != txn_b) + return false; + + if (SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b)) + return false; + + return true; +} + +static __inline int meta_recent(const bool roolback2steady) { + + if (meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, roolback2steady)) + return meta_ot(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, + envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, + roolback2steady) + ? 1 + : 2; + + return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, roolback2steady) + ? 2 + : 0; +} + +static __inline int meta_tail(int head) { + + if (head == 0) + return meta_ot(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, + envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true) + ? 1 + : 2; + if (head == 1) + return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true) + ? 0 + : 2; + if (head == 2) + return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, true) + ? 0 + : 1; + assert(false); + return -1; +} + +static int meta_steady(void) { return meta_recent(true); } + +static int meta_head(void) { return meta_recent(false); } + +void verbose_meta(int num, txnid_t txnid, uint64_t sign) { + print(" - meta-%d: %s %" PRIu64, num, meta_synctype(sign), txnid); + bool stay = true; + + const int steady = meta_steady(); + const int head = meta_head(); + if (num == steady && num == head) { + print(", head"); + stay = false; + } else if (num == steady) { + print(", head-steady"); + stay = false; + } else if (num == head) { + print(", head-weak"); + stay = false; + } + if (num == meta_tail(head)) { + print(", tail"); + stay = false; + } + if (stay) + print(", stay"); + + if (txnid > envinfo.mi_recent_txnid && + (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE) + print(", rolled-back %" PRIu64 " (%" PRIu64 " >>> %" PRIu64 ")", + txnid - envinfo.mi_recent_txnid, txnid, envinfo.mi_recent_txnid); + print("\n"); +} + +static int check_meta_head(bool steady) { + switch (meta_recent(steady)) { + default: + assert(false); + error("unexpected internal error (%s)\n", + steady ? "meta_steady_head" : "meta_weak_head"); + __fallthrough; + case 0: + if (envinfo.mi_meta0_txnid != envinfo.mi_recent_txnid) { + print(" - meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 + ")\n", + 0, envinfo.mi_meta0_txnid, envinfo.mi_recent_txnid); + return 1; + } + break; + case 1: + if (envinfo.mi_meta1_txnid != envinfo.mi_recent_txnid) { + print(" - meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 + ")\n", + 1, envinfo.mi_meta1_txnid, envinfo.mi_recent_txnid); + return 1; + } + break; + case 2: + if (envinfo.mi_meta2_txnid != envinfo.mi_recent_txnid) { + print(" - meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 + ")\n", + 2, envinfo.mi_meta2_txnid, envinfo.mi_recent_txnid); + return 1; + } + } + return 0; +} + +static void print_size(const char *prefix, const uint64_t value, + const char *suffix) { + const char sf[] = + "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */ + double k = 1024.0; + size_t i; + for (i = 0; sf[i + 1] && value / k > 1000.0; ++i) + k *= 1024; + print("%s%" PRIu64 " (%.2f %cb)%s", prefix, value, value / k, sf[i], suffix); +} + +int main(int argc, char *argv[]) { + int rc; + char *prog = argv[0]; + char *envname; + int problems_maindb = 0, problems_freedb = 0, problems_meta = 0; + bool dont_traversal = false; + bool locked = false; + + double elapsed; +#if defined(_WIN32) || defined(_WIN64) + uint64_t timestamp_start, timestamp_finish; + timestamp_start = GetTickCount64(); +#else + struct timespec timestamp_start, timestamp_finish; + if (clock_gettime(CLOCK_MONOTONIC, ×tamp_start)) { + rc = errno; + error("clock_gettime failed, error %d %s\n", rc, mdbx_strerror(rc)); + return EXIT_FAILURE_SYS; + } +#endif + + dbi_meta.name = "@META"; + dbi_free.name = "@GC"; + dbi_main.name = "@MAIN"; + atexit(pagemap_cleanup); + + if (argc < 2) + usage(prog); + + for (int i; (i = getopt(argc, argv, "Vvqnwcdsi:")) != EOF;) { + switch (i) { + case 'V': + printf("mdbx_chk version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + case 'v': + verbose++; + break; + case 'q': + quiet = true; + break; + case 'n': + envflags |= MDBX_NOSUBDIR; + break; + case 'w': + envflags &= ~MDBX_RDONLY; + break; + case 'c': + envflags &= ~MDBX_EXCLUSIVE; + break; + case 'd': + dont_traversal = true; + break; + case 's': + if (only_subdb && strcmp(only_subdb, optarg)) + usage(prog); + only_subdb = optarg; + break; + case 'i': + ignore_wrong_order = true; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + envname = argv[optind]; + print("mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, envname, + (envflags & MDBX_RDONLY) ? "only" : "write"); + fflush(NULL); + + rc = mdbx_env_create(&env); + if (rc) { + error("mdbx_env_create failed, error %d %s\n", rc, mdbx_strerror(rc)); + return rc < 0 ? EXIT_FAILURE_MDB : EXIT_FAILURE_SYS; + } + + rc = mdbx_env_set_maxdbs(env, MAX_DBI); + if (rc) { + error("mdbx_env_set_maxdbs failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + rc = mdbx_env_open(env, envname, envflags, 0664); + if ((envflags & MDBX_EXCLUSIVE) && + (rc == MDBX_BUSY || +#if defined(_WIN32) || defined(_WIN64) + rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION +#else + rc == EBUSY || rc == EAGAIN +#endif + )) { + envflags &= ~MDBX_EXCLUSIVE; + rc = mdbx_env_open(env, envname, envflags, 0664); + } + + if (rc) { + error("mdbx_env_open failed, error %d %s\n", rc, mdbx_strerror(rc)); + if (rc == MDBX_WANNA_RECOVERY && (envflags & MDBX_RDONLY)) + print("Please run %s in the read-write mode (with '-w' option).\n", prog); + goto bailout; + } + if (verbose) + print(" - %s mode\n", + (envflags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative"); + + if ((envflags & MDBX_RDONLY) == 0) { + rc = mdbx_txn_lock(env, false); + if (rc != MDBX_SUCCESS) { + error("mdbx_txn_lock failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + locked = true; + } + + rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn); + if (rc) { + error("mdbx_txn_begin() failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + rc = mdbx_env_get_maxkeysize(env); + if (rc < 0) { + error("mdbx_env_get_maxkeysize failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto bailout; + } + maxkeysize = rc; + + rc = mdbx_env_info_ex(env, txn, &envinfo, sizeof(envinfo)); + if (rc) { + error("mdbx_env_info failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + rc = mdbx_env_stat_ex(env, txn, &envstat, sizeof(envstat)); + if (rc) { + error("mdbx_env_stat failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + mdbx_filehandle_t dxb_fd; + rc = mdbx_env_get_fd(env, &dxb_fd); + if (rc) { + error("mdbx_env_get_fd failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + uint64_t dxb_filesize = 0; +#if defined(_WIN32) || defined(_WIN64) + { + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(dxb_fd, &info)) + rc = GetLastError(); + else + dxb_filesize = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; + } +#else + { + struct stat st; + STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), + "libmdbx requires 64-bit file I/O on 64-bit systems"); + if (fstat(dxb_fd, &st)) + rc = errno; + else + dxb_filesize = st.st_size; + } +#endif + if (rc) { + error("mdbx_filesize failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + errno = 0; + const uint64_t dxbfile_pages = dxb_filesize / envinfo.mi_dxb_pagesize; + alloc_pages = txn->mt_next_pgno; + backed_pages = envinfo.mi_geo.current / envinfo.mi_dxb_pagesize; + if (backed_pages > dxbfile_pages) { + print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n", + backed_pages, dxbfile_pages); + ++problems_meta; + } + if (dxbfile_pages < NUM_METAS) + print(" ! file-pages %" PRIu64 " < %u\n", dxbfile_pages, NUM_METAS); + if (backed_pages < NUM_METAS) + print(" ! backed-pages %" PRIu64 " < %u\n", backed_pages, NUM_METAS); + if (backed_pages < NUM_METAS || dxbfile_pages < NUM_METAS) + goto bailout; + if (backed_pages > MAX_PAGENO) { + print(" ! backed-pages %" PRIu64 " > max-pages %" PRIaPGNO "\n", + backed_pages, MAX_PAGENO); + ++problems_meta; + backed_pages = MAX_PAGENO; + } + + if ((envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { + if (backed_pages > dxbfile_pages) { + print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n", + backed_pages, dxbfile_pages); + ++problems_meta; + backed_pages = dxbfile_pages; + } + if (alloc_pages > backed_pages) { + print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n", + alloc_pages, backed_pages); + ++problems_meta; + alloc_pages = backed_pages; + } + } else { + /* LY: DB may be shrinked by writer downto the allocated pages. */ + if (alloc_pages > backed_pages) { + print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n", + alloc_pages, backed_pages); + ++problems_meta; + alloc_pages = backed_pages; + } + if (alloc_pages > dxbfile_pages) { + print(" ! alloc-pages %" PRIu64 " > file-pages %" PRIu64 "\n", + alloc_pages, dxbfile_pages); + ++problems_meta; + alloc_pages = dxbfile_pages; + } + if (backed_pages > dxbfile_pages) + backed_pages = dxbfile_pages; + } + + if (verbose) { + print(" - pagesize %u (%u system), max keysize %" PRIuPTR + ", max readers %u\n", + envinfo.mi_dxb_pagesize, envinfo.mi_sys_pagesize, maxkeysize, + envinfo.mi_maxreaders); + print_size(" - mapsize ", envinfo.mi_mapsize, "\n"); + if (envinfo.mi_geo.lower == envinfo.mi_geo.upper) + print_size(" - fixed datafile: ", envinfo.mi_geo.current, ""); + else { + print_size(" - dynamic datafile: ", envinfo.mi_geo.lower, ""); + print_size(" .. ", envinfo.mi_geo.upper, ", "); + print_size("+", envinfo.mi_geo.grow, ", "); + print_size("-", envinfo.mi_geo.shrink, "\n"); + print_size(" - current datafile: ", envinfo.mi_geo.current, ""); + } + printf(", %" PRIu64 " pages\n", + envinfo.mi_geo.current / envinfo.mi_dxb_pagesize); + print(" - transactions: recent %" PRIu64 ", latter reader %" PRIu64 + ", lag %" PRIi64 "\n", + envinfo.mi_recent_txnid, envinfo.mi_latter_reader_txnid, + envinfo.mi_recent_txnid - envinfo.mi_latter_reader_txnid); + + verbose_meta(0, envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign); + verbose_meta(1, envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign); + verbose_meta(2, envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign); + } + + if (verbose) + print(" - performs check for meta-pages clashes\n"); + if (meta_eq(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, + envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign)) { + print(" ! meta-%d and meta-%d are clashed\n", 0, 1); + ++problems_meta; + } + if (meta_eq(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, + envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign)) { + print(" ! meta-%d and meta-%d are clashed\n", 1, 2); + ++problems_meta; + } + if (meta_eq(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, + envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign)) { + print(" ! meta-%d and meta-%d are clashed\n", 2, 0); + ++problems_meta; + } + + if (envflags & MDBX_EXCLUSIVE) { + if (verbose) + print(" - performs full check recent-txn-id with meta-pages\n"); + problems_meta += check_meta_head(true); + } else if (locked) { + if (verbose) + print(" - performs lite check recent-txn-id with meta-pages (not a " + "monopolistic mode)\n"); + problems_meta += check_meta_head(false); + } else if (verbose) { + print(" - skip check recent-txn-id with meta-pages (monopolistic or " + "read-write mode only)\n"); + } + + if (!dont_traversal) { + struct problem *saved_list; + size_t traversal_problems; + uint64_t empty_pages, lost_bytes; + + print("Traversal b-tree by txn#%" PRIaTXN "...\n", txn->mt_txnid); + fflush(NULL); + walk.pagemap = mdbx_calloc((size_t)backed_pages, sizeof(*walk.pagemap)); + if (!walk.pagemap) { + rc = errno ? errno : MDBX_ENOMEM; + error("calloc failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto bailout; + } + + saved_list = problems_push(); + rc = mdbx_env_pgwalk(txn, pgvisitor, NULL); + traversal_problems = problems_pop(saved_list); + + if (rc) { + if (rc == MDBX_EINTR && user_break) { + print(" - interrupted by signal\n"); + fflush(NULL); + } else { + error("mdbx_env_pgwalk failed, error %d %s\n", rc, mdbx_strerror(rc)); + } + goto bailout; + } + + for (uint64_t n = 0; n < alloc_pages; ++n) + if (!walk.pagemap[n]) + unused_pages += 1; + + empty_pages = lost_bytes = 0; + for (walk_dbi_t *dbi = &dbi_main; dbi < ARRAY_END(walk.dbi) && dbi->name; + ++dbi) { + empty_pages += dbi->pages.empty; + lost_bytes += dbi->lost_bytes; + } + + if (verbose) { + uint64_t total_page_bytes = walk.pgcount * envstat.ms_psize; + print(" - pages: total %" PRIu64 ", unused %" PRIu64 "\n", walk.pgcount, + unused_pages); + if (verbose > 1) { + for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name; + ++dbi) { + print(" %s: subtotal %" PRIu64, dbi->name, dbi->pages.total); + if (dbi->pages.other && dbi->pages.other != dbi->pages.total) + print(", other %" PRIu64, dbi->pages.other); + if (dbi->pages.branch) + print(", branch %" PRIu64, dbi->pages.branch); + if (dbi->pages.large_count) + print(", large %" PRIu64, dbi->pages.large_count); + uint64_t all_leaf = dbi->pages.leaf + dbi->pages.leaf_dupfixed; + if (all_leaf) { + print(", leaf %" PRIu64, all_leaf); + if (verbose > 2 && + (dbi->pages.leaf_dupfixed | dbi->pages.subleaf_dupsort | + dbi->pages.subleaf_dupsort)) + print(" (usual %" PRIu64 ", sub-dupsort %" PRIu64 + ", dupfixed %" PRIu64 ", sub-dupfixed %" PRIu64 ")", + dbi->pages.leaf, dbi->pages.subleaf_dupsort, + dbi->pages.leaf_dupfixed, dbi->pages.subleaf_dupfixed); + } + print("\n"); + } + } + + if (verbose > 1) + print(" - usage: total %" PRIu64 " bytes, payload %" PRIu64 + " (%.1f%%), unused " + "%" PRIu64 " (%.1f%%)\n", + total_page_bytes, walk.total_payload_bytes, + walk.total_payload_bytes * 100.0 / total_page_bytes, + total_page_bytes - walk.total_payload_bytes, + (total_page_bytes - walk.total_payload_bytes) * 100.0 / + total_page_bytes); + if (verbose > 2) { + for (walk_dbi_t *dbi = walk.dbi; dbi < ARRAY_END(walk.dbi) && dbi->name; + ++dbi) + if (dbi->pages.total) { + uint64_t dbi_bytes = dbi->pages.total * envstat.ms_psize; + print(" %s: subtotal %" PRIu64 " bytes (%.1f%%)," + " payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)", + dbi->name, dbi_bytes, dbi_bytes * 100.0 / total_page_bytes, + dbi->payload_bytes, dbi->payload_bytes * 100.0 / dbi_bytes, + dbi_bytes - dbi->payload_bytes, + (dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes); + if (dbi->pages.empty) + print(", %" PRIu64 " empty pages", dbi->pages.empty); + if (dbi->lost_bytes) + print(", %" PRIu64 " bytes lost", dbi->lost_bytes); + print("\n"); + } else + print(" %s: empty\n", dbi->name); + } + print(" - summary: average fill %.1f%%", + walk.total_payload_bytes * 100.0 / total_page_bytes); + if (empty_pages) + print(", %" PRIu64 " empty pages", empty_pages); + if (lost_bytes) + print(", %" PRIu64 " bytes lost", lost_bytes); + print(", %" PRIuPTR " problems\n", traversal_problems); + } + } else if (verbose) { + print("Skipping b-tree walk...\n"); + fflush(NULL); + } + + if (!verbose) + print("Iterating DBIs...\n"); + problems_maindb = process_db(~0u, /* MAIN_DBI */ NULL, NULL, false); + problems_freedb = process_db(FREE_DBI, "@GC", handle_freedb, false); + + if (verbose) { + uint64_t value = envinfo.mi_mapsize / envstat.ms_psize; + double percent = value / 100.0; + print(" - space: %" PRIu64 " total pages", value); + print(", backed %" PRIu64 " (%.1f%%)", backed_pages, + backed_pages / percent); + print(", allocated %" PRIu64 " (%.1f%%)", alloc_pages, + alloc_pages / percent); + + if (verbose > 1) { + value = envinfo.mi_mapsize / envstat.ms_psize - alloc_pages; + print(", remained %" PRIu64 " (%.1f%%)", value, value / percent); + + value = alloc_pages - gc_pages; + print(", used %" PRIu64 " (%.1f%%)", value, value / percent); + + print(", gc %" PRIu64 " (%.1f%%)", gc_pages, gc_pages / percent); + + value = gc_pages - reclaimable_pages; + print(", detained %" PRIu64 " (%.1f%%)", value, value / percent); + + print(", reclaimable %" PRIu64 " (%.1f%%)", reclaimable_pages, + reclaimable_pages / percent); + } + + value = + envinfo.mi_mapsize / envstat.ms_psize - alloc_pages + reclaimable_pages; + print(", available %" PRIu64 " (%.1f%%)\n", value, value / percent); + } + + if (problems_maindb == 0 && problems_freedb == 0) { + if (!dont_traversal && + (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { + if (walk.pgcount != alloc_pages - gc_pages) { + error("used pages mismatch (%" PRIu64 "(walked) != %" PRIu64 + "(allocated - GC))\n", + walk.pgcount, alloc_pages - gc_pages); + } + if (unused_pages != gc_pages) { + error("gc pages mismatch (%" PRIu64 "(walked) != %" PRIu64 "(GC))\n", + unused_pages, gc_pages); + } + } else if (verbose) { + print(" - skip check used and gc pages (btree-traversal with " + "monopolistic or read-write mode only)\n"); + } + + if (!process_db(MAIN_DBI, NULL, handle_maindb, true)) { + if (!userdb_count && verbose) + print(" - does not contain multiple databases\n"); + } + } + +bailout: + if (txn) + mdbx_txn_abort(txn); + if (locked) + mdbx_txn_unlock(env); + if (env) + mdbx_env_close(env); + fflush(NULL); + if (rc) { + if (rc < 0) + return (user_break) ? EXIT_INTERRUPTED : EXIT_FAILURE_SYS; + return EXIT_FAILURE_MDB; + } + +#if defined(_WIN32) || defined(_WIN64) + timestamp_finish = GetTickCount64(); + elapsed = (timestamp_finish - timestamp_start) * 1e-3; +#else + if (clock_gettime(CLOCK_MONOTONIC, ×tamp_finish)) { + rc = errno; + error("clock_gettime failed, error %d %s\n", rc, mdbx_strerror(rc)); + return EXIT_FAILURE_SYS; + } + elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec + + (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9; +#endif /* !WINDOWS */ + + total_problems += problems_meta; + if (total_problems || problems_maindb || problems_freedb) { + print("Total %" PRIu64 " error%s detected, elapsed %.3f seconds.\n", + total_problems, (total_problems > 1) ? "s are" : " is", elapsed); + if (problems_meta || problems_maindb || problems_freedb) + return EXIT_FAILURE_CHECK_MAJOR; + return EXIT_FAILURE_CHECK_MINOR; + } + print("No error is detected, elapsed %.3f seconds\n", elapsed); + return EXIT_SUCCESS; +} diff --git a/src/contrib/db/libmdbx/src/tools/mdbx_copy.c b/src/contrib/db/libmdbx/src/tools/mdbx_copy.c new file mode 100644 index 0000000..a58fd74 --- /dev/null +++ b/src/contrib/db/libmdbx/src/tools/mdbx_copy.c @@ -0,0 +1,130 @@ +/* mdbx_copy.c - memory-mapped database backup tool */ + +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#include "../elements/internals.h" + +#if defined(_WIN32) || defined(_WIN64) +#include "wingetopt.h" + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +int main(int argc, char *argv[]) { + int rc; + MDBX_env *env = NULL; + const char *progname = argv[0], *act; + unsigned flags = MDBX_RDONLY; + unsigned cpflags = 0; + bool quiet = false; + + for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { + if (argv[1][1] == 'n' && argv[1][2] == '\0') + flags |= MDBX_NOSUBDIR; + else if (argv[1][1] == 'c' && argv[1][2] == '\0') + cpflags |= MDBX_CP_COMPACT; + else if (argv[1][1] == 'q' && argv[1][2] == '\0') + quiet = true; + else if (argv[1][1] == 'V' && argv[1][2] == '\0') { + printf("mdbx_copy version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + } else + argc = 0; + } + + if (argc < 2 || argc > 3) { + fprintf(stderr, "usage: %s [-V] [-q] [-c] [-n] srcpath [dstpath]\n", + progname); + exit(EXIT_FAILURE); + } + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + if (!quiet) { + fprintf((argc == 2) ? stderr : stdout, + "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]); + fflush(NULL); + } + + act = "opening environment"; + rc = mdbx_env_create(&env); + if (rc == MDBX_SUCCESS) { + rc = mdbx_env_open(env, argv[1], flags, 0640); + } + if (rc == MDBX_SUCCESS) { + act = "copying"; + if (argc == 2) { + mdbx_filehandle_t fd; +#if defined(_WIN32) || defined(_WIN64) + fd = GetStdHandle(STD_OUTPUT_HANDLE); +#else + fd = fileno(stdout); +#endif + rc = mdbx_env_copy2fd(env, fd, cpflags); + } else + rc = mdbx_env_copy(env, argv[2], cpflags); + } + if (rc) + fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc, + mdbx_strerror(rc)); + mdbx_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/contrib/db/libmdbx/src/tools/mdbx_dump.c b/src/contrib/db/libmdbx/src/tools/mdbx_dump.c new file mode 100644 index 0000000..20cd376 --- /dev/null +++ b/src/contrib/db/libmdbx/src/tools/mdbx_dump.c @@ -0,0 +1,352 @@ +/* mdbx_dump.c - memory-mapped database dump tool */ + +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#include "../elements/internals.h" + +#include + +#define PRINT 1 +static int mode; + +typedef struct flagbit { + int bit; + char *name; +} flagbit; + +flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"}, + {MDBX_DUPSORT, "dupsort"}, + {MDBX_INTEGERKEY, "integerkey"}, + {MDBX_DUPFIXED, "dupfixed"}, + {MDBX_INTEGERDUP, "integerdup"}, + {MDBX_REVERSEDUP, "reversedup"}, + {0, NULL}}; + +#if defined(_WIN32) || defined(_WIN64) +#include "wingetopt.h" + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +static const char hexc[] = "0123456789abcdef"; + +static void dumpbyte(unsigned char c) { + putchar(hexc[c >> 4]); + putchar(hexc[c & 0xf]); +} + +static void text(MDBX_val *v) { + unsigned char *c, *end; + + putchar(' '); + c = v->iov_base; + end = c + v->iov_len; + while (c < end) { + if (isprint(*c) && *c != '\\') { + putchar(*c); + } else { + putchar('\\'); + dumpbyte(*c); + } + c++; + } + putchar('\n'); +} + +static void dumpval(MDBX_val *v) { + unsigned char *c, *end; + + putchar(' '); + c = v->iov_base; + end = c + v->iov_len; + while (c < end) { + dumpbyte(*c++); + } + putchar('\n'); +} + +/* Dump in BDB-compatible format */ +static int dumpit(MDBX_txn *txn, MDBX_dbi dbi, char *name) { + MDBX_cursor *mc; + MDBX_stat ms; + MDBX_val key, data; + MDBX_envinfo info; + unsigned int flags; + int rc, i; + + rc = mdbx_dbi_flags(txn, dbi, &flags); + if (rc) + return rc; + + rc = mdbx_dbi_stat(txn, dbi, &ms, sizeof(ms)); + if (rc) + return rc; + + rc = mdbx_env_info(mdbx_txn_env(txn), &info, sizeof(info)); + if (rc) + return rc; + + printf("VERSION=3\n"); + printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); + if (name) + printf("database=%s\n", name); + printf("type=btree\n"); + printf("mapsize=%" PRIu64 "\n", info.mi_mapsize); + printf("maxreaders=%u\n", info.mi_maxreaders); + + for (i = 0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + printf("%s=1\n", dbflags[i].name); + + printf("db_pagesize=%d\n", ms.ms_psize); + printf("HEADER=END\n"); + + rc = mdbx_cursor_open(txn, dbi, &mc); + if (rc) + return rc; + + while ((rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT)) == MDBX_SUCCESS) { + if (user_break) { + rc = MDBX_EINTR; + break; + } + if (mode & PRINT) { + text(&key); + text(&data); + } else { + dumpval(&key); + dumpval(&data); + } + } + printf("DATA=END\n"); + if (rc == MDBX_NOTFOUND) + rc = MDBX_SUCCESS; + + return rc; +} + +static void usage(char *prog) { + fprintf(stderr, + "usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n", + prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) { + int i, rc; + MDBX_env *env; + MDBX_txn *txn; + MDBX_dbi dbi; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envflags = 0, list = 0; + + if (argc < 2) + usage(prog); + + /* -a: dump main DB and all subDBs + * -s: dump only the named subDB + * -n: use NOSUBDIR flag on env_open + * -p: use printable characters + * -f: write to file instead of stdout + * -V: print version and exit + * (default) dump only the main DB + */ + while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) { + switch (i) { + case 'V': + printf("mdbx_dump version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + case 'l': + list = 1; + /*FALLTHROUGH*/; + __fallthrough; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'f': + if (freopen(optarg, "w", stdout) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, + mdbx_strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDBX_NOSUBDIR; + break; + case 'p': + mode |= PRINT; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + envname = argv[optind]; + printf("mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, envname); + fflush(NULL); + + rc = mdbx_env_create(&env); + if (rc) { + fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc, + mdbx_strerror(rc)); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + mdbx_env_set_maxdbs(env, 2); + } + + rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664); + if (rc) { + fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + + rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn); + if (rc) { + fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + + rc = mdbx_dbi_open(txn, subname, 0, &dbi); + if (rc) { + fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto txn_abort; + } + + if (alldbs) { + MDBX_cursor *cursor; + MDBX_val key; + int count = 0; + + rc = mdbx_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) { + if (user_break) { + rc = MDBX_EINTR; + break; + } + char *str; + MDBX_dbi db2; + if (memchr(key.iov_base, '\0', key.iov_len)) + continue; + count++; + str = mdbx_malloc(key.iov_len + 1); + memcpy(str, key.iov_base, key.iov_len); + str[key.iov_len] = '\0'; + rc = mdbx_dbi_open(txn, str, 0, &db2); + if (rc == MDBX_SUCCESS) { + if (list) { + printf("%s\n", str); + list++; + } else { + rc = dumpit(txn, db2, str); + if (rc) + break; + } + mdbx_dbi_close(env, db2); + } + mdbx_free(str); + if (rc) + continue; + } + mdbx_cursor_close(cursor); + if (!count) { + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, + envname); + rc = MDBX_NOTFOUND; + } else if (rc == MDBX_INCOMPATIBLE) { + /* LY: the record it not a named sub-db. */ + rc = MDBX_SUCCESS; + } + } else { + rc = dumpit(txn, dbi, subname); + } + if (rc && rc != MDBX_NOTFOUND) + fprintf(stderr, "%s: %s: %s\n", prog, envname, mdbx_strerror(rc)); + + mdbx_dbi_close(env, dbi); +txn_abort: + mdbx_txn_abort(txn); +env_close: + mdbx_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/contrib/db/libmdbx/src/tools/mdbx_load.c b/src/contrib/db/libmdbx/src/tools/mdbx_load.c new file mode 100644 index 0000000..00bfa87 --- /dev/null +++ b/src/contrib/db/libmdbx/src/tools/mdbx_load.c @@ -0,0 +1,567 @@ +/* mdbx_load.c - memory-mapped database load tool */ + +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#include "../elements/internals.h" + +#include + +#if defined(_WIN32) || defined(_WIN64) +#include "wingetopt.h" + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +#define PRINT 1 +#define NOHDR 2 +static int mode; + +static char *subname = NULL; +static size_t lineno; +static int version; + +static int dbi_flags; +static char *prog; +static int Eof; + +static MDBX_envinfo envinfo; +static MDBX_val kbuf, dbuf; +static MDBX_val k0buf; + +#define STRLENOF(s) (sizeof(s) - 1) + +typedef struct flagbit { + int bit; + char *name; + int len; +} flagbit; + +#define S(s) s, STRLENOF(s) + +flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")}, + {MDBX_DUPSORT, S("dupsort")}, + {MDBX_INTEGERKEY, S("integerkey")}, + {MDBX_DUPFIXED, S("dupfixed")}, + {MDBX_INTEGERDUP, S("integerdup")}, + {MDBX_REVERSEDUP, S("reversedup")}, + {0, NULL, 0}}; + +static void readhdr(void) { + char *ptr; + + dbi_flags = 0; + while (fgets(dbuf.iov_base, (int)dbuf.iov_len, stdin) != NULL) { + lineno++; + if (!strncmp(dbuf.iov_base, "db_pagesize=", STRLENOF("db_pagesize=")) || + !strncmp(dbuf.iov_base, "duplicates=", STRLENOF("duplicates="))) { + /* LY: silently ignore information fields. */ + continue; + } else if (!strncmp(dbuf.iov_base, "VERSION=", STRLENOF("VERSION="))) { + version = atoi((char *)dbuf.iov_base + STRLENOF("VERSION=")); + if (version > 3) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported VERSION %d\n", + prog, lineno, version); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.iov_base, "HEADER=END", STRLENOF("HEADER=END"))) { + break; + } else if (!strncmp(dbuf.iov_base, "format=", STRLENOF("format="))) { + if (!strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "print", + STRLENOF("print"))) + mode |= PRINT; + else if (strncmp((char *)dbuf.iov_base + STRLENOF("FORMAT="), "bytevalue", + STRLENOF("bytevalue"))) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported FORMAT %s\n", prog, + lineno, (char *)dbuf.iov_base + STRLENOF("FORMAT=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.iov_base, "database=", STRLENOF("database="))) { + ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len); + if (ptr) + *ptr = '\0'; + if (subname) + mdbx_free(subname); + subname = mdbx_strdup((char *)dbuf.iov_base + STRLENOF("database=")); + } else if (!strncmp(dbuf.iov_base, "type=", STRLENOF("type="))) { + if (strncmp((char *)dbuf.iov_base + STRLENOF("type="), "btree", + STRLENOF("btree"))) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported type %s\n", prog, + lineno, (char *)dbuf.iov_base + STRLENOF("type=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.iov_base, "mapaddr=", STRLENOF("mapaddr="))) { + int i; + ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len); + if (ptr) + *ptr = '\0'; + void *unused; + i = sscanf((char *)dbuf.iov_base + STRLENOF("mapaddr="), "%p", &unused); + if (i != 1) { + fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapaddr %s\n", prog, + lineno, (char *)dbuf.iov_base + STRLENOF("mapaddr=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.iov_base, "mapsize=", STRLENOF("mapsize="))) { + int i; + ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len); + if (ptr) + *ptr = '\0'; + i = sscanf((char *)dbuf.iov_base + STRLENOF("mapsize="), "%" PRIu64, + &envinfo.mi_mapsize); + if (i != 1) { + fprintf(stderr, "%s: line %" PRIiSIZE ": invalid mapsize %s\n", prog, + lineno, (char *)dbuf.iov_base + STRLENOF("mapsize=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.iov_base, + "maxreaders=", STRLENOF("maxreaders="))) { + int i; + ptr = memchr(dbuf.iov_base, '\n', dbuf.iov_len); + if (ptr) + *ptr = '\0'; + i = sscanf((char *)dbuf.iov_base + STRLENOF("maxreaders="), "%u", + &envinfo.mi_maxreaders); + if (i != 1) { + fprintf(stderr, "%s: line %" PRIiSIZE ": invalid maxreaders %s\n", prog, + lineno, (char *)dbuf.iov_base + STRLENOF("maxreaders=")); + exit(EXIT_FAILURE); + } + } else { + int i; + for (i = 0; dbflags[i].bit; i++) { + if (!strncmp(dbuf.iov_base, dbflags[i].name, dbflags[i].len) && + ((char *)dbuf.iov_base)[dbflags[i].len] == '=') { + if (((char *)dbuf.iov_base)[dbflags[i].len + 1] == '1') + dbi_flags |= dbflags[i].bit; + break; + } + } + if (!dbflags[i].bit) { + ptr = memchr(dbuf.iov_base, '=', dbuf.iov_len); + if (!ptr) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected format\n", prog, + lineno); + exit(EXIT_FAILURE); + } else { + *ptr = '\0'; + fprintf(stderr, + "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", + prog, lineno, (char *)dbuf.iov_base); + } + } + } + } +} + +static void badend(void) { + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, + lineno); +} + +static int unhex(unsigned char *c2) { + int x, c; + x = *c2++ & 0x4f; + if (x & 0x40) + x -= 55; + c = x << 4; + x = *c2 & 0x4f; + if (x & 0x40) + x -= 55; + c |= x; + return c; +} + +static int readline(MDBX_val *out, MDBX_val *buf) { + unsigned char *c1, *c2, *end; + size_t len, l2; + int c; + + if (!(mode & NOHDR)) { + c = fgetc(stdin); + if (c == EOF) { + Eof = 1; + return EOF; + } + if (c != ' ') { + lineno++; + if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) { + badend: + Eof = 1; + badend(); + return EOF; + } + if (c == 'D' && !strncmp(buf->iov_base, "ATA=END", STRLENOF("ATA=END"))) + return EOF; + goto badend; + } + } + if (fgets(buf->iov_base, (int)buf->iov_len, stdin) == NULL) { + Eof = 1; + return EOF; + } + lineno++; + + c1 = buf->iov_base; + len = strlen((char *)c1); + l2 = len; + + /* Is buffer too short? */ + while (c1[len - 1] != '\n') { + buf->iov_base = mdbx_realloc(buf->iov_base, buf->iov_len * 2); + if (!buf->iov_base) { + Eof = 1; + fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", + prog, lineno); + return EOF; + } + c1 = buf->iov_base; + c1 += l2; + if (fgets((char *)c1, (int)buf->iov_len + 1, stdin) == NULL) { + Eof = 1; + badend(); + return EOF; + } + buf->iov_len *= 2; + len = strlen((char *)c1); + l2 += len; + } + c1 = c2 = buf->iov_base; + len = l2; + c1[--len] = '\0'; + end = c1 + len; + + if (mode & PRINT) { + while (c2 < end) { + if (unlikely(*c2 == '\\')) { + if (c2[1] == '\\') { + *c1++ = '\\'; + } else { + if (c2 + 3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { + Eof = 1; + badend(); + return EOF; + } + *c1++ = (char)unhex(++c2); + } + c2 += 2; + } else { + /* copies are redundant when no escapes were used */ + *c1++ = *c2++; + } + } + } else { + /* odd length not allowed */ + if (len & 1) { + Eof = 1; + badend(); + return EOF; + } + while (c2 < end) { + if (!isxdigit(*c2) || !isxdigit(c2[1])) { + Eof = 1; + badend(); + return EOF; + } + *c1++ = (char)unhex(c2); + c2 += 2; + } + } + c2 = out->iov_base = buf->iov_base; + out->iov_len = c1 - c2; + + return 0; +} + +static void usage(void) { + fprintf(stderr, + "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", + prog); + exit(EXIT_FAILURE); +} + +static int anyway_greater(const MDBX_val *a, const MDBX_val *b) { + (void)a; + (void)b; + return 1; +} + +int main(int argc, char *argv[]) { + int i, rc; + MDBX_env *env = NULL; + MDBX_txn *txn = NULL; + MDBX_cursor *mc = NULL; + MDBX_dbi dbi; + char *envname = NULL; + int envflags = MDBX_UTTERLY_NOSYNC, putflags = 0; + int append = 0; + MDBX_val prevk; + + prog = argv[0]; + if (argc < 2) + usage(); + + /* -a: append records in input order + * -f: load file instead of stdin + * -n: use NOSUBDIR flag on env_open + * -s: load into named subDB + * -N: use NOOVERWRITE on puts + * -T: read plaintext + * -V: print version and exit + */ + while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) { + switch (i) { + case 'V': + printf("mdbx_load version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + case 'a': + append = 1; + break; + case 'f': + if (freopen(optarg, "r", stdin) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, + mdbx_strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDBX_NOSUBDIR; + break; + case 's': + subname = mdbx_strdup(optarg); + break; + case 'N': + putflags = MDBX_NOOVERWRITE | MDBX_NODUPDATA; + break; + case 'T': + mode |= NOHDR | PRINT; + break; + default: + usage(); + } + } + + if (optind != argc - 1) + usage(); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + envname = argv[optind]; + printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, envname); + fflush(NULL); + + dbuf.iov_len = 4096; + dbuf.iov_base = mdbx_malloc(dbuf.iov_len); + + /* read first header for mapsize= */ + if (!(mode & NOHDR)) + readhdr(); + + rc = mdbx_env_create(&env); + if (rc) { + fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc, + mdbx_strerror(rc)); + return EXIT_FAILURE; + } + + mdbx_env_set_maxdbs(env, 2); + + if (envinfo.mi_maxreaders) + mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders); + + if (envinfo.mi_mapsize) { + if (envinfo.mi_mapsize > SIZE_MAX) { + fprintf(stderr, "mdbx_env_set_mapsize failed, error %d %s\n", rc, + mdbx_strerror(MDBX_TOO_LARGE)); + return EXIT_FAILURE; + } + mdbx_env_set_mapsize(env, (size_t)envinfo.mi_mapsize); + } + +#ifdef MDBX_FIXEDMAP + if (info.mi_mapaddr) + envflags |= MDBX_FIXEDMAP; +#endif + + rc = mdbx_env_open(env, envname, envflags, 0664); + if (rc) { + fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + + kbuf.iov_len = mdbx_env_get_maxkeysize(env); + if (kbuf.iov_len >= SIZE_MAX / 4) { + fprintf(stderr, "mdbx_env_get_maxkeysize failed, returns %zu\n", + kbuf.iov_len); + goto env_close; + } + kbuf.iov_len = (kbuf.iov_len + 1) * 2; + kbuf.iov_base = malloc(kbuf.iov_len * 2); + k0buf.iov_len = kbuf.iov_len; + k0buf.iov_base = (char *)kbuf.iov_base + kbuf.iov_len; + prevk.iov_base = k0buf.iov_base; + + while (!Eof) { + if (user_break) { + rc = MDBX_EINTR; + break; + } + + rc = mdbx_txn_begin(env, NULL, 0, &txn); + if (rc) { + fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + + rc = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, + append ? anyway_greater : NULL, + append ? anyway_greater : NULL); + if (rc) { + fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto txn_abort; + } + + rc = mdbx_cursor_open(txn, dbi, &mc); + if (rc) { + fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + + int batch = 0; + prevk.iov_len = 0; + while (1) { + MDBX_val key; + rc = readline(&key, &kbuf); + if (rc) /* rc == EOF */ + break; + + MDBX_val data; + rc = readline(&data, &dbuf); + if (rc) { + fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", + prog, lineno); + goto txn_abort; + } + + int appflag = 0; + if (append) { + appflag = MDBX_APPEND; + if (dbi_flags & MDBX_DUPSORT) { + if (prevk.iov_len == key.iov_len && + memcmp(prevk.iov_base, key.iov_base, key.iov_len) == 0) + appflag = MDBX_APPEND | MDBX_APPENDDUP; + else + memcpy(prevk.iov_base, key.iov_base, prevk.iov_len = key.iov_len); + } + } + rc = mdbx_cursor_put(mc, &key, &data, putflags | appflag); + if (rc == MDBX_KEYEXIST && putflags) + continue; + if (rc) { + fprintf(stderr, "mdbx_cursor_put failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + batch++; + if (batch == 100) { + rc = mdbx_txn_commit(txn); + if (rc) { + fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog, + lineno, mdbx_strerror(rc)); + goto env_close; + } + rc = mdbx_txn_begin(env, NULL, 0, &txn); + if (rc) { + fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + rc = mdbx_cursor_open(txn, dbi, &mc); + if (rc) { + fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + batch = 0; + } + } + rc = mdbx_txn_commit(txn); + txn = NULL; + if (rc) { + fprintf(stderr, "%s: line %" PRIiSIZE ": txn_commit: %s\n", prog, lineno, + mdbx_strerror(rc)); + goto env_close; + } + mdbx_dbi_close(env, dbi); + + /* try read next header */ + if (!(mode & NOHDR)) + readhdr(); + } + +txn_abort: + mdbx_txn_abort(txn); +env_close: + mdbx_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/contrib/db/libmdbx/src/tools/mdbx_stat.c b/src/contrib/db/libmdbx/src/tools/mdbx_stat.c new file mode 100644 index 0000000..04c1fb9 --- /dev/null +++ b/src/contrib/db/libmdbx/src/tools/mdbx_stat.c @@ -0,0 +1,436 @@ +/* mdbx_stat.c - memory-mapped database status tool */ + +/* + * Copyright 2015-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . */ + +#ifdef _MSC_VER +#if _MSC_VER > 1800 +#pragma warning(disable : 4464) /* relative include path contains '..' */ +#endif +#pragma warning(disable : 4996) /* The POSIX name is deprecated... */ +#endif /* _MSC_VER (warnings) */ + +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#include "../elements/internals.h" + +#if defined(_WIN32) || defined(_WIN64) +#include "wingetopt.h" + +static volatile BOOL user_break; +static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { + (void)dwCtrlType; + user_break = true; + return true; +} + +#else /* WINDOWS */ + +static volatile sig_atomic_t user_break; +static void signal_handler(int sig) { + (void)sig; + user_break = 1; +} + +#endif /* !WINDOWS */ + +static void prstat(MDBX_stat *ms) { + printf(" Pagesize: %u\n", ms->ms_psize); + printf(" Tree depth: %u\n", ms->ms_depth); + printf(" Branch pages: %" PRIu64 "\n", ms->ms_branch_pages); + printf(" Leaf pages: %" PRIu64 "\n", ms->ms_leaf_pages); + printf(" Overflow pages: %" PRIu64 "\n", ms->ms_overflow_pages); + printf(" Entries: %" PRIu64 "\n", ms->ms_entries); +} + +static void usage(char *prog) { + fprintf(stderr, + "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n", + prog); + exit(EXIT_FAILURE); +} + +static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid, + mdbx_tid_t thread, uint64_t txnid, uint64_t lag, + size_t bytes_used, size_t bytes_retained) { + (void)ctx; + if (num == 1) + printf("Reader Table Status\n" + " #\tslot\t%6s %*s %20s %10s %13s %13s\n", + "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used", + "retained"); + + printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxSIZE, num, slot, (size_t)pid, + (int)sizeof(size_t) * 2, (size_t)thread); + if (txnid) + printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag, + bytes_used / 1048576.0, bytes_retained / 1048576.0); + else + printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0"); + + return user_break ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; +} + +int main(int argc, char *argv[]) { + int o, rc; + MDBX_env *env; + MDBX_txn *txn; + MDBX_dbi dbi; + MDBX_stat mst; + MDBX_envinfo mei; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0; + + if (argc < 2) + usage(prog); + + /* -a: print stat of main DB and all subDBs + * -s: print stat of only the named subDB + * -e: print env info + * -f: print freelist info + * -r: print reader info + * -n: use NOSUBDIR flag on env_open + * -V: print version and exit + * (default) print stat of only the main DB + */ + while ((o = getopt(argc, argv, "Vaefnrs:")) != EOF) { + switch (o) { + case 'V': + printf("mdbx_stat version %d.%d.%d.%d\n" + " - source: %s %s, commit %s, tree %s\n" + " - anchor: %s\n" + " - build: %s for %s by %s\n" + " - flags: %s\n" + " - options: %s\n", + mdbx_version.major, mdbx_version.minor, mdbx_version.release, + mdbx_version.revision, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, + mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, + mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, + mdbx_build.options); + return EXIT_SUCCESS; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'e': + envinfo++; + break; + case 'f': + freinfo++; + break; + case 'n': + envflags |= MDBX_NOSUBDIR; + break; + case 'r': + rdrinfo++; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + +#if defined(_WIN32) || defined(_WIN64) + SetConsoleCtrlHandler(ConsoleBreakHandlerRoutine, true); +#else +#ifdef SIGPIPE + signal(SIGPIPE, signal_handler); +#endif +#ifdef SIGHUP + signal(SIGHUP, signal_handler); +#endif + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); +#endif /* !WINDOWS */ + + envname = argv[optind]; + envname = argv[optind]; + printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, + mdbx_version.git.tree, envname); + fflush(NULL); + + rc = mdbx_env_create(&env); + if (rc) { + fprintf(stderr, "mdbx_env_create failed, error %d %s\n", rc, + mdbx_strerror(rc)); + return EXIT_FAILURE; + } + + if (alldbs || subname) + mdbx_env_set_maxdbs(env, 4); + + rc = mdbx_env_open(env, envname, envflags | MDBX_RDONLY, 0664); + if (rc) { + fprintf(stderr, "mdbx_env_open failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + + if (envinfo || freinfo) { + (void)mdbx_env_info(env, &mei, sizeof(mei)); + } else { + /* LY: zap warnings from gcc */ + memset(&mei, 0, sizeof(mei)); + } + + if (envinfo) { + (void)mdbx_env_stat(env, &mst, sizeof(mst)); + printf("Environment Info\n"); + printf(" Pagesize: %u\n", mst.ms_psize); + if (mei.mi_geo.lower != mei.mi_geo.upper) { + printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 + "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64 + "/-%" PRIu64 ")\n", + mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, + mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize, + mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize, + mei.mi_geo.shrink / mst.ms_psize); + printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", + mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); + } else { + printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", + mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); + } + printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", + mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize); + printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1); + printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid); + printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n", + mei.mi_latter_reader_txnid, + mei.mi_latter_reader_txnid - mei.mi_recent_txnid); + printf(" Max readers: %u\n", mei.mi_maxreaders); + printf(" Number of readers used: %u\n", mei.mi_numreaders); + } else { + /* LY: zap warnings from gcc */ + memset(&mst, 0, sizeof(mst)); + } + + if (rdrinfo) { + rc = mdbx_reader_list(env, reader_list_func, nullptr); + if (rc == MDBX_RESULT_TRUE) + printf("Reader Table is empty\n"); + else if (rc == MDBX_SUCCESS && rdrinfo > 1) { + int dead; + rc = mdbx_reader_check(env, &dead); + if (rc == MDBX_RESULT_TRUE) { + printf(" %d stale readers cleared.\n", dead); + rc = mdbx_reader_list(env, reader_list_func, nullptr); + if (rc == MDBX_RESULT_TRUE) + printf(" Now Reader Table is empty\n"); + } else + printf(" No stale readers.\n"); + } + if (MDBX_IS_ERROR(rc)) { + fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + if (!(subname || alldbs || freinfo)) + goto env_close; + } + + rc = mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn); + if (rc) { + fprintf(stderr, "mdbx_txn_begin failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto env_close; + } + + if (freinfo) { + MDBX_cursor *cursor; + MDBX_val key, data; + pgno_t pages = 0, *iptr; + pgno_t reclaimable = 0; + + printf("Freelist Status\n"); + dbi = 0; + rc = mdbx_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst)); + if (rc) { + fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == + MDBX_SUCCESS) { + if (user_break) { + rc = MDBX_EINTR; + break; + } + iptr = data.iov_base; + const pgno_t number = *iptr++; + + pages += number; + if (envinfo && mei.mi_latter_reader_txnid > *(size_t *)key.iov_base) + reclaimable += number; + + if (freinfo > 1) { + char *bad = ""; + pgno_t prev = + MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1; + pgno_t span = 1; + for (unsigned i = 0; i < number; ++i) { + pgno_t pg = iptr[i]; + if (MDBX_PNL_DISORDERED(prev, pg)) + bad = " [bad sequence]"; + prev = pg; + while (i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) + : pgno_sub(pg, span))) + ++span; + } + printf(" Transaction %" PRIaTXN ", %" PRIaPGNO + " pages, maxspan %" PRIaPGNO "%s\n", + *(txnid_t *)key.iov_base, number, span, bad); + if (freinfo > 2) { + for (unsigned i = 0; i < number; i += span) { + const pgno_t pg = iptr[i]; + for (span = 1; + i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) + : pgno_sub(pg, span)); + ++span) + ; + if (span > 1) + printf(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pg, span); + else + printf(" %9" PRIaPGNO "\n", pg); + } + } + } + } + mdbx_cursor_close(cursor); + + switch (rc) { + case MDBX_SUCCESS: + case MDBX_NOTFOUND: + break; + case MDBX_EINTR: + fprintf(stderr, "Interrupted by signal/user\n"); + goto txn_abort; + default: + fprintf(stderr, "mdbx_cursor_get failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + + if (envinfo) { + uint64_t value = mei.mi_mapsize / mst.ms_psize; + double percent = value / 100.0; + printf("Page Allocation Info\n"); + printf(" Max pages: %" PRIu64 " 100%%\n", value); + + value = mei.mi_last_pgno + 1; + printf(" Pages used: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1); + printf(" Remained: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = mei.mi_last_pgno + 1 - pages; + printf(" Used now: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = pages; + printf(" Unallocated: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = pages - reclaimable; + printf(" Detained: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = reclaimable; + printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent); + + value = + mei.mi_mapsize / mst.ms_psize - (mei.mi_last_pgno + 1) + reclaimable; + printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent); + } else + printf(" Free pages: %" PRIaPGNO "\n", pages); + } + + rc = mdbx_dbi_open(txn, subname, 0, &dbi); + if (rc) { + fprintf(stderr, "mdbx_open failed, error %d %s\n", rc, mdbx_strerror(rc)); + goto txn_abort; + } + + rc = mdbx_dbi_stat(txn, dbi, &mst, sizeof(mst)); + if (rc) { + fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + printf("Status of %s\n", subname ? subname : "Main DB"); + prstat(&mst); + + if (alldbs) { + MDBX_cursor *cursor; + MDBX_val key; + + rc = mdbx_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdbx_cursor_open failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + while ((rc = mdbx_cursor_get(cursor, &key, NULL, MDBX_NEXT_NODUP)) == 0) { + char *str; + MDBX_dbi db2; + if (memchr(key.iov_base, '\0', key.iov_len)) + continue; + str = mdbx_malloc(key.iov_len + 1); + memcpy(str, key.iov_base, key.iov_len); + str[key.iov_len] = '\0'; + rc = mdbx_dbi_open(txn, str, 0, &db2); + if (rc == MDBX_SUCCESS) + printf("Status of %s\n", str); + mdbx_free(str); + if (rc) + continue; + rc = mdbx_dbi_stat(txn, db2, &mst, sizeof(mst)); + if (rc) { + fprintf(stderr, "mdbx_dbi_stat failed, error %d %s\n", rc, + mdbx_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + mdbx_dbi_close(env, db2); + } + mdbx_cursor_close(cursor); + } + + if (rc == MDBX_NOTFOUND) + rc = MDBX_SUCCESS; + + mdbx_dbi_close(env, dbi); +txn_abort: + mdbx_txn_abort(txn); +env_close: + mdbx_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/contrib/db/libmdbx/src/tools/wingetopt.c b/src/contrib/db/libmdbx/src/tools/wingetopt.c new file mode 100644 index 0000000..7feb223 --- /dev/null +++ b/src/contrib/db/libmdbx/src/tools/wingetopt.c @@ -0,0 +1,95 @@ +/* + * POSIX getopt for Windows + * + * AT&T Public License + * + * Code given out at the 1985 UNIFORUM conference in Dallas. + */ + +/*----------------------------------------------------------------------------*/ +/* Microsoft compiler generates a lot of warning for self includes... */ + +#ifdef _MSC_VER +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + * semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ + * not guaranteed. Specify /EHsc */ +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif +#endif /* _MSC_VER (warnings) */ + +#include "wingetopt.h" +#include +#include + +#ifdef _MSC_VER +#pragma warning(pop) +#endif +/*----------------------------------------------------------------------------*/ + +#ifndef NULL +#define NULL 0 +#endif + +#ifndef EOF +#define EOF (-1) +#endif + +#define ERR(s, c) \ + if (opterr) { \ + fputs(argv[0], stderr); \ + fputs(s, stderr); \ + fputc(c, stderr); \ + } + +int opterr = 1; +int optind = 1; +int optopt; +char *optarg; + +int getopt(int argc, char *const argv[], const char *opts) { + static int sp = 1; + int c; + const char *cp; + + if (sp == 1) { + if (optind >= argc || argv[optind][0] != '-' || argv[optind][1] == '\0') + return EOF; + else if (strcmp(argv[optind], "--") == 0) { + optind++; + return EOF; + } + } + optopt = c = argv[optind][sp]; + if (c == ':' || (cp = strchr(opts, c)) == NULL) { + ERR(": illegal option -- ", c); + if (argv[optind][++sp] == '\0') { + optind++; + sp = 1; + } + return '?'; + } + if (*++cp == ':') { + if (argv[optind][sp + 1] != '\0') + optarg = &argv[optind++][sp + 1]; + else if (++optind >= argc) { + ERR(": option requires an argument -- ", c); + sp = 1; + return '?'; + } else + optarg = argv[optind++]; + sp = 1; + } else { + if (argv[optind][++sp] == '\0') { + sp = 1; + optind++; + } + optarg = NULL; + } + return c; +} diff --git a/src/contrib/db/libmdbx/src/tools/wingetopt.h b/src/contrib/db/libmdbx/src/tools/wingetopt.h new file mode 100644 index 0000000..d328e38 --- /dev/null +++ b/src/contrib/db/libmdbx/src/tools/wingetopt.h @@ -0,0 +1,30 @@ +/* + * POSIX getopt for Windows + * + * AT&T Public License + * + * Code given out at the 1985 UNIFORUM conference in Dallas. + */ + +#ifndef _WINGETOPT_H_ +#define _WINGETOPT_H_ + +/* Bit of madness for Windows console */ +#define mdbx_strerror mdbx_strerror_ANSI2OEM +#define mdbx_strerror_r mdbx_strerror_r_ANSI2OEM + +#ifdef __cplusplus +extern "C" { +#endif + +extern int opterr; +extern int optind; +extern int optopt; +extern char *optarg; +int getopt(int argc, char *const argv[], const char *optstring); + +#ifdef __cplusplus +} +#endif + +#endif /* _GETOPT_H_ */ diff --git a/src/contrib/db/libmdbx/test/CMakeLists.txt b/src/contrib/db/libmdbx/test/CMakeLists.txt new file mode 100644 index 0000000..0cc2218 --- /dev/null +++ b/src/contrib/db/libmdbx/test/CMakeLists.txt @@ -0,0 +1,53 @@ +if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set(TEST_OSAL windows) +else() + set(TEST_OSAL unix) +endif() + +add_executable(mdbx_test + base.h + cases.cc + chrono.cc + chrono.h + config.cc + config.h + copy.cc + dead.cc + hill.cc + jitter.cc + keygen.cc + keygen.h + log.cc + log.h + main.cc + osal.h + osal-${TEST_OSAL}.cc + test.cc + test.h + try.cc + utils.cc + utils.h + append.cc + ttl.cc + nested.cc + ) + +set_target_properties(mdbx_test PROPERTIES + INTERPROCEDURAL_OPTIMIZATION $ + CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) + +if(CC_HAS_FASTMATH) + target_compile_options(mdbx_test PRIVATE "-ffast-math") +endif() +if(CC_HAS_VISIBILITY AND (LTO_ENABLED OR INTERPROCEDURAL_OPTIMIZATION)) + set_target_properties(mdbx_test PROPERTIES LINK_FLAGS "-fvisibility=hidden") +endif() + +target_link_libraries(mdbx_test mdbx ${LIB_MATH} ${CMAKE_THREAD_LIBS_INIT}) +if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + target_link_libraries(mdbx_test winmm.lib) +endif() + +if(UNIX AND NOT SUBPROJECT) + add_subdirectory(pcrf) +endif() diff --git a/src/contrib/db/libmdbx/test/append.cc b/src/contrib/db/libmdbx/test/append.cc new file mode 100644 index 0000000..9f567c7 --- /dev/null +++ b/src/contrib/db/libmdbx/test/append.cc @@ -0,0 +1,164 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +bool testcase_append::run() { + int err = db_open__begin__table_create_open_clean(dbi); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("append: bailout-prepare due '%s'", mdbx_strerror(err)); + return true; + } + + keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); + /* LY: теÑÑ‚ Ð½Ð°Ð¿Ð¾Ð»Ð½ÐµÐ½Ð¸Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† в append-режиме, + * при котором запиÑи добавлÑÑŽÑ‚ÑÑ Ñтрого в конец (в порÑдке Ñортировки) */ + const unsigned flags = (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_APPEND | MDBX_APPENDDUP + : MDBX_APPEND; + keyvalue_maker.make_ordered(); + + key = keygen::alloc(config.params.keylen_max); + data = keygen::alloc(config.params.datalen_max); + keygen::buffer last_key = keygen::alloc(config.params.keylen_max); + keygen::buffer last_data = keygen::alloc(config.params.datalen_max); + last_key->value.iov_base = last_key->bytes; + last_key->value.iov_len = 0; + last_data->value.iov_base = last_data->bytes; + last_data->value.iov_len = 0; + + simple_checksum inserted_checksum; + uint64_t inserted_number = 0; + uint64_t serial_count = 0; + + unsigned txn_nops = 0; + uint64_t commited_inserted_number = inserted_number; + simple_checksum commited_inserted_checksum = inserted_checksum; + while (should_continue()) { + const keygen::serial_t serial = serial_count; + if (!keyvalue_maker.increment(serial_count, 1)) { + // дошли до границы проÑтранÑтва ключей + break; + } + + log_trace("append: append-a %" PRIu64, serial); + generate_pair(serial, key, data); + int cmp = inserted_number ? mdbx_cmp(txn_guard.get(), dbi, &key->value, + &last_key->value) + : 1; + if (cmp == 0 && (config.params.table_flags & MDBX_DUPSORT)) + cmp = mdbx_dcmp(txn_guard.get(), dbi, &data->value, &last_data->value); + + err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, flags); + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("append: bailout-insert due '%s'", mdbx_strerror(err)); + txn_end(true); + inserted_number = commited_inserted_number; + inserted_checksum = commited_inserted_checksum; + break; + } + + if (cmp > 0) { + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_put(appenda-a)", err); + + memcpy(last_key->value.iov_base, key->value.iov_base, + last_key->value.iov_len = key->value.iov_len); + memcpy(last_data->value.iov_base, data->value.iov_base, + last_data->value.iov_len = data->value.iov_len); + ++inserted_number; + inserted_checksum.push((uint32_t)inserted_number, key->value); + inserted_checksum.push(10639, data->value); + } else { + if (unlikely(err != MDBX_EKEYMISMATCH)) + failure_perror("mdbx_put(appenda-a) != MDBX_EKEYMISMATCH", err); + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("append: bailout-commit due '%s'", mdbx_strerror(err)); + inserted_number = commited_inserted_number; + inserted_checksum = commited_inserted_checksum; + break; + } + commited_inserted_number = inserted_number; + commited_inserted_checksum = inserted_checksum; + txn_nops = 0; + } + + report(1); + } + + if (txn_guard) { + err = breakable_commit(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("append: bailout-commit due '%s'", mdbx_strerror(err)); + inserted_number = commited_inserted_number; + inserted_checksum = commited_inserted_checksum; + } + } + //---------------------------------------------------------------------------- + txn_begin(true); + cursor_open(dbi); + + MDBX_val check_key, check_data; + err = + mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_FIRST); + if (likely(inserted_number)) { + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_cursor_get(MDBX_FIRST)", err); + } + + simple_checksum read_checksum; + uint64_t read_count = 0; + while (err == MDBX_SUCCESS) { + ++read_count; + read_checksum.push((uint32_t)read_count, check_key); + read_checksum.push(10639, check_data); + + err = + mdbx_cursor_get(cursor_guard.get(), &check_key, &check_data, MDBX_NEXT); + } + + if (unlikely(err != MDBX_NOTFOUND)) + failure_perror("mdbx_cursor_get(MDBX_NEXT) != EOF", err); + + if (unlikely(read_count != inserted_number)) + failure("read_count(%" PRIu64 ") != inserted_number(%" PRIu64 ")", + read_count, inserted_number); + + if (unlikely(read_checksum.value != inserted_checksum.value)) + failure("read_checksum(0x%016" PRIu64 ") " + "!= inserted_checksum(0x%016" PRIu64 ")", + read_checksum.value, inserted_checksum.value); + + cursor_close(); + txn_end(true); + //---------------------------------------------------------------------------- + + if (dbi) { + if (config.params.drop_table && !mode_readonly()) { + txn_begin(false); + db_table_drop(dbi); + err = breakable_commit(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("append: bailout-clean due '%s'", mdbx_strerror(err)); + return true; + } + } else + db_table_close(dbi); + } + return true; +} diff --git a/src/contrib/db/libmdbx/test/base.h b/src/contrib/db/libmdbx/test/base.h new file mode 100644 index 0000000..0494281 --- /dev/null +++ b/src/contrib/db/libmdbx/test/base.h @@ -0,0 +1,116 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#ifndef NOMINMAX +#define NOMINMAX +#endif + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) +#ifdef _MSC_VER +#define _CRT_SECURE_NO_WARNINGS +#pragma warning(push, 1) +#pragma warning(disable : 4548) /* expression before comma has no effect; \ + expected expression with side - effect */ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ + semantics are not enabled. Specify /EHsc */ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + mode specified; termination on exception \ + is not guaranteed. Specify /EHsc */ +#endif /* _MSC_VER (warnings) */ + +/* If you wish to build your application for a previous Windows platform, + * include WinSDKVer.h and set the _WIN32_WINNT macro to the platform you + * wish to support before including SDKDDKVer.h. + * + * TODO: #define _WIN32_WINNT WIN32_MUSTDIE */ +#include +#endif /* WINDOWS */ + +#ifdef __APPLE__ +#define _DARWIN_C_SOURCE +#endif + +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) +#include +#else +#include +#include +#include +#include +#include +#include +#endif + +#ifdef _BSD_SOURCE +#include +#endif + +#include +#include +#include // for PRId64, PRIu64 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MDBX_INTERNAL_FUNC +#define MDBX_INTERNAL_VAR extern +#define MDBX_TOOLS /* Avoid using internal mdbx_assert() */ +#include "../mdbx.h" +#include "../src/elements/defs.h" +#include "../src/elements/osal.h" + +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifdef _MSC_VER +#pragma warning(pop) +#pragma warning(disable : 4201) /* nonstandard extension used : \ + nameless struct / union */ +#pragma warning(disable : 4127) /* conditional expression is constant */ +#if _MSC_VER < 1900 +#pragma warning(disable : 4510) /* default constructor could \ + not be generated */ +#pragma warning(disable : 4512) /* assignment operator could \ + not be generated */ +#pragma warning(disable : 4610) /* user-defined constructor required */ +#ifndef snprintf +#define snprintf(buffer, buffer_size, format, ...) \ + _snprintf_s(buffer, buffer_size, _TRUNCATE, format, __VA_ARGS__) +#endif +#ifndef vsnprintf +#define vsnprintf(buffer, buffer_size, format, args) \ + _vsnprintf_s(buffer, buffer_size, _TRUNCATE, format, args) +#endif +#pragma warning(disable : 4996) /* 'vsnprintf': This function or variable \ + may be unsafe */ +#endif +#endif /* _MSC_VER */ diff --git a/src/contrib/db/libmdbx/test/cases.cc b/src/contrib/db/libmdbx/test/cases.cc new file mode 100644 index 0000000..df6d402 --- /dev/null +++ b/src/contrib/db/libmdbx/test/cases.cc @@ -0,0 +1,99 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +void configure_actor(unsigned &last_space_id, const actor_testcase testcase, + const char *space_id_cstr, const actor_params ¶ms) { + unsigned wait4id = 0; + + if (params.waitfor_nops) { + for (auto i = global::actors.rbegin(); i != global::actors.rend(); ++i) { + if (i->is_waitable(params.waitfor_nops)) { + if (i->signal_nops && i->signal_nops != params.waitfor_nops) + failure("Previous waitable actor (id=%u) already linked on %u-ops\n", + i->actor_id, i->signal_nops); + wait4id = i->actor_id; + i->signal_nops = params.waitfor_nops; + break; + } + } + if (!wait4id) + failure("No previous waitable actor for %u-ops\n", params.waitfor_nops); + } + + unsigned space_id = 0; + if (!space_id_cstr || strcmp(space_id_cstr, "auto") == 0) + space_id = last_space_id + 1; + else { + char *end = nullptr; + errno = 0; + space_id = strtoul(space_id_cstr, &end, 0); + if (errno) + failure_perror("Expects an integer value for space-id\n", errno); + if (end && *end) + failure("The '%s' is unexpected for space-id\n", end); + } + + if (space_id > ACTOR_ID_MAX) + failure("Invalid space-id %u\n", space_id); + last_space_id = space_id; + + log_trace("configure_actor: space %u for %s", space_id, + testcase2str(testcase)); + global::actors.emplace_back( + actor_config(testcase, params, space_id, wait4id)); + global::databases.insert(params.pathname_db); +} + +void testcase_setup(const char *casename, actor_params ¶ms, + unsigned &last_space_id) { + if (strcmp(casename, "basic") == 0) { + log_notice(">>> testcase_setup(%s)", casename); + configure_actor(last_space_id, ac_nested, nullptr, params); + configure_actor(last_space_id, ac_hill, nullptr, params); + configure_actor(last_space_id, ac_ttl, nullptr, params); + configure_actor(last_space_id, ac_copy, nullptr, params); + configure_actor(last_space_id, ac_append, nullptr, params); + configure_actor(last_space_id, ac_jitter, nullptr, params); + configure_actor(last_space_id, ac_try, nullptr, params); + configure_actor(last_space_id, ac_jitter, nullptr, params); + configure_actor(last_space_id, ac_try, nullptr, params); + log_notice("<<< testcase_setup(%s): done", casename); + } else { + failure("unknown testcase `%s`", casename); + } +} + +void keycase_setup(const char *casename, actor_params ¶ms) { + if (strcmp(casename, "random") == 0 || strcmp(casename, "prng") == 0) { + log_notice(">>> keycase_setup(%s)", casename); + params.keygen.keycase = kc_random; + // TODO + log_notice("<<< keycase_setup(%s): done", casename); + } else if (strcmp(casename, "dashes") == 0 || + strcmp(casename, "aside") == 0) { + log_notice(">>> keycase_setup(%s)", casename); + params.keygen.keycase = kc_dashes; + // TODO + log_notice("<<< keycase_setup(%s): done", casename); + } else if (strcmp(casename, "custom") == 0) { + log_notice("=== keycase_setup(%s): skip", casename); + params.keygen.keycase = kc_custom; + } else { + failure("unknown keycase `%s`", casename); + } +} + +/* TODO */ diff --git a/src/contrib/db/libmdbx/test/chrono.cc b/src/contrib/db/libmdbx/test/chrono.cc new file mode 100644 index 0000000..315d379 --- /dev/null +++ b/src/contrib/db/libmdbx/test/chrono.cc @@ -0,0 +1,136 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +namespace chrono { + +#ifndef NSEC_PER_SEC +#define NSEC_PER_SEC 1000000000u +#endif /* NSEC_PER_SEC */ + +uint32_t ns2fractional(uint32_t ns) { + assert(ns < NSEC_PER_SEC); + /* LY: здеÑÑŒ и далее иÑпользуетÑÑ "длинное деление", которое + * Ð´Ð»Ñ ÑÑноÑти кода оÑтавлено как еÑÑ‚ÑŒ (без ручной оптимизации). Так как + * GCC, Clang и даже MSVC Ñами давно умеют конвертировать деление на + * конÑтанту в быÑтрую reciprocal-форму. */ + return ((uint64_t)ns << 32) / NSEC_PER_SEC; +} + +uint32_t fractional2ns(uint32_t fractional) { + return (fractional * (uint64_t)NSEC_PER_SEC) >> 32; +} + +#ifndef USEC_PER_SEC +#define USEC_PER_SEC 1000000u +#endif /* USEC_PER_SEC */ +uint32_t us2fractional(uint32_t us) { + assert(us < USEC_PER_SEC); + return ((uint64_t)us << 32) / USEC_PER_SEC; +} + +uint32_t fractional2us(uint32_t fractional) { + return (fractional * (uint64_t)USEC_PER_SEC) >> 32; +} + +#ifndef MSEC_PER_SEC +#define MSEC_PER_SEC 1000u +#endif /* MSEC_PER_SEC */ +uint32_t ms2fractional(uint32_t ms) { + assert(ms < MSEC_PER_SEC); + return ((uint64_t)ms << 32) / MSEC_PER_SEC; +} + +uint32_t fractional2ms(uint32_t fractional) { + return (fractional * (uint64_t)MSEC_PER_SEC) >> 32; +} + +time from_ns(uint64_t ns) { + time result; + result.fixedpoint = ((ns / NSEC_PER_SEC) << 32) | + ns2fractional((uint32_t)(ns % NSEC_PER_SEC)); + return result; +} + +time from_us(uint64_t us) { + time result; + result.fixedpoint = ((us / USEC_PER_SEC) << 32) | + us2fractional((uint32_t)(us % USEC_PER_SEC)); + return result; +} + +time from_ms(uint64_t ms) { + time result; + result.fixedpoint = ((ms / MSEC_PER_SEC) << 32) | + ms2fractional((uint32_t)(ms % MSEC_PER_SEC)); + return result; +} + +time now_realtime() { +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + static void(WINAPI * query_time)(LPFILETIME); + if (!query_time) { + query_time = (void(WINAPI *)(LPFILETIME))GetProcAddress( + GetModuleHandle(TEXT("kernel32.dll")), + "GetSystemTimePreciseAsFileTime"); + if (!query_time) + query_time = GetSystemTimeAsFileTime; + } + + FILETIME filetime; + query_time(&filetime); + uint64_t ns100 = + (uint64_t)filetime.dwHighDateTime << 32 | filetime.dwLowDateTime; + return from_ns((ns100 - UINT64_C(116444736000000000)) * 100u); +#else + struct timespec ts; + if (unlikely(clock_gettime(CLOCK_REALTIME, &ts))) + failure_perror("clock_gettime(CLOCK_REALTIME", errno); + + return from_timespec(ts); +#endif +} + +time now_motonic() { +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + static uint64_t reciprocal; + static LARGE_INTEGER Frequency; + if (reciprocal == 0) { + if (!QueryPerformanceFrequency(&Frequency)) + failure_perror("QueryPerformanceFrequency()", GetLastError()); + reciprocal = (((UINT64_C(1) << 48) + Frequency.QuadPart / 2 + 1) / + Frequency.QuadPart); + assert(reciprocal); + } + + LARGE_INTEGER Counter; + if (!QueryPerformanceCounter(&Counter)) + failure_perror("QueryPerformanceCounter()", GetLastError()); + + time result; + result.fixedpoint = (Counter.QuadPart / Frequency.QuadPart) << 32; + uint64_t mod = Counter.QuadPart % Frequency.QuadPart; + result.fixedpoint += (mod * reciprocal) >> 16; + return result; +#else + struct timespec ts; + if (unlikely(clock_gettime(CLOCK_MONOTONIC, &ts))) + failure_perror("clock_gettime(CLOCK_MONOTONIC)", errno); + + return from_timespec(ts); +#endif +} + +} /* namespace chrono */ diff --git a/src/contrib/db/libmdbx/test/chrono.h b/src/contrib/db/libmdbx/test/chrono.h new file mode 100644 index 0000000..07cdef6 --- /dev/null +++ b/src/contrib/db/libmdbx/test/chrono.h @@ -0,0 +1,99 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" +#include "utils.h" + +namespace chrono { + +#pragma pack(push, 1) + +typedef union time { + uint64_t fixedpoint; + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + uint32_t fractional; + union { + uint32_t utc; + uint32_t integer; + }; +#else + union { + uint32_t utc; + uint32_t integer; + }; + uint32_t fractional; +#endif + }; + + void reset() { fixedpoint = 0; } + uint32_t seconds() const { return utc; } +} time; + +#pragma pack(pop) + +uint32_t ns2fractional(uint32_t); +uint32_t fractional2ns(uint32_t); +uint32_t us2fractional(uint32_t); +uint32_t fractional2us(uint32_t); +uint32_t ms2fractional(uint32_t); +uint32_t fractional2ms(uint32_t); + +time from_ns(uint64_t us); +time from_us(uint64_t ns); +time from_ms(uint64_t ms); + +inline time from_seconds(uint64_t seconds) { + assert(seconds < UINT32_MAX); + time result; + result.fixedpoint = seconds << 32; + return result; +} + +inline time from_utc(time_t utc) { + assert(utc >= 0); + return from_seconds((uint64_t)utc); +} + +inline time infinite() { + time result; + result.fixedpoint = UINT64_MAX; + return result; +} + +#if defined(HAVE_TIMESPEC_TV_NSEC) || defined(__timespec_defined) || \ + defined(CLOCK_REALTIME) +inline time from_timespec(const struct timespec &ts) { + time result; + result.fixedpoint = + ((uint64_t)ts.tv_sec << 32) | ns2fractional((uint32_t)ts.tv_nsec); + return result; +} +#endif /* HAVE_TIMESPEC_TV_NSEC */ + +#if defined(HAVE_TIMEVAL_TV_USEC) || defined(_STRUCT_TIMEVAL) +inline time from_timeval(const struct timeval &tv) { + time result; + result.fixedpoint = + ((uint64_t)tv.tv_sec << 32) | us2fractional((uint32_t)tv.tv_usec); + return result; +} +#endif /* HAVE_TIMEVAL_TV_USEC */ + +time now_realtime(); +time now_motonic(); + +} /* namespace chrono */ diff --git a/src/contrib/db/libmdbx/test/config.cc b/src/contrib/db/libmdbx/test/config.cc new file mode 100644 index 0000000..e900d4a --- /dev/null +++ b/src/contrib/db/libmdbx/test/config.cc @@ -0,0 +1,602 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +#if defined(_MSC_VER) && !defined(strcasecmp) +#define strcasecmp(str, len) _stricmp(str, len) +#endif /* _MSC_VER && strcasecmp() */ + +namespace config { + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + const char **value, const char *default_value) { + assert(narg < argc); + const char *current = argv[narg]; + const size_t optlen = strlen(option); + + if (strncmp(current, "--", 2) || strncmp(current + 2, option, optlen)) + return false; + + if (!value) { + if (current[optlen + 2] == '=') + failure("Option '--%s' doen't accept any value\n", option); + return true; + } + + *value = nullptr; + if (current[optlen + 2] == '=') { + *value = ¤t[optlen + 3]; + return true; + } + + if (narg + 1 < argc && strncmp("--", argv[narg + 1], 2) != 0) { + *value = argv[narg + 1]; + if (strcmp(*value, "default") == 0) { + if (!default_value) + failure("Option '--%s' doen't accept default value\n", option); + *value = default_value; + } + ++narg; + return true; + } + + if (default_value) { + *value = default_value; + return true; + } + + failure("No value given for '--%s' option\n", option); +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + std::string &value, bool allow_empty) { + return parse_option(argc, argv, narg, option, value, allow_empty, + allow_empty ? "" : nullptr); +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + std::string &value, bool allow_empty, + const char *default_value) { + const char *value_cstr; + if (!parse_option(argc, argv, narg, option, &value_cstr, default_value)) + return false; + + if (!allow_empty && strlen(value_cstr) == 0) + failure("Value for option '--%s' could't be empty\n", option); + + value = value_cstr; + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + unsigned &mask, const option_verb *verbs) { + const char *list; + if (!parse_option(argc, argv, narg, option, &list)) + return false; + + unsigned clear = 0; + while (*list) { + if (*list == ',' || *list == ' ' || *list == '\t') { + ++list; + continue; + } + + const char *const comma = strchr(list, ','); + const bool strikethrough = *list == '-' || *list == '~'; + if (strikethrough || *list == '+') + ++list; + else + mask = clear; + const size_t len = (comma) ? comma - list : strlen(list); + const option_verb *scan = verbs; + + while (true) { + if (!scan->verb) + failure("Unknown verb '%.*s', for option '==%s'\n", (int)len, list, + option); + if (strlen(scan->verb) == len && strncmp(list, scan->verb, len) == 0) { + mask = strikethrough ? mask & ~scan->mask : mask | scan->mask; + clear = strikethrough ? clear & ~scan->mask : clear | scan->mask; + list += len; + break; + } + ++scan; + } + } + + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + uint64_t &value, const scale_mode scale, + const uint64_t minval, const uint64_t maxval, + const uint64_t default_value) { + + const char *value_cstr; + if (!parse_option(argc, argv, narg, option, &value_cstr)) + return false; + + if (default_value && strcmp(value_cstr, "default") == 0) { + value = default_value; + return true; + } + + if (strcmp(value_cstr, "min") == 0 || strcmp(value_cstr, "minimal") == 0) { + value = minval; + return true; + } + + if (strcmp(value_cstr, "max") == 0 || strcmp(value_cstr, "maximal") == 0) { + value = maxval; + return true; + } + + char *suffix = nullptr; + errno = 0; + unsigned long long raw = strtoull(value_cstr, &suffix, 0); + if ((suffix && *suffix) || errno) { + suffix = nullptr; + errno = 0; + raw = strtoull(value_cstr, &suffix, 10); + } + if (errno) + failure("Option '--%s' expects a numeric value (%s)\n", option, + test_strerror(errno)); + + uint64_t multipler = 1; + if (suffix && *suffix) { + if (scale == no_scale) + failure("Option '--%s' doen't accepts suffixes, so '%s' is unexpected\n", + option, suffix); + if (strcmp(suffix, "K") == 0 || strcasecmp(suffix, "Kilo") == 0) + multipler = (scale == decimal) ? UINT64_C(1000) : UINT64_C(1024); + else if (strcmp(suffix, "M") == 0 || strcasecmp(suffix, "Mega") == 0) + multipler = + (scale == decimal) ? UINT64_C(1000) * 1000 : UINT64_C(1024) * 1024; + else if (strcmp(suffix, "G") == 0 || strcasecmp(suffix, "Giga") == 0) + multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 + : UINT64_C(1024) * 1024 * 1024; + else if (strcmp(suffix, "T") == 0 || strcasecmp(suffix, "Tera") == 0) + multipler = (scale == decimal) ? UINT64_C(1000) * 1000 * 1000 * 1000 + : UINT64_C(1024) * 1024 * 1024 * 1024; + else if (scale == duration && + (strcmp(suffix, "s") == 0 || strcasecmp(suffix, "Seconds") == 0)) + multipler = 1; + else if (scale == duration && + (strcmp(suffix, "m") == 0 || strcasecmp(suffix, "Minutes") == 0)) + multipler = 60; + else if (scale == duration && + (strcmp(suffix, "h") == 0 || strcasecmp(suffix, "Hours") == 0)) + multipler = 3600; + else if (scale == duration && + (strcmp(suffix, "d") == 0 || strcasecmp(suffix, "Days") == 0)) + multipler = 3600 * 24; + else + failure( + "Option '--%s' expects a numeric value with Kilo/Mega/Giga/Tera %s" + "suffixes, but '%s' is unexpected\n", + option, (scale == duration) ? "or Seconds/Minutes/Hours/Days " : "", + suffix); + } + + if (raw >= UINT64_MAX / multipler) + failure("The value for option '--%s' is too huge\n", option); + + value = raw * multipler; + if (maxval && value > maxval) + failure("The maximal value for option '--%s' is %" PRIu64 "\n", option, + maxval); + if (value < minval) + failure("The minimal value for option '--%s' is %" PRIu64 "\n", option, + minval); + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + unsigned &value, const scale_mode scale, + const unsigned minval, const unsigned maxval, + const unsigned default_value) { + + uint64_t huge; + if (!parse_option(argc, argv, narg, option, huge, scale, minval, maxval, + default_value)) + return false; + value = (unsigned)huge; + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + uint8_t &value, const uint8_t minval, const uint8_t maxval, + const uint8_t default_value) { + + uint64_t huge; + if (!parse_option(argc, argv, narg, option, huge, no_scale, minval, maxval, + default_value)) + return false; + value = (uint8_t)huge; + return true; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + int64_t &value, const int64_t minval, const int64_t maxval, + const int64_t default_value) { + uint64_t proxy = (uint64_t)value; + if (parse_option(argc, argv, narg, option, proxy, config::binary, + (uint64_t)minval, (uint64_t)maxval, + (uint64_t)default_value)) { + value = (int64_t)proxy; + return true; + } + return false; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + int32_t &value, const int32_t minval, const int32_t maxval, + const int32_t default_value) { + uint64_t proxy = (uint64_t)value; + if (parse_option(argc, argv, narg, option, proxy, config::binary, + (uint64_t)minval, (uint64_t)maxval, + (uint64_t)default_value)) { + value = (int32_t)proxy; + return true; + } + return false; +} + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + bool &value) { + const char *value_cstr = nullptr; + if (!parse_option(argc, argv, narg, option, &value_cstr, "yes")) { + const char *current = argv[narg]; + if (strncmp(current, "--no-", 5) == 0 && strcmp(current + 5, option) == 0) { + value = false; + return true; + } + if (strncmp(current, "--dont-", 7) == 0 && + strcmp(current + 7, option) == 0) { + value = false; + return true; + } + return false; + } + + if (!value_cstr) { + value = true; + return true; + } + + if (strcasecmp(value_cstr, "yes") == 0 || strcasecmp(value_cstr, "1") == 0) { + value = true; + return true; + } + + if (strcasecmp(value_cstr, "no") == 0 || strcasecmp(value_cstr, "0") == 0) { + value = false; + return true; + } + + failure( + "Option '--%s' expects a 'boolean' value Yes/No, so '%s' is unexpected\n", + option, value_cstr); +} + +//----------------------------------------------------------------------------- + +const struct option_verb mode_bits[] = { + {"rdonly", MDBX_RDONLY}, {"mapasync", MDBX_MAPASYNC}, + {"utterly", MDBX_UTTERLY_NOSYNC}, {"nosubdir", MDBX_NOSUBDIR}, + {"nosync", MDBX_NOSYNC}, {"nometasync", MDBX_NOMETASYNC}, + {"writemap", MDBX_WRITEMAP}, {"notls", MDBX_NOTLS}, + {"nordahead", MDBX_NORDAHEAD}, {"nomeminit", MDBX_NOMEMINIT}, + {"coalesce", MDBX_COALESCE}, {"lifo", MDBX_LIFORECLAIM}, + {"perturb", MDBX_PAGEPERTURB}, {nullptr, 0}}; + +const struct option_verb table_bits[] = { + {"key.reverse", MDBX_REVERSEKEY}, + {"key.integer", MDBX_INTEGERKEY}, + {"data.integer", MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT}, + {"data.fixed", MDBX_DUPFIXED | MDBX_DUPSORT}, + {"data.reverse", MDBX_REVERSEDUP | MDBX_DUPSORT}, + {"data.dups", MDBX_DUPSORT}, + {nullptr, 0}}; + +static void dump_verbs(const char *caption, size_t bits, + const struct option_verb *verbs) { + log_verbose("%s: 0x%" PRIx64 " = ", caption, (uint64_t)bits); + + const char *comma = ""; + while (verbs->mask && bits) { + if ((bits & verbs->mask) == verbs->mask) { + logging::feed("%s%s", comma, verbs->verb); + bits -= verbs->mask; + comma = ", "; + } + ++verbs; + } + + logging::feed("%s\n", (*comma == '\0') ? "none" : ""); +} + +static void dump_duration(const char *caption, unsigned duration) { + log_verbose("%s: ", caption); + if (duration) { + if (duration > 24 * 3600) + logging::feed("%u_", duration / (24 * 3600)); + if (duration > 3600) + logging::feed("%02u:", (duration % (24 * 3600)) / 3600); + logging::feed("%02u:%02u", (duration % 3600) / 60, duration % 60); + } else { + logging::feed("INFINITE"); + } + logging::feed("\n"); +} + +void dump(const char *title) { + logging::local_suffix indent(title); + + for (auto i = global::actors.begin(); i != global::actors.end(); ++i) { + log_verbose("#%u, testcase %s, space_id/table %u\n", i->actor_id, + testcase2str(i->testcase), i->space_id); + indent.push(); + + if (i->params.loglevel) { + log_verbose("log: level %u, %s\n", i->params.loglevel, + i->params.pathname_log.empty() + ? "console" + : i->params.pathname_log.c_str()); + } + + log_verbose("database: %s, size %" PRIuPTR "[%" PRIiPTR "..%" PRIiPTR + ", %i %i, %i]\n", + i->params.pathname_db.c_str(), i->params.size_now, + i->params.size_lower, i->params.size_upper, + i->params.shrink_threshold, i->params.growth_step, + i->params.pagesize); + + dump_verbs("mode", i->params.mode_flags, mode_bits); + dump_verbs("table", i->params.table_flags, table_bits); + + if (i->params.test_nops) + log_verbose("iterations/records %u\n", i->params.test_nops); + else + dump_duration("duration", i->params.test_duration); + + if (i->params.nrepeat) + log_verbose("repeat %u\n", i->params.nrepeat); + else + log_verbose("repeat ETERNALLY\n"); + + log_verbose("threads %u\n", i->params.nthreads); + + log_verbose( + "keygen.params: case %s, width %u, mesh %u, rotate %u, offset %" PRIu64 + ", split %u/%u\n", + keygencase2str(i->params.keygen.keycase), i->params.keygen.width, + i->params.keygen.mesh, i->params.keygen.rotate, i->params.keygen.offset, + i->params.keygen.split, + i->params.keygen.width - i->params.keygen.split); + log_verbose("keygen.seed: %u\n", i->params.keygen.seed); + log_verbose("key: minlen %u, maxlen %u\n", i->params.keylen_min, + i->params.keylen_max); + log_verbose("data: minlen %u, maxlen %u\n", i->params.datalen_min, + i->params.datalen_max); + + log_verbose("batch: read %u, write %u\n", i->params.batch_read, + i->params.batch_write); + + if (i->params.waitfor_nops) + log_verbose("wait: actor %u for %u ops\n", i->wait4id, + i->params.waitfor_nops); + else if (i->params.delaystart) + dump_duration("delay", i->params.delaystart); + else + log_verbose("no-delay\n"); + + if (i->params.inject_writefaultn) + log_verbose("inject-writefault on %u ops\n", + i->params.inject_writefaultn); + else + log_verbose("no-inject-writefault\n"); + + log_verbose("limits: readers %u, tables %u\n", i->params.max_readers, + i->params.max_tables); + + log_verbose("drop table: %s\n", i->params.drop_table ? "Yes" : "No"); + log_verbose("ignore MDBX_MAP_FULL error: %s\n", + i->params.ignore_dbfull ? "Yes" : "No"); + log_verbose("verifying by speculum: %s\n", + i->params.speculum ? "Yes" : "No"); + + indent.pop(); + } + + dump_duration("timeout", global::config::timeout_duration_seconds); + log_verbose("cleanup: before %s, after %s\n", + global::config::cleanup_before ? "Yes" : "No", + global::config::cleanup_after ? "Yes" : "No"); + + log_verbose("failfast: %s\n", global::config::failfast ? "Yes" : "No"); + log_verbose("progress indicator: %s\n", + global::config::progress_indicator ? "Yes" : "No"); + log_verbose("console mode: %s\n", + global::config::console_mode ? "Yes" : "No"); +} + +} /* namespace config */ + +//----------------------------------------------------------------------------- + +using namespace config; + +actor_config::actor_config(actor_testcase testcase, const actor_params ¶ms, + unsigned space_id, unsigned wait4id) + : params(params) { + this->space_id = space_id; + this->actor_id = 1 + (unsigned)global::actors.size(); + this->testcase = testcase; + this->wait4id = wait4id; + signal_nops = 0; +} + +const std::string actor_config::serialize(const char *prefix) const { + simple_checksum checksum; + + std::string result; + if (prefix) + result.append(prefix); + + checksum.push(params.pathname_db); + result.append(params.pathname_db); + result.push_back('|'); + + checksum.push(params.pathname_log); + result.append(params.pathname_log); + result.push_back('|'); + + static_assert(std::is_pod::value, + "actor_params_pod should by POD"); + result.append(data2hex(static_cast(¶ms), + sizeof(actor_params_pod), checksum)); + result.push_back('|'); + + static_assert(std::is_pod::value, + "actor_config_pod should by POD"); + result.append(data2hex(static_cast(this), + sizeof(actor_config_pod), checksum)); + result.push_back('|'); + result.push_back(global::config::progress_indicator ? 'Y' : 'N'); + checksum.push(global::config::progress_indicator); + result.push_back(global::config::console_mode ? 'Y' : 'N'); + checksum.push(global::config::console_mode); + result.push_back('|'); + + result.append(osal_serialize(checksum)); + result.push_back('|'); + + result.append(std::to_string(checksum.value)); + return result; +} + +bool actor_config::deserialize(const char *str, actor_config &config) { + simple_checksum checksum; + + TRACE(">> actor_config::deserialize: %s\n", str); + + const char *slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-1\n"); + return false; + } + config.params.pathname_db.assign(str, slash - str); + checksum.push(config.params.pathname_db); + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-2\n"); + return false; + } + config.params.pathname_log.assign(str, slash - str); + checksum.push(config.params.pathname_log); + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-3\n"); + return false; + } + static_assert(std::is_pod::value, + "actor_params_pod should by POD"); + if (!hex2data(str, slash, static_cast(&config.params), + sizeof(actor_params_pod), checksum)) { + TRACE("<< actor_config::deserialize: actor_params_pod(%.*s)\n", + (int)(slash - str), str); + return false; + } + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-4\n"); + return false; + } + static_assert(std::is_pod::value, + "actor_config_pod should by POD"); + if (!hex2data(str, slash, static_cast(&config), + sizeof(actor_config_pod), checksum)) { + TRACE("<< actor_config::deserialize: actor_config_pod(%.*s)\n", + (int)(slash - str), str); + return false; + } + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-5\n"); + return false; + } + if ((str[0] == 'Y' || str[0] == 'N') && (str[1] == 'Y' || str[1] == 'N')) { + global::config::progress_indicator = str[0] == 'Y'; + checksum.push(global::config::progress_indicator); + global::config::console_mode = str[1] == 'Y'; + checksum.push(global::config::console_mode); + str = slash + 1; + + slash = strchr(str, '|'); + if (!slash) { + TRACE("<< actor_config::deserialize: slash-6\n"); + return false; + } + } + + if (!config.osal_deserialize(str, slash, checksum)) { + TRACE("<< actor_config::deserialize: osal\n"); + return false; + } + str = slash + 1; + + uint64_t verify = std::stoull(std::string(str)); + if (checksum.value != verify) { + TRACE("<< actor_config::deserialize: checksum mismatch\n"); + return false; + } + + TRACE("<< actor_config::deserialize: OK\n"); + return true; +} + +unsigned actor_params::mdbx_keylen_min() const { + return (table_flags & MDBX_INTEGERKEY) ? 4 : 0; +} + +unsigned actor_params::mdbx_keylen_max() const { + return (table_flags & MDBX_INTEGERKEY) + ? 8 + : std::min((unsigned)mdbx_limits_keysize_max(pagesize), + (unsigned)UINT16_MAX); +} + +unsigned actor_params::mdbx_datalen_min() const { + return (table_flags & MDBX_INTEGERDUP) ? 4 : 0; +} + +unsigned actor_params::mdbx_datalen_max() const { + return (table_flags & MDBX_INTEGERDUP) + ? 8 + : std::min((table_flags & MDBX_DUPSORT) + ? (unsigned)mdbx_limits_keysize_max(pagesize) + : (unsigned)MDBX_MAXDATASIZE, + (unsigned)UINT16_MAX); +} diff --git a/src/contrib/db/libmdbx/test/config.h b/src/contrib/db/libmdbx/test/config.h new file mode 100644 index 0000000..3703d3f --- /dev/null +++ b/src/contrib/db/libmdbx/test/config.h @@ -0,0 +1,326 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" +#include "log.h" +#include "utils.h" + +#define ACTOR_ID_MAX INT16_MAX + +enum actor_testcase { + ac_none, + ac_hill, + ac_deadread, + ac_deadwrite, + ac_jitter, + ac_try, + ac_copy, + ac_append, + ac_ttl, + ac_nested +}; + +enum actor_status { + as_unknown, + as_debuging, + as_running, + as_successful, + as_killed, + as_failed, + as_coredump, +}; + +const char *testcase2str(const actor_testcase); +const char *status2str(actor_status status); + +enum keygen_case { + kc_random, /* [ 6.. 2.. 7.. 4.. 0.. 1.. 5.. 3.. ] */ + kc_dashes, /* [ 0123.. 4567.. ] */ + kc_custom, + /* TODO: more cases */ +}; + +const char *keygencase2str(const keygen_case); + +//----------------------------------------------------------------------------- + +namespace config { + +enum scale_mode { no_scale, decimal, binary, duration }; + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + const char **value, const char *default_value = nullptr); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + std::string &value, bool allow_empty = false); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + std::string &value, bool allow_empty, + const char *default_value); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + bool &value); + +struct option_verb { + const char *const verb; + unsigned mask; +}; + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + unsigned &mask, const option_verb *verbs); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + uint64_t &value, const scale_mode scale, + const uint64_t minval = 0, const uint64_t maxval = INT64_MAX, + const uint64_t default_value = 0); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + unsigned &value, const scale_mode scale, + const unsigned minval = 0, const unsigned maxval = INT32_MAX, + const unsigned default_value = 0); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + uint8_t &value, const uint8_t minval = 0, + const uint8_t maxval = 255, const uint8_t default_value = 0); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + int64_t &value, const int64_t minval, const int64_t maxval, + const int64_t default_value = -1); + +bool parse_option(int argc, char *const argv[], int &narg, const char *option, + int32_t &value, const int32_t minval, const int32_t maxval, + const int32_t default_value = -1); + +inline bool parse_option_intptr(int argc, char *const argv[], int &narg, + const char *option, intptr_t &value, + const intptr_t minval, const intptr_t maxval, + const intptr_t default_value = -1) { + static_assert(sizeof(intptr_t) == 4 || sizeof(intptr_t) == 8, "WTF?"); + if (sizeof(intptr_t) == 8) + return parse_option(argc, argv, narg, option, + *reinterpret_cast(&value), int64_t(minval), + int64_t(maxval), int64_t(default_value)); + else + return parse_option(argc, argv, narg, option, + *reinterpret_cast(&value), int32_t(minval), + int32_t(maxval), int32_t(default_value)); +} + +//----------------------------------------------------------------------------- + +#pragma pack(push, 1) + +struct keygen_params_pod { + /* Параметры генератора пар key-value. + * + * Ключи и Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð³ÐµÐ½ÐµÑ€Ð¸Ñ€ÑƒÑŽÑ‚ÑÑ Ð¿Ð¾ задаваемым параметрам на оÑнове "плоÑкой" + * иÑходной координаты. При Ñтом, в общем Ñлучае, в процеÑÑе теÑтов иÑÑ…Ð¾Ð´Ð½Ð°Ñ + * координата поÑледовательно итерируетÑÑ Ð² заданном диапазоне, а необходимые + * паттерны/поÑледовательноÑти/узоры получаютÑÑ Ð·Ð° Ñчет Ð¿Ñ€ÐµÐ¾Ð±Ñ€Ð°Ð·Ð¾Ð²Ð°Ð½Ð¸Ñ + * иÑходной координаты, ÑоглаÑно опиÑанным ниже параметрам. + * + * Стоит отметить, что порÑдок опиÑÐ°Ð½Ð¸Ñ Ð¿Ð°Ñ€Ð°Ð¼ÐµÑ‚Ñ€Ð¾Ð² Ð´Ð»Ñ ÑƒÐ´Ð¾Ð±Ñтва Ñовпадает Ñ + * порÑдком их иÑпользованиÑ, Ñ‚.е. Ñ Ð¿Ð¾Ñ€Ñдком ÑоответÑтвующих преобразований. + * + * Второе важное замечание каÑаетÑÑ Ð¾Ð³Ñ€Ð°Ð½Ð¸Ñ‡ÐµÐ½Ð¸Ð¹ одновременной координированной + * генерации паттеров как Ð´Ð»Ñ ÐºÐ»ÑŽÑ‡ÐµÐ¹, так и Ð´Ð»Ñ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ð¹. Суть в том, что + * Ñ‚Ð°ÐºÐ°Ñ Ð²Ð¾Ð·Ð¼Ð¾Ð¶Ð½Ð¾ÑÑ‚ÑŒ не нужна по Ñледующим причинам: + * - libmdbx поддерживает два ÑущеÑтвенно различающихÑÑ Ð²Ð¸Ð´Ð° таблиц, + * "уникальные" (без дубликатов и без multi-value), и так называемые + * "Ñ Ð´ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ‚Ð°Ð¼Ð¸" (c multi-value). + * - Ð”Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† "без дубликатов" только размер ÑвÑзанных к ключами значений + * (данных) оказывает влиÑÐ½Ð¸Ñ Ð½Ð° работу движка, непоÑредÑтвенно Ñодержимое + * данных не анализируетÑÑ Ð´Ð²Ð¸Ð¶ÐºÐ¾Ð¼ и не оказывает влиÑÐ½Ð¸Ñ Ð½Ð° его работу. + * - Ð”Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† "Ñ Ð´ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ‚Ð°Ð¼Ð¸", при наличии более одного Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð´Ð»Ñ + * некоторого ключа, формируетÑÑ Ð´Ð¾Ñ‡ÐµÑ€Ð½ÐµÐµ btree-поддерево. Это дерево + * формируетÑÑ Ð² отдельном "куÑте" Ñтраниц и обÑлуживаетÑÑ Ð½ÐµÐ·Ð°Ð²Ð¸Ñимо + * от Ð¾ÐºÑ€ÑƒÐ¶ÐµÐ½Ð¸Ñ Ñ€Ð¾Ð´Ð¸Ñ‚ÐµÐ»ÑŒÑкого ключа. + * - Таким образом, паттерн генерации значений имеет ÑмыÑл только Ð´Ð»Ñ + * таблиц "Ñ Ð´ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ‚Ð°Ð¼Ð¸" и только в контекÑте одного Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ ÐºÐ»ÑŽÑ‡Ð°. + * Иначе говорÑ, нет ÑмыÑла в Ñо-координации генерации паттернов Ð´Ð»Ñ + * ключей и значений. Более того, генерацию значений вÑегда необходимо + * раÑÑматривать в контекÑте ÑвÑзки Ñ Ð¾Ð´Ð½Ð¸Ð¼ значением ключа. + * - Тем не менее, во вÑех ÑлучаÑÑ… доÑтаточно важным ÑвлÑетÑÑ Ñ€Ð°Ð²Ð½Ð¾Ð¼ÐµÑ€Ð½Ð°Ñ + * вÑех возможных Ñочетаний длин ключей и данных. + * + * width: + * БольшинÑтво теÑтов предполагают Ñоздание или итерирование некоторого + * количеÑтва запиÑей. При Ñтом требуетÑÑ Ð¸Ñ‚ÐµÑ€Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ðµ или Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ + * значений и ключей из некоторого ограниченного проÑтранÑтва вариантов. + * + * Параметр width задает такую ширину проÑтранÑтва вариантов в битах. + * Таким образом мощноÑÑ‚ÑŒ проÑтранÑтва вариантов (пока) вÑегда равна + * Ñтепени двойки. Это ограничение можно ÑнÑÑ‚ÑŒ, но ценой ÑƒÐ²ÐµÐ»Ð¸Ñ‡ÐµÐ½Ð¸Ñ + * вычиÑлительной ÑложноÑти, Ð²ÐºÐ»ÑŽÑ‡Ð°Ñ Ð¿Ð¾Ñ‚ÐµÑ€ÑŽ проÑтоты и прозрачноÑти. + * + * С другой Ñтороны, не-битовый width может быть полезен: + * - Позволит генерировать ключи/Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð² точно задаваемом диапазоне. + * Ðапример, перебрать в пÑевдо-Ñлучайном порÑдке 10001 значение. + * - Позволит поровну разделÑÑ‚ÑŒ заданное проÑтранÑтво (диапазон) + * ключей/значений между количеÑтвом потоков некратным Ñтепени двойки. + * + * mesh и seed: + * ПозволÑÑŽÑ‚ получить пÑевдо-Ñлучайные поÑледовательноÑти ключей/значений. + * Параметр mesh задает Ñколько младших бит иÑходной плоÑкой координаты + * будет "перемешано" (инъективно отображено), а параметр seed позволÑет + * выбрать конкретный вариант "перемешиваниÑ". + * + * Перемешивание выполнÑетÑÑ Ð¿Ñ€Ð¸ ненулевом значении mesh. Перемешивание + * реализуетÑÑ Ð¿Ð¾ÑредÑтвом Ð¿Ñ€Ð¸Ð¼ÐµÐ½ÐµÐ½Ð¸Ñ Ð´Ð²ÑƒÑ… инъективных функций Ð´Ð»Ñ + * заданного количеÑтва бит: + * - применÑетÑÑ Ð¿ÐµÑ€Ð²Ð°Ñ Ð¸Ð½ÑŠÐµÐºÑ‚Ð¸Ð²Ð½Ð°Ñ Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ; + * - к результату добавлÑетÑÑ salt полученный из seed; + * - применÑетÑÑ Ð²Ñ‚Ð¾Ñ€Ð°Ñ Ð¸Ð½ÑŠÐµÐºÑ‚Ð¸Ð²Ð½Ð°Ñ Ñ„ÑƒÐ½ÐºÑ†Ð¸Ñ; + * + * Следует отметить, что mesh умышленно позволÑет перемешать только младшую + * чаÑÑ‚ÑŒ, что при ненулевом значении split (Ñм далее) не позволÑет получать + * пÑевдо-Ñлучайные значений ключей без пÑевдо-ÑлучайноÑти в значениÑÑ…. + * + * Такое ограничение ÑоответÑтвуют внутренней алгоритмике libmdbx. Проще + * говорÑ, мы можем проверить движок пÑевдо-Ñлучайной поÑледовательноÑтью + * ключей на таблицах без дубликатов (без multi-value), а затем проверить + * корректноÑÑ‚ÑŒ работу пÑевдо-Ñлучайной поÑледовательноÑтью значений на + * таблицах Ñ Ð´ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ‚Ð°Ð¼Ð¸ (Ñ multi-value), опционально добавлÑÑ + * пÑевдо-ÑлучайноÑти к поÑледовательноÑти ключей. Однако, нет ÑмыÑла + * генерировать пÑевдо-Ñлучайные ключи, одновременно Ñ Ñ„Ð¾Ñ€Ð¼Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸ÐµÐ¼ + * какого-либо паттерна в значениÑÑ…, так как Ñодержимое в данных либо + * не будет иметь Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ (Ð´Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† без дубликатов), либо будет + * обрабатыватьÑÑ Ð² отдельных btree-поддеревьÑÑ…. + * + * rotate и offset: + * Ð”Ð»Ñ Ð¿Ñ€Ð¾Ð²ÐµÑ€ÐºÐ¸ ÑлиÑÐ½Ð¸Ñ Ð¸ Ñ€Ð°Ð·Ð´ÐµÐ»ÐµÐ½Ð¸Ñ Ñтраниц внутри движка требуютÑÑ + * Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ ÐºÐ»ÑŽÑ‡ÐµÐ¹/значений в виде не-Ñмежных поÑледовательноÑтей, как-бы + * в виде "пунктира", который поÑтепенно заполнÑет веÑÑŒ заданных диапазон. + * + * Параметры позволÑÑŽÑ‚ генерировать такой "пунктир". СоответÑтвенно rotate + * задает цикличеÑкий Ñдвиг вправо, а offset задает Ñмещение, точнее Ð³Ð¾Ð²Ð¾Ñ€Ñ + * Ñложение по модулю внутри диапазона заданного поÑредÑтвом width. + * + * Ðапример, при rotate равном 1 (цикличеÑкий Ñдвиг вправо на 1 бит), + * четные и нечетные иÑходные Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ ÑложатÑÑ Ð² две линейные + * поÑледовательноÑти, которые поÑтепенно закроют Ñтаршую и младшую + * половины диапазона. + * + * split: + * Ð”Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† без дубликатов (без multi-value ключей) фактичеÑки требуетÑÑ + * Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ñ‚Ð¾Ð»ÑŒÐºÐ¾ ключей, а данные могут быть поÑтоÑнным. Ðо Ð´Ð»Ñ Ñ‚Ð°Ð±Ð»Ð¸Ñ† Ñ + * дубликатами (Ñ multi-value ключами) также требуетÑÑ Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ð¹. + * + * Ðенулевое значение параметра split фактичеÑки включает генерацию значений, + * при Ñтом значение split определÑет Ñколько бит иÑходного абÑтрактного + * номера будет отрезано Ð´Ð»Ñ Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ð¸ значениÑ. + */ + + uint8_t width; + uint8_t mesh; + uint8_t rotate; + uint8_t split; + uint32_t seed; + uint64_t offset; + keygen_case keycase; +}; + +struct actor_params_pod { + unsigned mode_flags; + unsigned table_flags; + intptr_t size_lower; + intptr_t size_now; + intptr_t size_upper; + int shrink_threshold; + int growth_step; + int pagesize; + + unsigned test_duration; + unsigned test_nops; + unsigned nrepeat; + unsigned nthreads; + + unsigned keylen_min, keylen_max; + unsigned datalen_min, datalen_max; + + unsigned batch_read; + unsigned batch_write; + + unsigned delaystart; + unsigned waitfor_nops; + unsigned inject_writefaultn; + + unsigned max_readers; + unsigned max_tables; + keygen_params_pod keygen; + + uint8_t loglevel; + bool drop_table; + bool ignore_dbfull; + bool speculum; +}; + +struct actor_config_pod { + unsigned actor_id, space_id; + actor_testcase testcase; + unsigned wait4id; + unsigned signal_nops; +}; + +#pragma pack(pop) + +extern const struct option_verb mode_bits[]; +extern const struct option_verb table_bits[]; +void dump(const char *title = "config-dump: "); + +} /* namespace config */ + +struct actor_params : public config::actor_params_pod { + std::string pathname_log; + std::string pathname_db; + void set_defaults(const std::string &tmpdir); + + unsigned mdbx_keylen_min() const; + unsigned mdbx_keylen_max() const; + unsigned mdbx_datalen_min() const; + unsigned mdbx_datalen_max() const; +}; + +struct actor_config : public config::actor_config_pod { + actor_params params; + + bool wanna_event4signalling() const { return true /* TODO ? */; } + + actor_config(actor_testcase testcase, const actor_params ¶ms, + unsigned space_id, unsigned wait4id); + + actor_config(const char *str) { + if (!deserialize(str, *this)) + failure("Invalid internal parameter '%s'\n", str); + } + + const std::string osal_serialize(simple_checksum &) const; + bool osal_deserialize(const char *str, const char *end, simple_checksum &); + + const std::string serialize(const char *prefix) const; + static bool deserialize(const char *str, actor_config &config); + + bool is_waitable(size_t nops) const { + switch (testcase) { + case ac_hill: + if (!params.test_nops || params.test_nops >= nops) + return true; + __fallthrough; + default: + return false; + } + } +}; diff --git a/src/contrib/db/libmdbx/test/copy.cc b/src/contrib/db/libmdbx/test/copy.cc new file mode 100644 index 0000000..ff53153 --- /dev/null +++ b/src/contrib/db/libmdbx/test/copy.cc @@ -0,0 +1,26 @@ +#include "test.h" + +void testcase_copy::copy_db(const bool with_compaction) { + int err = osal_removefile(copy_pathname); + if (err != MDBX_SUCCESS && err != MDBX_ENOFILE) + failure_perror("mdbx_removefile()", err); + + err = mdbx_env_copy(db_guard.get(), copy_pathname.c_str(), + with_compaction ? MDBX_CP_COMPACT : 0); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror(with_compaction ? "mdbx_env_copy(MDBX_CP_COMPACT)" + : "mdbx_env_copy(MDBX_CP_ASIS)", + err); +} + +bool testcase_copy::run() { + jitter_delay(); + db_open(); + assert(!txn_guard); + const bool order = flipcoin(); + jitter_delay(); + copy_db(order); + jitter_delay(); + copy_db(!order); + return true; +} diff --git a/src/contrib/db/libmdbx/test/darwin/LICENSE b/src/contrib/db/libmdbx/test/darwin/LICENSE new file mode 100644 index 0000000..6a0dd30 --- /dev/null +++ b/src/contrib/db/libmdbx/test/darwin/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2015, Aleksey Demakov +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/contrib/db/libmdbx/test/darwin/README.md b/src/contrib/db/libmdbx/test/darwin/README.md new file mode 100644 index 0000000..a6a8fd1 --- /dev/null +++ b/src/contrib/db/libmdbx/test/darwin/README.md @@ -0,0 +1,8 @@ +# DarwinPthreadBarrier + +A pthread_barrier_t implementation for Mac OS/X + +There is no pthread_barrier_t in Mac OS/X pthreads. This project fixes +this omission by providing a simple-minded barrier implementation based +on a pair of pthread_mutex_t and pthread_cond_t. + diff --git a/src/contrib/db/libmdbx/test/darwin/pthread_barrier.c b/src/contrib/db/libmdbx/test/darwin/pthread_barrier.c new file mode 100644 index 0000000..054aa00 --- /dev/null +++ b/src/contrib/db/libmdbx/test/darwin/pthread_barrier.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2015, Aleksey Demakov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "pthread_barrier.h" + +#include + +#ifdef __APPLE__ + +int pthread_barrierattr_init(pthread_barrierattr_t *attr) { + memset(attr, 0, sizeof(pthread_barrierattr_t)); + int m = pthread_mutexattr_init(&attr->mattr); + int c = pthread_condattr_init(&attr->cattr); + return m ? m : c; +} + +int pthread_barrierattr_destroy(pthread_barrierattr_t *attr) { + int c = pthread_condattr_destroy(&attr->cattr); + int m = pthread_mutexattr_destroy(&attr->mattr); + return m ? m : c; +} + +int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, + int *__restrict pshared) { + return pthread_condattr_getpshared(&attr->cattr, pshared); +} + +int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) { + int m = pthread_mutexattr_setpshared(&attr->mattr, pshared); + int c = pthread_condattr_setpshared(&attr->cattr, pshared); + return m ? m : c; +} + +int pthread_barrier_init(pthread_barrier_t *__restrict barrier, + const pthread_barrierattr_t *__restrict attr, + unsigned count) { + if (count == 0) + return errno = EINVAL; + + int rc = pthread_mutex_init(&barrier->mutex, attr ? &attr->mattr : 0); + if (rc) + return rc; + + rc = pthread_cond_init(&barrier->cond, attr ? &attr->cattr : 0); + if (rc) { + int errno_save = errno; + pthread_mutex_destroy(&barrier->mutex); + errno = errno_save; + return rc; + } + + barrier->limit = count; + barrier->count = 0; + barrier->phase = 0; + return 0; +} + +int pthread_barrier_destroy(pthread_barrier_t *barrier) { + pthread_mutex_destroy(&barrier->mutex); + pthread_cond_destroy(&barrier->cond); + return 0; +} + +int pthread_barrier_wait(pthread_barrier_t *barrier) { + int rc = pthread_mutex_lock(&barrier->mutex); + if (rc) + return rc; + + barrier->count++; + if (barrier->count >= barrier->limit) { + barrier->phase++; + barrier->count = 0; + pthread_cond_broadcast(&barrier->cond); + pthread_mutex_unlock(&barrier->mutex); + return PTHREAD_BARRIER_SERIAL_THREAD; + } else { + unsigned phase = barrier->phase; + do + pthread_cond_wait(&barrier->cond, &barrier->mutex); + while (phase == barrier->phase); + pthread_mutex_unlock(&barrier->mutex); + return 0; + } +} + +#endif /* __APPLE__ */ diff --git a/src/contrib/db/libmdbx/test/darwin/pthread_barrier.h b/src/contrib/db/libmdbx/test/darwin/pthread_barrier.h new file mode 100644 index 0000000..efa9b9b --- /dev/null +++ b/src/contrib/db/libmdbx/test/darwin/pthread_barrier.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2015, Aleksey Demakov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PTHREAD_BARRIER_H +#define PTHREAD_BARRIER_H + +#include + +#ifdef __APPLE__ + +#ifdef __cplusplus +extern "C" { +#endif + +#if !defined(PTHREAD_BARRIER_SERIAL_THREAD) +#define PTHREAD_BARRIER_SERIAL_THREAD (1) +#endif + +#if !defined(PTHREAD_PROCESS_PRIVATE) +#define PTHREAD_PROCESS_PRIVATE (42) +#endif +#if !defined(PTHREAD_PROCESS_SHARED) +#define PTHREAD_PROCESS_SHARED (43) +#endif + +typedef struct { + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; +} pthread_barrierattr_t; + +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; + unsigned int limit; + unsigned int count; + unsigned int phase; +} pthread_barrier_t; + +int pthread_barrierattr_init(pthread_barrierattr_t *attr); +int pthread_barrierattr_destroy(pthread_barrierattr_t *attr); + +int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, + int *__restrict pshared); +int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared); + +int pthread_barrier_init(pthread_barrier_t *__restrict barrier, + const pthread_barrierattr_t *__restrict attr, + unsigned int count); +int pthread_barrier_destroy(pthread_barrier_t *barrier); + +int pthread_barrier_wait(pthread_barrier_t *barrier); + +#ifdef __cplusplus +} +#endif + +#endif /* __APPLE__ */ + +#endif /* PTHREAD_BARRIER_H */ diff --git a/src/contrib/db/libmdbx/test/dead.cc b/src/contrib/db/libmdbx/test/dead.cc new file mode 100644 index 0000000..8f83bbe --- /dev/null +++ b/src/contrib/db/libmdbx/test/dead.cc @@ -0,0 +1,35 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +bool testcase_deadread::run() { + db_open(); + txn_begin(true); + cursor_guard.reset(); + txn_guard.reset(); + db_guard.reset(); + return true; +} + +//----------------------------------------------------------------------------- + +bool testcase_deadwrite::run() { + db_open(); + txn_begin(false); + cursor_guard.reset(); + txn_guard.reset(); + db_guard.reset(); + return true; +} diff --git a/src/contrib/db/libmdbx/test/hill.cc b/src/contrib/db/libmdbx/test/hill.cc new file mode 100644 index 0000000..9d98909 --- /dev/null +++ b/src/contrib/db/libmdbx/test/hill.cc @@ -0,0 +1,409 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +bool testcase_hill::run() { + int err = db_open__begin__table_create_open_clean(dbi); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("hill: bailout-prepare due '%s'", mdbx_strerror(err)); + return false; + } + speculum.clear(); + speculum_commited.clear(); + + /* LY: теÑÑ‚ "холмиком": + * - Ñначала наполнÑем таблицу цикличеÑкими CRUD-манипулÑциÑми, + * которые в каждом цикле делают неÑколько операций, Ð²ÐºÐ»ÑŽÑ‡Ð°Ñ ÑƒÐ´Ð°Ð»ÐµÐ½Ð¸Ðµ, + * но в результате добавлÑÑŽÑ‚ запиÑи. + * - затем очищаем таблицу также CRUD-манипулÑциÑми, но уже Ñ Ð´Ñ€ÑƒÐ³Ð¾Ð¹ + * пропорцией удалений. + * + * При Ñтом очень многое завиÑит от порÑдка перебора ключей: + * - (пÑевдо)Ñлучайное раÑпределение требуетÑÑ Ð»Ð¸ÑˆÑŒ Ð´Ð»Ñ Ð¿Ð¾Ð»Ð½Ð¾Ñ‚Ñ‹ картины, + * но в целом не покрывает важных кейÑов. + * - кроме (пÑевдо)Ñлучайного перебора требуетÑÑ Ð¿Ð¾Ñледовательное + * итерирование ключей интервалами различной ширины, Ñ Ñ‚ÐµÐ¼ чтобы + * проверить различные варианты как разделениÑ, так и ÑлиÑÐ½Ð¸Ñ Ñтраниц + * внутри движка. + * - при не-уникальных ключах (MDBX_DUPSORT Ñ Ð¿Ð¾Ð´Ð²Ð°Ñ€Ð¸Ð°Ð½Ñ‚Ð°Ð¼Ð¸), Ð´Ð»Ñ ÐºÐ°Ð¶Ð´Ð¾Ð³Ð¾ + * повтора внутри движка формируетÑÑ Ð²Ð»Ð¾Ð¶ÐµÐ½Ð½Ð¾Ðµ btree-дерево, + * ÑоответÑтвенно требуетÑÑ Ñоблюдение аналогичных принципов + * Ð¸Ñ‚ÐµÑ€Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð´Ð»Ñ Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ð¹. + */ + + /* TODO: работа в неÑколько потоков */ + keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); + + keygen::buffer a_key = keygen::alloc(config.params.keylen_max); + keygen::buffer a_data_0 = keygen::alloc(config.params.datalen_max); + keygen::buffer a_data_1 = keygen::alloc(config.params.datalen_max); + keygen::buffer b_key = keygen::alloc(config.params.keylen_max); + keygen::buffer b_data = keygen::alloc(config.params.datalen_max); + + const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_NODUPDATA + : MDBX_NODUPDATA | MDBX_NOOVERWRITE; + const unsigned update_flags = + (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_CURRENT | MDBX_NODUPDATA | MDBX_NOOVERWRITE + : MDBX_NODUPDATA; + + uint64_t serial_count = 0; + uint64_t commited_serial = serial_count; + unsigned txn_nops = 0; + + bool rc = false; + while (should_continue()) { + const keygen::serial_t a_serial = serial_count; + if (unlikely(!keyvalue_maker.increment(serial_count, 1))) { + log_notice("uphill: unexpected key-space overflow"); + break; + } + + const keygen::serial_t b_serial = serial_count; + assert(b_serial > a_serial); + + // Ñоздаем первую запиÑÑŒ из пары + const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31); + log_trace("uphill: insert-a (age %" PRIu64 ") %" PRIu64, age_shift, + a_serial); + generate_pair(a_serial, a_key, a_data_1, age_shift); + + err = insert(a_key, a_data_1, insert_flags); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("uphill: bailout at insert-a due '%s'", mdbx_strerror(err)); + txn_restart(true, false); + serial_count = commited_serial; + speculum = speculum_commited; + break; + } + failure_perror("mdbx_put(insert-a.1)", err); + } + if (!speculum_verify()) { + log_notice("uphill: bailout after insert-a, before commit"); + goto bailout; + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); + serial_count = commited_serial; + speculum = speculum_commited; + break; + } + speculum_commited = speculum; + commited_serial = a_serial; + txn_nops = 0; + if (!speculum_verify()) { + log_notice("uphill: bailout after insert-a, after commit"); + goto bailout; + } + } + + // Ñоздаем вторую запиÑÑŒ из пары + log_trace("uphill: insert-b %" PRIu64, b_serial); + generate_pair(b_serial, b_key, b_data, 0); + err = insert(b_key, b_data, insert_flags); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("uphill: bailout at insert-b due '%s'", mdbx_strerror(err)); + txn_restart(true, false); + serial_count = commited_serial; + speculum = speculum_commited; + break; + } + failure_perror("mdbx_put(insert-b)", err); + } + if (!speculum_verify()) { + log_notice("uphill: bailout after insert-b, before commit"); + goto bailout; + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); + serial_count = commited_serial; + speculum = speculum_commited; + break; + } + speculum_commited = speculum; + commited_serial = a_serial; + txn_nops = 0; + if (!speculum_verify()) { + log_notice("uphill: bailout after insert-b, after commit"); + goto bailout; + } + } + + // обновлÑем данные в первой запиÑи + log_trace("uphill: update-a (age %" PRIu64 "->0) %" PRIu64, age_shift, + a_serial); + generate_pair(a_serial, a_key, a_data_0, 0); + checkdata("uphill: update-a", dbi, a_key->value, a_data_1->value); + err = replace(a_key, a_data_0, a_data_1, update_flags); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("uphill: bailout at update-a due '%s'", mdbx_strerror(err)); + txn_restart(true, false); + serial_count = commited_serial; + speculum = speculum_commited; + break; + } + failure_perror("mdbx_replace(update-a: 1->0)", err); + } + if (!speculum_verify()) { + log_notice("uphill: bailout after update-a, before commit"); + goto bailout; + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); + serial_count = commited_serial; + speculum = speculum_commited; + break; + } + speculum_commited = speculum; + commited_serial = a_serial; + txn_nops = 0; + if (!speculum_verify()) { + log_notice("uphill: bailout after update-a, after commit"); + goto bailout; + } + } + + // удалÑем вторую запиÑÑŒ + log_trace("uphill: delete-b %" PRIu64, b_serial); + checkdata("uphill: delete-b", dbi, b_key->value, b_data->value); + err = remove(b_key, b_data); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("uphill: bailout at delete-b due '%s'", mdbx_strerror(err)); + txn_restart(true, false); + serial_count = commited_serial; + speculum = speculum_commited; + break; + } + failure_perror("mdbx_del(b)", err); + } + if (!speculum_verify()) { + log_notice("uphill: bailout after delete-b, before commit"); + goto bailout; + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("uphill: bailout at commit due '%s'", mdbx_strerror(err)); + serial_count = commited_serial; + speculum = speculum_commited; + break; + } + speculum_commited = speculum; + commited_serial = a_serial; + txn_nops = 0; + if (!speculum_verify()) { + log_notice("uphill: bailout after delete-b, after commit"); + goto bailout; + } + } + + report(1); + if (!keyvalue_maker.increment(serial_count, 1)) { + // дошли до границы проÑтранÑтва ключей + serial_count = a_serial; + goto overflow; + } + } + + while (serial_count > 1) { + if (unlikely(!keyvalue_maker.increment(serial_count, -2))) + failure("downhill: unexpected key-space underflow"); + + overflow: + const keygen::serial_t a_serial = serial_count; + const keygen::serial_t b_serial = a_serial + 1; + assert(b_serial > a_serial); + + // обновлÑем первую запиÑÑŒ из пары + const keygen::serial_t age_shift = UINT64_C(1) << (a_serial % 31); + log_trace("downhill: update-a (age 0->%" PRIu64 ") %" PRIu64, age_shift, + a_serial); + generate_pair(a_serial, a_key, a_data_0, 0); + generate_pair(a_serial, a_key, a_data_1, age_shift); + checkdata("downhill: update-a", dbi, a_key->value, a_data_0->value); + err = replace(a_key, a_data_1, a_data_0, update_flags); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("downhill: bailout at update-a due '%s'", + mdbx_strerror(err)); + txn_end(true); + speculum = speculum_commited; + break; + } + failure_perror("mdbx_put(update-a: 0->1)", err); + } + if (!speculum_verify()) { + log_notice("downhill: bailout after update-a, before commit"); + break; + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + speculum = speculum_commited; + break; + } + speculum_commited = speculum; + txn_nops = 0; + if (!speculum_verify()) { + log_notice("downhill: bailout after update-a, after commit"); + break; + } + } + + // Ñоздаем вторую запиÑÑŒ из пары + log_trace("downhill: insert-b %" PRIu64, b_serial); + generate_pair(b_serial, b_key, b_data, 0); + err = insert(b_key, b_data, insert_flags); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("downhill: bailout at insert-a due '%s'", + mdbx_strerror(err)); + txn_end(true); + speculum = speculum_commited; + break; + } + failure_perror("mdbx_put(insert-b)", err); + } + if (!speculum_verify()) { + log_notice("downhill: bailout after insert-b, before commit"); + break; + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + speculum = speculum_commited; + break; + } + speculum_commited = speculum; + txn_nops = 0; + if (!speculum_verify()) { + log_notice("downhill: bailout after insert-b, after commit"); + break; + } + } + + // удалÑем первую запиÑÑŒ + log_trace("downhill: delete-a (age %" PRIu64 ") %" PRIu64, age_shift, + a_serial); + checkdata("downhill: delete-a", dbi, a_key->value, a_data_1->value); + err = remove(a_key, a_data_1); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("downhill: bailout at delete-a due '%s'", + mdbx_strerror(err)); + txn_end(true); + speculum = speculum_commited; + break; + } + failure_perror("mdbx_del(a)", err); + } + if (!speculum_verify()) { + log_notice("downhill: bailout after delete-a, before commit"); + break; + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + speculum = speculum_commited; + break; + } + speculum_commited = speculum; + txn_nops = 0; + if (!speculum_verify()) { + log_notice("downhill: bailout after delete-a, after commit"); + break; + } + } + + // удалÑем вторую запиÑÑŒ + log_trace("downhill: delete-b %" PRIu64, b_serial); + checkdata("downhill: delete-b", dbi, b_key->value, b_data->value); + err = remove(b_key, b_data); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("downhill: bailout at delete-b due '%s'", + mdbx_strerror(err)); + txn_end(true); + speculum = speculum_commited; + break; + } + failure_perror("mdbx_del(b)", err); + } + if (!speculum_verify()) { + log_notice("downhill: bailout after delete-b, before commit"); + break; + } + + if (++txn_nops >= config.params.batch_write) { + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + speculum = speculum_commited; + break; + } + speculum_commited = speculum; + txn_nops = 0; + if (!speculum_verify()) { + log_notice("downhill: bailout after delete-b, after commit"); + goto bailout; + } + } + + report(1); + } + + rc = speculum_verify(); +bailout: + if (txn_guard) { + err = breakable_commit(); + if (unlikely(err != MDBX_SUCCESS)) + log_notice("downhill: bailout at commit due '%s'", mdbx_strerror(err)); + } + + if (dbi) { + if (config.params.drop_table && !mode_readonly()) { + txn_begin(false); + db_table_drop(dbi); + err = breakable_commit(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("hill: bailout-clean due '%s'", mdbx_strerror(err)); + return rc; + } + } else + db_table_close(dbi); + } + return rc; +} diff --git a/src/contrib/db/libmdbx/test/jitter.cc b/src/contrib/db/libmdbx/test/jitter.cc new file mode 100644 index 0000000..38caf33 --- /dev/null +++ b/src/contrib/db/libmdbx/test/jitter.cc @@ -0,0 +1,59 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +bool testcase_jitter::run() { + while (should_continue()) { + jitter_delay(); + db_open(); + + if (flipcoin()) { + jitter_delay(); + txn_begin(true); + fetch_canary(); + jitter_delay(); + txn_end(flipcoin()); + } + + jitter_delay(); + txn_begin(mode_readonly()); + jitter_delay(); + if (!mode_readonly()) { + fetch_canary(); + update_canary(1); + /* TODO: + * - db_setsize() + * ... + */ + } + txn_end(flipcoin()); + + if (flipcoin()) { + jitter_delay(); + txn_begin(true); + jitter_delay(); + txn_end(flipcoin()); + } + + jitter_delay(); + db_close(); + + /* just 'align' nops with other tests with batching */ + const auto batching = + std::max(config.params.batch_read, config.params.batch_write); + report(std::max(1u, batching / 2)); + } + return true; +} diff --git a/src/contrib/db/libmdbx/test/keygen.cc b/src/contrib/db/libmdbx/test/keygen.cc new file mode 100644 index 0000000..e156142 --- /dev/null +++ b/src/contrib/db/libmdbx/test/keygen.cc @@ -0,0 +1,275 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +namespace keygen { + +static inline __pure_function serial_t mask(unsigned bits) { + assert(bits > 0 && bits <= serial_maxwith); + return serial_allones >> (serial_maxwith - bits); +} + +/* LY: https://en.wikipedia.org/wiki/Injective_function */ +serial_t injective(const serial_t serial, + const unsigned bits /* at least serial_minwith (8) */, + const serial_t salt) { + assert(bits > serial_minwith && bits <= serial_maxwith); + + /* LY: All these "magic" prime numbers were found + * and verified with a bit of brute force. */ + + static const uint64_t m[64 - serial_minwith + 1] = { + /* 8 - 24 */ + 113, 157, 397, 653, 1753, 5641, 9697, 23873, 25693, 80833, 105953, 316937, + 309277, 834497, 1499933, 4373441, 10184137, + /* 25 - 64 */ + 10184137, 17279209, 33990377, 67295161, 284404553, 1075238767, 6346721573, + 6924051577, 19204053433, 45840188887, 53625693977, 73447827913, + 141638870249, 745683604649, 1283334050489, 1100828289853, 2201656586197, + 5871903036137, 11238507001417, 45264020802263, 105008404482889, + 81921776907059, 199987980256399, 307207457507641, 946769023178273, + 2420886491930041, 3601632139991929, 11984491914483833, 21805846439714153, + 23171543400565993, 53353226456762893, 155627817337932409, + 227827205384840249, 816509268558278821, 576933057762605689, + 2623957345935638441, 5048241705479929949, 4634245581946485653, + 4613509448041658233, 4952535426879925961}; + static const uint8_t s[64 - serial_minwith + 1] = { + /* 8 - 24 */ + 2, 3, 4, 4, 2, 4, 3, 3, 7, 3, 3, 4, 8, 3, 10, 3, 11, + /* 25 - 64 */ + 11, 9, 9, 9, 11, 10, 5, 14, 11, 16, 14, 12, 13, 16, 19, 10, 10, 21, 7, 20, + 10, 14, 22, 19, 3, 21, 18, 19, 26, 24, 2, 21, 25, 29, 24, 10, 11, 14, 20, + 19}; + + const auto mult = m[bits - 8]; + const auto shift = s[bits - 8]; + serial_t result = serial * mult; + if (salt) { + const unsigned left = bits / 2; + const unsigned right = bits - left; + result = (result << left) | ((result & mask(bits)) >> right); + result = (result ^ salt) * mult; + } + + result ^= result << shift; + result &= mask(bits); + log_trace("keygen-injective: serial %" PRIu64 "/%u @%" PRIx64 ",%u,%" PRIu64 + " => %" PRIu64 "/%u", + serial, bits, mult, shift, salt, result, bits); + return result; +} + +void __hot maker::pair(serial_t serial, const buffer &key, buffer &value, + serial_t value_age) { + assert(mapping.width >= serial_minwith && mapping.width <= serial_maxwith); + assert(mapping.split <= mapping.width); + assert(mapping.mesh <= mapping.width); + assert(mapping.rotate <= mapping.width); + assert(mapping.offset <= mask(mapping.width)); + assert(!(key_essentials.flags & + ~(MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT))); + assert(!(value_essentials.flags & ~(MDBX_INTEGERDUP | MDBX_REVERSEDUP))); + + log_trace("keygen-pair: serial %" PRIu64 ", data-age %" PRIu64, serial, + value_age); + + if (mapping.mesh >= serial_minwith) { + serial = + (serial & ~mask(mapping.mesh)) | injective(serial, mapping.mesh, salt); + log_trace("keygen-pair: mesh@%u => %" PRIu64, mapping.mesh, serial); + } + + if (mapping.rotate) { + const unsigned right = mapping.rotate; + const unsigned left = mapping.width - right; + serial = (serial << left) | ((serial & mask(mapping.width)) >> right); + log_trace("keygen-pair: rotate@%u => %" PRIu64 ", 0x%" PRIx64, + mapping.rotate, serial, serial); + } + + if (mapping.offset) { + serial = (serial + mapping.offset) & mask(mapping.width); + log_trace("keygen-pair: offset@%" PRIu64 " => %" PRIu64, mapping.offset, + serial); + } + if (base) { + serial += base; + log_trace("keygen-pair: base@%" PRIu64 " => %" PRIu64, base, serial); + } + + serial_t key_serial = serial; + serial_t value_serial = value_age << mapping.split; + if (mapping.split) { + if (key_essentials.flags & MDBX_DUPSORT) { + key_serial >>= mapping.split; + value_serial += serial & mask(mapping.split); + } else { + /* Без MDBX_DUPSORT требуетÑÑ ÑƒÐ½Ð¸ÐºÐ°Ð»ÑŒÐ½Ð¾ÑÑ‚ÑŒ ключей, а Ð´Ð»Ñ Ñтого Ð½ÐµÐ»ÑŒÐ·Ñ + * отбраÑывать какие-либо биты serial поÑле инъективного преобразованиÑ. + * ПоÑтому key_serial не трогаем, а в value_serial нелинейно вмешиваем + * запрошенное количеÑтво бит из serial */ + value_serial += + (serial ^ (serial >> mapping.split)) & mask(mapping.split); + } + + value_serial |= value_age << mapping.split; + log_trace("keygen-pair: split@%u => k%" PRIu64 ", v%" PRIu64, mapping.split, + key_serial, value_serial); + } + + log_trace("keygen-pair: key %" PRIu64 ", value %" PRIu64, key_serial, + value_serial); + mk(key_serial, key_essentials, *key); + mk(value_serial, value_essentials, *value); + + if (log_enabled(logging::trace)) { + char dump_key[128], dump_value[128]; + log_trace("keygen-pair: key %s, value %s", + mdbx_dump_val(&key->value, dump_key, sizeof(dump_key)), + mdbx_dump_val(&value->value, dump_value, sizeof(dump_value))); + } +} + +void maker::setup(const config::actor_params_pod &actor, unsigned actor_id, + unsigned thread_number) { + key_essentials.flags = + actor.table_flags & (MDBX_INTEGERKEY | MDBX_REVERSEKEY | MDBX_DUPSORT); + assert(actor.keylen_min <= UINT8_MAX); + key_essentials.minlen = (uint8_t)actor.keylen_min; + assert(actor.keylen_max <= UINT16_MAX); + key_essentials.maxlen = (uint16_t)actor.keylen_max; + + value_essentials.flags = + actor.table_flags & (MDBX_INTEGERDUP | MDBX_REVERSEDUP); + assert(actor.datalen_min <= UINT8_MAX); + value_essentials.minlen = (uint8_t)actor.datalen_min; + assert(actor.datalen_max <= UINT16_MAX); + value_essentials.maxlen = (uint16_t)actor.datalen_max; + + assert(thread_number < 2); + (void)thread_number; + mapping = actor.keygen; + salt = (actor.keygen.seed + actor_id) * UINT64_C(14653293970879851569); + + // FIXME: TODO + base = 0; +} + +void maker::make_ordered() { + mapping.mesh = 0; + mapping.rotate = 0; +} + +bool maker::is_unordered() const { + return (mapping.mesh >= serial_minwith || mapping.rotate) != 0; +} + +bool maker::increment(serial_t &serial, int delta) const { + if (serial > mask(mapping.width)) { + log_extra("keygen-increment: %" PRIu64 " > %" PRIu64 ", overflow", serial, + mask(mapping.width)); + return false; + } + + serial_t target = serial + (int64_t)delta; + if (target > mask(mapping.width) || + ((delta > 0) ? target < serial : target > serial)) { + log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", overflow", + serial, delta, target); + return false; + } + + log_extra("keygen-increment: %" PRIu64 "%-d => %" PRIu64 ", continue", serial, + delta, target); + serial = target; + return true; +} + +//----------------------------------------------------------------------------- + +static size_t length(serial_t serial) { + size_t n = 0; + if (serial > UINT32_MAX) { + n = 4; + serial >>= 32; + } + if (serial > UINT16_MAX) { + n += 2; + serial >>= 16; + } + if (serial > UINT8_MAX) { + n += 1; + serial >>= 8; + } + return (serial > 0) ? n + 1 : n; +} + +buffer alloc(size_t limit) { + result *ptr = (result *)malloc(sizeof(result) + limit); + if (unlikely(ptr == nullptr)) + failure_perror("malloc(keyvalue_buffer)", errno); + ptr->value.iov_base = ptr->bytes; + ptr->value.iov_len = 0; + ptr->limit = limit; + return buffer(ptr); +} + +void __hot maker::mk(const serial_t serial, const essentials ¶ms, + result &out) { + assert(out.limit >= params.maxlen); + assert(params.maxlen >= params.minlen); + assert(params.maxlen >= length(serial)); + + out.value.iov_base = out.bytes; + out.value.iov_len = + (params.maxlen > params.minlen) + ? params.minlen + serial % (params.maxlen - params.minlen) + : params.minlen; + + if (params.flags & (MDBX_INTEGERKEY | MDBX_INTEGERDUP)) { + assert(params.maxlen == params.minlen); + assert(params.minlen == 4 || params.minlen == 8); + if (is_byteorder_le() || params.minlen == 8) + out.u64 = serial; + else + out.u32 = (uint32_t)serial; + } else if (params.flags & (MDBX_REVERSEKEY | MDBX_REVERSEDUP)) { + if (out.value.iov_len > 8) { + memset(out.bytes, '\0', out.value.iov_len - 8); + unaligned::store(out.bytes + out.value.iov_len - 8, htobe64(serial)); + } else { + out.u64 = htobe64(serial); + if (out.value.iov_len < 8) { + out.value.iov_len = std::max(length(serial), out.value.iov_len); + out.value.iov_base = out.bytes + 8 - out.value.iov_len; + } + } + } else { + out.u64 = htole64(serial); + if (out.value.iov_len > 8) + memset(out.bytes + 8, '\0', out.value.iov_len - 8); + else + out.value.iov_len = std::max(length(serial), out.value.iov_len); + } + + assert(out.value.iov_len >= params.minlen); + assert(out.value.iov_len <= params.maxlen); + assert(out.value.iov_len >= length(serial)); + assert(out.value.iov_base >= out.bytes); + assert((uint8_t *)out.value.iov_base + out.value.iov_len <= + out.bytes + out.limit); +} + +} /* namespace keygen */ diff --git a/src/contrib/db/libmdbx/test/keygen.h b/src/contrib/db/libmdbx/test/keygen.h new file mode 100644 index 0000000..0403ab8 --- /dev/null +++ b/src/contrib/db/libmdbx/test/keygen.h @@ -0,0 +1,130 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" +#include "config.h" +#include "log.h" +#include "utils.h" + +namespace keygen { + +/* Под "генерацией ключей" здеÑÑŒ понимаетÑÑ Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ð¾Ð±Ð¾Ð¸Ñ… значений Ð´Ð»Ñ + * пар key-value, Ñ‚.е. не только ключей, но и аÑÑоциированных Ñ Ð½Ð¸Ð¼Ð¸ данных. + */ + +/* Генерацию ключей Ð½ÐµÐ»ÑŒÐ·Ñ Ð¾Ñ‚Ð½ÐµÑти к проÑтым задачам, так как Ñ‚Ñ€ÐµÐ±Ð¾Ð²Ð°Ð½Ð¸Ñ + * примерно Ñледующие: + * - Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ Ñ€Ð°Ð·Ð½Ð¾Ð³Ð¾ количеÑтва уникальных ключей различной длины + * в задаваемом диапазоне; + * - возможноÑÑ‚ÑŒ выбора как пÑевдо-Ñлучайного порÑдка ключей, + * так и по некоторым ÑпецифичеÑким законам (ограниченными упорÑдоченными + * поÑледовательноÑÑ‚Ñми, в шахматном порÑдке по граница диапазона и Ñ‚.д.); + * - возможноÑÑ‚ÑŒ генерации дубликатов Ñ Ð·Ð°Ð´Ð°Ð²Ð°ÐµÐ¼Ñ‹Ð¼ законом раÑпределениÑ; + * - возможноÑÑ‚ÑŒ генерации непереÑекающимиÑÑ ÐºÐ»Ð°Ñтерами Ð´Ð»Ñ Ð¿Ð°Ñ€Ð°Ð»Ð»ÐµÐ»ÑŒÐ½Ð¾Ð³Ð¾ + * иÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ð½Ð¸Ñ Ð² неÑкольких потоках; + * - иÑпользовать минимум реÑурÑов, как CPU, так и RAM, в том чиÑле + * Ð²ÐºÐ»ÑŽÑ‡Ð°Ñ cache pollution и ram bandwidth. + * + * При Ñтом заведомо извеÑтно, что Ð´Ð»Ñ MDBX не имеет значениÑ: + * - иÑпользуемый алфавит (Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð±Ð°Ð¹Ñ‚Ð¾Ð²); + * - чаÑтотное раÑпределение по алфавиту; + * - абÑолютное значение ключей или разноÑÑ‚ÑŒ между отдельными значениÑми; + * + * СоответÑтвенно, в общих чертах, Ñхема генерации ÑледующаÑ: + * - вводитÑÑ Ð¿Ð»Ð¾ÑÐºÐ°Ñ Ð¾Ð´Ð½Ð¾Ð¼ÐµÑ€Ð½Ð°Ñ "координата" serial (uint64_t); + * - Ð³ÐµÐ½ÐµÑ€Ð°Ñ†Ð¸Ñ ÑпецифичеÑких паттернов (поÑледовательноÑтей) + * реализуетÑÑ Ð¿Ð¾ÑредÑтвом ÑоответÑтвующих преобразований "координат", при + * Ñтом вÑе подобные Ð¿Ñ€ÐµÐ¾Ð±Ñ€Ð°Ð·Ð¾Ð²Ð°Ð½Ð¸Ñ Ð²Ñ‹Ð¿Ð¾Ð»Ð½ÑÑŽÑ‚ÑÑ Ñ‚Ð¾Ð»ÑŒÐºÐ¾ над "координатой"; + * - Ð¸Ñ‚Ð¾Ð³Ð¾Ð²Ð°Ñ "координата" преобразуетÑÑ Ð² 8-байтное Ñуррогатное значение + * ключа; + * - Ð´Ð»Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ ÐºÐ»ÑŽÑ‡ÐµÐ¹ длиной МЕÐЕЕ 8 байт Ñуррогат может уÑекатьÑÑ + * до ненулевых байт, в том чиÑле до нулевой длины; + * - Ð´Ð»Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ ÐºÐ»ÑŽÑ‡ÐµÐ¹ длиной БОЛЕЕ 8 байт Ñуррогат дополнÑетÑÑ + * нулÑми или пÑевдоÑлучайной поÑледовательноÑтью; + * + * Механизм генерации паттернов: + * - реализованный механизм ÑвлÑетÑÑ ÐºÐ¾Ð¼Ð¿Ñ€Ð¾Ð¼Ð¸ÑÑом между ÑкороÑтью/проÑтотой + * и гибкоÑтью, необходимой Ð´Ð»Ñ Ð¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸Ñ Ð¿Ð¾ÑледовательноÑтей, которых + * будет доÑтаточно Ð´Ð»Ñ Ð¿Ñ€Ð¾Ð²ÐµÑ€ÐºÐ¸ Ñценариев Ñ€Ð°Ð·Ð´ÐµÐ»ÐµÐ½Ð¸Ñ Ð¸ ÑлиÑÐ½Ð¸Ñ Ñтраниц + * Ñ Ð´Ð°Ð½Ð½Ñ‹Ð¼Ð¸ внутри mdbx; + * - пÑевдо-Ñлучайные паттерны реализуютÑÑ Ð¿Ð¾ÑредÑтвом набора инъективных + * отображающих функций; + * - не-пÑевдо-Ñлучайные паттерны реализуютÑÑ Ð¿Ð¾ÑредÑтвом параметризируемого + * трех-Ñтапного преобразованиÑ: + * 1) Ñмещение (Ñложение) по модулю; + * 2) цикличеÑкий Ñдвиг; + * 3) добавление абÑолютного ÑÐ¼ÐµÑ‰ÐµÐ½Ð¸Ñ (базы); + */ + +typedef uint64_t serial_t; + +enum : serial_t { + serial_minwith = 8, + serial_maxwith = sizeof(serial_t) * 8, + serial_allones = ~(serial_t)0u +}; + +struct result { + MDBX_val value; + size_t limit; + union { + uint8_t bytes[sizeof(uint64_t)]; + uint32_t u32; + uint64_t u64; + }; + + std::string as_string() const { + return std::string((const char *)value.iov_base, value.iov_len); + } +}; + +//----------------------------------------------------------------------------- + +struct buffer_deleter /* : public std::unary_function */ { + void operator()(result *buffer) const { free(buffer); } +}; + +typedef std::unique_ptr buffer; + +buffer alloc(size_t limit); + +class maker { + config::keygen_params_pod mapping; + serial_t base; + serial_t salt; + + struct essentials { + uint8_t minlen; + uint8_t flags; + uint16_t maxlen; + } key_essentials, value_essentials; + + static void mk(const serial_t serial, const essentials ¶ms, result &out); + +public: + maker() { memset(this, 0, sizeof(*this)); } + + void pair(serial_t serial, const buffer &key, buffer &value, + serial_t value_age); + void setup(const config::actor_params_pod &actor, unsigned actor_id, + unsigned thread_number); + void make_ordered(); + bool is_unordered() const; + + bool increment(serial_t &serial, int delta) const; +}; + +} /* namespace keygen */ diff --git a/src/contrib/db/libmdbx/test/log.cc b/src/contrib/db/libmdbx/test/log.cc new file mode 100644 index 0000000..57139ea --- /dev/null +++ b/src/contrib/db/libmdbx/test/log.cc @@ -0,0 +1,371 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +static void fflushall() { fflush(nullptr); } + +void failure(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + fflushall(); + logging::output_nocheckloglevel_ap(logging::failure, fmt, ap); + va_end(ap); + fflushall(); + exit(EXIT_FAILURE); +} + +const char *test_strerror(int errnum) { + static __thread char buf[1024]; + return mdbx_strerror_r(errnum, buf, sizeof(buf)); +} + +void __noreturn failure_perror(const char *what, int errnum) { + failure("%s failed: %s (%d)\n", what, test_strerror(errnum), errnum); +} + +//----------------------------------------------------------------------------- + +static void mdbx_logger(int priority, const char *function, int line, + const char *msg, va_list args) { + if (!function) + function = "unknown"; + + if (priority == MDBX_LOG_FATAL) + log_error("mdbx: fatal failure: %s, %d", function, line); + + logging::output_nocheckloglevel( + logging::loglevel(priority), + strncmp(function, "mdbx_", 5) == 0 ? "%s: " : "mdbx %s: ", function); + logging::feed_ap(msg, args); +} + +namespace logging { + +static std::string prefix; +static std::string suffix; +static loglevel level; +static FILE *last; + +void setlevel(loglevel priority) { + level = priority; + int rc = mdbx_setup_debug(priority, + MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER | + MDBX_DBG_DUMP, + mdbx_logger); + log_trace("set mdbx debug-opts: 0x%02x", rc); +} + +void setup(loglevel priority, const std::string &_prefix) { + setlevel(priority); + prefix = _prefix; +} + +void setup(const std::string &_prefix) { prefix = _prefix; } + +const char *level2str(const loglevel alevel) { + switch (alevel) { + default: + return "invalid/unknown"; + case extra: + return "extra"; + case trace: + return "trace"; + case debug: + return "debug"; + case verbose: + return "verbose"; + case notice: + return "notice"; + case warning: + return "warning"; + case error: + return "error"; + case failure: + return "failure"; + } +} + +bool output(const loglevel priority, const char *format, ...) { + if (lower(priority, level)) + return false; + + va_list ap; + va_start(ap, format); + output_nocheckloglevel_ap(priority, format, ap); + va_end(ap); + return true; +} + +void output_nocheckloglevel_ap(const logging::loglevel priority, + const char *format, va_list ap) { + if (last) { + putc('\n', last); + fflush(last); + if (last == stderr) { + putc('\n', stdout); + fflush(stdout); + } + last = nullptr; + } + + chrono::time now = chrono::now_realtime(); + struct tm tm; +#ifdef _MSC_VER + int rc = _localtime32_s(&tm, (const __time32_t *)&now.utc); +#else + time_t time = now.utc; + int rc = localtime_r(&time, &tm) ? MDBX_SUCCESS : errno; +#endif + if (rc != MDBX_SUCCESS) + failure_perror("localtime_r()", rc); + + last = stdout; + fprintf(last, + "[ %02d%02d%02d-%02d:%02d:%02d.%06d_%05u %-10s %.4s ] %s" /* TODO */, + tm.tm_year - 100, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, + tm.tm_sec, chrono::fractional2us(now.fractional), osal_getpid(), + prefix.c_str(), level2str(priority), suffix.c_str()); + + va_list ones; + memset(&ones, 0, sizeof(ones)) /* zap MSVC and other stupid compilers */; + if (same_or_higher(priority, error)) + va_copy(ones, ap); + vfprintf(last, format, ap); + + size_t len = strlen(format); + char end = len ? format[len - 1] : '\0'; + + switch (end) { + default: + putc('\n', last); + // fall through + case '\n': + fflush(last); + last = nullptr; + // fall through + case ' ': + case '_': + case ':': + case '|': + case ',': + case '\t': + case '\b': + case '\r': + case '\0': + break; + } + + if (same_or_higher(priority, error)) { + if (last != stderr) { + fprintf(stderr, "[ %05u %-10s %.4s ] %s", osal_getpid(), prefix.c_str(), + level2str(priority), suffix.c_str()); + vfprintf(stderr, format, ones); + if (end == '\n') + fflush(stderr); + else + last = stderr; + } + va_end(ones); + } +} + +bool feed_ap(const char *format, va_list ap) { + if (!last) + return false; + + if (last == stderr) { + va_list ones; + va_copy(ones, ap); + vfprintf(stdout, format, ones); + va_end(ones); + } + vfprintf(last, format, ap); + size_t len = strlen(format); + if (len && format[len - 1] == '\n') { + fflush(last); + if (last == stderr) + fflush(stdout); + last = nullptr; + } + return true; +} + +bool feed(const char *format, ...) { + if (!last) + return false; + + va_list ap; + va_start(ap, format); + feed_ap(format, ap); + va_end(ap); + return true; +} + +local_suffix::local_suffix(const char *c_str) + : trim_pos(suffix.size()), indent(0) { + suffix.append(c_str); +} + +local_suffix::local_suffix(const std::string &str) + : trim_pos(suffix.size()), indent(0) { + suffix.append(str); +} + +void local_suffix::push() { + indent += 1; + suffix.push_back('\t'); +} + +void local_suffix::pop() { + assert(indent > 0); + if (indent > 0) { + indent -= 1; + suffix.pop_back(); + } +} + +local_suffix::~local_suffix() { suffix.erase(trim_pos); } + +void progress_canary(bool active) { + static chrono::time progress_timestamp; + chrono::time now = chrono::now_motonic(); + + if (now.fixedpoint - progress_timestamp.fixedpoint < + chrono::from_ms(42).fixedpoint) + return; + + if (osal_progress_push(active)) { + progress_timestamp = now; + return; + } + + if (progress_timestamp.fixedpoint == 0) { + putc('>', stderr); + progress_timestamp = now; + } else if (global::config::console_mode) { + if (active) { + static int last_point = -1; + int point = (now.fixedpoint >> 29) & 3; + if (point != last_point) { + progress_timestamp = now; + fprintf(stderr, "%c\b", "-\\|/"[last_point = point]); + } + } else if (now.fixedpoint - progress_timestamp.fixedpoint > + chrono::from_seconds(2).fixedpoint) { + progress_timestamp = now; + fprintf(stderr, "%c\b", "@*"[now.utc & 1]); + } + } else { + static int count; + if (active && now.fixedpoint - progress_timestamp.fixedpoint > + chrono::from_seconds(1).fixedpoint) { + putc('.', stderr); + progress_timestamp = now; + ++count; + } else if (now.fixedpoint - progress_timestamp.fixedpoint > + chrono::from_seconds(5).fixedpoint) { + putc("@*"[now.utc & 1], stderr); + progress_timestamp = now; + ++count; + } + if (count == 60) { + count = 0; + putc('\n', stderr); + } + } + fflush(stderr); +} + +} // namespace logging + +void log_extra(const char *msg, ...) { + if (logging::same_or_higher(logging::extra, logging::level)) { + va_list ap; + va_start(ap, msg); + logging::output_nocheckloglevel_ap(logging::extra, msg, ap); + va_end(ap); + } else + logging::last = nullptr; +} + +void log_trace(const char *msg, ...) { + if (logging::same_or_higher(logging::trace, logging::level)) { + va_list ap; + va_start(ap, msg); + logging::output_nocheckloglevel_ap(logging::trace, msg, ap); + va_end(ap); + } else + logging::last = nullptr; +} + +void log_debug(const char *msg, ...) { + if (logging::same_or_higher(logging::debug, logging::level)) { + va_list ap; + va_start(ap, msg); + logging::output_nocheckloglevel_ap(logging::debug, msg, ap); + va_end(ap); + } else + logging::last = nullptr; +} + +void log_verbose(const char *msg, ...) { + if (logging::same_or_higher(logging::verbose, logging::level)) { + va_list ap; + va_start(ap, msg); + logging::output_nocheckloglevel_ap(logging::verbose, msg, ap); + va_end(ap); + } else + logging::last = nullptr; +} + +void log_notice(const char *msg, ...) { + if (logging::same_or_higher(logging::notice, logging::level)) { + va_list ap; + va_start(ap, msg); + logging::output_nocheckloglevel_ap(logging::notice, msg, ap); + va_end(ap); + } else + logging::last = nullptr; +} + +void log_warning(const char *msg, ...) { + if (logging::same_or_higher(logging::warning, logging::level)) { + va_list ap; + va_start(ap, msg); + logging::output_nocheckloglevel_ap(logging::warning, msg, ap); + va_end(ap); + } else + logging::last = nullptr; +} + +void log_error(const char *msg, ...) { + if (logging::same_or_higher(logging::error, logging::level)) { + va_list ap; + va_start(ap, msg); + logging::output_nocheckloglevel_ap(logging::error, msg, ap); + va_end(ap); + } else + logging::last = nullptr; +} + +void log_trouble(const char *where, const char *what, int errnum) { + log_error("%s: %s %s", where, what, test_strerror(errnum)); +} + +bool log_enabled(const logging::loglevel priority) { + return logging::same_or_higher(priority, logging::level); +} + +void log_flush(void) { fflushall(); } diff --git a/src/contrib/db/libmdbx/test/log.h b/src/contrib/db/libmdbx/test/log.h new file mode 100644 index 0000000..bb8f997 --- /dev/null +++ b/src/contrib/db/libmdbx/test/log.h @@ -0,0 +1,104 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" +#include "chrono.h" + +void __noreturn usage(void); +void __noreturn __printf_args(1, 2) failure(const char *fmt, ...); +void __noreturn failure_perror(const char *what, int errnum); +const char *test_strerror(int errnum); + +namespace logging { + +enum loglevel { + extra = MDBX_LOG_EXTRA, + trace = MDBX_LOG_TRACE, + debug = MDBX_LOG_DEBUG, + verbose = MDBX_LOG_VERBOSE, + notice = MDBX_LOG_NOTICE, + warning = MDBX_LOG_WARN, + error = MDBX_LOG_ERROR, + failure = MDBX_LOG_FATAL +}; + +inline bool lower(loglevel left, loglevel right) { + static_assert(MDBX_LOG_EXTRA > MDBX_LOG_FATAL, "WTF?"); + return left > right; +} + +inline bool same_or_higher(loglevel left, loglevel right) { + return left <= right; +} + +const char *level2str(const loglevel level); +void setup(loglevel priority, const std::string &prefix); +void setup(const std::string &prefix); +void setlevel(loglevel priority); + +void output_nocheckloglevel_ap(const loglevel priority, const char *format, + va_list ap); +bool __printf_args(2, 3) + output(const loglevel priority, const char *format, ...); +bool feed_ap(const char *format, va_list ap); +bool __printf_args(1, 2) feed(const char *format, ...); + +void inline __printf_args(2, 3) + output_nocheckloglevel(const loglevel priority, const char *format, ...) { + va_list ap; + va_start(ap, format); + output_nocheckloglevel_ap(priority, format, ap); + va_end(ap); +} + +void progress_canary(bool active); + +class local_suffix { +protected: + size_t trim_pos; + int indent; + +public: + local_suffix(const local_suffix &) = delete; + local_suffix(const local_suffix &&) = delete; + const local_suffix &operator=(const local_suffix &) = delete; + + local_suffix(const char *c_str); + local_suffix(const std::string &str); + void push(); + void pop(); + ~local_suffix(); +}; + +} // namespace logging + +void __printf_args(1, 2) log_extra(const char *msg, ...); +void __printf_args(1, 2) log_trace(const char *msg, ...); +void __printf_args(1, 2) log_debug(const char *msg, ...); +void __printf_args(1, 2) log_verbose(const char *msg, ...); +void __printf_args(1, 2) log_notice(const char *msg, ...); +void __printf_args(1, 2) log_warning(const char *msg, ...); +void __printf_args(1, 2) log_error(const char *msg, ...); + +void log_trouble(const char *where, const char *what, int errnum); +void log_flush(void); +bool log_enabled(const logging::loglevel priority); + +#ifdef _DEBUG +#define TRACE(...) log_trace(__VA_ARGS__) +#else +#define TRACE(...) __noop(__VA_ARGS__) +#endif diff --git a/src/contrib/db/libmdbx/test/long_stochastic.sh b/src/contrib/db/libmdbx/test/long_stochastic.sh new file mode 100644 index 0000000..7b9f90f --- /dev/null +++ b/src/contrib/db/libmdbx/test/long_stochastic.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +if ! which make cc c++ tee lz4 >/dev/null; then + echo "Please install the following prerequisites: make cc c++ tee lz4" >&2 + exit 1 +fi + +set -euo pipefail + +UNAME="$(uname -s 2>/dev/null || echo Unknown)" + +## NOTE: Valgrind could produce some false-positive warnings +## in multi-process environment with shared memory. +## For instance, when the process "A" explicitly marks a memory +## region as "undefined", the process "B" fill it, +## and after this process "A" read such region, etc. +#VALGRIND="valgrind --trace-children=yes --log-file=valgrind-%p.log --leak-check=full --track-origins=yes --error-exitcode=42 --suppressions=test/valgrind_suppress.txt" + +############################################################################### +# 1. clean data from prev runs and examine available RAM + +if [[ -v VALGRIND && ! -z "$VALGRIND" ]]; then + rm -f valgrind-*.log +else + VALGRIND=time +fi + +WANNA_MOUNT=0 +case ${UNAME} in + Linux) + MAKE=make + if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then + for old_test_dir in $(ls -d /dev/shm/mdbx-test.[0-9]*); do + rm -rf $old_test_dir + done + TESTDB_DIR="/dev/shm/mdbx-test.$$" + fi + mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* + + if LC_ALL=C free | grep -q -i available; then + ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 7) / 1024)) + else + ram_avail_mb=$(($(LC_ALL=C free | grep -i Mem: | tr -s [:blank:] ' ' | cut -d ' ' -f 4) / 1024)) + fi + ;; + + FreeBSD) + MAKE=gmake + if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then + for old_test_dir in $(ls -d /tmp/mdbx-test.[0-9]*); do + umount $old_test_dir && rm -r $old_test_dir + done + TESTDB_DIR="/tmp/mdbx-test.$$" + rm -rf $TESTDB_DIR && mkdir -p $TESTDB_DIR + WANNA_MOUNT=1 + else + mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* + fi + + ram_avail_mb=$(($(LC_ALL=C vmstat -s | grep -ie '[0-9] pages free$' | cut -d p -f 1) * ($(LC_ALL=C vmstat -s | grep -ie '[0-9] bytes per page$' | cut -d b -f 1) / 1024) / 1024)) + ;; + + Darwin) + MAKE=make + if [[ ! -v TESTDB_DIR || -z "$TESTDB_DIR" ]]; then + for vol in $(ls -d /Volumes/mdx[0-9]*[0-9]tst); do + disk=$(mount | grep $vol | cut -d ' ' -f 1) + echo "umount: volume $vol disk $disk" + hdiutil unmount $vol -force + hdiutil detach $disk + done + TESTDB_DIR="/Volumes/mdx$$tst" + WANNA_MOUNT=1 + else + mkdir -p $TESTDB_DIR && rm -f $TESTDB_DIR/* + fi + + pagesize=$(($(LC_ALL=C vm_stat | grep -o 'page size of [0-9]\+ bytes' | cut -d' ' -f 4) / 1024)) + freepages=$(LC_ALL=C vm_stat | grep '^Pages free:' | grep -o '[0-9]\+\.$' | cut -d'.' -f 1) + ram_avail_mb=$((pagesize * freepages / 1024)) + echo "pagesize ${pagesize}K, freepages ${freepages}, ram_avail_mb ${ram_avail_mb}" + + ;; + + *) + echo "FIXME: ${UNAME} not supported by this script" + exit 2 + ;; +esac + +############################################################################### +# 2. estimate reasonable RAM space for test-db + +echo "=== ${ram_avail_mb}M RAM available" +ram_reserve4logs_mb=1234 +if [ $ram_avail_mb -lt $ram_reserve4logs_mb ]; then + echo "=== At least ${ram_reserve4logs_mb}Mb RAM required" + exit 3 +fi + +# +# Ð’ режимах отличных от MDBX_WRITEMAP Ð¸Ð·Ð¼ÐµÐ½ÐµÐ½Ð¸Ñ Ð´Ð¾ запиÑи в файл +# будут накапливатьÑÑ Ð² памÑти, что может потребовать Ñвободной +# памÑти размером Ñ Ð‘Ð”. Кроме Ñтого, в теÑÑ‚ входит Ñценарий +# ÑÐ¾Ð·Ð´Ð°Ð½Ð¸Ñ ÐºÐ¾Ð¿Ð¸Ñ Ð‘Ð” на ходу. ПоÑтому БД не может быть больше 1/3 +# от доÑтупной памÑти. Однако, Ñледует учеÑÑ‚ÑŒ что malloc() будет +# не Ñразу возвращать выделенную памÑÑ‚ÑŒ ÑиÑтеме, а также +# предуÑмотреть меÑта Ð´Ð»Ñ Ð»Ð¾Ð³Ð¾Ð². +# +# In non-MDBX_WRITEMAP modes, updates (dirty pages) will +# accumulate in memory before writing to the disk, which may +# require a free memory up to the size of a whole database. In +# addition, the test includes a script create a copy of the +# database on the go. Therefore, the database cannot be more 1/3 +# of available memory. Moreover, should be taken into account +# that malloc() will not return the allocated memory to the +# system immediately, as well some space is required for logs. +# +db_size_mb=$(((ram_avail_mb - ram_reserve4logs_mb) / 4)) +if [ $db_size_mb -gt 3072 ]; then + db_size_mb=3072 +fi +echo "=== use ${db_size_mb}M for DB" + +############################################################################### +# 3. Create test-directory in ramfs/tmpfs, i.e. create/format/mount if required +case ${UNAME} in + Linux) + ;; + + FreeBSD) + if [[ WANNA_MOUNT ]]; then + mount -t tmpfs tmpfs $TESTDB_DIR + fi + ;; + + Darwin) + if [[ WANNA_MOUNT ]]; then + ramdisk_size_mb=$((42 + db_size_mb * 2 + ram_reserve4logs_mb)) + number_of_sectors=$((ramdisk_size_mb * 2048)) + ramdev=$(hdiutil attach -nomount ram://${number_of_sectors}) + diskutil erasevolume ExFAT "mdx$$tst" ${ramdev} + fi + ;; + + *) + echo "FIXME: ${UNAME} not supported by this script" + exit 2 + ;; +esac + +############################################################################### +# 4. Run basic test, i.e. `make check` + +${MAKE} TEST_DB=${TESTDB_DIR}/smoke.db TEST_LOG=${TESTDB_DIR}/smoke.log check +rm -f ${TESTDB_DIR}/* + +############################################################################### +# 5. run stochastic iterations + +function rep9 { printf "%*s" $1 '' | tr ' ' '9'; } +function join { local IFS="$1"; shift; echo "$*"; } +function bit2option { local -n arr=$1; (( ($2&(1<<$3)) != 0 )) && echo -n '+' || echo -n '-'; echo "${arr[$3]}"; } + +options=(writemap coalesce lifo) + +function bits2list { + local -n arr=$1 + local i + local list=() + for ((i=0; i<${#arr[@]}; ++i)) do + list[$i]=$(bit2option $1 $2 $i) + done + join , "${list[@]}" +} + +function probe { + echo "=============================================== $(date)" + echo "${caption}: $*" + rm -f ${TESTDB_DIR}/* \ + && ${VALGRIND} ./mdbx_test --ignore-dbfull --repeat=42 --pathname=${TESTDB_DIR}/long.db "$@" | lz4 > ${TESTDB_DIR}/long.log.lz4 \ + && ${VALGRIND} ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db | tee ${TESTDB_DIR}/long-chk.log \ + && ([ ! -e ${TESTDB_DIR}/long.db-copy ] || ${VALGRIND} ./mdbx_chk -nvvv ${TESTDB_DIR}/long.db-copy | tee ${TESTDB_DIR}/long-chk-copy.log) \ + || (echo "FAILED"; exit 1) +} + +#------------------------------------------------------------------------------ + +count=0 +for nops in $(seq 2 6); do + for ((wbatch=nops-1; wbatch > 0; --wbatch)); do + loops=$(((111 >> nops) / nops + 3)) + for ((rep=0; rep++ < loops; )); do + for ((bits=2**${#options[@]}; --bits >= 0; )); do + seed=$(($(date +%s) + RANDOM)) + caption="Probe #$((++count)) int-key,w/o-dups, repeat ${rep} of ${loops}" probe \ + --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \ + --keygen.seed=${seed} basic + caption="Probe #$((++count)) int-key,with-dups, repeat ${rep} of ${loops}" probe \ + --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \ + --keygen.seed=${seed} basic + caption="Probe #$((++count)) int-key,int-data, repeat ${rep} of ${loops}" probe \ + --pagesize=min --size-upper=${db_size_mb}M --table=+key.integer,+data.integer --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \ + --keygen.seed=${seed} basic + caption="Probe #$((++count)) w/o-dups, repeat ${rep} of ${loops}" probe \ + --pagesize=min --size-upper=${db_size_mb}M --table=-data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=1111 \ + --nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \ + --keygen.seed=${seed} basic + caption="Probe #$((++count)) with-dups, repeat ${rep} of ${loops}" probe \ + --pagesize=min --size-upper=${db_size_mb}M --table=+data.dups --keylen.min=min --keylen.max=max --datalen.min=min --datalen.max=max \ + --nops=$( rep9 $nops ) --batch.write=$( rep9 $wbatch ) --mode=$(bits2list options $bits) \ + --keygen.seed=${seed} basic + done + done + done +done + +echo "=== ALL DONE ====================== $(date)" diff --git a/src/contrib/db/libmdbx/test/main.cc b/src/contrib/db/libmdbx/test/main.cc new file mode 100644 index 0000000..fe5243e --- /dev/null +++ b/src/contrib/db/libmdbx/test/main.cc @@ -0,0 +1,617 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +#if !(defined(_WIN32) || defined(_WIN64)) +#include +#include +#endif /* !Windows */ + +void __noreturn usage(void) { + puts( + "usage:\n" + " --help or -h Show this text\n" + "Common parameters:\n" + " --pathname=... Path and/or name of database files\n" + " --repeat=N Set repeat counter\n" + " --threads=N Number of thread (unsunpported for now)\n" + " --timeout=N[s|m|h|d] Set timeout in seconds/minutes/hours/days\n" + " --failfast[=YES/no] Lill all actors on first failure/error\n" + " --max-readers=N See mdbx_env_set_maxreaders() description\n" + " --max-tables=N Se mdbx_env_set_maxdbs() description\n" + " --dump-config[=YES/no] Dump entire test config before run\n" + " --progress[=YES/no] Enable/disable progress `canary`\n" + " --console[=yes/no] Enable/disable console-like output\n" + " --cleanup-before[=YES/no] Cleanup/remove and re-create database\n" + " --cleanup-after[=YES/no] Cleanup/remove database after completion\n" + "Database size control:\n" + " --pagesize=... Database page size: min, max, 256..65536\n" + " --size-lower=N[K|M|G|T] Lower-bound of size in Kb/Mb/Gb/Tb\n" + " --size-upper Upper-bound of size in Kb/Mb/Gb/Tb\n" + " --size Initial size in Kb/Mb/Gb/Tb\n" + " --shrink-threshold Shrink threshold in Kb/Mb/Gb/Tb\n" + " --growth-step Grow step in Kb/Mb/Gb/Tb\n" + "Predefined complext scenarios/cases:\n" + " --case=... Only `basic` scenario implemented for now\n" + " basic == Simultaneous multi-process execution\n" + " of test-actors: nested,hill,ttl,copy,append,jitter,try\n" + "Test actors:\n" + " --hill Fill-up and empty-down\n" + " by CRUD-operation quads\n" + " --ttl Stochastic time-to-live simulation\n" + " --nested Nested transactionÑ‹\n" + " with stochastic-size bellows\n" + " --jitter Jitter/delays simulation\n" + " --try Try write-transaction, no more\n" + " --copy Online copy/backup\n" + " --append Append-mode insertions\n" + " --dead.reader Dead-reader simulator\n" + " --dead.writer Dead-writer simulator\n" + "Actor options:\n" + " --batch.read=N Read-operations batch size\n" + " --batch.write=N Write-operations batch size\n" + " --delay=N | --no-delay (no)Delay test-actor before start\n" + " --wait4ops=N | --no-wait4ops (no)Wait for previous test-actor\n" + " completes # ops before start\n" + " --duration=N[s|m|h|d] Define running duration\n" + " --nops=N[K|M|G|T] Define number of operations/steps\n" + " --inject-writefault[=yes|NO] TBD (see the source code)\n" + " --drop[=yes|NO] Drop key-value space/table on " + "completion\n" + " --ignore-dbfull[=yes|NO] Ignore MDBX_MAP_FULL error\n" + " --speculum[=yes|NO] Use internal `speculum` to check " + "dataset\n" + "Keys and Value:\n" + " --keygen.min=N Minimal keys length\n" + " --keygen.max=N Miximal keys length\n" + " --datalen.min=N Minimal data length\n" + " --datalen.max=N Miximal data length\n" + " --keygen.width=N TBD (see the source code)\n" + " --keygen.mesh=N TBD (see the source code)\n" + " --keygen.seed=N TBD (see the source code)\n" + " --keygen.split=N TBD (see the source code)\n" + " --keygen.rotate=N TBD (see the source code)\n" + " --keygen.offset=N TBD (see the source code)\n" + " --keygen.case=random Generator case (only `random` for now)\n" + "Database operation mode:\n" + " --mode={[+-]FLAG}[,[+-]FLAG]...\n" + " nosubdir == MDBX_NOSUBDIR\n" + " rdonly == MDBX_RDONLY\n" + " nometasync == MDBX_NOMETASYNC\n" + " lifo == MDBX_LIFORECLAIM\n" + " coalesce == MDBX_COALESCE\n" + " nosync == MDBX_NOSYNC\n" + " writemap == MDBX_WRITEMAP\n" + " mapasync == MDBX_MAPASYNC\n" + " utterly == MDBX_UTTERLY_NOSYNC\n" + " perturb == MDBX_PAGEPERTURB\n" + " notls == MDBX_NOTLS\n" + " nordahead == MDBX_NORDAHEAD\n" + " nomeminit == MDBX_NOMEMINIT\n" + "Key-value space/table options:\n" + " --table={[+-]FLAG}[,[+-]FLAG]...\n" + " key.reverse == MDBX_REVERSEKEY\n" + " key.integer == MDBX_INTEGERKEY\n" + " data.dups == MDBX_DUPSORT\n" + " data.integer == MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT\n" + " data.fixed == MDBX_DUPFIXED | MDBX_DUPSORT\n" + " data.reverse == MDBX_REVERSEDUP | MDBX_DUPSORT\n"); + exit(EXIT_FAILURE); +} + +//----------------------------------------------------------------------------- + +void actor_params::set_defaults(const std::string &tmpdir) { + pathname_log = ""; + loglevel = +#if defined(NDEBUG) || defined(_WIN32) || defined(_WIN64) + logging::verbose; +#else + logging::trace; +#endif + + pathname_db = tmpdir + "mdbx-test.db"; + mode_flags = MDBX_NOSUBDIR | MDBX_WRITEMAP | MDBX_MAPASYNC | MDBX_NOMEMINIT | + MDBX_COALESCE | MDBX_LIFORECLAIM; + table_flags = MDBX_DUPSORT; + + size_lower = -1; + size_now = + intptr_t(1024) * 1024 * ((table_flags & MDBX_DUPSORT) ? 256 : 1024); + size_upper = -1; + shrink_threshold = -1; + growth_step = -1; + pagesize = -1; + + keygen.seed = 1; + keygen.keycase = kc_random; + keygen.width = (table_flags & MDBX_DUPSORT) ? 32 : 64; + keygen.mesh = keygen.width; + keygen.split = keygen.width / 2; + keygen.rotate = 3; + keygen.offset = 41; + + test_duration = 0; + test_nops = 1000; + nrepeat = 1; + nthreads = 1; + + keylen_min = mdbx_keylen_min(); + keylen_max = mdbx_keylen_max(); + datalen_min = mdbx_datalen_min(); + datalen_max = std::min(mdbx_datalen_max(), 256u * 1024 + 42); + + batch_read = 42; + batch_write = 42; + + delaystart = 0; + waitfor_nops = 0; + inject_writefaultn = 0; + + drop_table = false; + ignore_dbfull = false; + speculum = false; + + max_readers = 42; + max_tables = 42; + + global::config::timeout_duration_seconds = 0 /* infinite */; + global::config::dump_config = true; + global::config::cleanup_before = true; + global::config::cleanup_after = true; + global::config::failfast = true; + global::config::progress_indicator = true; + global::config::console_mode = osal_istty(STDERR_FILENO); +} + +namespace global { + +std::vector actors; +std::unordered_map events; +std::unordered_map pid2actor; +std::set databases; +unsigned nactors; +chrono::time start_motonic; +chrono::time deadline_motonic; +bool singlemode; + +namespace config { +unsigned timeout_duration_seconds; +bool dump_config; +bool cleanup_before; +bool cleanup_after; +bool failfast; +bool progress_indicator; +bool console_mode; +} /* namespace config */ + +} /* namespace global */ + +//----------------------------------------------------------------------------- + +const char global::thunk_param_prefix[] = "--execute="; + +std::string thunk_param(const actor_config &config) { + return config.serialize(global::thunk_param_prefix); +} + +void cleanup() { + log_trace(">> cleanup"); + /* TODO: remove each database */ + log_trace("<< cleanup"); +} + +int main(int argc, char *const argv[]) { + +#ifdef _DEBUG + log_trace("#argc = %d", argc); + for (int i = 0; i < argc; ++i) + log_trace("#argv[%d] = %s", i, argv[i]); +#endif /* _DEBUG */ + + if (argc < 2) + failure("No parameters given. Try --help\n"); + + if (argc == 2 && strncmp(argv[1], global::thunk_param_prefix, + strlen(global::thunk_param_prefix)) == 0) + return test_execute( + actor_config(argv[1] + strlen(global::thunk_param_prefix))) + ? EXIT_SUCCESS + : EXIT_FAILURE; + + if (argc == 2 && + (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0)) + usage(); + + actor_params params; + params.set_defaults(osal_tempdir()); + global::config::dump_config = true; + logging::setup((logging::loglevel)params.loglevel, "main"); + unsigned last_space_id = 0; + + for (int narg = 1; narg < argc; ++narg) { + const char *value = nullptr; + + if (config::parse_option(argc, argv, narg, "case", &value)) { + testcase_setup(value, params, last_space_id); + continue; + } + if (config::parse_option(argc, argv, narg, "pathname", params.pathname_db)) + continue; + if (config::parse_option(argc, argv, narg, "mode", params.mode_flags, + config::mode_bits)) + continue; + if (config::parse_option(argc, argv, narg, "table", params.table_flags, + config::table_bits)) { + if ((params.table_flags & MDBX_DUPFIXED) == 0) + params.table_flags &= ~MDBX_INTEGERDUP; + if ((params.table_flags & MDBX_DUPSORT) == 0) + params.table_flags &= + ~(MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP); + continue; + } + + if (config::parse_option(argc, argv, narg, "pagesize", params.pagesize, + int(mdbx_limits_pgsize_min()), + int(mdbx_limits_pgsize_max()))) { + const unsigned keylen_max = params.mdbx_keylen_max(); + if (params.keylen_min > keylen_max) + params.keylen_min = keylen_max; + if (params.keylen_max > keylen_max) + params.keylen_max = keylen_max; + const unsigned datalen_max = params.mdbx_datalen_max(); + if (params.datalen_min > datalen_max) + params.datalen_min = datalen_max; + if (params.datalen_max > datalen_max) + params.datalen_max = datalen_max; + continue; + } + if (config::parse_option(argc, argv, narg, "repeat", params.nrepeat, + config::no_scale)) + continue; + if (config::parse_option(argc, argv, narg, "threads", params.nthreads, + config::no_scale, 1, 64)) + continue; + if (config::parse_option(argc, argv, narg, "timeout", + global::config::timeout_duration_seconds, + config::duration, 1)) + continue; + + if (config::parse_option_intptr(argc, argv, narg, "size-lower", + params.size_lower, + mdbx_limits_dbsize_min(params.pagesize), + mdbx_limits_dbsize_max(params.pagesize))) + continue; + if (config::parse_option_intptr(argc, argv, narg, "size-upper", + params.size_upper, + mdbx_limits_dbsize_min(params.pagesize), + mdbx_limits_dbsize_max(params.pagesize))) + continue; + if (config::parse_option_intptr(argc, argv, narg, "size", params.size_now, + mdbx_limits_dbsize_min(params.pagesize), + mdbx_limits_dbsize_max(params.pagesize))) + continue; + if (config::parse_option( + argc, argv, narg, "shrink-threshold", params.shrink_threshold, 0, + (int)std::min((intptr_t)INT_MAX, + mdbx_limits_dbsize_max(params.pagesize) - + mdbx_limits_dbsize_min(params.pagesize)))) + continue; + if (config::parse_option( + argc, argv, narg, "growth-step", params.growth_step, 0, + (int)std::min((intptr_t)INT_MAX, + mdbx_limits_dbsize_max(params.pagesize) - + mdbx_limits_dbsize_min(params.pagesize)))) + continue; + + if (config::parse_option(argc, argv, narg, "keygen.width", + params.keygen.width, 1, 64)) + continue; + if (config::parse_option(argc, argv, narg, "keygen.mesh", + params.keygen.mesh, 1, 64)) + continue; + if (config::parse_option(argc, argv, narg, "keygen.seed", + params.keygen.seed, config::no_scale)) + continue; + if (config::parse_option(argc, argv, narg, "keygen.split", + params.keygen.split, 1, 64)) + continue; + if (config::parse_option(argc, argv, narg, "keygen.rotate", + params.keygen.rotate, 1, 64)) + continue; + if (config::parse_option(argc, argv, narg, "keygen.offset", + params.keygen.offset, config::binary)) + continue; + if (config::parse_option(argc, argv, narg, "keygen.case", &value)) { + keycase_setup(value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "keylen.min", params.keylen_min, + config::no_scale, params.mdbx_keylen_min(), + params.mdbx_keylen_max())) { + if ((params.table_flags & MDBX_INTEGERKEY) || + params.keylen_max < params.keylen_min) + params.keylen_max = params.keylen_min; + continue; + } + if (config::parse_option(argc, argv, narg, "keylen.max", params.keylen_max, + config::no_scale, params.mdbx_keylen_min(), + params.mdbx_keylen_max())) { + if ((params.table_flags & MDBX_INTEGERKEY) || + params.keylen_min > params.keylen_max) + params.keylen_min = params.keylen_max; + continue; + } + if (config::parse_option(argc, argv, narg, "datalen.min", + params.datalen_min, config::no_scale, + params.mdbx_datalen_min(), + params.mdbx_datalen_max())) { + if ((params.table_flags & MDBX_DUPFIXED) || + params.datalen_max < params.datalen_min) + params.datalen_max = params.datalen_min; + continue; + } + if (config::parse_option(argc, argv, narg, "datalen.max", + params.datalen_max, config::no_scale, + params.mdbx_datalen_min(), + params.mdbx_datalen_max())) { + if ((params.table_flags & MDBX_DUPFIXED) || + params.datalen_min > params.datalen_max) + params.datalen_min = params.datalen_max; + continue; + } + if (config::parse_option(argc, argv, narg, "batch.read", params.batch_read, + config::no_scale, 1)) + continue; + if (config::parse_option(argc, argv, narg, "batch.write", + params.batch_write, config::no_scale, 1)) + continue; + if (config::parse_option(argc, argv, narg, "delay", params.delaystart, + config::duration)) + continue; + if (config::parse_option(argc, argv, narg, "wait4ops", params.waitfor_nops, + config::decimal)) + continue; + if (config::parse_option(argc, argv, narg, "inject-writefault", + params.inject_writefaultn, config::decimal)) + continue; + if (config::parse_option(argc, argv, narg, "drop", params.drop_table)) + continue; + if (config::parse_option(argc, argv, narg, "ignore-dbfull", + params.ignore_dbfull)) + continue; + if (config::parse_option(argc, argv, narg, "speculum", params.speculum)) + continue; + if (config::parse_option(argc, argv, narg, "dump-config", + global::config::dump_config)) + continue; + if (config::parse_option(argc, argv, narg, "cleanup-before", + global::config::cleanup_before)) + continue; + if (config::parse_option(argc, argv, narg, "cleanup-after", + global::config::cleanup_after)) + continue; + if (config::parse_option(argc, argv, narg, "max-readers", + params.max_readers, config::no_scale, 1, 255)) + continue; + if (config::parse_option(argc, argv, narg, "max-tables", params.max_tables, + config::no_scale, 1, INT16_MAX)) + continue; + + if (config::parse_option(argc, argv, narg, "no-delay", nullptr)) { + params.delaystart = 0; + continue; + } + if (config::parse_option(argc, argv, narg, "no-wait4ops", nullptr)) { + params.waitfor_nops = 0; + continue; + } + if (config::parse_option(argc, argv, narg, "duration", params.test_duration, + config::duration, 1)) { + params.test_nops = 0; + continue; + } + if (config::parse_option(argc, argv, narg, "nops", params.test_nops, + config::decimal, 1)) { + params.test_duration = 0; + continue; + } + if (config::parse_option(argc, argv, narg, "hill", &value, "auto")) { + configure_actor(last_space_id, ac_hill, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "jitter", nullptr)) { + configure_actor(last_space_id, ac_jitter, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "dead.reader", nullptr)) { + configure_actor(last_space_id, ac_deadread, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "dead.writer", nullptr)) { + configure_actor(last_space_id, ac_deadwrite, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "try", nullptr)) { + configure_actor(last_space_id, ac_try, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "copy", nullptr)) { + configure_actor(last_space_id, ac_copy, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "append", nullptr)) { + configure_actor(last_space_id, ac_append, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "ttl", nullptr)) { + configure_actor(last_space_id, ac_ttl, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "nested", nullptr)) { + configure_actor(last_space_id, ac_nested, value, params); + continue; + } + if (config::parse_option(argc, argv, narg, "failfast", + global::config::failfast)) + continue; + if (config::parse_option(argc, argv, narg, "progress", + global::config::progress_indicator)) + continue; + if (config::parse_option(argc, argv, narg, "console", + global::config::console_mode)) + continue; + + if (*argv[narg] != '-') + testcase_setup(argv[narg], params, last_space_id); + else + failure("Unknown option '%s'. Try --help\n", argv[narg]); + } + + if (global::config::dump_config) + config::dump(); + + //-------------------------------------------------------------------------- + + if (global::actors.empty()) { + log_notice("no testcase(s) configured, exiting"); + return EXIT_SUCCESS; + } + + bool failed = false; + global::start_motonic = chrono::now_motonic(); + global::deadline_motonic.fixedpoint = + (global::config::timeout_duration_seconds == 0) + ? chrono::infinite().fixedpoint + : global::start_motonic.fixedpoint + + chrono::from_seconds(global::config::timeout_duration_seconds) + .fixedpoint; + + if (global::config::cleanup_before) + cleanup(); + + log_trace(">> probe entropy_ticks()"); + entropy_ticks(); + log_trace("<< probe entropy_ticks()"); + + if (global::actors.size() == 1) { + logging::setup("main"); + global::singlemode = true; + if (!test_execute(global::actors.front())) + failed = true; + } else { + logging::setup("overlord"); + + log_trace("=== preparing..."); + log_trace(">> osal_setup"); + osal_setup(global::actors); + log_trace("<< osal_setup"); + + for (auto &a : global::actors) { + mdbx_pid_t pid; + log_trace(">> actor_start"); + int rc = osal_actor_start(a, pid); + log_trace("<< actor_start"); + if (rc) { + log_trace(">> killall_actors: (%s)", "start failed"); + osal_killall_actors(); + log_trace("<< killall_actors"); + failure("Failed to start actor #%u (%s)\n", a.actor_id, + test_strerror(rc)); + } + global::pid2actor[pid] = &a; + } + + log_trace("=== ready to start..."); + atexit(osal_killall_actors); + log_trace(">> wait4barrier"); + osal_wait4barrier(); + log_trace("<< wait4barrier"); + + size_t left = global::actors.size(); + log_trace("=== polling..."); + while (left > 0) { + unsigned timeout_seconds_left = INT_MAX; + chrono::time now_motonic = chrono::now_motonic(); + if (now_motonic.fixedpoint >= global::deadline_motonic.fixedpoint) + timeout_seconds_left = 0; + else { + chrono::time left_motonic; + left_motonic.fixedpoint = + global::deadline_motonic.fixedpoint - now_motonic.fixedpoint; + timeout_seconds_left = left_motonic.seconds(); + } + + mdbx_pid_t pid; + int rc = osal_actor_poll(pid, timeout_seconds_left); + if (rc) + failure("Poll error: %s (%d)\n", test_strerror(rc), rc); + + if (pid) { + actor_status status = osal_actor_info(pid); + actor_config *actor = global::pid2actor.at(pid); + if (!actor) + continue; + + log_verbose("actor #%u, id %d, pid %u: %s\n", actor->actor_id, + actor->space_id, pid, status2str(status)); + if (status > as_running) { + left -= 1; + if (status != as_successful) { + if (global::config::failfast && !failed) { + log_trace(">> killall_actors: (%s)", "failfast"); + osal_killall_actors(); + log_trace("<< killall_actors"); + } + failed = true; + } + } + } else { + if (timeout_seconds_left == 0) + failure("Timeout\n"); + } + } + log_trace("=== done..."); + } + + log_notice("RESULT: %s\n", failed ? "Failed" : "Successful"); + if (global::config::cleanup_before) { + if (failed) + log_verbose("skip cleanup"); + else + cleanup(); + } + +#if !(defined(_WIN32) || defined(_WIN64)) + struct rusage spent; + if (getrusage(RUSAGE_CHILDREN, &spent) == 0) { + log_notice("%6s: user %f, system %f", "CPU", + spent.ru_utime.tv_sec + spent.ru_utime.tv_usec * 1e-6, + spent.ru_stime.tv_sec + spent.ru_stime.tv_usec * 1e-6); +#if defined(__linux__) || defined(__gnu_linux__) || defined(__FreeBSD__) || \ + defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || \ + defined(__NETBSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ + defined(__APPLE__) || defined(__MACH__) + log_notice("%6s: read %ld, write %ld", "IOPs", spent.ru_inblock, + spent.ru_oublock); + log_notice("%6s: %ld Kb", "RAM", spent.ru_maxrss); + log_notice("%6s: reclaims %ld, faults %ld, swaps %ld", "Paging", + spent.ru_minflt, spent.ru_majflt, spent.ru_nswap); +#endif /* Linux */ + } +#endif /* !Windows */ + + return failed ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/src/contrib/db/libmdbx/test/nested.cc b/src/contrib/db/libmdbx/test/nested.cc new file mode 100644 index 0000000..e02fe8b --- /dev/null +++ b/src/contrib/db/libmdbx/test/nested.cc @@ -0,0 +1,284 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" +#include + +bool testcase_nested::setup() { + if (!inherited::setup()) + return false; + int err = db_open__begin__table_create_open_clean(dbi); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("nested: bailout-prepare due '%s'", mdbx_strerror(err)); + return false; + } + + keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); + key = keygen::alloc(config.params.keylen_max); + data = keygen::alloc(config.params.datalen_max); + serial = 0; + fifo.clear(); + speculum.clear(); + assert(stack.empty()); + stack.emplace(nullptr, serial, fifo, speculum); + return true; +} + +bool testcase_nested::teardown() { + while (!stack.empty()) + pop_txn(true); + + bool ok = true; + if (dbi) { + if (config.params.drop_table && !mode_readonly()) { + txn_begin(false); + db_table_drop(dbi); + int err = breakable_commit(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("nested: bailout-clean due '%s'", mdbx_strerror(err)); + ok = false; + } + } else + db_table_close(dbi); + dbi = 0; + } + return inherited::teardown() && ok; +} + +static unsigned edge2window(uint64_t edge, unsigned window_max) { + const double rnd = u64_to_double1(bleach64(edge)); + const unsigned window = window_max - std::lrint(std::pow(window_max, rnd)); + return window; +} + +static unsigned edge2count(uint64_t edge, unsigned count_max) { + const double rnd = u64_to_double1(prng64_map1_white(edge)); + const unsigned count = std::lrint(std::pow(count_max, rnd)); + return count; +} + +void testcase_nested::push_txn() { + MDBX_txn *txn; + int err = mdbx_txn_begin( + db_guard.get(), txn_guard.get(), + prng32() & (MDBX_NOSYNC | MDBX_NOMETASYNC | MDBX_MAPASYNC), &txn); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_txn_begin(nested)", err); +#if __cplusplus >= 201703L + stack.emplace(txn, serial, fifo, speculum); +#else + stack.push(std::make_tuple(scoped_txn_guard(txn), serial, fifo, speculum)); +#endif + std::swap(txn_guard, std::get<0>(stack.top())); + log_verbose("begin level#%zu txn, serial %" PRIu64, stack.size(), serial); +} + +bool testcase_nested::pop_txn(bool abort) { + assert(txn_guard && !stack.empty()); + bool should_continue = true; + MDBX_txn *txn = txn_guard.release(); + bool commited = false; + if (abort) { + log_verbose("abort level#%zu txn, undo serial %" PRIu64 " <- %" PRIu64, + stack.size(), serial, std::get<1>(stack.top())); + int err = mdbx_txn_abort(txn); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_txn_abort()", err); + } else { + log_verbose("commit level#%zu txn, nested serial %" PRIu64 " -> %" PRIu64, + stack.size(), serial, std::get<1>(stack.top())); + int err = mdbx_txn_commit(txn); + if (likely(err == MDBX_SUCCESS)) + commited = true; + else { + should_continue = false; + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + err = mdbx_txn_abort(txn); + if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH && + err != MDBX_BAD_TXN)) + failure_perror("mdbx_txn_abort()", err); + } else + failure_perror("mdbx_txn_commit()", err); + } + } + + std::swap(txn_guard, std::get<0>(stack.top())); + if (!commited) { + serial = std::get<1>(stack.top()); + std::swap(fifo, std::get<2>(stack.top())); + std::swap(speculum, std::get<3>(stack.top())); + } + stack.pop(); + return should_continue; +} + +bool testcase_nested::stochastic_breakable_restart_with_nested( + bool force_restart) { + log_trace(">> stochastic_breakable_restart_with_nested%s", + force_restart ? ": force_restart" : ""); + + if (force_restart) + while (txn_guard) + pop_txn(true); + + bool should_continue = true; + while (!stack.empty() && + (flipcoin() || txn_underutilization_x256(txn_guard.get()) < 42)) + should_continue &= pop_txn(); + + if (should_continue) + while (stack.empty() || + (is_nested_txn_available() && flipcoin() && stack.size() < 5)) + push_txn(); + + log_trace("<< stochastic_breakable_restart_with_nested: should_continue=%s", + should_continue ? "yes" : "no"); + return should_continue; +} + +bool testcase_nested::trim_tail(unsigned window_width) { + if (window_width) { + while (fifo.size() > window_width) { + uint64_t tail_serial = fifo.back().first; + const unsigned tail_count = fifo.back().second; + log_trace("nested: pop-tail (serial %" PRIu64 ", count %u)", tail_serial, + tail_count); + fifo.pop_back(); + for (unsigned n = 0; n < tail_count; ++n) { + log_trace("nested: remove-tail %" PRIu64, tail_serial); + generate_pair(tail_serial); + int err = remove(key, data); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("nested: tail-bailout due '%s'", mdbx_strerror(err)); + return false; + } + failure_perror("mdbx_del(tail)", err); + } + if (unlikely(!keyvalue_maker.increment(tail_serial, 1))) + failure("nested: unexpected key-space overflow on the tail"); + } + } + } else { + log_trace("nested: purge state"); + db_table_clear(dbi, txn_guard.get()); + fifo.clear(); + speculum.clear(); + } + return true; +} + +bool testcase_nested::grow_head(unsigned head_count) { + const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_NODUPDATA + : MDBX_NODUPDATA | MDBX_NOOVERWRITE; +retry: + fifo.push_front(std::make_pair(serial, head_count)); + for (unsigned n = 0; n < head_count; ++n) { + log_trace("nested: insert-head %" PRIu64, serial); + generate_pair(serial); + int err = insert(key, data, insert_flags); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("nested: head-insert skip due '%s'", mdbx_strerror(err)); + head_count = n; + stochastic_breakable_restart_with_nested(true); + goto retry; + } + failure_perror("mdbx_put(head)", err); + } + + if (unlikely(!keyvalue_maker.increment(serial, 1))) { + log_notice("nested: unexpected key-space overflow"); + return false; + } + } + + return true; +} + +bool testcase_nested::run() { + /* LY: теÑÑ‚ "ÑмулÑцией time-to-live" Ñ Ð²Ð»Ð¾Ð¶ÐµÐ½Ð½Ñ‹Ð¼Ð¸ транзакциÑми: + * - организуетÑÑ "ÑкользÑщее окно", которое каждую транзакцию ÑдвигаетÑÑ + * вперед вдоль чиÑловой оÑи. + * - по переднему краю "ÑкользÑщего окна" запиÑи добавлÑÑŽÑ‚ÑÑ Ð² таблицу, + * а по заднему удалÑÑŽÑ‚ÑÑ. + * - количеÑтво добавлÑемых/удалÑемых запиÑей пÑевдоÑлучайно завиÑит + * от номера транзакции, но Ñ ÑкÑпоненциальным раÑпределением. + * - размер "ÑкользÑщего окна" также пÑевдоÑлучайно завиÑит от номера + * транзакции Ñ "отрицательным" ÑкÑпоненциальным раÑпределением + * MAX_WIDTH - exp(rnd(N)), при уменьшении окна ÑдвигаетÑÑ Ð·Ð°Ð´Ð½Ð¸Ð¹ + * край и удалÑÑŽÑ‚ÑÑ Ð·Ð°Ð¿Ð¸Ñи позади него. + * - групповое добавление данных в начало окна и групповое уделение в конце, + * в половине Ñлучаев выполнÑÑŽÑ‚ÑÑ Ð²Ð¾ вложенных транзакциÑÑ…. + * - половина запуÑкаемых вложенных транзакций отменÑетÑÑ, поÑледуюим + * повтором групповой операции. + * + * Таким образом имитируетÑÑ Ð¿Ð¾Ð²ÐµÐ´ÐµÐ½Ð¸Ðµ таблицы Ñ TTL: запиÑи ÑтохаÑтичеÑки + * добавлÑÑŽÑ‚ÑÑ Ð¸ удалÑÑŽÑ‚ÑÑ, но изредка проиÑходÑÑ‚ маÑÑивные удалениÑ. */ + + /* LY: Ð´Ð»Ñ Ð¿Ð°Ñ€Ð°Ð¼ÐµÑ‚Ñ€Ð¸Ð·Ð°Ñ†Ð¸Ð¸ иÑпользуем подходÑщие параметры, которые не имеют + * здеÑÑŒ ÑмыÑла в первоначальном значении. */ + const unsigned window_max_lower = 333; + const unsigned count_max_lower = 333; + + const unsigned window_max = (config.params.batch_read > window_max_lower) + ? config.params.batch_read + : window_max_lower; + const unsigned count_max = (config.params.batch_write > count_max_lower) + ? config.params.batch_write + : count_max_lower; + log_verbose("nested: using `batch_read` value %u for window_max", window_max); + log_verbose("nested: using `batch_write` value %u for count_max", count_max); + + uint64_t seed = + prng64_map2_white(config.params.keygen.seed) + config.actor_id; + + while (should_continue()) { + const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; + const unsigned window_width = + flipcoin_x4() ? 0 : edge2window(salt, window_max); + const unsigned head_count = edge2count(salt, count_max); + log_debug("nested: step #%zu (serial %" PRIu64 + ", window %u, count %u) salt %" PRIu64, + nops_completed, serial, window_width, head_count, salt); + + if (!trim_tail(window_width)) + return false; + if (!stochastic_breakable_restart_with_nested()) { + log_notice("nested: bailout at commit/restart after tail-trim"); + return false; + } + if (!speculum_verify()) { + log_notice("nested: bailout after tail-trim"); + return false; + } + + if (!grow_head(head_count)) + return false; + if (!stochastic_breakable_restart_with_nested()) + log_notice("nested: skip commit/restart after head-grow"); + if (!speculum_verify()) { + log_notice("nested: bailout after head-grow"); + return false; + } + + report(1); + } + + while (!stack.empty()) + pop_txn(false); + + return speculum_verify(); +} diff --git a/src/contrib/db/libmdbx/test/osal-unix.cc b/src/contrib/db/libmdbx/test/osal-unix.cc new file mode 100644 index 0000000..d954cea --- /dev/null +++ b/src/contrib/db/libmdbx/test/osal-unix.cc @@ -0,0 +1,368 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include "darwin/pthread_barrier.c" +#endif + +struct shared_t { + pthread_barrier_t barrier; + pthread_mutex_t mutex; + size_t conds_size; + pthread_cond_t conds[1]; +}; + +static shared_t *shared; + +void osal_wait4barrier(void) { + assert(shared != nullptr && shared != MAP_FAILED); + int rc = pthread_barrier_wait(&shared->barrier); + if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { + failure_perror("pthread_barrier_wait(shared)", rc); + } +} + +void osal_setup(const std::vector &actors) { + assert(shared == nullptr); + + pthread_mutexattr_t mutexattr; + int rc = pthread_mutexattr_init(&mutexattr); + if (rc) + failure_perror("pthread_mutexattr_init()", rc); + rc = pthread_mutexattr_setpshared(&mutexattr, PTHREAD_PROCESS_SHARED); + if (rc) + failure_perror("pthread_mutexattr_setpshared()", rc); + + pthread_barrierattr_t barrierattr; + rc = pthread_barrierattr_init(&barrierattr); + if (rc) + failure_perror("pthread_barrierattr_init()", rc); + rc = pthread_barrierattr_setpshared(&barrierattr, PTHREAD_PROCESS_SHARED); + if (rc) + failure_perror("pthread_barrierattr_setpshared()", rc); + + pthread_condattr_t condattr; + rc = pthread_condattr_init(&condattr); + if (rc) + failure_perror("pthread_condattr_init()", rc); + rc = pthread_condattr_setpshared(&condattr, PTHREAD_PROCESS_SHARED); + if (rc) + failure_perror("pthread_condattr_setpshared()", rc); + + shared = (shared_t *)mmap( + nullptr, sizeof(shared_t) + actors.size() * sizeof(pthread_cond_t), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (MAP_FAILED == (void *)shared) + failure_perror("mmap(shared_conds)", errno); + + rc = pthread_mutex_init(&shared->mutex, &mutexattr); + if (rc) + failure_perror("pthread_mutex_init(shared)", rc); + + rc = pthread_barrier_init(&shared->barrier, &barrierattr, actors.size() + 1); + if (rc) + failure_perror("pthread_barrier_init(shared)", rc); + + const size_t n = actors.size() + 1; + for (size_t i = 0; i < n; ++i) { + pthread_cond_t *event = &shared->conds[i]; + rc = pthread_cond_init(event, &condattr); + if (rc) + failure_perror("pthread_cond_init(shared)", rc); + log_trace("osal_setup: event(shared pthread_cond) %" PRIuPTR " -> %p", i, + __Wpedantic_format_voidptr(event)); + } + shared->conds_size = actors.size() + 1; + + pthread_barrierattr_destroy(&barrierattr); + pthread_condattr_destroy(&condattr); + pthread_mutexattr_destroy(&mutexattr); +} + +void osal_broadcast(unsigned id) { + assert(shared != nullptr && shared != MAP_FAILED); + log_trace("osal_broadcast: event %u", id); + if (id >= shared->conds_size) + failure("osal_broadcast: id > limit"); + int rc = pthread_cond_broadcast(shared->conds + id); + if (rc) + failure_perror("sem_post(shared)", rc); +} + +int osal_waitfor(unsigned id) { + assert(shared != nullptr && shared != MAP_FAILED); + + log_trace("osal_waitfor: event %u", id); + if (id >= shared->conds_size) + failure("osal_waitfor: id > limit"); + + int rc = pthread_mutex_lock(&shared->mutex); + if (rc != 0) + failure_perror("pthread_mutex_lock(shared)", rc); + + rc = pthread_cond_wait(shared->conds + id, &shared->mutex); + if (rc && rc != EINTR) + failure_perror("pthread_cond_wait(shared)", rc); + + rc = pthread_mutex_unlock(&shared->mutex); + if (rc != 0) + failure_perror("pthread_mutex_unlock(shared)", rc); + + return (rc == 0) ? true : false; +} + +//----------------------------------------------------------------------------- + +const std::string +actor_config::osal_serialize(simple_checksum &checksum) const { + (void)checksum; + /* not used in workload, but just for testing */ + return "unix.fork"; +} + +bool actor_config::osal_deserialize(const char *str, const char *end, + simple_checksum &checksum) { + (void)checksum; + /* not used in workload, but just for testing */ + return strncmp(str, "unix.fork", 9) == 0 && str + 9 == end; +} + +//----------------------------------------------------------------------------- + +static pid_t overlord_pid; + +static volatile sig_atomic_t sigusr1_head, sigusr2_head; +static void handler_SIGUSR(int signum) { + switch (signum) { + case SIGUSR1: + sigusr1_head += 1; + return; + case SIGUSR2: + sigusr2_head += 1; + return; + default: + abort(); + } +} + +bool osal_progress_push(bool active) { + if (overlord_pid) { + if (kill(overlord_pid, active ? SIGUSR1 : SIGUSR2)) + failure_perror("osal_progress_push: kill(overload)", errno); + return true; + } + + return false; +} + +//----------------------------------------------------------------------------- + +static std::unordered_map childs; + +static volatile sig_atomic_t sigalarm_head; +static void handler_SIGCHLD(int signum) { + if (signum == SIGALRM) + sigalarm_head += 1; +} + +mdbx_pid_t osal_getpid(void) { return getpid(); } + +int osal_delay(unsigned seconds) { return sleep(seconds) ? errno : 0; } + +int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { + if (childs.empty()) { + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = handler_SIGCHLD; + sigaction(SIGCHLD, &act, nullptr); + sigaction(SIGALRM, &act, nullptr); + act.sa_handler = handler_SIGUSR; + sigaction(SIGUSR1, &act, nullptr); + sigaction(SIGUSR2, &act, nullptr); + + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGCHLD); + sigaddset(&mask, SIGUSR1); + sigaddset(&mask, SIGUSR2); + sigprocmask(SIG_UNBLOCK, &mask, nullptr); + } + + pid = fork(); + + if (pid == 0) { + overlord_pid = getppid(); + const bool result = test_execute(config); + exit(result ? EXIT_SUCCESS : EXIT_FAILURE); + } + + if (pid < 0) + return errno; + + log_trace("osal_actor_start: fork pid %i for %u", pid, config.actor_id); + childs[pid] = as_running; + return 0; +} + +actor_status osal_actor_info(const mdbx_pid_t pid) { return childs.at(pid); } + +void osal_killall_actors(void) { + for (auto &pair : childs) { + kill(pair.first, SIGKILL); + pair.second = as_killed; + } +} + +int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { + static sig_atomic_t sigalarm_tail; + alarm(0) /* cancel prev timeout */; + sigalarm_tail = sigalarm_head /* reset timeout flag */; + + int options = WNOHANG; + if (timeout) { + alarm((timeout > INT_MAX) ? INT_MAX : timeout); + options = 0; + } + +#ifdef WUNTRACED + options |= WUNTRACED; +#endif +#ifdef WCONTINUED + options |= WCONTINUED; +#endif + + while (sigalarm_tail == sigalarm_head) { + int status; + pid = waitpid(0, &status, options); + + if (pid > 0) { + if (WIFEXITED(status)) + childs[pid] = + (WEXITSTATUS(status) == EXIT_SUCCESS) ? as_successful : as_failed; + else if (WCOREDUMP(status)) + childs[pid] = as_coredump; + else if (WIFSIGNALED(status)) + childs[pid] = as_killed; + else if (WIFSTOPPED(status)) + childs[pid] = as_debuging; + else if (WIFCONTINUED(status)) + childs[pid] = as_running; + else { + assert(false); + } + return 0; + } + + static sig_atomic_t sigusr1_tail, sigusr2_tail; + if (sigusr1_tail != sigusr1_head) { + sigusr1_tail = sigusr1_head; + logging::progress_canary(true); + if (pid < 0 && errno == EINTR) + continue; + } + if (sigusr2_tail != sigusr2_head) { + sigusr2_tail = sigusr2_head; + logging::progress_canary(false); + if (pid < 0 && errno == EINTR) + continue; + } + + if (pid == 0) + break; + + int err = errno; + if (err != EINTR) + return err; + } + return 0 /* timeout */; +} + +void osal_yield(void) { + if (sched_yield()) + failure_perror("sched_yield()", errno); +} + +void osal_udelay(unsigned us) { + chrono::time until, now = chrono::now_motonic(); + until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint; + struct timespec ts; + + static unsigned threshold_us; + if (threshold_us == 0) { + if (clock_getres(CLOCK_PROCESS_CPUTIME_ID, &ts)) { + int rc = errno; + failure_perror("clock_getres(CLOCK_PROCESS_CPUTIME_ID)", rc); + } + chrono::time threshold = chrono::from_timespec(ts); + assert(threshold.seconds() == 0); + + threshold_us = chrono::fractional2us(threshold.fractional); + if (threshold_us < 1000) + threshold_us = 1000; + } + + ts.tv_sec = ts.tv_nsec = 0; + if (us > threshold_us) { + ts.tv_sec = us / 1000000u; + ts.tv_nsec = (us % 1000000u) * 1000u; + } + + do { + if (us > threshold_us) { + if (nanosleep(&ts, &ts)) { + int rc = errno; + /* if (rc == EINTR) { ... } ? */ + failure_perror("usleep()", rc); + } + us = ts.tv_sec * 1000000u + ts.tv_nsec / 1000u; + } + cpu_relax(); + + now = chrono::now_motonic(); + } while (until.fixedpoint > now.fixedpoint); +} + +bool osal_istty(int fd) { return isatty(fd) == 1; } + +std::string osal_tempdir(void) { + const char *tempdir = getenv("TMPDIR"); + if (!tempdir) + tempdir = getenv("TMP"); + if (!tempdir) + tempdir = getenv("TEMPDIR"); + if (!tempdir) + tempdir = getenv("TEMP"); + if (tempdir) { + std::string dir(tempdir); + if (!dir.empty() && dir.at(dir.length() - 1) != '/') + dir.append("/"); + return dir; + } + if (access("/dev/shm/", R_OK | W_OK | X_OK) == 0) + return "/dev/shm/"; + return ""; +} + +int osal_removefile(const std::string &pathname) { + return unlink(pathname.c_str()) ? errno : MDBX_SUCCESS; +} diff --git a/src/contrib/db/libmdbx/test/osal-windows.cc b/src/contrib/db/libmdbx/test/osal-windows.cc new file mode 100644 index 0000000..0b158c0 --- /dev/null +++ b/src/contrib/db/libmdbx/test/osal-windows.cc @@ -0,0 +1,461 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +static std::unordered_map events; +static HANDLE hBarrierSemaphore, hBarrierEvent; +static HANDLE hProgressActiveEvent, hProgressPassiveEvent; + +static int waitstatus2errcode(DWORD result) { + switch (result) { + case WAIT_OBJECT_0: + return MDBX_SUCCESS; + case WAIT_FAILED: + return GetLastError(); + case WAIT_ABANDONED: + return ERROR_ABANDONED_WAIT_0; + case WAIT_IO_COMPLETION: + return ERROR_USER_APC; + case WAIT_TIMEOUT: + return ERROR_TIMEOUT; + default: + return ERROR_UNHANDLED_ERROR; + } +} + +void osal_wait4barrier(void) { + DWORD rc = WaitForSingleObject(hBarrierSemaphore, 0); + switch (rc) { + default: + failure_perror("WaitForSingleObject(BarrierSemaphore)", + waitstatus2errcode(rc)); + case WAIT_OBJECT_0: + rc = WaitForSingleObject(hBarrierEvent, INFINITE); + if (rc != WAIT_OBJECT_0) + failure_perror("WaitForSingleObject(BarrierEvent)", + waitstatus2errcode(rc)); + break; + case WAIT_TIMEOUT: + if (!SetEvent(hBarrierEvent)) + failure_perror("SetEvent(BarrierEvent)", GetLastError()); + break; + } +} + +static HANDLE make_inheritable(HANDLE hHandle) { + assert(hHandle != NULL && hHandle != INVALID_HANDLE_VALUE); + if (!DuplicateHandle(GetCurrentProcess(), hHandle, GetCurrentProcess(), + &hHandle, 0, TRUE, + DUPLICATE_CLOSE_SOURCE | DUPLICATE_SAME_ACCESS)) + failure_perror("DuplicateHandle()", GetLastError()); + return hHandle; +} + +void osal_setup(const std::vector &actors) { + assert(events.empty()); + const size_t n = actors.size() + 1; + events.reserve(n); + + for (unsigned i = 0; i < n; ++i) { + HANDLE hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if (!hEvent) + failure_perror("CreateEvent()", GetLastError()); + hEvent = make_inheritable(hEvent); + log_trace("osal_setup: event %" PRIuPTR " -> %p", i, hEvent); + events[i] = hEvent; + } + + hBarrierSemaphore = CreateSemaphore(NULL, 0, (LONG)actors.size(), NULL); + if (!hBarrierSemaphore) + failure_perror("CreateSemaphore(BarrierSemaphore)", GetLastError()); + hBarrierSemaphore = make_inheritable(hBarrierSemaphore); + + hBarrierEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + if (!hBarrierEvent) + failure_perror("CreateEvent(BarrierEvent)", GetLastError()); + hBarrierEvent = make_inheritable(hBarrierEvent); + + hProgressActiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!hProgressActiveEvent) + failure_perror("CreateEvent(ProgressActiveEvent)", GetLastError()); + hProgressActiveEvent = make_inheritable(hProgressActiveEvent); + + hProgressPassiveEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!hProgressPassiveEvent) + failure_perror("CreateEvent(ProgressPassiveEvent)", GetLastError()); + hProgressPassiveEvent = make_inheritable(hProgressPassiveEvent); +} + +void osal_broadcast(unsigned id) { + log_trace("osal_broadcast: event %u", id); + if (!SetEvent(events.at(id))) + failure_perror("SetEvent()", GetLastError()); +} + +int osal_waitfor(unsigned id) { + log_trace("osal_waitfor: event %u", id); + DWORD rc = WaitForSingleObject(events.at(id), INFINITE); + return waitstatus2errcode(rc); +} + +mdbx_pid_t osal_getpid(void) { return GetCurrentProcessId(); } + +int osal_delay(unsigned seconds) { + Sleep(seconds * 1000u); + return 0; +} + +//----------------------------------------------------------------------------- + +const std::string +actor_config::osal_serialize(simple_checksum &checksum) const { + checksum.push(hBarrierSemaphore); + checksum.push(hBarrierEvent); + checksum.push(hProgressActiveEvent); + checksum.push(hProgressPassiveEvent); + + HANDLE hWait = INVALID_HANDLE_VALUE; + if (wait4id) { + hWait = events.at(wait4id); + checksum.push(hWait); + } + + HANDLE hSignal = INVALID_HANDLE_VALUE; + if (wanna_event4signalling()) { + hSignal = events.at(actor_id); + checksum.push(hSignal); + } + + return format("%p.%p.%p.%p.%p.%p", hBarrierSemaphore, hBarrierEvent, hWait, + hSignal, hProgressActiveEvent, hProgressPassiveEvent); +} + +bool actor_config::osal_deserialize(const char *str, const char *end, + simple_checksum &checksum) { + + std::string copy(str, end - str); + TRACE(">> osal_deserialize(%s)\n", copy.c_str()); + + assert(hBarrierSemaphore == 0); + assert(hBarrierEvent == 0); + assert(hProgressActiveEvent == 0); + assert(hProgressPassiveEvent == 0); + assert(events.empty()); + + HANDLE hWait, hSignal; + if (sscanf_s(copy.c_str(), "%p.%p.%p.%p.%p.%p", &hBarrierSemaphore, + &hBarrierEvent, &hWait, &hSignal, &hProgressActiveEvent, + &hProgressPassiveEvent) != 6) { + TRACE("<< osal_deserialize: failed\n"); + return false; + } + + checksum.push(hBarrierSemaphore); + checksum.push(hBarrierEvent); + checksum.push(hProgressActiveEvent); + checksum.push(hProgressPassiveEvent); + + if (wait4id) { + checksum.push(hWait); + events[wait4id] = hWait; + } + + if (wanna_event4signalling()) { + checksum.push(hSignal); + events[actor_id] = hSignal; + } + + TRACE("<< osal_deserialize: OK\n"); + return true; +} + +//----------------------------------------------------------------------------- + +typedef std::pair child; +static std::unordered_map childs; + +bool osal_progress_push(bool active) { + if (childs.empty()) { + if (!SetEvent(active ? hProgressActiveEvent : hProgressPassiveEvent)) + failure_perror("osal_progress_push: SetEvent(overlord.progress)", + GetLastError()); + return true; + } + + return false; +} + +static void ArgvQuote(std::string &CommandLine, const std::string &Argument, + bool Force = false) + +/*++ + +https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + +Routine Description: + + This routine appends the given argument to a command line such + that CommandLineToArgvW will return the argument string unchanged. + Arguments in a command line should be separated by spaces; this + function does not add these spaces. + +Arguments: + + Argument - Supplies the argument to encode. + + CommandLine - Supplies the command line to which we append the encoded +argument string. + + Force - Supplies an indication of whether we should quote + the argument even if it does not contain any characters that would + ordinarily require quoting. + +Return Value: + + None. + +Environment: + + Arbitrary. + +--*/ + +{ + // + // Unless we're told otherwise, don't quote unless we actually + // need to do so --- hopefully avoid problems if programs won't + // parse quotes properly + // + + if (Force == false && Argument.empty() == false && + Argument.find_first_of(" \t\n\v\"") == Argument.npos) { + CommandLine.append(Argument); + } else { + CommandLine.push_back('"'); + + for (auto It = Argument.begin();; ++It) { + unsigned NumberBackslashes = 0; + + while (It != Argument.end() && *It == '\\') { + ++It; + ++NumberBackslashes; + } + + if (It == Argument.end()) { + // + // Escape all backslashes, but let the terminating + // double quotation mark we add below be interpreted + // as a metacharacter. + // + CommandLine.append(NumberBackslashes * 2, '\\'); + break; + } else if (*It == L'"') { + // + // Escape all backslashes and the following + // double quotation mark. + // + CommandLine.append(NumberBackslashes * 2 + 1, '\\'); + CommandLine.push_back(*It); + } else { + // + // Backslashes aren't special here. + // + CommandLine.append(NumberBackslashes, '\\'); + CommandLine.push_back(*It); + } + } + + CommandLine.push_back('"'); + } +} + +int osal_actor_start(const actor_config &config, mdbx_pid_t &pid) { + if (childs.size() == MAXIMUM_WAIT_OBJECTS) + failure("Could't manage more that %u actors on Windows\n", + MAXIMUM_WAIT_OBJECTS); + + _flushall(); + + STARTUPINFOA StartupInfo; + GetStartupInfoA(&StartupInfo); + + char exename[_MAX_PATH + 1]; + DWORD exename_size = sizeof(exename); + if (!QueryFullProcessImageNameA(GetCurrentProcess(), 0, exename, + &exename_size)) + failure_perror("QueryFullProcessImageName()", GetLastError()); + + if (exename[1] != ':') { + exename_size = GetModuleFileName(NULL, exename, sizeof(exename)); + if (exename_size >= sizeof(exename)) + return ERROR_BAD_LENGTH; + } + + std::string cmdline = "$ "; + ArgvQuote(cmdline, thunk_param(config)); + + if (cmdline.size() >= 32767) + return ERROR_BAD_LENGTH; + + PROCESS_INFORMATION ProcessInformation; + if (!CreateProcessA(exename, const_cast(cmdline.c_str()), + NULL, // Retuned process handle is not inheritable. + NULL, // Retuned thread handle is not inheritable. + TRUE, // Child inherits all inheritable handles. + NORMAL_PRIORITY_CLASS | INHERIT_PARENT_AFFINITY, + NULL, // Inherit the parent's environment. + NULL, // Inherit the parent's current directory. + &StartupInfo, &ProcessInformation)) + failure_perror(exename, GetLastError()); + + CloseHandle(ProcessInformation.hThread); + pid = ProcessInformation.dwProcessId; + childs[pid] = std::make_pair(ProcessInformation.hProcess, as_running); + return 0; +} + +actor_status osal_actor_info(const mdbx_pid_t pid) { + actor_status status = childs.at(pid).second; + if (status > as_running) + return status; + + DWORD ExitCode; + if (!GetExitCodeProcess(childs.at(pid).first, &ExitCode)) + failure_perror("GetExitCodeProcess()", GetLastError()); + + switch (ExitCode) { + case STILL_ACTIVE: + return as_running; + case EXIT_SUCCESS: + status = as_successful; + break; + case EXCEPTION_BREAKPOINT: + case EXCEPTION_SINGLE_STEP: + status = as_debuging; + break; + case STATUS_CONTROL_C_EXIT: + status = as_killed; + break; + case EXCEPTION_ACCESS_VIOLATION: + case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: + case EXCEPTION_DATATYPE_MISALIGNMENT: + case EXCEPTION_STACK_OVERFLOW: + case EXCEPTION_INVALID_DISPOSITION: + case EXCEPTION_ILLEGAL_INSTRUCTION: + case EXCEPTION_NONCONTINUABLE_EXCEPTION: + status = as_coredump; + break; + default: + status = as_failed; + break; + } + + childs.at(pid).second = status; + return status; +} + +void osal_killall_actors(void) { + for (auto &pair : childs) + TerminateProcess(pair.second.first, STATUS_CONTROL_C_EXIT); +} + +int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout) { + std::vector handles; + handles.reserve(childs.size() + 2); + handles.push_back(hProgressActiveEvent); + handles.push_back(hProgressPassiveEvent); + for (const auto &pair : childs) + if (pair.second.second <= as_running) + handles.push_back(pair.second.first); + + while (true) { + DWORD rc = + MsgWaitForMultipleObjectsEx((DWORD)handles.size(), &handles[0], + (timeout > 60) ? 60 * 1000 : timeout * 1000, + QS_ALLINPUT | QS_ALLPOSTMESSAGE, 0); + + if (rc == WAIT_OBJECT_0) { + logging::progress_canary(true); + continue; + } + if (rc == WAIT_OBJECT_0 + 1) { + logging::progress_canary(false); + continue; + } + + if (rc >= WAIT_OBJECT_0 + 2 && rc < WAIT_OBJECT_0 + handles.size()) { + pid = 0; + for (const auto &pair : childs) + if (pair.second.first == handles[rc - WAIT_OBJECT_0]) { + pid = pair.first; + break; + } + return 0; + } + + if (rc == WAIT_TIMEOUT) { + pid = 0; + return 0; + } + + return waitstatus2errcode(rc); + } +} + +void osal_yield(void) { SwitchToThread(); } + +void osal_udelay(unsigned us) { + chrono::time until, now = chrono::now_motonic(); + until.fixedpoint = now.fixedpoint + chrono::from_us(us).fixedpoint; + + static unsigned threshold_us; + if (threshold_us == 0) { +#if 1 + unsigned timeslice_ms = 1; + while (timeBeginPeriod(timeslice_ms) == TIMERR_NOCANDO) + ++timeslice_ms; + threshold_us = timeslice_ms * 1500u; +#else + ULONGLONG InterruptTimePrecise_100ns; + QueryInterruptTimePrecise(&InterruptTimePrecise_100ns); + threshold_us = InterruptTimePrecise_100ns / 5; +#endif + assert(threshold_us > 0); + } + + do { + if (us > threshold_us && us > 1000) { + DWORD rc = SleepEx(us / 1000, TRUE); + if (rc) + failure_perror("SleepEx()", waitstatus2errcode(rc)); + us = 0; + } + + YieldProcessor(); + now = chrono::now_motonic(); + } while (now.fixedpoint < until.fixedpoint); +} + +bool osal_istty(int fd) { return _isatty(fd) != 0; } + +std::string osal_tempdir(void) { + char buf[MAX_PATH + 1]; + DWORD len = GetTempPathA(sizeof(buf), buf); + return std::string(buf, len); +} + +int osal_removefile(const std::string &pathname) { + return DeleteFileA(pathname.c_str()) ? MDBX_SUCCESS : GetLastError(); +} diff --git a/src/contrib/db/libmdbx/test/osal.h b/src/contrib/db/libmdbx/test/osal.h new file mode 100644 index 0000000..6d0e1c4 --- /dev/null +++ b/src/contrib/db/libmdbx/test/osal.h @@ -0,0 +1,49 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" + +void osal_setup(const std::vector &actors); +void osal_broadcast(unsigned id); +int osal_waitfor(unsigned id); + +int osal_actor_start(const actor_config &config, mdbx_pid_t &pid); +actor_status osal_actor_info(const mdbx_pid_t pid); +void osal_killall_actors(void); +int osal_actor_poll(mdbx_pid_t &pid, unsigned timeout); +void osal_wait4barrier(void); + +bool osal_progress_push(bool active); + +mdbx_pid_t osal_getpid(void); +int osal_delay(unsigned seconds); +void osal_udelay(unsigned us); +void osal_yield(void); +bool osal_istty(int fd); +std::string osal_tempdir(void); +int osal_removefile(const std::string &pathname); + +#ifdef _MSC_VER +#ifndef STDIN_FILENO +#define STDIN_FILENO _fileno(stdin) +#endif +#ifndef STDOUT_FILENO +#define STDOUT_FILENO _fileno(stdout) +#endif +#ifndef STDERR_FILENO +#define STDERR_FILENO _fileno(stderr) +#endif +#endif /* _MSC_VER */ diff --git a/src/contrib/db/libmdbx/test/pcrf/CMakeLists.txt b/src/contrib/db/libmdbx/test/pcrf/CMakeLists.txt new file mode 100644 index 0000000..8bd3e3d --- /dev/null +++ b/src/contrib/db/libmdbx/test/pcrf/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET pcrf_test) +add_executable(${TARGET} pcrf_test.c) +target_include_directories(${TARGET} PRIVATE "${PROJECT_SOURCE_DIR}") +target_link_libraries(${TARGET} mdbx) + diff --git a/src/contrib/db/libmdbx/test/pcrf/README.md b/src/contrib/db/libmdbx/test/pcrf/README.md new file mode 100644 index 0000000..b2c9b5c --- /dev/null +++ b/src/contrib/db/libmdbx/test/pcrf/README.md @@ -0,0 +1,2 @@ +PCRF Session DB emulation test + diff --git a/src/contrib/db/libmdbx/test/pcrf/pcrf_test.c b/src/contrib/db/libmdbx/test/pcrf/pcrf_test.c new file mode 100644 index 0000000..90e0e28 --- /dev/null +++ b/src/contrib/db/libmdbx/test/pcrf/pcrf_test.c @@ -0,0 +1,413 @@ +/* + * Copyright 2016-2019 Leonid Yuriev . + * Copyright 2015 Vladimir Romanov + * , Yota Lab. + * + * This file is part of libmdbx. + * + * ReOpenMDBX is free software; you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * ReOpenMDBX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include "mdbx.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define IP_PRINTF_ARG_HOST(addr) \ + (int)((addr) >> 24), (int)((addr) >> 16 & 0xff), (int)((addr) >> 8 & 0xff), \ + (int)((addr)&0xff) + +char opt_db_path[PATH_MAX] = "./mdbx_bench2"; +static MDBX_env *env; +#define REC_COUNT 10240000 +int64_t ids[REC_COUNT * 10]; +int32_t ids_count = 0; + +int64_t mdbx_add_count = 0; +int64_t mdbx_del_count = 0; +uint64_t mdbx_add_time = 0; +uint64_t mdbx_del_time = 0; +int64_t obj_id = 0; +int64_t mdbx_data_size = 0; +int64_t mdbx_key_size = 0; + +typedef struct { + char session_id1[100]; + char session_id2[100]; + char ip[20]; + uint8_t fill[100]; +} session_data_t; + +typedef struct { + int64_t obj_id; + int8_t event_type; +} __attribute__((__packed__)) event_data_t; + +static void add_id_to_pool(int64_t id) { + ids[ids_count] = id; + ids_count++; +} + +static inline int64_t getClockUs(void) { + struct timespec val; +#ifdef CYGWIN + clock_gettime(CLOCK_REALTIME, &val); +#else + clock_gettime(CLOCK_MONOTONIC, &val); +#endif + return val.tv_sec * ((int64_t)1000000) + val.tv_nsec / 1000; +} + +static int64_t get_id_from_pool() { + if (ids_count == 0) { + return -1; + } + int32_t index = rand() % ids_count; + int64_t id = ids[index]; + ids[index] = ids[ids_count - 1]; + ids_count--; + return id; +} + +#define MDBX_CHECK(x) \ + do { \ + const int rc = (x); \ + if (rc != MDBX_SUCCESS) { \ + printf("Error [%d] %s in %s at %s:%d\n", rc, mdbx_strerror(rc), #x, \ + __FILE__, __LINE__); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +static void db_connect() { + MDBX_dbi dbi_session; + MDBX_dbi dbi_session_id; + MDBX_dbi dbi_event; + MDBX_dbi dbi_ip; + + MDBX_CHECK(mdbx_env_create(&env)); + MDBX_CHECK( + mdbx_env_set_mapsize(env, REC_COUNT * sizeof(session_data_t) * 10)); + MDBX_CHECK(mdbx_env_set_maxdbs(env, 30)); + MDBX_CHECK(mdbx_env_open(env, opt_db_path, + MDBX_CREATE | MDBX_WRITEMAP | MDBX_MAPASYNC | + MDBX_NOSYNC | MDBX_LIFORECLAIM, + 0664)); + MDBX_txn *txn; + + // transaction init + MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); + // open database in read-write mode + MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session)); + MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id)); + MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event)); + MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip)); + // transaction commit + MDBX_CHECK(mdbx_txn_commit(txn)); + printf("Connection open\n"); +} + +static void create_record(uint64_t record_id) { + MDBX_dbi dbi_session; + MDBX_dbi dbi_session_id; + MDBX_dbi dbi_event; + MDBX_dbi dbi_ip; + event_data_t event; + MDBX_txn *txn; + session_data_t data; + // transaction init + snprintf(data.session_id1, sizeof(data.session_id1), + "prefix%02u_%02u.fill.fill.fill.fill.fill.fill;%" PRIu64, + (unsigned)(record_id % 3) + 1, (unsigned)(record_id % 9) + 1, + record_id); + snprintf(data.session_id2, sizeof(data.session_id2), + "dprefix%" PRIu64 ";%" PRIu64 ".fill.fill.;suffix", record_id, + (record_id + UINT64_C(1442695040888963407)) % + UINT64_C(6364136223846793005)); + snprintf(data.ip, sizeof(data.ip), "%d.%d.%d.%d", + IP_PRINTF_ARG_HOST(record_id & 0xFFFFFFFF)); + event.obj_id = record_id; + event.event_type = 1; + + MDBX_val _session_id1_rec = {data.session_id1, strlen(data.session_id1)}; + MDBX_val _session_id2_rec = {data.session_id2, strlen(data.session_id2)}; + MDBX_val _ip_rec = {data.ip, strlen(data.ip)}; + MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)}; + MDBX_val _data_rec = {&data, offsetof(session_data_t, fill) + + (rand() % sizeof(data.fill))}; + MDBX_val _event_rec = {&event, sizeof(event)}; + + uint64_t start = getClockUs(); + MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); + MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session)); + MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id)); + MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event)); + MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip)); + MDBX_CHECK(mdbx_put(txn, dbi_session, &_obj_id_rec, &_data_rec, + MDBX_NOOVERWRITE | MDBX_NODUPDATA)); + MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id1_rec, &_obj_id_rec, + MDBX_NOOVERWRITE | MDBX_NODUPDATA)); + MDBX_CHECK(mdbx_put(txn, dbi_session_id, &_session_id2_rec, &_obj_id_rec, + MDBX_NOOVERWRITE | MDBX_NODUPDATA)); + MDBX_CHECK(mdbx_put(txn, dbi_ip, &_ip_rec, &_obj_id_rec, 0)); + MDBX_CHECK(mdbx_put(txn, dbi_event, &_event_rec, &_obj_id_rec, 0)); + MDBX_CHECK(mdbx_txn_commit(txn)); + + mdbx_data_size += (_data_rec.iov_len + _obj_id_rec.iov_len * 4); + mdbx_key_size += + (_obj_id_rec.iov_len + _session_id1_rec.iov_len + + _session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len); + + // transaction commit + mdbx_add_count++; + mdbx_add_time += (getClockUs() - start); +} + +static void delete_record(int64_t record_id) { + MDBX_dbi dbi_session; + MDBX_dbi dbi_session_id; + MDBX_dbi dbi_event; + MDBX_dbi dbi_ip; + event_data_t event; + MDBX_txn *txn; + + // transaction init + uint64_t start = getClockUs(); + MDBX_CHECK(mdbx_txn_begin(env, NULL, 0, &txn)); + // open database in read-write mode + MDBX_CHECK(mdbx_dbi_open(txn, "session", MDBX_CREATE, &dbi_session)); + MDBX_CHECK(mdbx_dbi_open(txn, "session_id", MDBX_CREATE, &dbi_session_id)); + MDBX_CHECK(mdbx_dbi_open(txn, "event", MDBX_CREATE, &dbi_event)); + MDBX_CHECK(mdbx_dbi_open(txn, "ip", MDBX_CREATE, &dbi_ip)); + // put data + MDBX_val _obj_id_rec = {&record_id, sizeof(record_id)}; + MDBX_val _data_rec; + // get data + MDBX_CHECK(mdbx_get(txn, dbi_session, &_obj_id_rec, &_data_rec)); + session_data_t *data = (session_data_t *)_data_rec.iov_base; + + MDBX_val _session_id1_rec = {data->session_id1, strlen(data->session_id1)}; + MDBX_val _session_id2_rec = {data->session_id2, strlen(data->session_id2)}; + MDBX_val _ip_rec = {data->ip, strlen(data->ip)}; + MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id1_rec, NULL)); + MDBX_CHECK(mdbx_del(txn, dbi_session_id, &_session_id2_rec, NULL)); + MDBX_CHECK(mdbx_del(txn, dbi_ip, &_ip_rec, NULL)); + event.obj_id = record_id; + event.event_type = 1; + MDBX_val _event_rec = {&event, sizeof(event)}; + MDBX_CHECK(mdbx_del(txn, dbi_event, &_event_rec, NULL)); + MDBX_CHECK(mdbx_del(txn, dbi_session, &_obj_id_rec, NULL)); + + mdbx_data_size -= (_data_rec.iov_len + _obj_id_rec.iov_len * 4); + mdbx_key_size -= + (_obj_id_rec.iov_len + _session_id1_rec.iov_len + + _session_id2_rec.iov_len + _ip_rec.iov_len + _event_rec.iov_len); + + // transaction commit + MDBX_CHECK(mdbx_txn_commit(txn)); + mdbx_del_count++; + mdbx_del_time += (getClockUs() - start); +} + +static void db_disconnect() { + mdbx_env_close(env); + printf("Connection closed\n"); +} + +static void get_db_stat(const char *db, int64_t *ms_branch_pages, + int64_t *ms_leaf_pages) { + MDBX_txn *txn; + MDBX_stat stat; + MDBX_dbi dbi; + + MDBX_CHECK(mdbx_txn_begin(env, NULL, MDBX_RDONLY, &txn)); + MDBX_CHECK(mdbx_dbi_open(txn, db, MDBX_CREATE, &dbi)); + MDBX_CHECK(mdbx_dbi_stat(txn, dbi, &stat, sizeof(stat))); + mdbx_txn_abort(txn); + printf("%15s | %15" PRIu64 " | %5u | %10" PRIu64 " | %10" PRIu64 + " | %11" PRIu64 " |\n", + db, stat.ms_branch_pages, stat.ms_depth, stat.ms_entries, + stat.ms_leaf_pages, stat.ms_overflow_pages); + (*ms_branch_pages) += stat.ms_branch_pages; + (*ms_leaf_pages) += stat.ms_leaf_pages; +} + +static void periodic_stat(void) { + int64_t ms_branch_pages = 0; + int64_t ms_leaf_pages = 0; + MDBX_stat mst; + MDBX_envinfo mei; + MDBX_CHECK(mdbx_env_stat(env, &mst, sizeof(mst))); + MDBX_CHECK(mdbx_env_info(env, &mei, sizeof(mei))); + printf("Environment Info\n"); + printf(" Pagesize: %u\n", mst.ms_psize); + if (mei.mi_geo.lower != mei.mi_geo.upper) { + printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 + "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64 + "/-%" PRIu64 ")\n", + mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, + mei.mi_geo.shrink, mei.mi_geo.lower / mst.ms_psize, + mei.mi_geo.upper / mst.ms_psize, mei.mi_geo.grow / mst.ms_psize, + mei.mi_geo.shrink / mst.ms_psize); + printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", + mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); + } else { + printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", + mei.mi_geo.current, mei.mi_geo.current / mst.ms_psize); + } + printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", + mei.mi_mapsize, mei.mi_mapsize / mst.ms_psize); + printf(" Number of pages used: %" PRIu64 "\n", mei.mi_last_pgno + 1); + printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid); + printf(" Tail transaction ID: %" PRIu64 " (%" PRIi64 ")\n", + mei.mi_latter_reader_txnid, + mei.mi_latter_reader_txnid - mei.mi_recent_txnid); + printf(" Max readers: %u\n", mei.mi_maxreaders); + printf(" Number of readers used: %u\n", mei.mi_numreaders); + + printf(" Name | ms_branch_pages | depth | entries | leaf_pages " + "| overf_pages |\n"); + get_db_stat("session", &ms_branch_pages, &ms_leaf_pages); + get_db_stat("session_id", &ms_branch_pages, &ms_leaf_pages); + get_db_stat("event", &ms_branch_pages, &ms_leaf_pages); + get_db_stat("ip", &ms_branch_pages, &ms_leaf_pages); + printf("%15s | %15" PRIu64 " | %5s | %10s | %10" PRIu64 " | %11s |\n", "", + ms_branch_pages, "", "", ms_leaf_pages, ""); + + static int64_t prev_add_count; + static int64_t prev_del_count; + static uint64_t prev_add_time; + static uint64_t prev_del_time; + static int64_t t = -1; + if (t > 0) { + int64_t delta = (getClockUs() - t); + printf("CPS: add %" PRIu64 ", delete %" PRIu64 + ", items processed - %" PRIu64 "K data=%" PRIu64 "K key=%" PRIu64 + "K\n", + (mdbx_add_count - prev_add_count) * 1000000 / delta, + (mdbx_del_count - prev_del_count) * 1000000 / delta, obj_id / 1024, + mdbx_data_size / 1024, mdbx_key_size / 1024); + printf("usage data=%" PRIu64 "%%", + ((mdbx_data_size + mdbx_key_size) * 100) / + ((ms_leaf_pages + ms_branch_pages) * 4096)); + if (prev_add_time != mdbx_add_time) { + printf(" Add : %" PRIu64 " c/s", (mdbx_add_count - prev_add_count) * + 1000000 / + (mdbx_add_time - prev_add_time)); + } + if (prev_del_time != mdbx_del_time) { + printf(" Del : %" PRIu64 " c/s", (mdbx_del_count - prev_del_count) * + 1000000 / + (mdbx_del_time - prev_del_time)); + } + if (mdbx_add_time) { + printf(" tAdd : %" PRIu64 " c/s", + mdbx_add_count * 1000000 / mdbx_add_time); + } + if (mdbx_del_time) { + printf(" tDel : %" PRIu64 " c/s", + mdbx_del_count * 1000000 / mdbx_del_time); + } + puts(""); + } + t = getClockUs(); + prev_add_count = mdbx_add_count; + prev_del_count = mdbx_del_count; + prev_add_time = mdbx_add_time; + prev_del_time = mdbx_del_time; +} + +// static void periodic_add_rec() { +// for (int i = 0; i < 10240; i++) { +// if (ids_count <= REC_COUNT) { +// int64_t id = obj_id++; +// create_record(id); +// add_id_to_pool(id); +// } +// if (ids_count > REC_COUNT) { +// int64_t id = get_id_from_pool(); +// delete_record(id); +// } +// } +// periodic_stat(); +//} + +int main(int argc, char **argv) { + (void)argc; + (void)argv; + + char filename[PATH_MAX]; + int i; + + mkdir(opt_db_path, 0775); + + strcpy(filename, opt_db_path); + strcat(filename, "/mdbx.dat"); + remove(filename); + + strcpy(filename, opt_db_path); + strcat(filename, "/mdbx.lck"); + remove(filename); + + puts("Open DB..."); + db_connect(); + puts("Create data..."); + int64_t t = getClockUs(); + for (i = 0; i < REC_COUNT; i++) { + int64_t id = obj_id++; + create_record(id); + add_id_to_pool(id); + if (i % 1000 == 0) { + int64_t now = getClockUs(); + if ((now - t) > 1000000L) { + periodic_stat(); + t = now; + } + } + } + periodic_stat(); + while (1) { + int i; + for (i = 0; i < 1000; i++) { + int64_t id = obj_id++; + create_record(id); + add_id_to_pool(id); + id = get_id_from_pool(); + delete_record(id); + } + // for (i = 0; i < 50; i++) { + // int64_t id = obj_id++; + // create_record(id); + // add_id_to_pool(id); + // } + // int64_t id = obj_id++; + // create_record(id); + // add_id_to_pool(id); + int64_t now = getClockUs(); + if ((now - t) > 10000000L) { + periodic_stat(); + t = now; + } + } + db_disconnect(); + return 0; +} diff --git a/src/contrib/db/libmdbx/test/test.cc b/src/contrib/db/libmdbx/test/test.cc new file mode 100644 index 0000000..806e0b3 --- /dev/null +++ b/src/contrib/db/libmdbx/test/test.cc @@ -0,0 +1,736 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" + +const char *testcase2str(const actor_testcase testcase) { + switch (testcase) { + default: + assert(false); + return "?!"; + case ac_none: + return "none"; + case ac_hill: + return "hill"; + case ac_deadread: + return "deadread"; + case ac_deadwrite: + return "deadwrite"; + case ac_jitter: + return "jitter"; + case ac_try: + return "try"; + case ac_copy: + return "copy"; + case ac_append: + return "append"; + case ac_ttl: + return "ttl"; + case ac_nested: + return "nested"; + } +} + +const char *status2str(actor_status status) { + switch (status) { + default: + assert(false); + return "?!"; + case as_debuging: + return "debuging"; + case as_running: + return "running"; + case as_successful: + return "successful"; + case as_killed: + return "killed"; + case as_failed: + return "failed"; + case as_coredump: + return "coredump"; + } +} + +const char *keygencase2str(const keygen_case keycase) { + switch (keycase) { + default: + assert(false); + return "?!"; + case kc_random: + return "random"; + case kc_dashes: + return "dashes"; + case kc_custom: + return "custom"; + } +} + +//----------------------------------------------------------------------------- + +int testcase::oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, + uint64_t txn, unsigned gap, size_t space, + int retry) { + + testcase *self = (testcase *)mdbx_env_get_userctx(env); + + if (retry == 0) + log_notice("oom_callback: waitfor pid %u, thread %" PRIuPTR + ", txn #%" PRIu64 ", gap %d, scape %zu", + pid, (size_t)tid, txn, gap, space); + + if (self->should_continue(true)) { + osal_yield(); + if (retry > 0) + osal_udelay(retry * 100); + return 0 /* always retry */; + } + + return -1; +} + +void testcase::db_prepare() { + log_trace(">> db_prepare"); + assert(!db_guard); + + MDBX_env *env = nullptr; + int rc = mdbx_env_create(&env); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_env_create()", rc); + + assert(env != nullptr); + db_guard.reset(env); + + rc = mdbx_env_set_userctx(env, this); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_env_set_userctx()", rc); + + rc = mdbx_env_set_maxreaders(env, config.params.max_readers); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_env_set_maxreaders()", rc); + + rc = mdbx_env_set_maxdbs(env, config.params.max_tables); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_env_set_maxdbs()", rc); + + rc = mdbx_env_set_oomfunc(env, testcase::oom_callback); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_env_set_oomfunc()", rc); + + rc = mdbx_env_set_geometry( + env, config.params.size_lower, config.params.size_now, + config.params.size_upper, config.params.growth_step, + config.params.shrink_threshold, config.params.pagesize); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_env_set_mapsize()", rc); + + log_trace("<< db_prepare"); +} + +void testcase::db_open() { + log_trace(">> db_open"); + + if (!db_guard) + db_prepare(); + + jitter_delay(true); + int rc = mdbx_env_open(db_guard.get(), config.params.pathname_db.c_str(), + (unsigned)config.params.mode_flags, 0640); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_env_open()", rc); + + log_trace("<< db_open"); +} + +void testcase::db_close() { + log_trace(">> db_close"); + cursor_guard.reset(); + txn_guard.reset(); + db_guard.reset(); + log_trace("<< db_close"); +} + +void testcase::txn_begin(bool readonly, unsigned flags) { + assert((flags & MDBX_RDONLY) == 0); + log_trace(">> txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write", + flags); + assert(!txn_guard); + + MDBX_txn *txn = nullptr; + int rc = mdbx_txn_begin(db_guard.get(), nullptr, + readonly ? flags | MDBX_RDONLY : flags, &txn); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_txn_begin()", rc); + txn_guard.reset(txn); + + log_trace("<< txn_begin(%s, 0x%04X)", readonly ? "read-only" : "read-write", + flags); +} + +int testcase::breakable_commit() { + int rc = MDBX_SUCCESS; + log_trace(">> txn_commit"); + assert(txn_guard); + + MDBX_txn *txn = txn_guard.release(); + txn_inject_writefault(txn); + int err = mdbx_txn_commit(txn); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + rc = err; + err = mdbx_txn_abort(txn); + if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH && + err != MDBX_BAD_TXN)) + failure_perror("mdbx_txn_abort()", err); + } else + failure_perror("mdbx_txn_commit()", err); + } + + log_trace("<< txn_commit: %s", rc ? "failed" : "Ok"); + return rc; +} + +unsigned testcase::txn_underutilization_x256(MDBX_txn *txn) const { + if (txn) { + MDBX_txn_info info; + int err = mdbx_txn_info(txn, &info, false); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_txn_info()", err); + const size_t left = size_t(info.txn_space_leftover); + const size_t total = + size_t(info.txn_space_leftover) + size_t(info.txn_space_dirty); + return (unsigned)(left / (total >> 8)); + } + return 0; +} + +void testcase::txn_end(bool abort) { + log_trace(">> txn_end(%s)", abort ? "abort" : "commit"); + assert(txn_guard); + + MDBX_txn *txn = txn_guard.release(); + if (abort) { + int err = mdbx_txn_abort(txn); + if (unlikely(err != MDBX_SUCCESS && err != MDBX_THREAD_MISMATCH && + err != MDBX_BAD_TXN)) + failure_perror("mdbx_txn_abort()", err); + } else { + txn_inject_writefault(txn); + int err = mdbx_txn_commit(txn); + if (unlikely(err != MDBX_SUCCESS)) + failure_perror("mdbx_txn_commit()", err); + } + + log_trace("<< txn_end(%s)", abort ? "abort" : "commit"); +} + +void testcase::cursor_open(MDBX_dbi handle) { + log_trace(">> cursor_open(%u)", handle); + assert(!cursor_guard); + assert(txn_guard); + + MDBX_cursor *cursor = nullptr; + int rc = mdbx_cursor_open(txn_guard.get(), handle, &cursor); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_cursor_open()", rc); + cursor_guard.reset(cursor); + + log_trace("<< cursor_open(%u)", handle); +} + +void testcase::cursor_close() { + log_trace(">> cursor_close()"); + assert(cursor_guard); + MDBX_cursor *cursor = cursor_guard.release(); + mdbx_cursor_close(cursor); + log_trace("<< cursor_close()"); +} + +int testcase::breakable_restart() { + int rc = MDBX_SUCCESS; + if (txn_guard) + rc = breakable_commit(); + if (cursor_guard) + cursor_close(); + txn_begin(false, 0); + return rc; +} + +void testcase::txn_restart(bool abort, bool readonly, unsigned flags) { + if (txn_guard) + txn_end(abort); + if (cursor_guard) + cursor_close(); + txn_begin(readonly, flags); +} + +void testcase::txn_inject_writefault(void) { + if (txn_guard) + txn_inject_writefault(txn_guard.get()); +} + +void testcase::txn_inject_writefault(MDBX_txn *txn) { + if (config.params.inject_writefaultn && txn) { + if (config.params.inject_writefaultn <= nops_completed && + (mdbx_txn_flags(txn) & MDBX_RDONLY) == 0) { + log_verbose( + "== txn_inject_writefault(): got %u nops or more, inject FAULT", + config.params.inject_writefaultn); + log_flush(); +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + TerminateProcess(GetCurrentProcess(), 42); +#else + raise(SIGKILL); +#endif + } + } +} + +bool testcase::wait4start() { + if (config.wait4id) { + log_trace(">> wait4start(%u)", config.wait4id); + assert(!global::singlemode); + int rc = osal_waitfor(config.wait4id); + if (rc) { + log_trace("<< wait4start(%u), failed %s", config.wait4id, + test_strerror(rc)); + return false; + } + } else { + log_trace("== skip wait4start: not needed"); + } + + if (config.params.delaystart) { + int rc = osal_delay(config.params.delaystart); + if (rc) { + log_trace("<< delay(%u), failed %s", config.params.delaystart, + test_strerror(rc)); + return false; + } + } else { + log_trace("== skip delay: not needed"); + } + + return true; +} + +void testcase::kick_progress(bool active) const { + if (!global::config::progress_indicator) + return; + logging::progress_canary(active); +} + +void testcase::report(size_t nops_done) { + assert(nops_done > 0); + if (!nops_done) + return; + + nops_completed += nops_done; + log_debug("== complete +%" PRIuPTR " iteration, total %" PRIuPTR " done", + nops_done, nops_completed); + + kick_progress(true); + + if (config.signal_nops && !signalled && + config.signal_nops <= nops_completed) { + log_trace(">> signal(n-ops %" PRIuPTR ")", nops_completed); + if (!global::singlemode) + osal_broadcast(config.actor_id); + signalled = true; + log_trace("<< signal(n-ops %" PRIuPTR ")", nops_completed); + } +} + +void testcase::signal() { + if (!signalled) { + log_trace(">> signal(forced)"); + if (!global::singlemode) + osal_broadcast(config.actor_id); + signalled = true; + log_trace("<< signal(forced)"); + } +} + +bool testcase::setup() { + db_prepare(); + if (!wait4start()) + return false; + + start_timestamp = chrono::now_motonic(); + nops_completed = 0; + return true; +} + +bool testcase::teardown() { + log_trace(">> testcase::teardown"); + signal(); + db_close(); + log_trace("<< testcase::teardown"); + return true; +} + +bool testcase::should_continue(bool check_timeout_only) const { + bool result = true; + + if (config.params.test_duration) { + chrono::time since; + since.fixedpoint = + chrono::now_motonic().fixedpoint - start_timestamp.fixedpoint; + if (since.seconds() >= config.params.test_duration) + result = false; + } + + if (!check_timeout_only && config.params.test_nops && + nops_completed >= config.params.test_nops) + result = false; + + if (result) + kick_progress(false); + + return result; +} + +void testcase::fetch_canary() { + mdbx_canary canary_now; + log_trace(">> fetch_canary"); + + int rc = mdbx_canary_get(txn_guard.get(), &canary_now); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_canary_get()", rc); + + if (canary_now.v < last.canary.v) + failure("fetch_canary: %" PRIu64 "(canary-now.v) < %" PRIu64 + "(canary-last.v)", + canary_now.v, last.canary.v); + if (canary_now.y < last.canary.y) + failure("fetch_canary: %" PRIu64 "(canary-now.y) < %" PRIu64 + "(canary-last.y)", + canary_now.y, last.canary.y); + + last.canary = canary_now; + log_trace("<< fetch_canary: db-sequence %" PRIu64 + ", db-sequence.txnid %" PRIu64, + last.canary.y, last.canary.v); +} + +void testcase::update_canary(uint64_t increment) { + mdbx_canary canary_now = last.canary; + + log_trace(">> update_canary: sequence %" PRIu64 " += %" PRIu64, canary_now.y, + increment); + canary_now.y += increment; + + int rc = mdbx_canary_put(txn_guard.get(), &canary_now); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_canary_put()", rc); + + log_trace("<< update_canary: sequence = %" PRIu64, canary_now.y); +} + +int testcase::db_open__begin__table_create_open_clean(MDBX_dbi &handle) { + db_open(); + + int err, retry_left = 42; + for (;;) { + txn_begin(false); + handle = db_table_open(true); + db_table_clear(handle); + err = breakable_commit(); + if (likely(err == MDBX_SUCCESS)) { + txn_begin(false); + return MDBX_SUCCESS; + } + if (--retry_left == 0) + break; + jitter_delay(true); + } + log_notice("db_begin_table_create_open_clean: bailout due '%s'", + mdbx_strerror(err)); + return err; +} + +MDBX_dbi testcase::db_table_open(bool create) { + log_trace(">> testcase::db_table_create"); + + char tablename_buf[16]; + const char *tablename = nullptr; + if (config.space_id) { + int rc = snprintf(tablename_buf, sizeof(tablename_buf), "TBL%04u", + config.space_id); + if (rc < 4 || rc >= (int)sizeof(tablename_buf) - 1) + failure("snprintf(tablename): %d", rc); + tablename = tablename_buf; + } + log_debug("use %s table", tablename ? tablename : "MAINDB"); + + MDBX_dbi handle = 0; + int rc = mdbx_dbi_open(txn_guard.get(), tablename, + (create ? MDBX_CREATE : 0) | config.params.table_flags, + &handle); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_dbi_open()", rc); + + log_trace("<< testcase::db_table_create, handle %u", handle); + return handle; +} + +void testcase::db_table_drop(MDBX_dbi handle) { + log_trace(">> testcase::db_table_drop, handle %u", handle); + + if (config.params.drop_table) { + int rc = mdbx_drop(txn_guard.get(), handle, true); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_drop(delete=true)", rc); + log_trace("<< testcase::db_table_drop"); + } else { + log_trace("<< testcase::db_table_drop: not needed"); + } +} + +void testcase::db_table_clear(MDBX_dbi handle, MDBX_txn *txn) { + log_trace(">> testcase::db_table_clear, handle %u", handle); + int rc = mdbx_drop(txn ? txn : txn_guard.get(), handle, false); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_drop(delete=false)", rc); + log_trace("<< testcase::db_table_clear"); +} + +void testcase::db_table_close(MDBX_dbi handle) { + log_trace(">> testcase::db_table_close, handle %u", handle); + assert(!txn_guard); + int rc = mdbx_dbi_close(db_guard.get(), handle); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_dbi_close()", rc); + log_trace("<< testcase::db_table_close"); +} + +void testcase::checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, + MDBX_val expected_valued) { + MDBX_val actual_value = expected_valued; + int rc = mdbx_get_nearest(txn_guard.get(), handle, &key2check, &actual_value); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror(step, rc); + if (!is_samedata(&actual_value, &expected_valued)) + failure("%s data mismatch", step); +} + +//----------------------------------------------------------------------------- + +bool test_execute(const actor_config &config_const) { + const mdbx_pid_t pid = osal_getpid(); + actor_config config = config_const; + + if (global::singlemode) { + logging::setup(format("single_%s", testcase2str(config.testcase))); + } else { + logging::setup((logging::loglevel)config.params.loglevel, + format("child_%u.%u", config.actor_id, config.space_id)); + log_trace(">> wait4barrier"); + osal_wait4barrier(); + log_trace("<< wait4barrier"); + } + + try { + std::unique_ptr test; + switch (config.testcase) { + case ac_hill: + test.reset(new testcase_hill(config, pid)); + break; + case ac_deadread: + test.reset(new testcase_deadread(config, pid)); + break; + case ac_deadwrite: + test.reset(new testcase_deadwrite(config, pid)); + break; + case ac_jitter: + test.reset(new testcase_jitter(config, pid)); + break; + case ac_try: + test.reset(new testcase_try(config, pid)); + break; + case ac_copy: + test.reset(new testcase_copy(config, pid)); + break; + case ac_append: + test.reset(new testcase_append(config, pid)); + break; + case ac_ttl: + test.reset(new testcase_ttl(config, pid)); + break; + case ac_nested: + test.reset(new testcase_nested(config, pid)); + break; + default: + test.reset(new testcase(config, pid)); + break; + } + + size_t iter = 0; + do { + iter++; + if (!test->setup()) { + log_notice("test setup failed"); + return false; + } + if (!test->run()) { + log_notice("test failed"); + return false; + } + if (!test->teardown()) { + log_notice("test teardown failed"); + return false; + } + + if (config.params.nrepeat == 1) + log_verbose("test successed"); + else { + if (config.params.nrepeat) + log_verbose("test successed (iteration %zi of %zi)", iter, + size_t(config.params.nrepeat)); + else + log_verbose("test successed (iteration %zi)", iter); + config.params.keygen.seed += INT32_C(0xA4F4D37B); + } + + } while (config.params.nrepeat == 0 || iter < config.params.nrepeat); + return true; + } catch (const std::exception &pipets) { + failure("***** Exception: %s *****", pipets.what()); + return false; + } +} + +//----------------------------------------------------------------------------- + +int testcase::insert(const keygen::buffer &akey, const keygen::buffer &adata, + unsigned flags) { + int err = mdbx_put(txn_guard.get(), dbi, &akey->value, &adata->value, flags); + if (err == MDBX_SUCCESS && config.params.speculum) { + const auto S_key = S(akey); + const auto S_data = S(adata); + const bool inserted = speculum.emplace(S_key, S_data).second; + assert(inserted); + (void)inserted; + } + return err; +} + +int testcase::replace(const keygen::buffer &akey, + const keygen::buffer &new_data, + const keygen::buffer &old_data, unsigned flags) { + if (config.params.speculum) { + const auto S_key = S(akey); + const auto S_old = S(old_data); + const auto S_new = S(new_data); + const auto removed = speculum.erase(SET::key_type(S_key, S_old)); + assert(removed == 1); + (void)removed; + const bool inserted = speculum.emplace(S_key, S_new).second; + assert(inserted); + (void)inserted; + } + return mdbx_replace(txn_guard.get(), dbi, &akey->value, &new_data->value, + &old_data->value, flags); +} + +int testcase::remove(const keygen::buffer &akey, const keygen::buffer &adata) { + if (config.params.speculum) { + const auto S_key = S(akey); + const auto S_data = S(adata); + const auto removed = speculum.erase(SET::key_type(S_key, S_data)); + assert(removed == 1); + (void)removed; + } + return mdbx_del(txn_guard.get(), dbi, &akey->value, &adata->value); +} + +bool testcase::speculum_verify() const { + if (!config.params.speculum) + return true; + + char dump_key[128], dump_value[128]; + char dump_mkey[128], dump_mvalue[128]; + + MDBX_cursor *cursor; + int err = mdbx_cursor_open(txn_guard.get(), dbi, &cursor); + if (err != MDBX_SUCCESS) + failure_perror("mdbx_cursor_open()", err); + + bool rc = true; + MDBX_val akey, avalue; + MDBX_val mkey, mvalue; + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_FIRST); + + assert(std::is_sorted(speculum.cbegin(), speculum.cend(), ItemCompare(this))); + auto it = speculum.cbegin(); + while (true) { + if (err != MDBX_SUCCESS) { + akey.iov_len = avalue.iov_len = 0; + akey.iov_base = avalue.iov_base = nullptr; + } + const auto S_key = S(akey); + const auto S_data = S(avalue); + if (it != speculum.cend()) { + mkey.iov_base = (void *)it->first.c_str(); + mkey.iov_len = it->first.size(); + mvalue.iov_base = (void *)it->second.c_str(); + mvalue.iov_len = it->second.size(); + } + if (err == MDBX_SUCCESS && it != speculum.cend() && S_key == it->first && + S_data == it->second) { + ++it; + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); + } else if (err == MDBX_SUCCESS && + (it == speculum.cend() || S_key < it->first || + (S_key == it->first && S_data < it->second))) { + if (it != speculum.cend()) { + log_error("extra pair: db{%s, %s} < mi{%s, %s}", + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } else { + log_error("extra pair: db{%s, %s} < mi.END", + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value))); + } + err = mdbx_cursor_get(cursor, &akey, &avalue, MDBX_NEXT); + rc = false; + } else if (it != speculum.cend() && + (err == MDBX_NOTFOUND || S_key > it->first || + (S_key == it->first && S_data > it->second))) { + if (err == MDBX_NOTFOUND) { + log_error("lost pair: db.END > mi{%s, %s}", + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } else { + log_error("lost pair: db{%s, %s} > mi{%s, %s}", + mdbx_dump_val(&akey, dump_key, sizeof(dump_key)), + mdbx_dump_val(&avalue, dump_value, sizeof(dump_value)), + mdbx_dump_val(&mkey, dump_mkey, sizeof(dump_mkey)), + mdbx_dump_val(&mvalue, dump_mvalue, sizeof(dump_mvalue))); + } + ++it; + rc = false; + } else if (err == MDBX_NOTFOUND && it == speculum.cend()) { + break; + } else if (err != MDBX_SUCCESS) { + failure_perror("mdbx_cursor_get()", err); + } else { + assert(!"WTF?"); + } + } + + mdbx_cursor_close(cursor); + return rc; +} diff --git a/src/contrib/db/libmdbx/test/test.h b/src/contrib/db/libmdbx/test/test.h new file mode 100644 index 0000000..178100e --- /dev/null +++ b/src/contrib/db/libmdbx/test/test.h @@ -0,0 +1,314 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once + +#include "base.h" +#include "chrono.h" +#include "config.h" +#include "keygen.h" +#include "log.h" +#include "osal.h" +#include "utils.h" + +#include +#include +#include +#include + +#ifndef HAVE_cxx17_std_string_view +#if __cplusplus >= 201703L && __has_include() +#include +#define HAVE_cxx17_std_string_view 1 +#else +#define HAVE_cxx17_std_string_view 0 +#endif +#endif /* HAVE_cxx17_std_string_view */ + +#if HAVE_cxx17_std_string_view +#include +#endif + +bool test_execute(const actor_config &config); +std::string thunk_param(const actor_config &config); +void testcase_setup(const char *casename, actor_params ¶ms, + unsigned &last_space_id); +void configure_actor(unsigned &last_space_id, const actor_testcase testcase, + const char *space_id_cstr, const actor_params ¶ms); +void keycase_setup(const char *casename, actor_params ¶ms); + +namespace global { + +extern const char thunk_param_prefix[]; +extern std::vector actors; +extern std::unordered_map events; +extern std::unordered_map pid2actor; +extern std::set databases; +extern unsigned nactors; +extern chrono::time start_motonic; +extern chrono::time deadline_motonic; +extern bool singlemode; + +namespace config { +extern unsigned timeout_duration_seconds; +extern bool dump_config; +extern bool cleanup_before; +extern bool cleanup_after; +extern bool failfast; +extern bool progress_indicator; +extern bool console_mode; +} /* namespace config */ + +} /* namespace global */ + +//----------------------------------------------------------------------------- + +struct db_deleter /* : public std::unary_function */ { + void operator()(MDBX_env *env) const { mdbx_env_close(env); } +}; + +struct txn_deleter /* : public std::unary_function */ { + void operator()(MDBX_txn *txn) const { + int rc = mdbx_txn_abort(txn); + if (rc) + log_trouble(__func__, "mdbx_txn_abort()", rc); + } +}; + +struct cursor_deleter /* : public std::unary_function */ { + void operator()(MDBX_cursor *cursor) const { mdbx_cursor_close(cursor); } +}; + +typedef std::unique_ptr scoped_db_guard; +typedef std::unique_ptr scoped_txn_guard; +typedef std::unique_ptr scoped_cursor_guard; + +//----------------------------------------------------------------------------- + +class testcase { +protected: +#if HAVE_cxx17_std_string_view + using data_view = std::string_view; +#else + using data_view = std::string; +#endif + static inline data_view S(const MDBX_val &v) { + return data_view(static_cast(v.iov_base), v.iov_len); + } + static inline data_view S(const keygen::buffer &b) { return S(b->value); } + + using Item = std::pair; + struct ItemCompare { + const testcase *context; + ItemCompare(const testcase *owner) : context(owner) {} + + bool operator()(const Item &a, const Item &b) const { + MDBX_val va, vb; + va.iov_base = (void *)a.first.data(); + va.iov_len = a.first.size(); + vb.iov_base = (void *)b.first.data(); + vb.iov_len = b.first.size(); + int cmp = mdbx_cmp(context->txn_guard.get(), context->dbi, &va, &vb); + if (cmp == 0 && + (context->config.params.table_flags & MDBX_DUPSORT) != 0) { + va.iov_base = (void *)a.second.data(); + va.iov_len = a.second.size(); + vb.iov_base = (void *)b.second.data(); + vb.iov_len = b.second.size(); + cmp = mdbx_dcmp(context->txn_guard.get(), context->dbi, &va, &vb); + } + return cmp < 0; + } + }; + using SET = std::set; + + const actor_config &config; + const mdbx_pid_t pid; + + MDBX_dbi dbi; + scoped_db_guard db_guard; + scoped_txn_guard txn_guard; + scoped_cursor_guard cursor_guard; + bool signalled; + + size_t nops_completed; + chrono::time start_timestamp; + keygen::buffer key; + keygen::buffer data; + keygen::maker keyvalue_maker; + + struct { + mdbx_canary canary; + } last; + + SET speculum; + bool speculum_verify() const; + int insert(const keygen::buffer &akey, const keygen::buffer &adata, + unsigned flags); + int replace(const keygen::buffer &akey, const keygen::buffer &new_value, + const keygen::buffer &old_value, unsigned flags); + int remove(const keygen::buffer &akey, const keygen::buffer &adata); + + static int oom_callback(MDBX_env *env, mdbx_pid_t pid, mdbx_tid_t tid, + uint64_t txn, unsigned gap, size_t space, int retry); + + bool is_nested_txn_available() const { + return (config.params.mode_flags & MDBX_WRITEMAP) == 0; + } + void kick_progress(bool active) const; + void db_prepare(); + void db_open(); + void db_close(); + void txn_begin(bool readonly, unsigned flags = 0); + int breakable_commit(); + void txn_end(bool abort); + int breakable_restart(); + void txn_restart(bool abort, bool readonly, unsigned flags = 0); + void cursor_open(MDBX_dbi handle); + void cursor_close(); + void txn_inject_writefault(void); + void txn_inject_writefault(MDBX_txn *txn); + void fetch_canary(); + void update_canary(uint64_t increment); + void checkdata(const char *step, MDBX_dbi handle, MDBX_val key2check, + MDBX_val expected_valued); + unsigned txn_underutilization_x256(MDBX_txn *txn) const; + + MDBX_dbi db_table_open(bool create); + void db_table_drop(MDBX_dbi handle); + void db_table_clear(MDBX_dbi handle, MDBX_txn *txn = nullptr); + void db_table_close(MDBX_dbi handle); + int db_open__begin__table_create_open_clean(MDBX_dbi &handle); + + bool wait4start(); + void report(size_t nops_done); + void signal(); + bool should_continue(bool check_timeout_only = false) const; + + void generate_pair(const keygen::serial_t serial, keygen::buffer &out_key, + keygen::buffer &out_value, keygen::serial_t data_age = 0) { + keyvalue_maker.pair(serial, out_key, out_value, data_age); + } + + void generate_pair(const keygen::serial_t serial, + keygen::serial_t data_age = 0) { + generate_pair(serial, key, data, data_age); + } + + bool mode_readonly() const { + return (config.params.mode_flags & MDBX_RDONLY) ? true : false; + } + +public: + testcase(const actor_config &config, const mdbx_pid_t pid) + : config(config), pid(pid), signalled(false), nops_completed(0), + speculum(ItemCompare(this)) { + start_timestamp.reset(); + memset(&last, 0, sizeof(last)); + } + + virtual bool setup(); + virtual bool run() { return true; } + virtual bool teardown(); + virtual ~testcase() {} +}; + +class testcase_ttl : public testcase { +public: + testcase_ttl(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool run() override; +}; + +class testcase_hill : public testcase { + using inherited = testcase; + SET speculum_commited; + +public: + testcase_hill(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid), speculum_commited(ItemCompare(this)) {} + bool run() override; +}; + +class testcase_append : public testcase { +public: + testcase_append(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool run() override; +}; + +class testcase_deadread : public testcase { +public: + testcase_deadread(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool run() override; +}; + +class testcase_deadwrite : public testcase { +public: + testcase_deadwrite(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool run() override; +}; + +class testcase_jitter : public testcase { +public: + testcase_jitter(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool run() override; +}; + +class testcase_try : public testcase { +public: + testcase_try(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool run() override; +}; + +class testcase_copy : public testcase { + const std::string copy_pathname; + void copy_db(const bool with_compaction); + +public: + testcase_copy(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid), + copy_pathname(config.params.pathname_db + "-copy") {} + bool run() override; +}; + +class testcase_nested : public testcase { + using inherited = testcase; + using FIFO = std::deque>; + + uint64_t serial; + FIFO fifo; + std::stack> stack; + + bool trim_tail(unsigned window_width); + bool grow_head(unsigned head_count); + bool pop_txn(bool abort); + bool pop_txn() { + return pop_txn(inherited::is_nested_txn_available() ? flipcoin_x3() + : flipcoin_x2()); + } + void push_txn(); + bool stochastic_breakable_restart_with_nested(bool force_restart = false); + +public: + testcase_nested(const actor_config &config, const mdbx_pid_t pid) + : testcase(config, pid) {} + bool setup() override; + bool run() override; + bool teardown() override; +}; diff --git a/src/contrib/db/libmdbx/test/try.cc b/src/contrib/db/libmdbx/test/try.cc new file mode 100644 index 0000000..adb0113 --- /dev/null +++ b/src/contrib/db/libmdbx/test/try.cc @@ -0,0 +1,20 @@ +#include "test.h" + +bool testcase_try::run() { + db_open(); + assert(!txn_guard); + + MDBX_txn *txn = nullptr; + MDBX_txn *txn2 = nullptr; + int rc = mdbx_txn_begin(db_guard.get(), nullptr, 0, &txn); + if (unlikely(rc != MDBX_SUCCESS)) + failure_perror("mdbx_txn_begin(MDBX_TRYTXN)", rc); + else { + rc = mdbx_txn_begin(db_guard.get(), nullptr, MDBX_TRYTXN, &txn2); + if (unlikely(rc != MDBX_BUSY)) + failure_perror("mdbx_txn_begin(MDBX_TRYTXN)", rc); + } + + txn_guard.reset(txn); + return true; +} diff --git a/src/contrib/db/libmdbx/test/ttl.cc b/src/contrib/db/libmdbx/test/ttl.cc new file mode 100644 index 0000000..782a8b4 --- /dev/null +++ b/src/contrib/db/libmdbx/test/ttl.cc @@ -0,0 +1,172 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" +#include +#include + +static unsigned edge2window(uint64_t edge, unsigned window_max) { + const double rnd = u64_to_double1(bleach64(edge)); + const unsigned window = window_max - std::lrint(std::pow(window_max, rnd)); + return window; +} + +static unsigned edge2count(uint64_t edge, unsigned count_max) { + const double rnd = u64_to_double1(prng64_map1_white(edge)); + const unsigned count = std::lrint(std::pow(count_max, rnd)); + return count; +} + +bool testcase_ttl::run() { + int err = db_open__begin__table_create_open_clean(dbi); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("ttl: bailout-prepare due '%s'", mdbx_strerror(err)); + return false; + } + + /* LY: теÑÑ‚ "ÑмулÑцией time-to-live": + * - организуетÑÑ "ÑкользÑщее окно", которое двигаетÑÑ Ð²Ð¿ÐµÑ€ÐµÐ´ вдоль + * чиÑловой оÑи каждую транзакцию. + * - по переднему краю "ÑкользÑщего окна" запиÑи добавлÑÑŽÑ‚ÑÑ Ð² таблицу, + * а по заднему удалÑÑŽÑ‚ÑÑ. + * - количеÑтво добавлÑемых/удалÑемых запиÑей пÑевдоÑлучайно завиÑит + * от номера транзакции, но Ñ ÑкÑпоненциальным раÑпределением. + * - размер "ÑкользÑщего окна" также пÑевдоÑлучайно завиÑит от номера + * транзакции Ñ "отрицательным" ÑкÑпоненциальным раÑпределением + * MAX_WIDTH - exp(rnd(N)), при уменьшении окна ÑдвигаетÑÑ Ð·Ð°Ð´Ð½Ð¸Ð¹ + * край и удалÑÑŽÑ‚ÑÑ Ð·Ð°Ð¿Ð¸Ñи позади него. + * + * Таким образом имитируетÑÑ Ð¿Ð¾Ð²ÐµÐ´ÐµÐ½Ð¸Ðµ таблицы Ñ TTL: запиÑи ÑтохаÑтичеÑки + * добавлÑÑŽÑ‚ÑÑ Ð¸ удалÑÑŽÑ‚ÑÑ, но изредка проиÑходÑÑ‚ маÑÑивные удалениÑ. + */ + + /* LY: Ð´Ð»Ñ Ð¿Ð°Ñ€Ð°Ð¼ÐµÑ‚Ñ€Ð¸Ð·Ð°Ñ†Ð¸Ð¸ иÑпользуем подходÑщие параметры, которые не имеют + * здеÑÑŒ ÑмыÑла в первоначальном значении. */ + const unsigned window_max_lower = 333; + const unsigned count_max_lower = 333; + + const unsigned window_max = (config.params.batch_read > window_max_lower) + ? config.params.batch_read + : window_max_lower; + const unsigned count_max = (config.params.batch_write > count_max_lower) + ? config.params.batch_write + : count_max_lower; + log_verbose("ttl: using `batch_read` value %u for window_max", window_max); + log_verbose("ttl: using `batch_write` value %u for count_max", count_max); + + uint64_t seed = + prng64_map2_white(config.params.keygen.seed) + config.actor_id; + keyvalue_maker.setup(config.params, config.actor_id, 0 /* thread_number */); + key = keygen::alloc(config.params.keylen_max); + data = keygen::alloc(config.params.datalen_max); + const unsigned insert_flags = (config.params.table_flags & MDBX_DUPSORT) + ? MDBX_NODUPDATA + : MDBX_NODUPDATA | MDBX_NOOVERWRITE; + + std::deque> fifo; + uint64_t serial = 0; + bool rc = false; + while (should_continue()) { + const uint64_t salt = prng64_white(seed) /* mdbx_txn_id(txn_guard.get()) */; + + const unsigned window_width = + flipcoin_x4() ? 0 : edge2window(salt, window_max); + unsigned head_count = edge2count(salt, count_max); + log_debug("ttl: step #%zu (serial %" PRIu64 + ", window %u, count %u) salt %" PRIu64, + nops_completed, serial, window_width, head_count, salt); + + if (window_width) { + while (fifo.size() > window_width) { + uint64_t tail_serial = fifo.back().first; + const unsigned tail_count = fifo.back().second; + log_trace("ttl: pop-tail (serial %" PRIu64 ", count %u)", tail_serial, + tail_count); + fifo.pop_back(); + for (unsigned n = 0; n < tail_count; ++n) { + log_trace("ttl: remove-tail %" PRIu64, tail_serial); + generate_pair(tail_serial); + err = mdbx_del(txn_guard.get(), dbi, &key->value, &data->value); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("ttl: tail-bailout due '%s'", mdbx_strerror(err)); + goto bailout; + } + failure_perror("mdbx_del(tail)", err); + } + if (unlikely(!keyvalue_maker.increment(tail_serial, 1))) + failure("ttl: unexpected key-space overflow on the tail"); + } + } + } else { + log_trace("ttl: purge state"); + db_table_clear(dbi); + fifo.clear(); + } + + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("ttl: bailout at commit due '%s'", mdbx_strerror(err)); + break; + } + fifo.push_front(std::make_pair(serial, head_count)); + retry: + for (unsigned n = 0; n < head_count; ++n) { + log_trace("ttl: insert-head %" PRIu64, serial); + generate_pair(serial); + err = mdbx_put(txn_guard.get(), dbi, &key->value, &data->value, + insert_flags); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_MAP_FULL && config.params.ignore_dbfull) { + log_notice("ttl: head-insert skip due '%s'", mdbx_strerror(err)); + txn_restart(true, false); + serial = fifo.front().first; + fifo.front().second = head_count = n; + goto retry; + } + failure_perror("mdbx_put(head)", err); + } + + if (unlikely(!keyvalue_maker.increment(serial, 1))) { + log_notice("ttl: unexpected key-space overflow"); + goto bailout; + } + } + err = breakable_restart(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("ttl: head-commit skip due '%s'", mdbx_strerror(err)); + serial = fifo.front().first; + fifo.pop_front(); + } + + report(1); + rc = true; + } + +bailout: + txn_end(true); + if (dbi) { + if (config.params.drop_table && !mode_readonly()) { + txn_begin(false); + db_table_drop(dbi); + err = breakable_commit(); + if (unlikely(err != MDBX_SUCCESS)) { + log_notice("ttl: bailout-clean due '%s'", mdbx_strerror(err)); + return false; + } + } else + db_table_close(dbi); + } + return rc; +} diff --git a/src/contrib/db/libmdbx/test/utils.cc b/src/contrib/db/libmdbx/test/utils.cc new file mode 100644 index 0000000..311cf54 --- /dev/null +++ b/src/contrib/db/libmdbx/test/utils.cc @@ -0,0 +1,370 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "test.h" +#include +#if defined(HAVE_IEEE754_H) || __has_include() +#include +#endif +#if defined(__APPLE__) || defined(__MACH__) +#include +#endif /* defined(__APPLE__) || defined(__MACH__) */ + +std::string format(const char *fmt, ...) { + va_list ap, ones; + va_start(ap, fmt); + va_copy(ones, ap); +#ifdef _MSC_VER + int needed = _vscprintf(fmt, ap); +#else + int needed = vsnprintf(nullptr, 0, fmt, ap); +#endif + assert(needed >= 0); + va_end(ap); + std::string result; + result.reserve((size_t)needed + 1); + result.resize((size_t)needed, '\0'); + int actual = vsnprintf((char *)result.data(), result.capacity(), fmt, ones); + assert(actual == needed); + (void)actual; + va_end(ones); + return result; +} + +std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum) { + std::string result; + if (bytes > 0) { + const uint8_t *data = (const uint8_t *)ptr; + checksum.push(data, bytes); + result.reserve(bytes * 2); + const uint8_t *const end = data + bytes; + do { + char h = *data >> 4; + char l = *data & 15; + result.push_back((l < 10) ? l + '0' : l - 10 + 'a'); + result.push_back((h < 10) ? h + '0' : h - 10 + 'a'); + } while (++data < end); + } + assert(result.size() == bytes * 2); + return result; +} + +bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, + size_t bytes, simple_checksum &checksum) { + if (bytes * 2 != (size_t)(hex_end - hex_begin)) + return false; + + uint8_t *data = (uint8_t *)ptr; + for (const char *hex = hex_begin; hex != hex_end; hex += 2, ++data) { + unsigned l = hex[0], h = hex[1]; + + if (l >= '0' && l <= '9') + l = l - '0'; + else if (l >= 'A' && l <= 'F') + l = l - 'A' + 10; + else if (l >= 'a' && l <= 'f') + l = l - 'a' + 10; + else + return false; + + if (h >= '0' && h <= '9') + h = h - '0'; + else if (h >= 'A' && h <= 'F') + h = h - 'A' + 10; + else if (h >= 'a' && h <= 'f') + h = h - 'a' + 10; + else + return false; + + uint32_t c = l + (h << 4); + checksum.push(c); + *data = (uint8_t)c; + } + return true; +} + +bool is_samedata(const MDBX_val *a, const MDBX_val *b) { + return a->iov_len == b->iov_len && + memcmp(a->iov_base, b->iov_base, a->iov_len) == 0; +} + +//----------------------------------------------------------------------------- + +/* TODO: replace my 'libmera' from t1ha. */ +uint64_t entropy_ticks(void) { +#if defined(EMSCRIPTEN) + return (uint64_t)emscripten_get_now(); +#endif /* EMSCRIPTEN */ + +#if defined(__APPLE__) || defined(__MACH__) + return mach_absolute_time(); +#endif /* defined(__APPLE__) || defined(__MACH__) */ + +#if defined(__sun__) || defined(__sun) + return gethrtime(); +#endif /* __sun__ */ + +#if defined(__GNUC__) || defined(__clang__) + +#if defined(__ia64__) + uint64_t ticks; + __asm __volatile("mov %0=ar.itc" : "=r"(ticks)); + return ticks; +#elif defined(__hppa__) + uint64_t ticks; + __asm __volatile("mfctl 16, %0" : "=r"(ticks)); + return ticks; +#elif defined(__s390__) + uint64_t ticks; + __asm __volatile("stck 0(%0)" : : "a"(&(ticks)) : "memory", "cc"); + return ticks; +#elif defined(__alpha__) + uint64_t ticks; + __asm __volatile("rpcc %0" : "=r"(ticks)); + return ticks; +#elif defined(__sparc__) || defined(__sparc) || defined(__sparc64__) || \ + defined(__sparc64) || defined(__sparc_v8plus__) || \ + defined(__sparc_v8plus) || defined(__sparc_v8plusa__) || \ + defined(__sparc_v8plusa) || defined(__sparc_v9__) || defined(__sparc_v9) + + union { + uint64_t u64; + struct { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + uint32_t h, l; +#else + uint32_t l, h; +#endif + } u32; + } cycles; + +#if defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || \ + defined(__sparc_v9__) || defined(__sparc_v8plus) || \ + defined(__sparc_v8plusa) || defined(__sparc_v9) + +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || \ + defined(__sparc64__) || defined(__sparc64) + __asm __volatile("rd %%tick, %0" : "=r"(cycles.u64)); +#else + __asm __volatile("rd %%tick, %1; srlx %1, 32, %0" + : "=r"(cycles.u32.h), "=r"(cycles.u32.l)); +#endif /* __sparc64__ */ + +#else + __asm __volatile(".byte 0x83, 0x41, 0x00, 0x00; mov %%g1, %0" + : "=r"(cycles.u64) + : + : "%g1"); +#endif /* __sparc8plus__ || __sparc_v9__ */ + return cycles.u64; + +#elif (defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) || \ + defined(__powerpc64)) + uint64_t ticks; + __asm __volatile("mfspr %0, 268" : "=r"(ticks)); + return ticks; +#elif (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) || \ + defined(__ppc)) +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul + uint64_t ticks; + __asm __volatile("mftb %0" : "=r"(ticks)); + *now = ticks; +#else + uint64_t ticks; + uint32_t low, high_before, high_after; + __asm __volatile("mftbu %0; mftb %1; mftbu %2" + : "=r"(high_before), "=r"(low), "=r"(high_after)); + ticks = (uint64_t)high_after << 32; + ticks |= low & /* zeroes if high part has changed */ + ~(high_before - high_after); +#endif +#elif (defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 7)) && \ + !defined(MDBX_SAFE4QEMU) + uint64_t virtual_timer; + __asm __volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer)); + return virtual_timer; +#elif (defined(__ARM_ARCH) && __ARM_ARCH > 5 && __ARM_ARCH < 8) || \ + defined(_M_ARM) + static uint32_t pmcntenset = 0x00425B00; + if (unlikely(pmcntenset == 0x00425B00)) { + uint32_t pmuseren; +#ifdef _M_ARM + pmuseren = _MoveFromCoprocessor(15, 0, 9, 14, 0); +#else + __asm("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren)); +#endif + if (1 & pmuseren /* Is it allowed for user mode code? */) { +#ifdef _M_ARM + pmcntenset = _MoveFromCoprocessor(15, 0, 9, 12, 1); +#else + __asm("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset)); +#endif + } else + pmcntenset = 0; + } + if (pmcntenset & 0x80000000ul /* Is it counting? */) { +#ifdef _M_ARM + return __rdpmccntr64(); +#else + uint32_t pmccntr; + __asm __volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr)); + return pmccntr; +#endif + } +#elif defined(__mips__) || defined(__mips) || defined(_R4000) + unsigned count; + __asm __volatile("rdhwr %0, $2" : "=r"(count)); + return count; +#endif /* arch selector */ +#endif /* __GNUC__ || __clang__ */ + +#if defined(__e2k__) || defined(__ia32__) + return __rdtsc(); +#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + LARGE_INTEGER PerformanceCount; + if (QueryPerformanceCounter(&PerformanceCount)) + return PerformanceCount.QuadPart; + return GetTickCount64(); +#else + struct timespec ts; +#if defined(CLOCK_MONOTONIC_COARSE) + clockid_t clk_id = CLOCK_MONOTONIC_COARSE; +#elif defined(CLOCK_MONOTONIC_RAW) + clockid_t clk_id = CLOCK_MONOTONIC_RAW; +#else + clockid_t clk_id = CLOCK_MONOTONIC; +#endif + int rc = clock_gettime(clk_id, &ts); + if (unlikely(rc)) + failure_perror("clock_gettime()", rc); + + return (((uint64_t)ts.tv_sec) << 32) + ts.tv_nsec; +#endif +} + +//----------------------------------------------------------------------------- + +uint64_t prng64_white(uint64_t &state) { + state = prng64_map2_careless(state); + return bleach64(state); +} + +uint32_t prng32(uint64_t &state) { + return (uint32_t)(prng64_careless(state) >> 32); +} + +void prng_fill(uint64_t &state, void *ptr, size_t bytes) { + while (bytes >= 4) { + *((uint32_t *)ptr) = prng32(state); + ptr = (uint32_t *)ptr + 1; + bytes -= 4; + } + + switch (bytes & 3) { + case 3: { + uint32_t u32 = prng32(state); + memcpy(ptr, &u32, 3); + } break; + case 2: + *((uint16_t *)ptr) = (uint16_t)prng32(state); + break; + case 1: + *((uint8_t *)ptr) = (uint8_t)prng32(state); + break; + case 0: + break; + } +} + +static __thread uint64_t prng_state; + +void prng_seed(uint64_t seed) { prng_state = bleach64(seed); } + +uint32_t prng32(void) { return prng32(prng_state); } + +uint64_t prng64(void) { return prng64_white(prng_state); } + +void prng_fill(void *ptr, size_t bytes) { prng_fill(prng_state, ptr, bytes); } + +uint64_t entropy_white() { return bleach64(entropy_ticks()); } + +double double_from_lower(uint64_t salt) { +#ifdef IEEE754_DOUBLE_BIAS + ieee754_double r; + r.ieee.negative = 0; + r.ieee.exponent = IEEE754_DOUBLE_BIAS; + r.ieee.mantissa0 = (unsigned)(salt >> 32); + r.ieee.mantissa1 = (unsigned)salt; + return r.d; +#else + const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1; + const double scale = 1.0 / (double)top; + return (salt & top) * scale; +#endif +} + +double double_from_upper(uint64_t salt) { +#ifdef IEEE754_DOUBLE_BIAS + ieee754_double r; + r.ieee.negative = 0; + r.ieee.exponent = IEEE754_DOUBLE_BIAS; + salt >>= 64 - DBL_MANT_DIG; + r.ieee.mantissa0 = (unsigned)(salt >> 32); + r.ieee.mantissa1 = (unsigned)salt; + return r.d; +#else + const uint64_t top = (UINT64_C(1) << DBL_MANT_DIG) - 1; + const double scale = 1.0 / (double)top; + return (salt >> (64 - DBL_MANT_DIG)) * scale; +#endif +} + +bool flipcoin() { return bleach32((uint32_t)entropy_ticks()) & 1; } +bool flipcoin_x2() { return (bleach32((uint32_t)entropy_ticks()) & 3) == 0; } +bool flipcoin_x3() { return (bleach32((uint32_t)entropy_ticks()) & 7) == 0; } +bool flipcoin_x4() { return (bleach32((uint32_t)entropy_ticks()) & 15) == 0; } + +bool jitter(unsigned probability_percent) { + const uint32_t top = UINT32_MAX - UINT32_MAX % 100; + uint32_t dice, edge = (top) / 100 * probability_percent; + do + dice = bleach32((uint32_t)entropy_ticks()); + while (dice >= top); + return dice < edge; +} + +void jitter_delay(bool extra) { + unsigned dice = entropy_white() & 3; + if (dice == 0) { + log_trace("== jitter.no-delay"); + } else { + log_trace(">> jitter.delay: dice %u", dice); + do { + cpu_relax(); + memory_barrier(); + cpu_relax(); + if (dice > 1) { + osal_yield(); + cpu_relax(); + if (dice > 2) { + unsigned us = entropy_white() & + (extra ? 0xffff /* 656 ms */ : 0x3ff /* 1 ms */); + log_trace("== jitter.delay: %0.6f", us / 1000000.0); + osal_udelay(us); + } + } + } while (flipcoin()); + log_trace("<< jitter.delay: dice %u", dice); + } +} diff --git a/src/contrib/db/libmdbx/test/utils.h b/src/contrib/db/libmdbx/test/utils.h new file mode 100644 index 0000000..2a5a54d --- /dev/null +++ b/src/contrib/db/libmdbx/test/utils.h @@ -0,0 +1,362 @@ +/* + * Copyright 2017-2019 Leonid Yuriev + * and other libmdbx authors: please see AUTHORS file. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#pragma once +#include "base.h" + +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ + !defined(__ORDER_BIG_ENDIAN__) +#error __BYTE_ORDER__ should be defined. +#endif + +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ && \ + __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ +#error Unsupported byte order. +#endif + +#if __GNUC_PREREQ(4, 4) || defined(__clang__) +#ifndef bswap64 +#define bswap64(v) __builtin_bswap64(v) +#endif +#ifndef bswap32 +#define bswap32(v) __builtin_bswap32(v) +#endif +#if (__GNUC_PREREQ(4, 8) || __has_builtin(__builtin_bswap16)) && \ + !defined(bswap16) +#define bswap16(v) __builtin_bswap16(v) +#endif + +#elif defined(_MSC_VER) + +#if _MSC_FULL_VER < 190024215 +#pragma message( \ + "It is recommended to use Visual Studio 2015 (MSC 19.0) or newer.") +#endif + +#define bswap64(v) _byteswap_uint64(v) +#define bswap32(v) _byteswap_ulong(v) +#define bswap16(v) _byteswap_ushort(v) +#define rot64(v, s) _rotr64(v, s) +#define rot32(v, s) _rotr(v, s) + +#if defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) +#pragma intrinsic(_umul128) +#define mul_64x64_128(a, b, ph) _umul128(a, b, ph) +#pragma intrinsic(__umulh) +#define mul_64x64_high(a, b) __umulh(a, b) +#endif + +#if defined(_M_IX86) +#pragma intrinsic(__emulu) +#define mul_32x32_64(a, b) __emulu(a, b) +#elif defined(_M_ARM) +#define mul_32x32_64(a, b) _arm_umull(a, b) +#endif + +#endif /* compiler */ + +#ifndef bswap64 +#ifdef __bswap_64 +#define bswap64(v) __bswap_64(v) +#else +static __inline uint64_t bswap64(uint64_t v) { + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | + ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | + ((v >> 8) & UINT64_C(0x00000000ff0000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | + ((v >> 40) & UINT64_C(0x000000000000ff00)); +} +#endif +#endif /* bswap64 */ + +#ifndef bswap32 +#ifdef __bswap_32 +#define bswap32(v) __bswap_32(v) +#else +static __inline uint32_t bswap32(uint32_t v) { + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | + ((v >> 8) & UINT32_C(0x0000ff00)); +} +#endif +#endif /* bswap32 */ + +#ifndef bswap16 +#ifdef __bswap_16 +#define bswap16(v) __bswap_16(v) +#else +static __inline uint16_t bswap16(uint16_t v) { return v << 8 | v >> 8; } +#endif +#endif /* bswap16 */ + +#define is_byteorder_le() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#define is_byteorder_be() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + +#ifndef htole16 +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define htobe16(v) bswap16(v) +#define htole16(v) (v) +#define be16toh(v) bswap16(v) +#define le16toh(v) (v) +#else +#define htobe16(v) (v) +#define htole16(v) bswap16(v) +#define be16toh(v) (v) +#define le16toh(v) bswap16(v) +#endif +#endif /* htole16 */ + +#ifndef htole32 +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define htobe32(v) bswap32(v) +#define htole32(v) (v) +#define be32toh(v) bswap32(v) +#define le32toh(v) (v) +#else +#define htobe32(v) (v) +#define htole32(v) bswap32(v) +#define be32toh(v) (v) +#define le32toh(v) bswap32(v) +#endif +#endif /* htole32 */ + +#ifndef htole64 +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define htobe64(v) bswap64(v) +#define htole64(v) (v) +#define be64toh(v) bswap64(v) +#define le64toh(v) (v) +#else +#define htobe64(v) (v) +#define htole64(v) bswap_64(v) +#define be64toh(v) (v) +#define le64toh(v) bswap_64(v) +#endif +#endif /* htole64 */ + +namespace unaligned { + +template static __inline T load(const void *ptr) { +#if defined(_MSC_VER) && \ + (defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)) + return *(const T __unaligned *)ptr; +#elif MDBX_UNALIGNED_OK + return *(const T *)ptr; +#else + T local; +#if defined(__GNUC__) || defined(__clang__) + __builtin_memcpy(&local, (const T *)ptr, sizeof(T)); +#else + memcpy(&local, (const T *)ptr, sizeof(T)); +#endif /* __GNUC__ || __clang__ */ + return local; +#endif /* MDBX_UNALIGNED_OK */ +} + +template static __inline void store(void *ptr, const T &value) { +#if defined(_MSC_VER) && \ + (defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64)) + *((T __unaligned *)ptr) = value; +#elif MDBX_UNALIGNED_OK + *(volatile T *)ptr = value; +#else +#if defined(__GNUC__) || defined(__clang__) + __builtin_memcpy(ptr, &value, sizeof(T)); +#else + memcpy(ptr, &value, sizeof(T)); +#endif /* __GNUC__ || __clang__ */ +#endif /* MDBX_UNALIGNED_OK */ +} + +} /* namespace unaligned */ + +//----------------------------------------------------------------------------- + +#ifndef rot64 +static __inline uint64_t rot64(uint64_t v, unsigned s) { + return (v >> s) | (v << (64 - s)); +} +#endif /* rot64 */ + +static __inline bool is_power2(size_t x) { return (x & (x - 1)) == 0; } + +#undef roundup2 +static __inline size_t roundup2(size_t value, size_t granularity) { + assert(is_power2(granularity)); + return (value + granularity - 1) & ~(granularity - 1); +} + +//----------------------------------------------------------------------------- + +static __inline void memory_barrier(void) { +#if __has_extension(c_atomic) || __has_extension(cxx_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__ATOMIC_SEQ_CST) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif defined(__clang__) || defined(__GNUC__) + __sync_synchronize(); +#elif defined(_MSC_VER) + MemoryBarrier(); +#elif defined(__INTEL_COMPILER) /* LY: Intel Compiler may mimic GCC and MSC */ +#if defined(__ia64__) || defined(__ia64) || defined(_M_IA64) + __mf(); +#elif defined(__ia32__) + _mm_mfence(); +#else +#error "Unknown target for Intel Compiler, please report to us." +#endif +#elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) + __machine_rw_barrier(); +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ + (defined(HP_IA64) || defined(__ia64)) + _Asm_mf(); +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ + defined(__ppc64__) || defined(__powerpc64__) + __lwsync(); +#else +#error "Could not guess the kind of compiler, please report to us." +#endif +} + +static __inline void cpu_relax() { +#if defined(__ia32__) + _mm_pause(); +#elif defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) || \ + defined(YieldProcessor) + YieldProcessor(); +#else +/* nope */ +#endif +} + +//----------------------------------------------------------------------------- + +struct simple_checksum { + uint64_t value; + + simple_checksum() : value(0) {} + + void push(const uint32_t &data) { + value += data * UINT64_C(9386433910765580089) + 1; + value ^= value >> 41; + value *= UINT64_C(0xBD9CACC22C6E9571); + } + + void push(const uint64_t &data) { + push((uint32_t)data); + push((uint32_t)(data >> 32)); + } + + void push(const bool data) { + push(data ? UINT32_C(0x780E) : UINT32_C(0xFA18E)); + } + + void push(const void *ptr, size_t bytes) { + const uint8_t *data = (const uint8_t *)ptr; + for (size_t i = 0; i < bytes; ++i) + push((uint32_t)data[i]); + } + + void push(const double &data) { push(&data, sizeof(double)); } + void push(const char *cstr) { push(cstr, strlen(cstr)); } + void push(const std::string &str) { push(str.data(), str.size()); } + + void push(unsigned salt, const MDBX_val &val) { + push(unsigned(val.iov_len)); + push(salt); + push(val.iov_base, val.iov_len); + } + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + void push(const HANDLE &handle) { push(&handle, sizeof(handle)); } +#endif /* _WINDOWS */ +}; + +std::string data2hex(const void *ptr, size_t bytes, simple_checksum &checksum); +bool hex2data(const char *hex_begin, const char *hex_end, void *ptr, + size_t bytes, simple_checksum &checksum); +bool is_samedata(const MDBX_val *a, const MDBX_val *b); +std::string format(const char *fmt, ...); + +uint64_t entropy_ticks(void); +uint64_t entropy_white(void); +static inline uint64_t bleach64(uint64_t v) { + // Tommy Ettinger, https://www.blogger.com/profile/04953541827437796598 + // http://mostlymangling.blogspot.com/2019/01/better-stronger-mixer-and-test-procedure.html + v ^= rot64(v, 25) ^ rot64(v, 50); + v *= UINT64_C(0xA24BAED4963EE407); + v ^= rot64(v, 24) ^ rot64(v, 49); + v *= UINT64_C(0x9FB21C651E98DF25); + return v ^ v >> 28; +} + +static inline uint32_t bleach32(uint32_t x) { + // https://github.com/skeeto/hash-prospector + // exact bias: 0.17353355999581582 + x ^= x >> 16; + x *= UINT32_C(0x7feb352d); + x ^= 0x3027C563 ^ (x >> 15); + x *= UINT32_C(0x846ca68b); + x ^= x >> 16; + return x; +} + +static inline uint64_t prng64_map1_careless(uint64_t state) { + return state * UINT64_C(6364136223846793005) + 1; +} + +static inline uint64_t prng64_map2_careless(uint64_t state) { + return (state + UINT64_C(1442695040888963407)) * + UINT64_C(6364136223846793005); +} + +static inline uint64_t prng64_map1_white(uint64_t state) { + return bleach64(prng64_map1_careless(state)); +} + +static inline uint64_t prng64_map2_white(uint64_t state) { + return bleach64(prng64_map2_careless(state)); +} + +static inline uint64_t prng64_careless(uint64_t &state) { + state = prng64_map1_careless(state); + return state; +} + +static inline double u64_to_double1(uint64_t v) { + union { + uint64_t u64; + double d; + } casting; + + casting.u64 = UINT64_C(0x3ff) << 52 | (v >> 12); + assert(casting.d >= 1.0 && casting.d < 2.0); + return casting.d - 1.0; +} + +uint64_t prng64_white(uint64_t &state); +uint32_t prng32(uint64_t &state); +void prng_fill(uint64_t &state, void *ptr, size_t bytes); + +void prng_seed(uint64_t seed); +uint32_t prng32(void); +uint64_t prng64(void); +void prng_fill(void *ptr, size_t bytes); + +bool flipcoin(); +bool flipcoin_x2(); +bool flipcoin_x3(); +bool flipcoin_x4(); +bool jitter(unsigned probability_percent); +void jitter_delay(bool extra = false); diff --git a/src/contrib/db/libmdbx/test/valgrind_suppress.txt b/src/contrib/db/libmdbx/test/valgrind_suppress.txt new file mode 100644 index 0000000..98309ce --- /dev/null +++ b/src/contrib/db/libmdbx/test/valgrind_suppress.txt @@ -0,0 +1,24 @@ +{ + msync-whole-mmap-1 + Memcheck:Param + msync(start) + fun:msync + ... + fun:mdbx_sync_locked +} +{ + msync-whole-mmap-2 + Memcheck:Param + msync(start) + fun:msync + ... + fun:mdbx_env_sync_ex +} +{ + pwrite-page-flush + Memcheck:Param + pwritev(vector[...]) + fun:pwritev + ... + fun:mdbx_page_flush +} diff --git a/src/contrib/eos_portable_archive/eos/portable_archive_exception.hpp b/src/contrib/eos_portable_archive/eos/portable_archive_exception.hpp index 719fc00..5af9c80 100644 --- a/src/contrib/eos_portable_archive/eos/portable_archive_exception.hpp +++ b/src/contrib/eos_portable_archive/eos/portable_archive_exception.hpp @@ -35,11 +35,14 @@ namespace eos { // version of the linked boost archive library const archive_version_type archive_version( - #if BOOST_VERSION < 103700 - boost::archive::ARCHIVE_VERSION() - #else - boost::archive::BOOST_ARCHIVE_VERSION() - #endif + + 100 // explicitly fix archive version to make it independent from Boost version + + //#if BOOST_VERSION < 103700 + // boost::archive::ARCHIVE_VERSION() + //#else + // boost::archive::BOOST_ARCHIVE_VERSION() + //#endif ); /** diff --git a/src/contrib/eos_portable_archive/eos/portable_iarchive.hpp b/src/contrib/eos_portable_archive/eos/portable_iarchive.hpp index 6f60c93..cd087f3 100644 --- a/src/contrib/eos_portable_archive/eos/portable_iarchive.hpp +++ b/src/contrib/eos_portable_archive/eos/portable_iarchive.hpp @@ -120,20 +120,28 @@ #elif BOOST_VERSION < 106900 #include #include +#elif BOOST_VERSION >= 106900 +# define BOOST_MATH_DISABLE_STD_FPCLASSIFY +# include +# include #else -#include +# include +# include #endif - // namespace alias -#if BOOST_VERSION < 103800 || BOOST_VERSION >= 106900 +#if BOOST_VERSION < 103800 namespace fp = boost::math; +#elif BOOST_VERSION >= 106900 +namespace fp = boost::math; #else -namespace fp = boost::spirit::math; +namespace fp = boost::spirit::math; #endif // namespace alias endian #if BOOST_VERSION < 104800 namespace endian = boost::detail; +#elif BOOST_VERSION >= 106900 +namespace endian = boost::endian; #else namespace endian = boost::spirit::detail; #endif @@ -352,7 +360,11 @@ namespace eos { // load the value from little endian - it is then converted // to the target type T and fits it because size <= sizeof(T) +#if BOOST_VERSION >= 106900 + t = endian::little_to_native(temp); +#else t = endian::load_little_endian(&temp); +#endif } else t = 0; // zero optimization diff --git a/src/contrib/eos_portable_archive/eos/portable_oarchive.hpp b/src/contrib/eos_portable_archive/eos/portable_oarchive.hpp index 20d4c29..fce2bd2 100644 --- a/src/contrib/eos_portable_archive/eos/portable_oarchive.hpp +++ b/src/contrib/eos_portable_archive/eos/portable_oarchive.hpp @@ -123,20 +123,29 @@ #elif BOOST_VERSION < 106900 #include #include +#elif BOOST_VERSION >= 106900 +#define BOOST_MATH_DISABLE_STD_FPCLASSIFY +#include +#include #else #include +#include #endif // namespace alias fp_classify -#if BOOST_VERSION < 103800 || BOOST_VERSION >= 106900 +#if BOOST_VERSION < 103800 namespace fp = boost::math; +#elif BOOST_VERSION >= 106900 +namespace fp = boost::math; #else -namespace fp = boost::spirit::math; +namespace fp = boost::spirit::math; #endif // namespace alias endian #if BOOST_VERSION < 104800 namespace endian = boost::detail; +#elif BOOST_VERSION >= 106900 +namespace endian = boost::endian; #else namespace endian = boost::spirit::detail; #endif @@ -330,7 +339,11 @@ namespace eos { // we choose to use little endian because this way we just // save the first size bytes to the stream and skip the rest - endian::store_little_endian(&temp, t); +#if BOOST_VERSION >= 106900 + temp = endian::native_to_little(temp); +#else + endian::store_little_endian(&temp, t); +#endif save_binary(&temp, size); } // zero optimization @@ -388,7 +401,11 @@ namespace eos { switch (fp::fpclassify(t)) { //case FP_ZERO: bits = 0; break; - case FP_NAN: bits = traits::exponent | traits::mantissa; break; +#if BOOST_VERSION >= 106900 + case FP_NAN: bits = traits::exponent | traits::significand; break; +#else + case FP_NAN: bits = traits::exponent | traits::mantissa; break; +#endif case FP_INFINITE: bits = traits::exponent | (t<0) * traits::sign; break; case FP_SUBNORMAL: assert(std::numeric_limits::has_denorm); // pass case FP_ZERO: // note that floats can be ±0.0 diff --git a/src/contrib/epee/include/console_handler.h b/src/contrib/epee/include/console_handler.h index ef13169..4887c7c 100644 --- a/src/contrib/epee/include/console_handler.h +++ b/src/contrib/epee/include/console_handler.h @@ -422,7 +422,6 @@ namespace epee std::string get_usage() { std::stringstream ss; - ss << "Commands: " << ENDL; size_t max_command_len = 0; for(auto& x:m_command_handlers) if(x.first.size() > max_command_len) @@ -430,8 +429,7 @@ namespace epee for(auto& x:m_command_handlers) { - ss.width(max_command_len + 3); - ss << " " << std::left << x.first << " " << x.second.second << ENDL; + ss << " " << std::left << std::setw(max_command_len + 3) << x.first << " " << x.second.second << ENDL; } return ss.str(); } @@ -480,7 +478,7 @@ namespace epee bool run_handling(const std::string& prompt, const std::string& usage_string) { - return m_console_handler.run(boost::bind(&console_handlers_binder::process_command_str, this, _1), prompt, usage_string); + return m_console_handler.run(boost::bind(&console_handlers_binder::process_command_str, this, boost::placeholders::_1), prompt, usage_string); } bool help(const std::vector& /*args*/) diff --git a/src/contrib/epee/include/file_io_utils.h b/src/contrib/epee/include/file_io_utils.h index cb4e0a2..7ece78b 100644 --- a/src/contrib/epee/include/file_io_utils.h +++ b/src/contrib/epee/include/file_io_utils.h @@ -51,7 +51,8 @@ #include #endif -#include "include_base_utils.h" +//#include "include_base_utils.h" +#include "string_coding.h" namespace epee { @@ -78,7 +79,7 @@ namespace file_io_utils #ifdef BOOST_LEXICAL_CAST_INCLUDED inline - bool get_not_used_filename(const std::string& folder, OUT std::string& result_name) + bool get_not_used_filename(const std::string& folder, std::string& result_name) { DWORD folder_attr = ::GetFileAttributesA(folder.c_str()); if(folder_attr == INVALID_FILE_ATTRIBUTES) @@ -222,13 +223,23 @@ namespace file_io_utils return str_result; } #endif + + inline const std::wstring& convert_utf8_to_wstring_if_needed(const std::wstring& s) + { + return s; + } + + inline std::wstring convert_utf8_to_wstring_if_needed(const std::string& s) + { + return epee::string_encoding::utf8_to_wstring(s); + } template bool is_file_exist(const t_string& path) - { - boost::filesystem::path p(path); - return boost::filesystem::exists(p); - } + { + boost::filesystem::path p(convert_utf8_to_wstring_if_needed(path)); + return boost::filesystem::exists(p); + } /* inline @@ -261,19 +272,18 @@ namespace file_io_utils template bool save_string_to_file_throw(const t_string& path_to_file, const std::string& str) { - //std::ofstream fstream; - boost::filesystem::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); - fstream << str; - fstream.close(); - return true; + //std::ofstream fstream; + boost::filesystem::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(convert_utf8_to_wstring_if_needed(path_to_file), std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); + fstream << str; + fstream.close(); + return true; } template bool save_string_to_file(const t_string& path_to_file, const std::string& str) { - try { return save_string_to_file_throw(path_to_file, str); @@ -289,6 +299,39 @@ namespace file_io_utils } } + template + bool load_file_to_string(const t_string& path_to_file, std::string& target_str) + { + try + { + boost::filesystem::ifstream fstream; + //fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(convert_utf8_to_wstring_if_needed(path_to_file), std::ios_base::binary | std::ios_base::in | std::ios::ate); + if (!fstream.good()) + return false; + std::ifstream::pos_type file_size = fstream.tellg(); + + if (file_size > 1000000000) + return false;//don't get crazy + size_t file_size_t = static_cast(file_size); + + target_str.resize(file_size_t); + + fstream.seekg(0, std::ios::beg); + fstream.read((char*)target_str.data(), target_str.size()); + if (!fstream.good()) + return false; + + fstream.close(); + return true; + } + catch (...) + { + return false; + } + } + + /* inline bool load_form_handle(HANDLE hfile, std::string& str) @@ -316,10 +359,10 @@ namespace file_io_utils } */ inline - bool get_file_time(const std::string& path_to_file, OUT time_t& ft) + bool get_file_time(const std::string& path_to_file, time_t& ft) { boost::system::error_code ec; - ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); + ft = boost::filesystem::last_write_time(epee::string_encoding::utf8_to_wstring(path_to_file), ec); if(!ec) return true; else @@ -330,7 +373,7 @@ namespace file_io_utils bool set_file_time(const std::string& path_to_file, const time_t& ft) { boost::system::error_code ec; - boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); + boost::filesystem::last_write_time(epee::string_encoding::utf8_to_wstring(path_to_file), ft, ec); if(!ec) return true; else @@ -338,38 +381,7 @@ namespace file_io_utils } - inline - bool load_file_to_string(const std::string& path_to_file, std::string& target_str) - { - try - { - std::ifstream fstream; - //fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - if (!fstream.good()) - return false; - std::ifstream::pos_type file_size = fstream.tellg(); - if(file_size > 1000000000) - return false;//don't go crazy - size_t file_size_t = static_cast(file_size); - - target_str.resize(file_size_t); - - fstream.seekg (0, std::ios::beg); - fstream.read((char*)target_str.data(), target_str.size()); - if (!fstream.good()) - return false; - - fstream.close(); - return true; - } - - catch(...) - { - return false; - } - } #ifdef WIN32 typedef HANDLE native_filesystem_handle; @@ -377,16 +389,18 @@ namespace file_io_utils typedef int native_filesystem_handle; #endif + // uses UTF-8 for unicode names for all systems inline bool open_and_lock_file(const std::string file_path, native_filesystem_handle& h_file) { #ifdef WIN32 - h_file = ::CreateFileA(file_path.c_str(), // name of the write - GENERIC_WRITE, // open for writing - 0, // do not share - NULL, // default security - OPEN_ALWAYS, // create new file only - FILE_ATTRIBUTE_NORMAL, // normal file - NULL); // no attr. template + std::wstring file_path_w = epee::string_encoding::utf8_to_wstring(file_path); + h_file = ::CreateFileW(file_path_w.c_str(), // name of the file + GENERIC_WRITE, // open for writing + 0, // do not share + NULL, // default security + OPEN_ALWAYS, // create new file only + FILE_ATTRIBUTE_NORMAL, // normal file + NULL); // no attr. template if (h_file == INVALID_HANDLE_VALUE) return false; else @@ -462,20 +476,21 @@ namespace file_io_utils bool copy_file(const std::string& source, const std::string& destination) { boost::system::error_code ec; - boost::filesystem::copy_file(source, destination, ec); + boost::filesystem::copy_file(epee::string_encoding::utf8_to_wstring(source), epee::string_encoding::utf8_to_wstring(destination), ec); if (ec) return false; else return true; } + inline bool append_string_to_file(const std::string& path_to_file, const std::string& str) { try { - std::ofstream fstream; + boost::filesystem::ofstream fstream; fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); + fstream.open(epee::string_encoding::utf8_to_wstring(path_to_file), std::ios_base::binary | std::ios_base::out | std::ios_base::app); fstream << str; fstream.close(); return true; @@ -523,7 +538,7 @@ namespace file_io_utils } */ #ifdef WINDOWS_PLATFORM - inline bool get_folder_content(const std::string& path, std::list& OUT target_list) + inline bool get_folder_content(const std::string& path, std::list& target_list) { WIN32_FIND_DATAA find_data = {0}; HANDLE hfind = ::FindFirstFileA((path + "\\*.*").c_str(), &find_data); @@ -541,13 +556,13 @@ namespace file_io_utils return true; } #endif - inline bool get_folder_content(const std::string& path, std::list& OUT target_list, bool only_files = false) + inline bool get_folder_content(const std::string& path, std::list& target_list, bool only_files = false) { try { boost::filesystem::directory_iterator end_itr; // default construction yields past-the-end - for ( boost::filesystem::directory_iterator itr( path ); itr != end_itr; ++itr ) + for ( boost::filesystem::directory_iterator itr( epee::string_encoding::utf8_to_wstring(path) ); itr != end_itr; ++itr ) { if ( only_files && boost::filesystem::is_directory(itr->status()) ) { diff --git a/src/contrib/epee/include/gzip_encoding.h b/src/contrib/epee/include/gzip_encoding.h index 2be51e7..e2e1ea2 100644 --- a/src/contrib/epee/include/gzip_encoding.h +++ b/src/contrib/epee/include/gzip_encoding.h @@ -29,9 +29,9 @@ #ifndef _GZIP_ENCODING_H_ #define _GZIP_ENCODING_H_ +#include "boost/core/ignore_unused.hpp" #include "net/http_client_base.h" #include "zlib/zlib.h" -//#include "http.h" namespace epee @@ -41,186 +41,329 @@ namespace net_utils - class content_encoding_gzip: public i_sub_handler - { - public: - /*! \brief - * Function content_encoding_gzip : Constructor - * - */ - inline - content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false):m_powner_filter(powner_filter), - m_is_stream_ended(false), - m_is_deflate_mode(is_deflate_mode), - m_is_first_update_in(true) - { - memset(&m_zstream_in, 0, sizeof(m_zstream_in)); - memset(&m_zstream_out, 0, sizeof(m_zstream_out)); - int ret = 0; - if(is_deflate_mode) - { - ret = inflateInit(&m_zstream_in); - ret = deflateInit(&m_zstream_out, Z_DEFAULT_COMPRESSION); - }else - { - ret = inflateInit2(&m_zstream_in, 0x1F); - ret = deflateInit2(&m_zstream_out, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY); - } - } - /*! \brief - * Function content_encoding_gzip : Destructor - * - */ - inline - ~content_encoding_gzip() - { - inflateEnd(& m_zstream_in ); - deflateEnd(& m_zstream_out ); - } - /*! \brief - * Function update_in : Entry point for income data - * - */ - inline - virtual bool update_in( std::string& piece_of_transfer) - { + class content_encoding_gzip: public i_sub_handler + { + public: + /*! \brief + * Function content_encoding_gzip : Constructor + * + */ + inline + content_encoding_gzip(i_target_handler* powner_filter, bool is_deflate_mode = false, int compression_level = Z_DEFAULT_COMPRESSION) :m_powner_filter(powner_filter), + m_is_stream_ended(false), + m_is_deflate_mode(is_deflate_mode), + m_is_first_update_in(true) + { + memset(&m_zstream_in, 0, sizeof(m_zstream_in)); + memset(&m_zstream_out, 0, sizeof(m_zstream_out)); + int ret = 0; + boost::ignore_unused(ret); + if(is_deflate_mode) + { + ret = inflateInit(&m_zstream_in); + ret = deflateInit(&m_zstream_out, compression_level); + }else + { + ret = inflateInit2(&m_zstream_in, 0x1F); + ret = deflateInit2(&m_zstream_out, compression_level, Z_DEFLATED, 0x1F, 8, Z_DEFAULT_STRATEGY); + } + } + /*! \brief + * Function content_encoding_gzip : Destructor + * + */ + inline + ~content_encoding_gzip() + { + inflateEnd(& m_zstream_in ); + deflateEnd(& m_zstream_out ); + } + /*! \brief + * Function update_in : Entry point for income data + * + */ + inline + virtual bool update_in( std::string& piece_of_transfer) + { - bool is_first_time_here = m_is_first_update_in; - m_is_first_update_in = false; + bool is_first_time_here = m_is_first_update_in; + m_is_first_update_in = false; - if(m_pre_decode.size()) - m_pre_decode += piece_of_transfer; - else - m_pre_decode.swap(piece_of_transfer); - piece_of_transfer.clear(); + if(m_pre_decode.size()) + m_pre_decode += piece_of_transfer; + else + m_pre_decode.swap(piece_of_transfer); + piece_of_transfer.clear(); - std::string decode_summary_buff; + std::string decode_summary_buff; - size_t ungzip_size = m_pre_decode.size() * 0x30; - std::string current_decode_buff(ungzip_size, 'X'); + size_t ungzip_size = m_pre_decode.size() * 0x30; + std::string current_decode_buff(ungzip_size, 'X'); + auto slh = misc_utils::create_scope_leave_handler([&]() { m_zstream_in.next_out = nullptr; } ); // make sure local pointer to current_decode_buff.data() won't be used out of this scope - //Here the cycle is introduced where we unpack the buffer, the cycle is required - //because of the case where if after unpacking the data will exceed the awaited size, we will not halt with error - bool continue_unpacking = true; - bool first_step = true; - while(m_pre_decode.size() && continue_unpacking) - { + //Here the cycle is introduced where we unpack the buffer, the cycle is required + //because of the case where if after unpacking the data will exceed the awaited size, we will not halt with error + bool continue_unpacking = true; + bool first_step = true; + while(m_pre_decode.size() && continue_unpacking) + { - //fill buffers - m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); - m_zstream_in.avail_in = (uInt)m_pre_decode.size(); - m_zstream_in.next_out = (Bytef*)current_decode_buff.data(); - m_zstream_in.avail_out = (uInt)ungzip_size; + //fill buffers + m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); + m_zstream_in.avail_in = (uInt)m_pre_decode.size(); + m_zstream_in.next_out = (Bytef*)current_decode_buff.data(); + m_zstream_in.avail_out = (uInt)ungzip_size; - int flag = Z_SYNC_FLUSH; - int ret = inflate(&m_zstream_in, flag); - CHECK_AND_ASSERT_MES(ret>=0 || m_zstream_in.avail_out ||m_is_deflate_mode, false, "content_encoding_gzip::update_in() Failed to inflate. err = " << ret); + int flag = Z_NO_FLUSH; + int ret = inflate(&m_zstream_in, flag); + CHECK_AND_ASSERT_MES(ret>=0 || m_zstream_in.avail_out ||m_is_deflate_mode, false, "content_encoding_gzip::update_in() Failed to inflate. ret = " << ret << ", msg: " << (m_zstream_in.msg ? m_zstream_in.msg : "")); - if(Z_STREAM_END == ret) - m_is_stream_ended = true; - else if(Z_DATA_ERROR == ret && is_first_time_here && m_is_deflate_mode&& first_step) - { - // some servers (notably Apache with mod_deflate) don't generate zlib headers - // insert a dummy header and try again - static char dummy_head[2] = - { - 0x8 + 0x7 * 0x10, - (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, - }; - inflateReset(&m_zstream_in); - m_zstream_in.next_in = (Bytef*) dummy_head; - m_zstream_in.avail_in = sizeof(dummy_head); + if(Z_STREAM_END == ret) + m_is_stream_ended = true; + else if(Z_DATA_ERROR == ret && is_first_time_here && m_is_deflate_mode&& first_step) + { + // some servers (notably Apache with mod_deflate) don't generate zlib headers + // insert a dummy header and try again + static char dummy_head[2] = + { + 0x8 + 0x7 * 0x10, + (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, + }; + inflateReset(&m_zstream_in); + m_zstream_in.next_in = (Bytef*) dummy_head; + m_zstream_in.avail_in = sizeof(dummy_head); - ret = inflate(&m_zstream_in, Z_NO_FLUSH); - if (ret != Z_OK) - { - LOCAL_ASSERT(0); - m_pre_decode.swap(piece_of_transfer); - return false; - } - m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); - m_zstream_in.avail_in = (uInt)m_pre_decode.size(); + ret = inflate(&m_zstream_in, Z_NO_FLUSH); + if (ret != Z_OK) + { + LOCAL_ASSERT(0); + m_pre_decode.swap(piece_of_transfer); + return false; + } + m_zstream_in.next_in = (Bytef*)m_pre_decode.data(); + m_zstream_in.avail_in = (uInt)m_pre_decode.size(); - ret = inflate(&m_zstream_in, Z_NO_FLUSH); - if (ret != Z_OK) - { - LOCAL_ASSERT(0); - m_pre_decode.swap(piece_of_transfer); - return false; - } - } + ret = inflate(&m_zstream_in, Z_NO_FLUSH); + if (ret != Z_OK) + { + LOCAL_ASSERT(0); + m_pre_decode.swap(piece_of_transfer); + return false; + } + } + else + { + CHECK_AND_ASSERT_MES(Z_DATA_ERROR != ret, false, "content_encoding_gzip::update_in() Failed to inflate. err = Z_DATA_ERROR"); + } - //leave only unpacked part in the output buffer to start with it the next time - m_pre_decode.erase(0, m_pre_decode.size()-m_zstream_in.avail_in); - //if decoder gave nothing to return, then everything is ahead, now simply break - if(ungzip_size == m_zstream_in.avail_out) - break; + //leave only unpacked part in the output buffer to start with it the next time + m_pre_decode.erase(0, m_pre_decode.size()-m_zstream_in.avail_in); + //if decoder gave nothing to return, then everything is ahead, now simply break + if(ungzip_size == m_zstream_in.avail_out) + break; - //decode_buff currently stores data parts that were unpacked, fix this size - current_decode_buff.resize(ungzip_size - m_zstream_in.avail_out); - if(decode_summary_buff.size()) - decode_summary_buff += current_decode_buff; - else - current_decode_buff.swap(decode_summary_buff); + //decode_buff currently stores data parts that were unpacked, fix this size + current_decode_buff.resize(ungzip_size - m_zstream_in.avail_out); + if(decode_summary_buff.size()) + decode_summary_buff += current_decode_buff; + else + current_decode_buff.swap(decode_summary_buff); - current_decode_buff.resize(ungzip_size); - first_step = false; - } + current_decode_buff.resize(ungzip_size); + first_step = false; + } - //Process these data if required - bool res = true; + //Process these data if required + return m_powner_filter->handle_target_data(decode_summary_buff); + } + /*! \brief + * Function stop : Entry point for stop signal and flushing cached data buffer. + * + */ + inline + virtual void stop(std::string& OUT collect_remains) + { + } + protected: + private: + /*! \brief + * Pointer to parent HTTP-parser + */ + i_target_handler* m_powner_filter; + /*! \brief + * ZLIB object for income stream + */ + z_stream m_zstream_in; + /*! \brief + * ZLIB object for outcome stream + */ + z_stream m_zstream_out; + /*! \brief + * Data that could not be unpacked immediately, left to wait for the next packet of data + */ + std::string m_pre_decode; + /*! \brief + * The data are accumulated for a package in the buffer to send the web client + */ + std::string m_pre_encode; + /*! \brief + * Signals that stream looks like ended + */ + bool m_is_stream_ended; + /*! \brief + * If this flag is set, income data is in HTTP-deflate mode + */ + bool m_is_deflate_mode; + /*! \brief + * Marks that it is a first data packet + */ + bool m_is_first_update_in; + }; // class content_encoding_gzip - res = m_powner_filter->handle_target_data(decode_summary_buff); + struct abstract_callback_base + { + virtual bool do_call(const std::string& piece_of_transfer) = 0; + virtual ~abstract_callback_base() {} + }; - return true; + template + struct abstract_callback : public abstract_callback_base + { + callback_t m_cb; - } - /*! \brief - * Function stop : Entry point for stop signal and flushing cached data buffer. - * - */ - inline - virtual void stop(std::string& OUT collect_remains) - { - } - protected: - private: - /*! \brief - * Pointer to parent HTTP-parser - */ - i_target_handler* m_powner_filter; - /*! \brief - * ZLIB object for income stream - */ - z_stream m_zstream_in; - /*! \brief - * ZLIB object for outcome stream - */ - z_stream m_zstream_out; - /*! \brief - * Data that could not be unpacked immediately, left to wait for the next packet of data - */ - std::string m_pre_decode; - /*! \brief - * The data are accumulated for a package in the buffer to send the web client - */ - std::string m_pre_encode; - /*! \brief - * Signals that stream looks like ended - */ - bool m_is_stream_ended; - /*! \brief - * If this flag is set, income data is in HTTP-deflate mode - */ - bool m_is_deflate_mode; - /*! \brief - * Marks that it is a first data packet - */ - bool m_is_first_update_in; - }; -} -} + abstract_callback(callback_t cb) : m_cb(cb){} + virtual bool do_call(const std::string& piece_of_transfer) + { + return m_cb(piece_of_transfer); + } + }; + + class gzip_decoder_lambda : public content_encoding_gzip, + public i_target_handler + { + std::shared_ptr m_pcb; + + virtual bool handle_target_data(std::string& piece_of_transfer) + { + bool r = m_pcb->do_call(piece_of_transfer); + piece_of_transfer.clear(); + return r; + } + public: + gzip_decoder_lambda() : content_encoding_gzip(this, true, Z_BEST_COMPRESSION) + {} + + template + bool update_in(std::string& piece_of_transfer, callback_t cb) + { + m_pcb.reset(new abstract_callback(cb)); + return content_encoding_gzip::update_in(piece_of_transfer); + } + template + bool stop(callback_t cb) + {return true;} + }; // class gzip_decoder_lambda + + class gzip_encoder_lyambda + { + bool m_initialized; + z_stream m_zstream; + public: + gzip_encoder_lyambda(int compression_level = Z_DEFAULT_COMPRESSION) :m_initialized(false), m_zstream(AUTO_VAL_INIT(m_zstream)) + { + int ret = deflateInit(&m_zstream, compression_level); + if (ret == Z_OK) + m_initialized = true; + } + + ~gzip_encoder_lyambda() + { + deflateEnd(&m_zstream); + } + + template + bool update_in(const std::string& target, callback_t cb) + { + if (!m_initialized) + { + return false; + } + + if (!target.size()) + { + return true; + } + + std::string result_packed_buff; + auto slh = misc_utils::create_scope_leave_handler([&]() { m_zstream.next_out = nullptr; } ); // make sure local pointer to result_packed_buff.data() won't be used out of this scope + + //theoretically it supposed to be smaller + result_packed_buff.resize(target.size(), 'X'); + while (true) + { + m_zstream.next_in = (Bytef*)target.data(); + m_zstream.avail_in = (uInt)target.size(); + m_zstream.next_out = (Bytef*)result_packed_buff.data(); + m_zstream.avail_out = (uInt)result_packed_buff.size(); + + int ret = deflate(&m_zstream, Z_NO_FLUSH); + CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate. err = " << ret); + if (m_zstream.avail_out == 0) + { + //twice bigger buffer + result_packed_buff.resize(result_packed_buff.size()*2); + continue; + } + CHECK_AND_ASSERT_MES(result_packed_buff.size() >= m_zstream.avail_out, false, "result_packed_buff.size()=" << result_packed_buff.size() << " >= m_zstream.avail_out=" << m_zstream.avail_out); + result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out); + break; + } + + return cb(result_packed_buff); + } + + template + bool stop(callback_t cb) + { + if (!m_initialized) + { + return false; + } + + std::string result_packed_buff; + //theoretically it supposed to be smaller + result_packed_buff.resize(1000000, 'X'); + while (true) + { + m_zstream.next_in = nullptr; + m_zstream.avail_in = 0; + m_zstream.next_out = (Bytef*)result_packed_buff.data(); + m_zstream.avail_out = (uInt)result_packed_buff.size(); + + int ret = deflate(&m_zstream, Z_FINISH); + CHECK_AND_ASSERT_MES(ret >= 0, false, "Failed to deflate at finish. err = " << ret); + if (ret != Z_STREAM_END) + { + //twice bigger buffer + result_packed_buff.resize(result_packed_buff.size() * 2); + continue; + } + + CHECK_AND_ASSERT_MES(result_packed_buff.size() >= m_zstream.avail_out, false, "result_packed_buff.size()=" << result_packed_buff.size() << " >= m_zstream.avail_out=" << m_zstream.avail_out); + result_packed_buff.resize(result_packed_buff.size() - m_zstream.avail_out); + + m_initialized = false; + break; + } + + return cb(result_packed_buff); + } + + }; // class gzip_encoder_lyambda + +} // namespace net_utils +} // namespace epee diff --git a/src/contrib/epee/include/math_helper.h b/src/contrib/epee/include/math_helper.h index dc7d74d..9ae935a 100644 --- a/src/contrib/epee/include/math_helper.h +++ b/src/contrib/epee/include/math_helper.h @@ -32,7 +32,7 @@ #include #include -#include +#include #include #include @@ -240,7 +240,7 @@ namespace math_helper } } -PUSH_WARNINGS +PUSH_GCC_WARNINGS DISABLE_GCC_WARNING(strict-aliasing) inline uint64_t generated_random_uint64() @@ -248,7 +248,7 @@ DISABLE_GCC_WARNING(strict-aliasing) boost::uuids::uuid id___ = boost::uuids::random_generator()(); return *reinterpret_cast(&id___.data[0]); //(*reinterpret_cast(&id___.data[0]) ^ *reinterpret_cast(&id___.data[8])); } -POP_WARNINGS +POP_GCC_WARNINGS template class once_a_time_seconds { diff --git a/src/contrib/epee/include/misc_helpers.h b/src/contrib/epee/include/misc_helpers.h new file mode 100644 index 0000000..dd3067c --- /dev/null +++ b/src/contrib/epee/include/misc_helpers.h @@ -0,0 +1,116 @@ +// Copyright (c) 2019, Zano Project +// Copyright (c) 2006-2019, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +#pragma once + +#define COMBINE1(X,Y) X##Y // helper macro +#define COMBINE(X,Y) COMBINE1(X,Y) +#define _STR(X) #X +#define STR(X) _STR(X) + +#if defined(_MSC_VER) +#define LOCAL_FUNCTION_DEF__ __FUNCTION__ +#define UNUSED_ATTRIBUTE +#else +#define LOCAL_FUNCTION_DEF__ __FUNCTION__ +#define UNUSED_ATTRIBUTE __attribute__((unused)) +#endif + +#define LOCATION_SS "[" << LOCAL_FUNCTION_DEF__ << ("] @ " __FILE__ ":" STR(__LINE__)) +#define LOCATION_STR (std::string("[") + LOCAL_FUNCTION_DEF__ + "] @ " __FILE__ ":" STR(__LINE__)) + + +// +// Try-catch helpers +// + +#define TRY_ENTRY() try { +#define CATCH_ALL_DO_NOTHING() }catch(...) {} + +#define CATCH_ENTRY_CUSTOM(location, custom_code, return_val) } \ + catch(const std::exception& ex) \ +{ \ + (void)(ex); \ + LOG_ERROR("Exception at [" << location << "], what=" << ex.what()); \ + custom_code; \ + return return_val; \ +} \ + catch(...) \ +{ \ + LOG_ERROR("Exception at [" << location << "], generic exception \"...\""); \ + custom_code; \ + return return_val; \ +} +#define CATCH_ENTRY(location, return_val) CATCH_ENTRY_CUSTOM(location, (void)0, return_val) +#define CATCH_ENTRY2(return_val) CATCH_ENTRY_CUSTOM(LOCATION_SS, (void)0, return_val) + +#define CATCH_ENTRY_L0(location, return_val) CATCH_ENTRY(location, return_val) +#define CATCH_ENTRY_L1(location, return_val) CATCH_ENTRY(location, return_val) +#define CATCH_ENTRY_L2(location, return_val) CATCH_ENTRY(location, return_val) +#define CATCH_ENTRY_L3(location, return_val) CATCH_ENTRY(location, return_val) +#define CATCH_ENTRY_L4(location, return_val) CATCH_ENTRY(location, return_val) + +/// @brief Catches TRY_ENTRY without returning +/// @details Useful within a dtor - but only if nested within another try block +/// (since we can still potentially throw here). See NESTED_*ENTRY() +/// @todo Exception dispatcher class +#define CATCH_ENTRY_NO_RETURN_CUSTOM(location, custom_code) } \ + catch(const std::exception& ex) \ +{ \ + (void)(ex); \ + LOG_ERROR("Exception at [" << location << "], what=" << ex.what()); \ + custom_code; \ +} \ + catch(...) \ +{ \ + LOG_ERROR("Exception at [" << location << "], generic exception \"...\""); \ + custom_code; \ +} + +#define CATCH_ENTRY_NO_RETURN() CATCH_ENTRY_NO_RETURN_CUSTOM(LOCATION_SS, (void)0) + +#define CATCH_ENTRY_WITH_FORWARDING_EXCEPTION() } \ + catch(const std::exception& ex) \ +{ \ + LOG_ERROR("Exception at [" << LOCATION_SS << "], what=" << ex.what()); \ + throw std::runtime_error(std::string("[EXCEPTION FORWARDED]: ") + ex.what()); \ +} \ + catch(...) \ +{ \ + LOG_ERROR("Exception at [" << LOCATION_SS << "], generic unknown exception \"...\""); \ + throw std::runtime_error("[EXCEPTION FORWARDED]"); \ +} + + + +#define NESTED_TRY_ENTRY() try { TRY_ENTRY(); + +#define NESTED_CATCH_ENTRY(location) \ + CATCH_ENTRY_NO_RETURN_CUSTOM(location, {}); \ + } catch (...) {} + + + diff --git a/src/contrib/epee/include/misc_language.h b/src/contrib/epee/include/misc_language.h index b70052c..1a02c7c 100644 --- a/src/contrib/epee/include/misc_language.h +++ b/src/contrib/epee/include/misc_language.h @@ -1,3 +1,4 @@ +// Copyright (c) 2019, anonimal, // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -76,6 +77,32 @@ namespace epee namespace misc_utils { + template + struct triple + { // store a pair of values + typedef _Ty1 first_type; + typedef _Ty2 second_type; + typedef _Ty3 third_type; + + triple() + : first(), second(), third() + { // default construct + } + + triple(const _Ty1& _Val1, const _Ty2& _Val2, const _Ty3& _Val3) + : first(_Val1), second(_Val2), third(_Val3) + { // construct from specified values + } + + _Ty1 first; // the first stored value + _Ty2 second; // the second stored value + _Ty3 third; // the second stored value + }; + + + template t_type get_max_t_val(t_type t) { @@ -313,7 +340,11 @@ namespace misc_utils {} ~call_befor_die() { + NESTED_TRY_ENTRY(); + m_func(); + + NESTED_CATCH_ENTRY(__func__); } }; @@ -388,23 +419,42 @@ namespace misc_utils auto res = container.insert(typename t_container_type::value_type(key, AUTO_VAL_INIT(typename t_container_type::mapped_type()))); return res.first->second; - } -} -} + } + + template + typename t_container_type::iterator it_get_or_insert_value_initialized(t_container_type& container, const typename t_container_type::key_type& key) + { + auto it = container.find(key); + if (it != container.end()) + { + return it; + } + + auto res = container.insert(typename t_container_type::value_type(key, AUTO_VAL_INIT(typename t_container_type::mapped_type()))); + return res.first; + } + +} // namespace misc_utils +} // namespace epee template std::ostream& print_container_content(std::ostream& out, const T& v); -template -std::ostream& operator<< (std::ostream& out, const std::vector& v) +namespace std { - return print_container_content(out, v); -} -template -std::ostream& operator<< (std::ostream& out, const std::list& v) -{ - return print_container_content(out, v); -} + template + std::ostream& operator<< (std::ostream& out, const std::vector& v) + { + return print_container_content(out, v); + } + + template + std::ostream& operator<< (std::ostream& out, const std::list& v) + { + return print_container_content(out, v); + } + +} // namespace std template std::ostream& print_container_content(std::ostream& out, const T& v) diff --git a/src/contrib/epee/include/misc_log_ex.h b/src/contrib/epee/include/misc_log_ex.h index 412cb50..e0a5861 100644 --- a/src/contrib/epee/include/misc_log_ex.h +++ b/src/contrib/epee/include/misc_log_ex.h @@ -1,3 +1,5 @@ +// Copyright (c) 2019, Zano Project +// Copyright (c) 2019, anonimal // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -44,6 +46,7 @@ #include #include #include +#include #include #endif #if defined(WIN32) @@ -53,18 +56,20 @@ #endif #include "os_defenitions.h" #include "warnings.h" -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4100) -#include "static_initializer.h" +#include "misc_helpers.h" +#include "static_helpers.h" #include "string_tools.h" #include "time_helper.h" #include "misc_os_dependent.h" #include "syncobj.h" #include "sync_locked_object.h" - +#include "string_coding.h" +#include "file_io_utils.h" #define LOG_LEVEL_SILENT -1 #define LOG_LEVEL_0 0 @@ -103,22 +108,145 @@ DISABLE_VS_WARNINGS(4100) #endif -#define COMBINE1(X,Y) X##Y // helper macro -#define COMBINE(X,Y) COMBINE1(X,Y) -#define _STR(X) #X -#define STR(X) _STR(X) +#if !defined(DISABLE_RELEASE_LOGGING) + #define ENABLE_LOGGING_INTERNAL +#endif -#if defined(_MSC_VER) -#define LOCAL_FUNCTION_DEF__ __FUNCTION__ -#define UNUSED_ATTRIBUTE -#else -#define LOCAL_FUNCTION_DEF__ __PRETTY_FUNCTION__ -#define UNUSED_ATTRIBUTE __attribute__((unused)) -#endif +#define LOG_DEFAULT_CHANNEL NULL -#define LOCATION_SS "[" << LOCAL_FUNCTION_DEF__ << ("] @ " __FILE__ ":" STR(__LINE__)) +#define ENABLE_CHANNEL_BY_DEFAULT(ch_name) \ + static bool COMBINE(init_channel, __LINE__) UNUSED_ATTRIBUTE = epee::misc_utils::static_initializer([](){ \ + epee::log_space::log_singletone::enable_channel(ch_name); return true; \ +}); + +#if defined(ENABLE_LOGGING_INTERNAL) + +#define LOG_PRINT_CHANNEL_NO_PREFIX2(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ + {TRY_ENTRY();std::stringstream ss________; ss________ << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str() , y, epee::log_space::console_color_default, false, log_name);CATCH_ALL_DO_NOTHING();}} + +#define LOG_PRINT_CHANNEL_NO_PREFIX_NO_POSTFIX2(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ + {TRY_ENTRY();std::stringstream ss________; ss________ << x; epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, false, log_name);CATCH_ALL_DO_NOTHING();}} + +#define LOG_PRINT_CHANNEL_NO_POSTFIX2(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ + {TRY_ENTRY();std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x; epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, false, log_name);CATCH_ALL_DO_NOTHING();}} + +#define LOG_PRINT_CHANNEL2(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ + {TRY_ENTRY();std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, false, log_name);CATCH_ALL_DO_NOTHING();}} + +#define LOG_PRINT_CHANNEL_COLOR2(log_channel, log_name, x, y, color) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ + {TRY_ENTRY();std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, color, false, log_name);CATCH_ALL_DO_NOTHING();}} + +#define LOG_PRINT_CHANNEL_2_JORNAL(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ + {TRY_ENTRY();std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, true, log_name);CATCH_ALL_DO_NOTHING();}} + +#define LOG_ERROR2(log_name, x) { \ + TRY_ENTRY();std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "[ERROR] Location: " << std::endl << LOCATION_SS << epee::misc_utils::get_callstack() << " Message:" << std::endl << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), LOG_LEVEL_0, epee::log_space::console_color_red, true, log_name); LOCAL_ASSERT(0); epee::log_space::increase_error_count(LOG_DEFAULT_CHANNEL);CATCH_ALL_DO_NOTHING();} + +#define LOG_FRAME2(log_name, x, y) epee::log_space::log_frame frame(x, y, log_name) + +#else // #if defined(ENABLE_LOGGING_INTERNAL) + +#define LOG_PRINT_NO_PREFIX2(log_name, x, y) +#define LOG_PRINT_NO_PREFIX_NO_POSTFIX2(log_name, x, y) +#define LOG_PRINT_NO_POSTFIX2(log_name, x, y) +#define LOG_PRINT_COLOR2(log_name, x, y, color) +#define LOG_PRINT2_JORNAL(log_name, x, y) +#define LOG_PRINT2(log_name, x, y) +#define LOG_ERROR2(log_name, x) +#define LOG_FRAME2(log_name, x, y) + +#endif // #if defined(ENABLE_LOGGING_INTERNAL) + +#define LOG_PRINT_NO_PREFIX2(log_name, x, y) LOG_PRINT_CHANNEL_NO_PREFIX2(LOG_DEFAULT_CHANNEL, log_name, x, y) +#define LOG_PRINT_NO_PREFIX_NO_POSTFIX2(log_name, x, y) LOG_PRINT_CHANNEL_NO_PREFIX_NO_POSTFIX2(LOG_DEFAULT_CHANNEL, log_name, x, y) +#define LOG_PRINT_NO_POSTFIX2(log_name, x, y) LOG_PRINT_CHANNEL_NO_POSTFIX2(LOG_DEFAULT_CHANNEL, log_name, x, y) +#define LOG_PRINT2(log_name, x, y) LOG_PRINT_CHANNEL2(LOG_DEFAULT_CHANNEL, log_name, x, y) +#define LOG_PRINT_COLOR2(log_name, x, y, color) LOG_PRINT_CHANNEL_COLOR2(LOG_DEFAULT_CHANNEL, log_name, x, y, color) +#define LOG_PRINT2_JORNAL(log_name, x, y) LOG_PRINT_CHANNEL_2_JORNAL(LOG_DEFAULT_CHANNEL, log_name, x, y) + +#ifndef LOG_DEFAULT_TARGET + #define LOG_DEFAULT_TARGET NULL +#endif + +#define LOG_PRINT_NO_POSTFIX(mess, level) LOG_PRINT_NO_POSTFIX2(LOG_DEFAULT_TARGET, mess, level) +#define LOG_PRINT_NO_PREFIX(mess, level) LOG_PRINT_NO_PREFIX2(LOG_DEFAULT_TARGET, mess, level) +#define LOG_PRINT_NO_PREFIX_NO_POSTFIX(mess, level) LOG_PRINT_NO_PREFIX_NO_POSTFIX2(LOG_DEFAULT_TARGET, mess, level) +#define LOG_PRINT(mess, level) LOG_PRINT2(LOG_DEFAULT_TARGET, mess, level) + +#define LOG_PRINT_COLOR(mess, level, color) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, color) +#define LOG_PRINT_RED(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_red) +#define LOG_PRINT_GREEN(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_green) +#define LOG_PRINT_BLUE(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_blue) +#define LOG_PRINT_YELLOW(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_yellow) +#define LOG_PRINT_CYAN(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_cyan) +#define LOG_PRINT_MAGENTA(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_magenta) + +#define LOG_PRINT_RED_L0(mess) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, LOG_LEVEL_0, epee::log_space::console_color_red) + +#define LOG_PRINT_L0(mess) LOG_PRINT(mess, LOG_LEVEL_0) +#define LOG_PRINT_L1(mess) LOG_PRINT(mess, LOG_LEVEL_1) +#define LOG_PRINT_L2(mess) LOG_PRINT(mess, LOG_LEVEL_2) +#define LOG_PRINT_L3(mess) LOG_PRINT(mess, LOG_LEVEL_3) +#define LOG_PRINT_L4(mess) LOG_PRINT(mess, LOG_LEVEL_4) +#define LOG_PRINT_J(mess, level) LOG_PRINT2_JORNAL(LOG_DEFAULT_TARGET, mess, level) + +#define LOG_ERROR(mess) LOG_ERROR2(LOG_DEFAULT_TARGET, mess) +#define LOG_FRAME(mess, level) LOG_FRAME2(LOG_DEFAULT_TARGET, mess, level) +#define LOG_VALUE(mess, level) LOG_VALUE2(LOG_DEFAULT_TARGET, mess, level) +#define LOG_ARRAY(mess, level) LOG_ARRAY2(LOG_DEFAULT_TARGET, mess, level) +//#define LOGWIN_PLATFORM_ERROR(err_no) LOGWINDWOS_PLATFORM_ERROR2(LOG_DEFAULT_TARGET, err_no) +#define LOG_SOCKET_ERROR(err_no) LOG_SOCKET_ERROR2(LOG_DEFAULT_TARGET, err_no) +//#define LOGWIN_PLATFORM_ERROR_UNCRITICAL(mess) LOGWINDWOS_PLATFORM_ERROR_UNCRITICAL2(LOG_DEFAULT_TARGET, mess) + +#define ENDL std::endl + + +#define ASSERT_MES_AND_THROW(message) {LOG_ERROR(message); std::stringstream ss; ss << message; throw std::runtime_error(ss.str());} + +#define CHECK_AND_ASSERT_THROW_MES(expr, message) {if(!(expr)) ASSERT_MES_AND_THROW(message << ENDL << "thrown from " << LOCATION_SS);} +#define CHECK_AND_ASSERT_THROW(expr, exception_exp) {if(!(expr)) {LOG_ERROR("EXCEPTION is thrown from " << LOCATION_SS); throw exception_exp; };} + +#ifndef CHECK_AND_ASSERT +#define CHECK_AND_ASSERT(expr, fail_ret_val) do{if(!(expr)){LOCAL_ASSERT(expr); return fail_ret_val;};}while(0) +#endif + +#define NOTHING + +#ifndef CHECK_AND_ASSERT_MES +#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_ERROR(message); return fail_ret_val;};}while(0) +#endif + +#ifndef CHECK_AND_FORCE_ASSERT_MES +#define CHECK_AND_FORCE_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_ERROR(message); FORCE_ASSERT(expr); return fail_ret_val;};}while(0) +#endif + +#ifndef CHECK_AND_ASSERT_MES_CUSTOM +#define CHECK_AND_ASSERT_MES_CUSTOM(expr, fail_ret_val, custom_code, message) do{if(!(expr)) {LOG_ERROR(message); custom_code; return fail_ret_val;};}while(0) +#endif + +/*#ifndef CHECK_AND_ASSERT_MES_AND_THROW +#define CHECK_AND_ASSERT_MES_AND_THROW(expr, message) do{if(!(expr)) {LOG_ERROR(message); throw std::runtime_error(message);};}while(0) +#endif +*/ + +#ifndef CHECK_AND_NO_ASSERT_MES +#define CHECK_AND_NO_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_PRINT_MAGENTA(message, LOG_LEVEL_0); /*LOCAL_ASSERT(expr);*/ return fail_ret_val;};}while(0) +#endif + +#ifndef CHECK_AND_NO_ASSERT_MES_LEVEL +#define CHECK_AND_NO_ASSERT_MES_LEVEL(expr, fail_ret_val, message, log_level) do{if(!(expr)) {LOG_PRINT(message, log_level); return fail_ret_val;};}while(0) +#endif + +#ifndef CHECK_AND_ASSERT_MES_NO_RET +#define CHECK_AND_ASSERT_MES_NO_RET(expr, message) do{if(!(expr)) {LOG_ERROR(message);};}while(0) +#endif + +#ifndef CHECK_AND_ASSERT_MES2 +#define CHECK_AND_ASSERT_MES2(expr, message) do{if(!(expr)) {LOG_ERROR(message); };}while(0) +#endif + namespace epee { namespace debug @@ -162,6 +290,8 @@ namespace log_space virtual bool set_max_logfile_size(uint64_t max_size){return true;}; virtual bool set_log_rotate_cmd(const std::string& cmd){return true;}; + virtual bool truncate_log_files() { return true; } + virtual std::string copy_logs_to_buffer() { return ""; } }; /************************************************************************/ @@ -239,6 +369,7 @@ namespace log_space inline bool is_stdout_a_tty() { +#ifndef ANDROID_BUILD static std::atomic initialized(false); static std::atomic is_a_tty(false); @@ -253,6 +384,9 @@ namespace log_space } return is_a_tty.load(std::memory_order_relaxed); +#else + return false; +#endif } inline void set_console_color(int color, bool bright) @@ -458,7 +592,7 @@ namespace log_space std::string buf(buffer, buffer_len); for(size_t i = 0; i!= buf.size(); i++) { - if(buf[i] == 7 || buf[i] == -107) + if(static_cast(buf[i]) == 0x7 || static_cast(buf[i]) == 0x95) buf[i] = '^'; } @@ -497,13 +631,13 @@ namespace log_space class file_output_stream : public ibase_log_stream { public: - typedef std::map named_log_streams; + typedef std::map > named_log_streams; - file_output_stream( std::string default_log_file_name, std::string log_path ) + file_output_stream( const std::string& default_log_file_name, const std::string& log_path ) { m_default_log_filename = default_log_file_name; m_max_logfile_size = 0; - m_default_log_path = log_path; + m_default_log_path_w = epee::string_encoding::utf8_to_wstring(log_path); m_pdefault_file_stream = add_new_stream_and_open(default_log_file_name.c_str()); } @@ -511,33 +645,39 @@ namespace log_space { for(named_log_streams::iterator it = m_log_file_names.begin(); it!=m_log_file_names.end(); it++) { - if ( it->second->is_open() ) + if ( it->second.first->is_open() ) { - it->second->flush(); - it->second->close(); + it->second.first->flush(); + it->second.first->close(); } - delete it->second; + delete it->second.first; } } private: named_log_streams m_log_file_names; - std::string m_default_log_path; - std::ofstream* m_pdefault_file_stream; + std::wstring m_default_log_path_w; + boost::filesystem::ofstream* m_pdefault_file_stream; std::string m_log_rotate_cmd; std::string m_default_log_filename; uint64_t m_max_logfile_size; - std::ofstream* add_new_stream_and_open(const char* pstream_name) + // gets utf-8 encoded string + boost::filesystem::ofstream* add_new_stream_and_open(const char* pstream_name) { //log_space::rotate_log_file((m_default_log_path + "\\" + pstream_name).c_str()); boost::system::error_code ec; - boost::filesystem::create_directories(m_default_log_path, ec); - std::ofstream* pstream = (m_log_file_names[pstream_name] = new std::ofstream); - std::string target_path = m_default_log_path + "/" + pstream_name; + boost::filesystem::create_directories(m_default_log_path_w, ec); + boost::filesystem::ofstream* pstream = new boost::filesystem::ofstream; + + std::wstring target_path = epee::string_encoding::utf8_to_wstring(pstream_name); + if (!m_default_log_path_w.empty()) + target_path = m_default_log_path_w + L"/" + target_path; + pstream->open( target_path.c_str(), std::ios_base::out | std::ios::app /*ios_base::trunc */); if(pstream->fail()) return NULL; + m_log_file_names[pstream_name] = std::pair(pstream, target_path); return pstream; } @@ -553,18 +693,63 @@ namespace log_space return true; } + bool truncate_log_files() + { + for (named_log_streams::iterator it = m_log_file_names.begin(); it != m_log_file_names.end(); it++) + { + std::wstring target_path = it->second.second; + //close and delete current stream + if (it->second.first->is_open()) + { + it->second.first->flush(); + it->second.first->close(); + } + delete it->second.first; + it->second.first = nullptr; + //reopen it with truncate + boost::filesystem::ofstream* pstream = new boost::filesystem::ofstream; + pstream->open(target_path.c_str(), std::ios_base::out | std::ios::trunc ); + if (pstream->fail()) + { + throw std::runtime_error("Unexpected error: failed to re-open log stream on truncate"); + } + it->second.first = pstream; + } + return true; + } + + std::string copy_logs_to_buffer() + { + std::stringstream res; + + for (named_log_streams::iterator it = m_log_file_names.begin(); it != m_log_file_names.end(); it++) + { + std::wstring target_path = it->second.second; + res << "[" << epee::string_encoding::convert_to_ansii(target_path) << "]" << ENDL; + std::string res_buf; + if (!epee::file_io_utils::load_file_to_string(target_path, res_buf)) + { + res << "ERROR"; + } + else + { + res << res_buf; + } + } + return res.str(); + } virtual bool out_buffer( const char* buffer, int buffer_len, int log_level, int color, const char* plog_name = NULL ) { - std::ofstream* m_target_file_stream = m_pdefault_file_stream; + boost::filesystem::ofstream* m_target_file_stream = m_pdefault_file_stream; if(plog_name) { //find named stream named_log_streams::iterator it = m_log_file_names.find(plog_name); if(it == m_log_file_names.end()) m_target_file_stream = add_new_stream_and_open(plog_name); else - m_target_file_stream = it->second; + m_target_file_stream = it->second.first; } if(!m_target_file_stream || !m_target_file_stream->is_open()) return false;//TODO: add assert here @@ -572,9 +757,10 @@ namespace log_space m_target_file_stream->write(buffer, buffer_len ); m_target_file_stream->flush(); + /* if(m_max_logfile_size) { - std::ofstream::pos_type pt = m_target_file_stream->tellp(); + boost::filesystem::ofstream::pos_type pt = m_target_file_stream->tellp(); uint64_t current_sz = pt; if(current_sz > m_max_logfile_size) { @@ -621,12 +807,13 @@ namespace log_space misc_utils::call_sys_cmd(m_log_rotate_cmd_local_copy); } - m_target_file_stream->open( (m_default_log_path + "/" + log_file_name).c_str(), std::ios_base::out | std::ios::app /*ios_base::trunc */); + + m_target_file_stream->open( (m_default_log_path + "/" + log_file_name).c_str(), std::ios_base::out | std::ios::app / * ios_base::trunc * /); if(m_target_file_stream->fail()) return false; } } - + */ return true; } int get_type(){return LOGGER_FILE;} @@ -660,6 +847,22 @@ namespace log_space return true; } + bool truncate_log_files() + { + for (streams_container::iterator it = m_log_streams.begin(); it != m_log_streams.end(); it++) + it->first->truncate_log_files(); + return true; + } + + std::string copy_logs_to_buffer() + { + std::string res; + for (streams_container::iterator it = m_log_streams.begin(); it != m_log_streams.end(); it++) + res += it->first->copy_logs_to_buffer(); + return res; + } + + bool do_log_message(const std::string& rlog_mes, int log_level, int color, const char* plog_name = NULL) { std::string str_mess = rlog_mes; @@ -703,7 +906,7 @@ namespace log_space m_log_streams.push_back(streams_container::value_type(ls, log_level_limit)); return true; } - return ls ? true:false; + return false; } bool add_logger( ibase_log_stream* pstream, int log_level_limit = LOG_LEVEL_4 ) { @@ -745,8 +948,8 @@ namespace log_space typedef std::map channels_err_stat_container_type; inline epee::locked_object_proxy get_channels_errors_stat_container() { - static std::recursive_mutex cs; - static channels_err_stat_container_type errors_by_channel; + static epee::static_helpers::wrapper cs; + static epee::static_helpers::wrapper errors_by_channel; epee::locked_object_proxy res(errors_by_channel, cs); return res; } @@ -831,6 +1034,21 @@ namespace log_space return true; } + std::string copy_logs_to_buffer() + { + FAST_CRITICAL_REGION_BEGIN(m_critical_sec); + return m_log_target.copy_logs_to_buffer(); + FAST_CRITICAL_REGION_END(); + } + + + bool truncate_log_files() + { + FAST_CRITICAL_REGION_BEGIN(m_critical_sec); + return m_log_target.truncate_log_files(); + FAST_CRITICAL_REGION_END(); + } + bool take_away_journal(std::list& journal) { FAST_CRITICAL_REGION_BEGIN(m_critical_sec); @@ -978,7 +1196,7 @@ namespace log_space class log_singletone { public: - friend class initializer; + friend class static_helpers::initializer; friend class logger; static int get_log_detalisation_level() { @@ -989,7 +1207,7 @@ namespace log_space //get_enabled_channels not thread-safe, at the moment leave it like this because it's configured in main, before other threads started static std::set& get_enabled_channels() { - static std::set genabled_channels; + static epee::static_helpers::wrapper> genabled_channels; return genabled_channels; } @@ -1020,7 +1238,9 @@ namespace log_space std::set enabled_channels_local = genabled_channels; enabled_channels_local.insert(ch_name); genabled_channels.swap(enabled_channels_local); +#ifndef ANDROID_BUILD std::cout << "log channel '" << ch_name << "' enabled" << std::endl; +#endif } static void disable_channels(const std::string& channels_set) @@ -1116,6 +1336,23 @@ namespace log_space return plogger->set_log_rotate_cmd(cmd); } + + static std::string copy_logs_to_buffer() + { + logger* plogger = get_or_create_instance(); + if (!plogger) return ""; + return plogger->copy_logs_to_buffer(); + } + + + static bool truncate_log_files() + { + logger* plogger = get_or_create_instance(); + if (!plogger) return false; + return plogger->truncate_log_files(); + } + + static bool add_logger( int type, const char* pdefault_file_name, const char* pdefault_log_folder, int log_level_limit = LOG_LEVEL_4) { @@ -1172,7 +1409,7 @@ namespace log_space if(!plogger) return false; return plogger->remove_logger(type); } -PUSH_WARNINGS +PUSH_GCC_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) static int get_set_log_detalisation_level(bool is_need_set = false, int log_level_to_set = LOG_LEVEL_1) { @@ -1184,7 +1421,7 @@ DISABLE_GCC_WARNING(maybe-uninitialized) } return log_detalisation_level; } -POP_WARNINGS +POP_GCC_WARNINGS static int get_set_time_level(bool is_need_set = false, int time_log_level = LOG_LEVEL_0) { static int val_time_log_level = LOG_LEVEL_0; @@ -1376,7 +1613,7 @@ POP_WARNINGS //static int get_set_error_filter(bool is_need_set = false) }; - const static initializer log_initializer; + const static static_helpers::initializer log_initializer; /************************************************************************/ /* */ // /************************************************************************/ @@ -1427,10 +1664,10 @@ POP_WARNINGS } ~log_frame() { + NESTED_TRY_ENTRY(); #ifdef _MSC_VER int lasterr=::GetLastError(); #endif - if (m_level <= log_singletone::get_log_detalisation_level() ) { std::stringstream ss; @@ -1440,6 +1677,7 @@ POP_WARNINGS #ifdef _MSC_VER ::SetLastError(lasterr); #endif + NESTED_CATCH_ENTRY(__func__); } }; @@ -1500,193 +1738,8 @@ POP_WARNINGS } } -#if !defined(DISABLE_RELEASE_LOGGING) - #define ENABLE_LOGGING_INTERNAL -#endif +} // namespace epee - - -#define LOG_DEFAULT_CHANNEL NULL -#define ENABLE_CHANNEL_BY_DEFAULT(ch_name) \ - static bool COMBINE(init_channel, __LINE__) UNUSED_ATTRIBUTE = epee::misc_utils::static_initializer([](){ \ - epee::log_space::log_singletone::enable_channel(ch_name); return true; \ -}); - - - - - -#if defined(ENABLE_LOGGING_INTERNAL) - -#define LOG_PRINT_CHANNEL_NO_PREFIX2(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ - {std::stringstream ss________; ss________ << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str() , y, epee::log_space::console_color_default, false, log_name);}} - -#define LOG_PRINT_CHANNEL_NO_PREFIX_NO_POSTFIX2(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ - {std::stringstream ss________; ss________ << x; epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, false, log_name);}} - -#define LOG_PRINT_CHANNEL_NO_POSTFIX2(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ - {std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x; epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, false, log_name);}} - -#define LOG_PRINT_CHANNEL2(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ - {std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, false, log_name);}} - -#define LOG_PRINT_CHANNEL_COLOR2(log_channel, log_name, x, y, color) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ - {std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, color, false, log_name);}} - -#define LOG_PRINT_CHANNEL_2_JORNAL(log_channel, log_name, x, y) {if ( y <= epee::log_space::log_singletone::get_log_detalisation_level() && epee::log_space::log_singletone::channel_enabled(log_channel))\ - {std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << x << std::endl;epee::log_space::log_singletone::do_log_message(ss________.str(), y, epee::log_space::console_color_default, true, log_name);}} - - - -#define LOG_ERROR2(log_name, x) { \ - std::stringstream ss________; ss________ << epee::log_space::log_singletone::get_prefix_entry() << "[ERROR] Location: " << std::endl << LOCATION_SS << epee::misc_utils::print_trace() << " Message:" << std::endl << x << std::endl; epee::log_space::log_singletone::do_log_message(ss________.str(), LOG_LEVEL_0, epee::log_space::console_color_red, true, log_name); LOCAL_ASSERT(0); epee::log_space::increase_error_count(LOG_DEFAULT_CHANNEL); } - -#define LOG_FRAME2(log_name, x, y) epee::log_space::log_frame frame(x, y, log_name) - -#else // #if defined(ENABLE_LOGGING_INTERNAL) - - -#define LOG_PRINT_NO_PREFIX2(log_name, x, y) - -#define LOG_PRINT_NO_PREFIX_NO_POSTFIX2(log_name, x, y) - -#define LOG_PRINT_NO_POSTFIX2(log_name, x, y) - -#define LOG_PRINT_COLOR2(log_name, x, y, color) - -#define LOG_PRINT2_JORNAL(log_name, x, y) - -#define LOG_PRINT2(log_name, x, y) - -#define LOG_ERROR2(log_name, x) - - -#define LOG_FRAME2(log_name, x, y) - - -#endif // #if defined(ENABLE_LOGGING_INTERNAL) - -#define LOG_PRINT_NO_PREFIX2(log_name, x, y) LOG_PRINT_CHANNEL_NO_PREFIX2(LOG_DEFAULT_CHANNEL, log_name, x, y) -#define LOG_PRINT_NO_PREFIX_NO_POSTFIX2(log_name, x, y) LOG_PRINT_CHANNEL_NO_PREFIX_NO_POSTFIX2(LOG_DEFAULT_CHANNEL, log_name, x, y) -#define LOG_PRINT_NO_POSTFIX2(log_name, x, y) LOG_PRINT_CHANNEL_NO_POSTFIX2(LOG_DEFAULT_CHANNEL, log_name, x, y) -#define LOG_PRINT2(log_name, x, y) LOG_PRINT_CHANNEL2(LOG_DEFAULT_CHANNEL, log_name, x, y) -#define LOG_PRINT_COLOR2(log_name, x, y, color) LOG_PRINT_CHANNEL_COLOR2(LOG_DEFAULT_CHANNEL, log_name, x, y, color) -#define LOG_PRINT2_JORNAL(log_name, x, y) LOG_PRINT_CHANNEL_2_JORNAL(LOG_DEFAULT_CHANNEL, log_name, x, y) - - - -#ifndef LOG_DEFAULT_TARGET - #define LOG_DEFAULT_TARGET NULL -#endif - - -#define LOG_PRINT_NO_POSTFIX(mess, level) LOG_PRINT_NO_POSTFIX2(LOG_DEFAULT_TARGET, mess, level) -#define LOG_PRINT_NO_PREFIX(mess, level) LOG_PRINT_NO_PREFIX2(LOG_DEFAULT_TARGET, mess, level) -#define LOG_PRINT_NO_PREFIX_NO_POSTFIX(mess, level) LOG_PRINT_NO_PREFIX_NO_POSTFIX2(LOG_DEFAULT_TARGET, mess, level) -#define LOG_PRINT(mess, level) LOG_PRINT2(LOG_DEFAULT_TARGET, mess, level) - -#define LOG_PRINT_COLOR(mess, level, color) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, color) -#define LOG_PRINT_RED(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_red) -#define LOG_PRINT_GREEN(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_green) -#define LOG_PRINT_BLUE(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_blue) -#define LOG_PRINT_YELLOW(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_yellow) -#define LOG_PRINT_CYAN(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_cyan) -#define LOG_PRINT_MAGENTA(mess, level) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, level, epee::log_space::console_color_magenta) - -#define LOG_PRINT_RED_L0(mess) LOG_PRINT_COLOR2(LOG_DEFAULT_TARGET, mess, LOG_LEVEL_0, epee::log_space::console_color_red) - -#define LOG_PRINT_L0(mess) LOG_PRINT(mess, LOG_LEVEL_0) -#define LOG_PRINT_L1(mess) LOG_PRINT(mess, LOG_LEVEL_1) -#define LOG_PRINT_L2(mess) LOG_PRINT(mess, LOG_LEVEL_2) -#define LOG_PRINT_L3(mess) LOG_PRINT(mess, LOG_LEVEL_3) -#define LOG_PRINT_L4(mess) LOG_PRINT(mess, LOG_LEVEL_4) -#define LOG_PRINT_J(mess, level) LOG_PRINT2_JORNAL(LOG_DEFAULT_TARGET, mess, level) - -#define LOG_ERROR(mess) LOG_ERROR2(LOG_DEFAULT_TARGET, mess) -#define LOG_FRAME(mess, level) LOG_FRAME2(LOG_DEFAULT_TARGET, mess, level) -#define LOG_VALUE(mess, level) LOG_VALUE2(LOG_DEFAULT_TARGET, mess, level) -#define LOG_ARRAY(mess, level) LOG_ARRAY2(LOG_DEFAULT_TARGET, mess, level) -//#define LOGWIN_PLATFORM_ERROR(err_no) LOGWINDWOS_PLATFORM_ERROR2(LOG_DEFAULT_TARGET, err_no) -#define LOG_SOCKET_ERROR(err_no) LOG_SOCKET_ERROR2(LOG_DEFAULT_TARGET, err_no) -//#define LOGWIN_PLATFORM_ERROR_UNCRITICAL(mess) LOGWINDWOS_PLATFORM_ERROR_UNCRITICAL2(LOG_DEFAULT_TARGET, mess) - -#define ENDL std::endl - -#define TRY_ENTRY() try { -#define CATCH_ENTRY_CUSTOM(location, custom_code, return_val) } \ - catch(const std::exception& ex) \ -{ \ - (void)(ex); \ - LOG_ERROR("Exception at [" << location << "], what=" << ex.what()); \ - custom_code; \ - return return_val; \ -} \ - catch(...) \ -{ \ - LOG_ERROR("Exception at [" << location << "], generic exception \"...\""); \ - custom_code; \ - return return_val; \ -} -#define CATCH_ENTRY(location, return_val) CATCH_ENTRY_CUSTOM(location, (void)0, return_val) - -#define CATCH_ENTRY2(return_val) CATCH_ENTRY_CUSTOM(LOCATION_SS, (void)0, return_val) - -#define CATCH_ENTRY_L0(location, return_val) CATCH_ENTRY(location, return_val) -#define CATCH_ENTRY_L1(location, return_val) CATCH_ENTRY(location, return_val) -#define CATCH_ENTRY_L2(location, return_val) CATCH_ENTRY(location, return_val) -#define CATCH_ENTRY_L3(location, return_val) CATCH_ENTRY(location, return_val) -#define CATCH_ENTRY_L4(location, return_val) CATCH_ENTRY(location, return_val) - - -#define ASSERT_MES_AND_THROW(message) {LOG_ERROR(message); std::stringstream ss; ss << message; throw std::runtime_error(ss.str());} - -#define CHECK_AND_ASSERT_THROW_MES(expr, message) {if(!(expr)) ASSERT_MES_AND_THROW(message << ENDL << "thrown from " << LOCATION_SS);} -#define CHECK_AND_ASSERT_THROW(expr, exception_exp) {if(!(expr)) {LOG_ERROR("EXCEPTION is thrown from " << LOCATION_SS); throw exception_exp; };} - - -#ifndef CHECK_AND_ASSERT -#define CHECK_AND_ASSERT(expr, fail_ret_val) do{if(!(expr)){LOCAL_ASSERT(expr); return fail_ret_val;};}while(0) -#endif - -#define NOTHING - -#ifndef CHECK_AND_ASSERT_MES -#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_ERROR(message); return fail_ret_val;};}while(0) -#endif - -#ifndef CHECK_AND_FORCE_ASSERT_MES -#define CHECK_AND_FORCE_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_ERROR(message); FORCE_ASSERT(expr); return fail_ret_val;};}while(0) -#endif - - -#ifndef CHECK_AND_ASSERT_MES_CUSTOM -#define CHECK_AND_ASSERT_MES_CUSTOM(expr, fail_ret_val, custom_code, message) do{if(!(expr)) {LOG_ERROR(message); custom_code; return fail_ret_val;};}while(0) -#endif - -/*#ifndef CHECK_AND_ASSERT_MES_AND_THROW -#define CHECK_AND_ASSERT_MES_AND_THROW(expr, message) do{if(!(expr)) {LOG_ERROR(message); throw std::runtime_error(message);};}while(0) -#endif -*/ - -#ifndef CHECK_AND_NO_ASSERT_MES -#define CHECK_AND_NO_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_PRINT_MAGENTA(message, LOG_LEVEL_0); /*LOCAL_ASSERT(expr);*/ return fail_ret_val;};}while(0) -#endif - -#ifndef CHECK_AND_NO_ASSERT_MES_LEVEL -#define CHECK_AND_NO_ASSERT_MES_LEVEL(expr, fail_ret_val, message, log_level) do{if(!(expr)) {LOG_PRINT(message, log_level); return fail_ret_val;};}while(0) -#endif - -#ifndef CHECK_AND_ASSERT_MES_NO_RET -#define CHECK_AND_ASSERT_MES_NO_RET(expr, message) do{if(!(expr)) {LOG_ERROR(message);};}while(0) -#endif - - -#ifndef CHECK_AND_ASSERT_MES2 -#define CHECK_AND_ASSERT_MES2(expr, message) do{if(!(expr)) {LOG_ERROR(message); };}while(0) -#endif - -} - -POP_WARNINGS +POP_VS_WARNINGS #endif //_MISC_LOG_EX_H_ diff --git a/src/contrib/epee/include/misc_os_dependent.h b/src/contrib/epee/include/misc_os_dependent.h index 116826c..c06e5d9 100644 --- a/src/contrib/epee/include/misc_os_dependent.h +++ b/src/contrib/epee/include/misc_os_dependent.h @@ -109,14 +109,15 @@ namespace misc_utils #endif } -#if defined(__GNUC__) + +#if defined(__GNUC__) && !defined(__ANDROID__) #include #include #endif - inline std::string print_trace() + inline std::string print_trace_default() { std::stringstream ss; -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(__ANDROID__) ss << std::endl << "STACK" << std::endl; const size_t max_depth = 100; size_t stack_depth; @@ -134,5 +135,33 @@ namespace misc_utils #endif return ss.str(); } + + typedef std::string (stack_retrieving_function_t)(); + + // + // To get stack trace call it with the defaults. + // + inline std::string get_callstack(stack_retrieving_function_t* p_stack_retrieving_function_to_be_added = nullptr, bool remove_func = false) + { + static stack_retrieving_function_t* p_srf = nullptr; + + if (remove_func) + { + p_srf = nullptr; + return ""; + } + + if (p_stack_retrieving_function_to_be_added != nullptr) + { + p_srf = p_stack_retrieving_function_to_be_added; + return ""; + } + + if (p_srf != nullptr) + return p_srf(); + + return print_trace_default(); + } + } } diff --git a/src/contrib/epee/include/net/abstract_tcp_server2.h b/src/contrib/epee/include/net/abstract_tcp_server2.h index 4c274c0..be9fc2a 100644 --- a/src/contrib/epee/include/net/abstract_tcp_server2.h +++ b/src/contrib/epee/include/net/abstract_tcp_server2.h @@ -1,6 +1,7 @@ +// Copyright (c) 2019, anonimal, // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // * Redistributions of source code must retain the above copyright @@ -11,7 +12,7 @@ // * Neither the name of the Andrey N. Sabelnikov nor the // names of its contributors may be used to endorse or promote products // derived from this software without specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -22,13 +23,10 @@ // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - - - -#ifndef _ABSTRACT_TCP_SERVER2_H_ -#define _ABSTRACT_TCP_SERVER2_H_ +// +#ifndef _ABSTRACT_TCP_SERVER2_H_ +#define _ABSTRACT_TCP_SERVER2_H_ #include #include @@ -47,247 +45,278 @@ #include "net_utils_base.h" #include "syncobj.h" -#undef LOG_DEFAULT_CHANNEL +#undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "net_server" #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000 -namespace epee -{ -namespace net_utils -{ +namespace epee { +namespace net_utils { + +struct i_connection_filter { + virtual bool is_remote_ip_allowed(uint32_t adress) = 0; - struct i_connection_filter - { - virtual bool is_remote_ip_allowed(uint32_t adress)=0; protected: - virtual ~i_connection_filter(){} - }; - - /************************************************************************/ - /* */ - /************************************************************************/ - /// Represents a single connection from a client. - template - class connection - : public boost::enable_shared_from_this >, - private boost::noncopyable, - public i_service_endpoint + virtual ~i_connection_filter() { + } +}; + +/************************************************************************/ +/* */ +/************************************************************************/ +/// Represents a single connection from a client. +template +class connection + : public boost::enable_shared_from_this>, + private boost::noncopyable, + public i_service_endpoint { public: - typedef typename t_protocol_handler::connection_context t_connection_context; - /// Construct a connection with the given io_service. - explicit connection(boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter * &pfilter); + typedef typename t_protocol_handler::connection_context t_connection_context; + /// Construct a connection with the given io_service. + explicit connection(boost::asio::io_service& io_service, + typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter*& pfilter); - virtual ~connection(); - /// Get the socket associated with the connection. - boost::asio::ip::tcp::socket& socket(); + virtual ~connection(); + /// Get the socket associated with the connection. + boost::asio::ip::tcp::socket& socket(); - /// Start the first asynchronous operation for the connection. - bool start(bool is_income, bool is_multithreaded); + /// Start the first asynchronous operation for the connection. + bool start(bool is_income, bool is_multithreaded); - void get_context(t_connection_context& context_){context_ = context;} + void get_context(t_connection_context& context_) + { + context_ = context; + } + + void call_back_starter(); + bool is_shutdown() + { + return m_was_shutdown; + } + bool cancel(); - void call_back_starter(); - bool is_shutdown(){return m_was_shutdown;} - bool cancel(); private: - //----------------- i_service_endpoint --------------------- - virtual bool do_send(const void* ptr, size_t cb); - virtual bool close(); - virtual bool call_run_once_service_io(); - virtual bool request_callback(); - virtual boost::asio::io_service& get_io_service(); - virtual bool add_ref(); - virtual bool release(); - //------------------------------------------------------ - boost::shared_ptr > safe_shared_from_this(); - bool shutdown(); - /// Handle completion of a read operation. - void handle_read(const boost::system::error_code& e, - std::size_t bytes_transferred); + //----------------- i_service_endpoint --------------------- + virtual bool do_send(const void* ptr, size_t cb); + virtual bool close(); + virtual bool call_run_once_service_io(); + virtual bool request_callback(); + virtual boost::asio::io_service& get_io_service(); + virtual bool add_ref(); + virtual bool release(); + //------------------------------------------------------ + boost::shared_ptr> safe_shared_from_this(); + bool shutdown(); + /// Handle completion of a read operation. + void handle_read(const boost::system::error_code& e, + std::size_t bytes_transferred); - /// Handle completion of a write operation. - void handle_write(const boost::system::error_code& e, size_t cb); + /// Handle completion of a write operation. + void handle_write(const boost::system::error_code& e, size_t cb); - /// Strand to ensure the connection's handlers are not called concurrently. - boost::asio::io_service::strand strand_; + /// Strand to ensure the connection's handlers are not called concurrently. + boost::asio::io_service::strand strand_; - /// Socket for the connection. - boost::asio::ip::tcp::socket socket_; + /// Socket for the connection. + boost::asio::ip::tcp::socket socket_; - /// Buffer for incoming data. - boost::array buffer_; + /// Buffer for incoming data. + boost::array buffer_; - t_connection_context context; - volatile uint32_t m_want_close_connection; - std::atomic m_was_shutdown; - critical_section m_send_que_lock; - std::list m_send_que; - volatile uint32_t& m_ref_sockets_count; - i_connection_filter* &m_pfilter; - volatile bool m_is_multithreaded; + boost::asio::io_service& m_rio_service; + t_connection_context context; + volatile uint32_t m_want_close_connection; + std::atomic m_was_shutdown; + critical_section m_send_que_lock; + std::list m_send_que; + volatile uint32_t& m_ref_sockets_count; + i_connection_filter*& m_pfilter; + volatile bool m_is_multithreaded; + std::list>> m_self_refs; // add_ref/release support + critical_section m_self_refs_lock; + + t_protocol_handler m_protocol_handler; + //this should be the last line with m_protocol_handler, because it could be wait on destructor, while other activities possible on other threads + //DON'T ADD ANYTHING HERE!!! +}; - //this should be the last one, because it could be wait on destructor, while other activities possible on other threads - t_protocol_handler m_protocol_handler; - //typename t_protocol_handler::config_type m_dummy_config; - std::list > > m_self_refs; // add_ref/release support - critical_section m_self_refs_lock; - }; - - - /************************************************************************/ - /* */ - /************************************************************************/ - template - class boosted_tcp_server - : private boost::noncopyable - { +/************************************************************************/ +/* */ +/************************************************************************/ +template +class boosted_tcp_server + : private boost::noncopyable { public: - typedef boost::shared_ptr > connection_ptr; - typedef typename t_protocol_handler::connection_context t_connection_context; - /// Construct the server to listen on the specified TCP address and port, and - /// serve up files from the given directory. - boosted_tcp_server(); - explicit boosted_tcp_server(boost::asio::io_service& external_io_service); - ~boosted_tcp_server(); + typedef boost::shared_ptr> connection_ptr; + typedef typename t_protocol_handler::connection_context t_connection_context; + /// Construct the server to listen on the specified TCP address and port, and + /// serve up files from the given directory. + boosted_tcp_server(); + explicit boosted_tcp_server(boost::asio::io_service& external_io_service); + ~boosted_tcp_server(); - bool init_server(uint32_t port, const std::string address = "0.0.0.0"); - bool init_server(const std::string port, const std::string& address = "0.0.0.0"); + bool init_server(uint32_t port, const std::string address = "0.0.0.0"); + bool init_server(const std::string port, const std::string& address = "0.0.0.0"); - /// Run the server's io_service loop. - bool run_server(size_t threads_count, bool wait = true); + /// Run the server's io_service loop. + bool run_server(size_t threads_count, bool wait = true); - /// wait for service workers stop - bool timed_wait_server_stop(uint64_t wait_mseconds); + /// wait for service workers stop + bool timed_wait_server_stop(uint64_t wait_mseconds); - /// Stop the server. - void send_stop_signal(); + /// Stop the server. + void send_stop_signal(); - bool is_stop_signal_sent(); + bool is_stop_signal_sent(); - void set_threads_prefix(const std::string& prefix_name); + void set_threads_prefix(const std::string& prefix_name); - bool deinit_server(){return true;} + bool deinit_server() + { + return true; + } - size_t get_threads_count(){return m_threads_count;} + size_t get_threads_count() + { + return m_threads_count; + } - void set_connection_filter(i_connection_filter* pfilter); + void set_connection_filter(i_connection_filter* pfilter); - bool connect(const std::string& adr, const std::string& port, uint32_t conn_timeot, t_connection_context& cn, const std::string& bind_ip = "0.0.0.0"); - template - bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, t_callback cb, const std::string& bind_ip = "0.0.0.0"); + bool connect(const std::string& adr, const std::string& port, uint32_t conn_timeot, t_connection_context& cn, const std::string& bind_ip = "0.0.0.0"); + template + bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, const t_callback& cb, const std::string& bind_ip = "0.0.0.0"); - typename t_protocol_handler::config_type& get_config_object(){return m_config;} + typename t_protocol_handler::config_type& get_config_object() + { + return m_config; + } - int get_binded_port(){return m_port;} + int get_binded_port() + { + return m_port; + } - boost::asio::io_service& get_io_service(){return io_service_;} + boost::asio::io_service& get_io_service() + { + return io_service_; + } - struct idle_callback_conext_base + struct idle_callback_conext_base { + virtual ~idle_callback_conext_base() { - virtual ~idle_callback_conext_base(){} + } - virtual bool call_handler(){return true;} - - idle_callback_conext_base(boost::asio::io_service& io_serice): - m_timer(io_serice) - {} - boost::asio::deadline_timer m_timer; - uint64_t m_period; - }; - - template - struct idle_callback_conext: public idle_callback_conext_base + virtual bool call_handler() { - idle_callback_conext(boost::asio::io_service& io_serice, t_handler& h, uint64_t period): - idle_callback_conext_base(io_serice), - m_handler(h) - {this->m_period = period;} - - t_handler m_handler; - virtual bool call_handler() - { - return m_handler(); - } - }; - - template - bool add_idle_handler(t_handler t_callback, uint64_t timeout_ms) - { - boost::shared_ptr ptr(new idle_callback_conext(io_service_, t_callback, timeout_ms)); - //needed call handler here ?... - ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period)); - ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server::global_timer_handler, this, ptr)); - return true; - } - - bool global_timer_handler(/*const boost::system::error_code& err, */boost::shared_ptr ptr) - { - //if handler return false - he don't want to be called anymore - try{ - if (!ptr->call_handler()) - return true; - } - catch (...) - { - return true; - } - - - ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period)); - ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server::global_timer_handler, this, ptr)); return true; } - template - bool async_call(t_handler t_callback) + idle_callback_conext_base(boost::asio::io_service& io_serice): m_timer(io_serice), + m_period(0) { - io_service_.post(t_callback); + } + boost::asio::deadline_timer m_timer; + uint64_t m_period; + }; + + template + struct idle_callback_conext : public idle_callback_conext_base { + idle_callback_conext(boost::asio::io_service& io_serice, t_handler& h, uint64_t period) + : idle_callback_conext_base(io_serice), + m_handler(h) + { + this->m_period = period; + } + + t_handler m_handler; + virtual bool call_handler() + { + return m_handler(); + } + }; + + template + bool add_idle_handler(t_handler t_callback, uint64_t timeout_ms) + { + boost::shared_ptr ptr(new idle_callback_conext(io_service_, t_callback, timeout_ms)); + //needed call handler here ?... + ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period)); + ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server::global_timer_handler, this, ptr)); + return true; + } + + bool global_timer_handler(/*const boost::system::error_code& err, */ boost::shared_ptr ptr) + { + //if handler return false - he don't want to be called anymore + try { + if(!ptr->call_handler()) + return true; + } + catch(std::exception& e) + { + LOG_ERROR("exeption caught in boosted_tcp_server::global_timer_handler: " << e.what() << ENDL << "won't be called anymore"); + return true; + } + catch(...) + { + LOG_ERROR("unknown exeption caught in boosted_tcp_server::global_timer_handler, it won't be called anymore"); return true; } + ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period)); + ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server::global_timer_handler, this, ptr)); + return true; + } + + template + bool async_call(t_handler t_callback) + { + io_service_.post(t_callback); + return true; + } + protected: - typename t_protocol_handler::config_type m_config; + typename t_protocol_handler::config_type m_config; private: - /// Run the server's io_service loop. - bool worker_thread(); - /// Handle completion of an asynchronous accept operation. - void handle_accept(const boost::system::error_code& e); + /// Run the server's io_service loop. + bool worker_thread(); + /// Handle completion of an asynchronous accept operation. + void handle_accept(const boost::system::error_code& e); - bool is_thread_worker(); + bool is_thread_worker(); - /// The io_service used to perform asynchronous operations. - std::unique_ptr m_io_service_local_instance; - boost::asio::io_service& io_service_; + /// The io_service used to perform asynchronous operations. + std::unique_ptr m_io_service_local_instance; + boost::asio::io_service& io_service_; - /// Acceptor used to listen for incoming connections. - boost::asio::ip::tcp::acceptor acceptor_; + /// Acceptor used to listen for incoming connections. + boost::asio::ip::tcp::acceptor acceptor_; - /// The next connection to be accepted. - connection_ptr new_connection_; - //std::mutex connections_mutex; - //std::deque connections_; - std::atomic m_stop_signal_sent; - uint32_t m_port; - volatile uint32_t m_sockets_count; - std::string m_address; - std::string m_thread_name_prefix; - size_t m_threads_count; - i_connection_filter* m_pfilter; - std::vector > m_threads; - boost::thread::id m_main_thread_id; - critical_section m_threads_lock; - volatile uint32_t m_thread_index; - }; -} -} + /// The next connection to be accepted. + connection_ptr new_connection_; + //std::mutex connections_mutex; + //std::deque connections_; + std::atomic m_stop_signal_sent; + uint32_t m_port; + volatile uint32_t m_sockets_count; + std::string m_address; + std::string m_thread_name_prefix; + size_t m_threads_count; + i_connection_filter* m_pfilter; + std::vector> m_threads; + boost::thread::id m_main_thread_id; + critical_section m_threads_lock; + volatile uint32_t m_thread_index; +}; +} // namespace net_utils +} // namespace epee #include "abstract_tcp_server2.inl" -#undef LOG_DEFAULT_CHANNEL +#undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL NULL -#endif \ No newline at end of file +#endif diff --git a/src/contrib/epee/include/net/abstract_tcp_server2.inl b/src/contrib/epee/include/net/abstract_tcp_server2.inl index 15ce844..2bb45f6 100644 --- a/src/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/src/contrib/epee/include/net/abstract_tcp_server2.inl @@ -1,6 +1,7 @@ +// Copyright (c) 2019, anonimal // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // * Redistributions of source code must retain the above copyright @@ -11,7 +12,7 @@ // * Neither the name of the Andrey N. Sabelnikov nor the // names of its contributors may be used to endorse or promote products // derived from this software without specific prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -22,9 +23,7 @@ // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - - +// #include "net_utils_base.h" #include @@ -37,807 +36,770 @@ #include "misc_language.h" #include "warnings.h" -PUSH_WARNINGS -namespace epee -{ -namespace net_utils -{ - /************************************************************************/ - /* */ - /************************************************************************/ +PUSH_VS_WARNINGS +namespace epee { +namespace net_utils { +/************************************************************************/ +/* */ +/************************************************************************/ DISABLE_VS_WARNINGS(4355) - template - connection::connection(boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter* &pfilter) - : strand_(io_service), - socket_(io_service), - m_protocol_handler(this, config, context), - m_want_close_connection(0), - m_was_shutdown(0), - m_ref_sockets_count(sock_count), - m_pfilter(pfilter) - { - boost::interprocess::ipcdetail::atomic_inc32(&m_ref_sockets_count); - } +template +connection::connection(boost::asio::io_service& io_service, + typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter*& pfilter) + : m_rio_service(io_service), + strand_(io_service), + socket_(io_service), + m_protocol_handler(this, config, context), + m_want_close_connection(0), + m_was_shutdown(0), + m_ref_sockets_count(sock_count), + m_pfilter(pfilter), + m_is_multithreaded(false) +{ + boost::interprocess::ipcdetail::atomic_inc32(&m_ref_sockets_count); +} DISABLE_VS_WARNINGS(4355) - //--------------------------------------------------------------------------------- - template - connection::~connection() - { - if(!m_was_shutdown) - { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown."); - shutdown(); - } +//--------------------------------------------------------------------------------- +template +connection::~connection() +{ + NESTED_TRY_ENTRY(); - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed"); - boost::interprocess::ipcdetail::atomic_dec32(&m_ref_sockets_count); - VALIDATE_MUTEX_IS_FREE(m_send_que_lock); - VALIDATE_MUTEX_IS_FREE(m_self_refs_lock); - } - //--------------------------------------------------------------------------------- - template - boost::asio::ip::tcp::socket& connection::socket() - { - return socket_; - } - //--------------------------------------------------------------------------------- - template - boost::shared_ptr > connection::safe_shared_from_this() - { - try - { - return connection::shared_from_this(); - } - catch (const boost::bad_weak_ptr&) - { - // It happens when the connection is being deleted - return boost::shared_ptr >(); - } - } - //--------------------------------------------------------------------------------- - template - bool connection::start(bool is_income, bool is_multithreaded) - { - TRY_ENTRY(); - - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if(!self) - { - LOG_PRINT_RED("Failed to start conntection, failed to call safe_shared_from_this", LOG_LEVEL_2); - return false; - } - - m_is_multithreaded = is_multithreaded; - - boost::system::error_code ec; - auto remote_ep = socket_.remote_endpoint(ec); - CHECK_AND_NO_ASSERT_MES(!m_was_shutdown, false, "was shutdown on start connection"); - - CHECK_AND_NO_ASSERT_MES_LEVEL(!ec, false, "Failed to get remote endpoint(" << (is_income?"INT":"OUT") << "): " << ec.message() << ':' << ec.value(), LOG_LEVEL_2); - - auto local_ep = socket_.local_endpoint(ec); - CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get local endpoint: " << ec.message() << ':' << ec.value()); - - context = boost::value_initialized(); - long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); - - context.set_details(boost::uuids::random_generator()(), ip_, remote_ep.port(), is_income); - context.m_last_send = context.m_last_recv = time(NULL); - - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] new connection, remote end_point: " << print_connection_context_short(context) << - " local end_point: " << local_ep.address().to_string() << ':' << local_ep.port() << - ", total sockets objects " << m_ref_sockets_count); - - if(is_income && m_pfilter && !m_pfilter->is_remote_ip_allowed(context.m_remote_ip)) - { - LOG_PRINT_L0("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection"); - close(); - return false; - } - - m_protocol_handler.after_init_connection(); - - socket_.async_read_some(boost::asio::buffer(buffer_), - strand_.wrap( - boost::bind(&connection::handle_read, self, - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred))); - - - return true; - - CATCH_ENTRY_L0("connection::start()", false); - } - //--------------------------------------------------------------------------------- - template - bool connection::request_callback() - { - TRY_ENTRY(); - LOG_PRINT_L2("[" << print_connection_context_short(context) << "] request_callback"); - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if(!self) - return false; - - strand_.post(boost::bind(&connection::call_back_starter, self)); - CATCH_ENTRY_L0("connection::request_callback()", false); - return true; - } - //--------------------------------------------------------------------------------- - template - boost::asio::io_service& connection::get_io_service() - { - return socket_.get_io_service(); - } - //--------------------------------------------------------------------------------- - template - bool connection::add_ref() - { - TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] add_ref"); - CRITICAL_REGION_LOCAL(m_self_refs_lock); - - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if(!self) - return false; - if(m_was_shutdown) - return false; - m_self_refs.push_back(self); - return true; - CATCH_ENTRY_L0("connection::add_ref()", false); - } - //--------------------------------------------------------------------------------- - template - bool connection::release() - { - TRY_ENTRY(); - boost::shared_ptr > back_connection_copy; - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] release"); - CRITICAL_REGION_BEGIN(m_self_refs_lock); - CHECK_AND_ASSERT_MES(m_self_refs.size(), false, "[sock " << socket_.native_handle() << "] m_self_refs empty at connection::release() call"); - //erasing from container without additional copy can cause start deleting object, including m_self_refs - back_connection_copy = m_self_refs.back(); - m_self_refs.pop_back(); - CRITICAL_REGION_END(); - return true; - CATCH_ENTRY_L0("connection::release()", false); - } - //--------------------------------------------------------------------------------- - template - void connection::call_back_starter() - { - TRY_ENTRY(); - LOG_PRINT_L2("[" << print_connection_context_short(context) << "] fired_callback"); - m_protocol_handler.handle_qued_callback(); - CATCH_ENTRY_L0("connection::call_back_starter()", void()); - } - //--------------------------------------------------------------------------------- - template - void connection::handle_read(const boost::system::error_code& e, - std::size_t bytes_transferred) - { - TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Assync read calledback."); - - if (!e) - { - LOG_PRINT("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred, LOG_LEVEL_4); - context.m_last_recv = time(NULL); - context.m_recv_cnt += bytes_transferred; - bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred); - if(!recv_res) - { - LOG_PRINT("[sock " << socket_.native_handle() << "] protocol_want_close", LOG_LEVEL_4); - - //some error in protocol, protocol handler ask to close connection - boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); - bool do_shutdown = false; - CRITICAL_REGION_BEGIN(m_send_que_lock); - if(!m_send_que.size()) - do_shutdown = true; - CRITICAL_REGION_END(); - if(do_shutdown) - shutdown(); - }else - { - socket_.async_read_some(boost::asio::buffer(buffer_), - strand_.wrap( - boost::bind(&connection::handle_read, connection::shared_from_this(), - boost::asio::placeholders::error, - boost::asio::placeholders::bytes_transferred))); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "]Assync read requested."); - } - }else - { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); - if(e.value() != 2) - { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); - shutdown(); - } - } - // If an error occurs then no new asynchronous operations are started. This - // means that all shared_ptr references to the connection object will - // disappear and the object will be destroyed automatically after this - // handler returns. The connection class's destructor closes the socket. - CATCH_ENTRY_L0("connection::handle_read", void()); - } - //--------------------------------------------------------------------------------- - template - bool connection::call_run_once_service_io() - { - TRY_ENTRY(); - if(!m_is_multithreaded) - { - //single thread model, we can wait in blocked call - size_t cnt = socket_.get_io_service().run_one(); - if(!cnt)//service is going to quit - return false; - }else - { - //multi thread model, we can't(!) wait in blocked call - //so we make non blocking call and releasing CPU by calling sleep(0); - //if no handlers were called - //TODO: Maybe we need to have have critical section + event + callback to upper protocol to - //ask it inside(!) critical region if we still able to go in event wait... - size_t cnt = socket_.get_io_service().poll_one(); - if(!cnt) - misc_utils::sleep_no_w(0); - } - - return true; - CATCH_ENTRY_L0("connection::call_run_once_service_io", false); - } - //--------------------------------------------------------------------------------- - template - bool connection::do_send(const void* ptr, size_t cb) - { - TRY_ENTRY(); - // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted - auto self = safe_shared_from_this(); - if(!self) - return false; - if(m_was_shutdown) - return false; - - LOG_PRINT("[sock " << socket_.native_handle() << "] SEND " << cb, LOG_LEVEL_4); - context.m_last_send = time(NULL); - context.m_send_cnt += cb; - //some data should be wrote to stream - //request complete - - CRITICAL_REGION_LOCAL_VAR(m_send_que_lock, send_guard); - if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) - { - send_guard.unlock();//manual unlock - LOG_ERROR("send to [" << print_connection_context_short(context) << ", (" << (void*)this << ")] que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); - close(); - return false; - } - - m_send_que.resize(m_send_que.size()+1); - m_send_que.back().assign((const char*)ptr, cb); - - if(m_send_que.size() > 1) - { - //active operation should be in progress, nothing to do, just wait last operation callback - }else - { - //no active operation - if(m_send_que.size()!=1) - { - LOG_ERROR("Looks like no active operations, but send que size != 1!!"); - return false; - } - - boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), - //strand_.wrap( - boost::bind(&connection::handle_write, self, _1, _2) - //) - ); - - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Assync send requested " << m_send_que.front().size()); - } - - return true; - - CATCH_ENTRY_L0("connection::do_send", false); - } - //--------------------------------------------------------------------------------- - template - bool connection::shutdown() - { - // Initiate graceful connection closure. - boost::system::error_code ignored_ec; - socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); - m_was_shutdown = true; - m_protocol_handler.release_protocol(); - return true; - } - //--------------------------------------------------------------------------------- - template - bool connection::close() - { - TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Que Shutdown called."); - size_t send_que_size = 0; - CRITICAL_REGION_BEGIN(m_send_que_lock); - send_que_size = m_send_que.size(); - CRITICAL_REGION_END(); - boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); - if(!send_que_size) - { - shutdown(); - } - - return true; - CATCH_ENTRY_L0("connection::close", false); - } - //--------------------------------------------------------------------------------- - template - bool connection::cancel() - { - return close(); - } - //--------------------------------------------------------------------------------- - template - void connection::handle_write(const boost::system::error_code& e, size_t cb) - { - TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Assync send calledback " << cb); - - if (e) - { - LOG_PRINT_L0("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); - shutdown(); - return; - } - - bool do_shutdown = false; - CRITICAL_REGION_BEGIN(m_send_que_lock); - if(m_send_que.empty()) - { - LOG_ERROR("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); - return; - } - - m_send_que.pop_front(); - if(m_send_que.empty()) - { - if(boost::interprocess::ipcdetail::atomic_read32(&m_want_close_connection)) - { - do_shutdown = true; - } - }else - { - //have more data to send - boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), - //strand_.wrap( - boost::bind(&connection::handle_write, connection::shared_from_this(), _1, _2)); - //); - } - CRITICAL_REGION_END(); - - if(do_shutdown) - { - shutdown(); - } - CATCH_ENTRY_L0("connection::handle_write", void()); - } - /************************************************************************/ - /* */ - /************************************************************************/ - template - boosted_tcp_server::boosted_tcp_server(): - m_io_service_local_instance(new boost::asio::io_service()), - io_service_(*m_io_service_local_instance.get()), - acceptor_(io_service_), - new_connection_(new connection(io_service_, m_config, m_sockets_count, m_pfilter)), - m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) - { - m_thread_name_prefix = "NET"; + if(!m_was_shutdown) { + LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown."); + shutdown(); } - template - boosted_tcp_server::boosted_tcp_server(boost::asio::io_service& extarnal_io_service): - io_service_(extarnal_io_service), - acceptor_(io_service_), - new_connection_(new connection(io_service_, m_config, m_sockets_count, m_pfilter)), - m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) - { - m_thread_name_prefix = "NET"; - } - //--------------------------------------------------------------------------------- - template - boosted_tcp_server::~boosted_tcp_server() - { - this->send_stop_signal(); - timed_wait_server_stop(10000); - } - //--------------------------------------------------------------------------------- - template - bool boosted_tcp_server::init_server(uint32_t port, const std::string address) - { - TRY_ENTRY(); - m_stop_signal_sent = false; - m_port = port; - m_address = address; - // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). - boost::asio::ip::tcp::resolver resolver(io_service_); - boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast(port)); - boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); - acceptor_.open(endpoint.protocol()); - acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); - acceptor_.bind(endpoint); - acceptor_.listen(); - boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); - m_port = binded_endpoint.port(); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server::handle_accept, this, - boost::asio::placeholders::error)); + LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed"); + boost::interprocess::ipcdetail::atomic_dec32(&m_ref_sockets_count); + VALIDATE_MUTEX_IS_FREE(m_send_que_lock); + VALIDATE_MUTEX_IS_FREE(m_self_refs_lock); - return true; - CATCH_ENTRY_L0("boosted_tcp_server::init_server", false); + NESTED_CATCH_ENTRY(__func__); +} +//--------------------------------------------------------------------------------- +template +boost::asio::ip::tcp::socket& connection::socket() +{ + return socket_; +} +//--------------------------------------------------------------------------------- +template +boost::shared_ptr> connection::safe_shared_from_this() +{ + try { + return connection::shared_from_this(); } - //----------------------------------------------------------------------------- -PUSH_WARNINGS -DISABLE_GCC_WARNING(maybe-uninitialized) - template - bool boosted_tcp_server::init_server(const std::string port, const std::string& address) - { - uint32_t p = 0; + catch(const boost::bad_weak_ptr&) { + // It happens when the connection is being deleted + return boost::shared_ptr>(); + } +} +//--------------------------------------------------------------------------------- +template +bool connection::start(bool is_income, bool is_multithreaded) +{ + TRY_ENTRY(); - if (port.size() && !string_tools::get_xtype_from_string(p, port)) { - LOG_ERROR("Failed to convert port no = " << port); - return false; - } - return this->init_server(p, address); - } -POP_WARNINGS - //--------------------------------------------------------------------------------- - template - bool boosted_tcp_server::worker_thread() - { - TRY_ENTRY(); - uint32_t local_thr_index = boost::interprocess::ipcdetail::atomic_inc32(&m_thread_index); - std::string thread_name = std::string("[") + m_thread_name_prefix; - thread_name += boost::to_string(local_thr_index) + "]"; - log_space::log_singletone::set_thread_log_prefix(thread_name); - while(!m_stop_signal_sent) - { - try - { - io_service_.run(); - } - catch(const std::exception& ex) - { - LOG_ERROR("Exception at server worker thread, what=" << ex.what()); - } - catch(...) - { - LOG_ERROR("Exception at server worker thread, unknown execption"); - } - } - LOG_PRINT_L4("Worker thread finished"); - return true; - CATCH_ENTRY_L0("boosted_tcp_server::worker_thread", false); - } - //--------------------------------------------------------------------------------- - template - void boosted_tcp_server::set_threads_prefix(const std::string& prefix_name) - { - m_thread_name_prefix = prefix_name; - } - //--------------------------------------------------------------------------------- - template - void boosted_tcp_server::set_connection_filter(i_connection_filter* pfilter) - { - m_pfilter = pfilter; - } - //--------------------------------------------------------------------------------- - template - bool boosted_tcp_server::run_server(size_t threads_count, bool wait) - { - TRY_ENTRY(); - m_threads_count = threads_count; - m_main_thread_id = boost::this_thread::get_id(); - log_space::log_singletone::set_thread_log_prefix("[SRV_MAIN]"); - while(!m_stop_signal_sent) - { - - // Create a pool of threads to run all of the io_services. - CRITICAL_REGION_BEGIN(m_threads_lock); - for (std::size_t i = 0; i < threads_count; ++i) - { - boost::shared_ptr thread(new boost::thread( - boost::bind(&boosted_tcp_server::worker_thread, this))); - m_threads.push_back(thread); - } - CRITICAL_REGION_END(); - // Wait for all threads in the pool to exit. - if(wait) - { - for (std::size_t i = 0; i < m_threads.size(); ++i) - m_threads[i]->join(); - m_threads.clear(); - - }else - { - return true; - } - - if(wait && !m_stop_signal_sent) - { - //some problems with the listening socket ?.. - LOG_PRINT_L0("Net service stopped without stop request, restarting..."); - if(!this->init_server(m_port, m_address)) - { - LOG_PRINT_L0("Reiniting service failed, exit."); - return false; - }else - { - LOG_PRINT_L0("Reiniting OK."); - } - } - } - return true; - CATCH_ENTRY_L0("boosted_tcp_server::run_server", false); - } - //--------------------------------------------------------------------------------- - template - bool boosted_tcp_server::is_thread_worker() - { - TRY_ENTRY(); - CRITICAL_REGION_LOCAL(m_threads_lock); - BOOST_FOREACH(boost::shared_ptr& thp, m_threads) - { - if(thp->get_id() == boost::this_thread::get_id()) - return true; - } - if(m_threads_count == 1 && boost::this_thread::get_id() == m_main_thread_id) - return true; + // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted + auto self = safe_shared_from_this(); + if(!self) { + LOG_PRINT_RED("Failed to start conntection, failed to call safe_shared_from_this", LOG_LEVEL_2); return false; - CATCH_ENTRY_L0("boosted_tcp_server::is_thread_worker", false); } - //--------------------------------------------------------------------------------- - template - bool boosted_tcp_server::timed_wait_server_stop(uint64_t wait_mseconds) - { - TRY_ENTRY(); - boost::chrono::milliseconds ms(wait_mseconds); - for (std::size_t i = 0; i < m_threads.size(); ++i) - { - if(m_threads[i]->joinable() && !m_threads[i]->try_join_for(ms)) - { - LOG_PRINT_L0("Interrupting thread " << m_threads[i]->native_handle()); - m_threads[i]->interrupt(); - } + + m_is_multithreaded = is_multithreaded; + + boost::system::error_code ec; + auto remote_ep = socket_.remote_endpoint(ec); + CHECK_AND_NO_ASSERT_MES(!m_was_shutdown, false, "was shutdown on start connection"); + + CHECK_AND_NO_ASSERT_MES_LEVEL(!ec, false, "Failed to get remote endpoint(" << (is_income ? "INT" : "OUT") << "): " << ec.message() << ':' << ec.value(), LOG_LEVEL_2); + + auto local_ep = socket_.local_endpoint(ec); + CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get local endpoint: " << ec.message() << ':' << ec.value()); + + context = boost::value_initialized(); + long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); + + context.set_details(boost::uuids::random_generator()(), ip_, remote_ep.port(), is_income); + context.m_last_send = context.m_last_recv = time(NULL); + + LOG_PRINT_L3("[sock " << socket_.native_handle() << "] new connection, remote end_point: " << print_connection_context_short(context) << " local end_point: " << local_ep.address().to_string() << ':' << local_ep.port() << ", total sockets objects " << m_ref_sockets_count); + + if(is_income && m_pfilter && !m_pfilter->is_remote_ip_allowed(context.m_remote_ip)) { + LOG_PRINT_L0("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection"); + close(); + return false; + } + + m_protocol_handler.after_init_connection(); + + socket_.async_read_some(boost::asio::buffer(buffer_), + strand_.wrap( + boost::bind(&connection::handle_read, self, + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred))); + + return true; + + CATCH_ENTRY_L0("connection::start()", false); +} +//--------------------------------------------------------------------------------- +template +bool connection::request_callback() +{ + TRY_ENTRY(); + LOG_PRINT_L2("[" << print_connection_context_short(context) << "] request_callback"); + // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted + auto self = safe_shared_from_this(); + if(!self) + return false; + + strand_.post(boost::bind(&connection::call_back_starter, self)); + CATCH_ENTRY_L0("connection::request_callback()", false); + return true; +} +//--------------------------------------------------------------------------------- +template +boost::asio::io_service& connection::get_io_service() +{ + return m_rio_service; +} +//--------------------------------------------------------------------------------- +template +bool connection::add_ref() +{ + TRY_ENTRY(); + LOG_PRINT_L4("[sock " << socket_.native_handle() << "] add_ref"); + CRITICAL_REGION_LOCAL(m_self_refs_lock); + + // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted + auto self = safe_shared_from_this(); + if(!self) + return false; + if(m_was_shutdown) + return false; + m_self_refs.push_back(self); + return true; + CATCH_ENTRY_L0("connection::add_ref()", false); +} +//--------------------------------------------------------------------------------- +template +bool connection::release() +{ + TRY_ENTRY(); + boost::shared_ptr> back_connection_copy; + LOG_PRINT_L4("[sock " << socket_.native_handle() << "] release"); + CRITICAL_REGION_BEGIN(m_self_refs_lock); + CHECK_AND_ASSERT_MES(m_self_refs.size(), false, "[sock " << socket_.native_handle() << "] m_self_refs empty at connection::release() call"); + //erasing from container without additional copy can cause start deleting object, including m_self_refs + back_connection_copy = m_self_refs.back(); + m_self_refs.pop_back(); + CRITICAL_REGION_END(); + return true; + CATCH_ENTRY_L0("connection::release()", false); +} +//--------------------------------------------------------------------------------- +template +void connection::call_back_starter() +{ + TRY_ENTRY(); + LOG_PRINT_L2("[" << print_connection_context_short(context) << "] fired_callback"); + m_protocol_handler.handle_qued_callback(); + CATCH_ENTRY_L0("connection::call_back_starter()", void()); +} +//--------------------------------------------------------------------------------- +template +void connection::handle_read(const boost::system::error_code& e, + std::size_t bytes_transferred) +{ + TRY_ENTRY(); + LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Assync read calledback."); + + if(!e) { + LOG_PRINT("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred, LOG_LEVEL_4); + context.m_last_recv = time(NULL); + context.m_recv_cnt += bytes_transferred; + bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred); + if(!recv_res) { + LOG_PRINT("[sock " << socket_.native_handle() << "] protocol_want_close", LOG_LEVEL_4); + + //some error in protocol, protocol handler ask to close connection + boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); + bool do_shutdown = false; + CRITICAL_REGION_BEGIN(m_send_que_lock); + if(!m_send_que.size()) + do_shutdown = true; + CRITICAL_REGION_END(); + if(do_shutdown) + shutdown(); } - return true; - CATCH_ENTRY_L0("boosted_tcp_server::timed_wait_server_stop", false); - } - //--------------------------------------------------------------------------------- - template - void boosted_tcp_server::send_stop_signal() - { - m_stop_signal_sent = true; - TRY_ENTRY(); - m_config.on_send_stop_signal(); - - io_service_.stop(); - CATCH_ENTRY_L0("boosted_tcp_server::send_stop_signal()", void()); - } - //--------------------------------------------------------------------------------- - template - bool boosted_tcp_server::is_stop_signal_sent() - { - return m_stop_signal_sent; - } - //--------------------------------------------------------------------------------- - template - void boosted_tcp_server::handle_accept(const boost::system::error_code& e) - { - TRY_ENTRY(); - if (!e) - { - connection_ptr conn(std::move(new_connection_)); - - new_connection_.reset(new connection(io_service_, m_config, m_sockets_count, m_pfilter)); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server::handle_accept, this, - boost::asio::placeholders::error)); - - conn->start(true, 1 < m_threads_count); - }else - { - LOG_ERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_sockets_count); + else { + socket_.async_read_some(boost::asio::buffer(buffer_), + strand_.wrap( + boost::bind(&connection::handle_read, connection::shared_from_this(), + boost::asio::placeholders::error, + boost::asio::placeholders::bytes_transferred))); + LOG_PRINT_L4("[sock " << socket_.native_handle() << "]Assync read requested."); } - CATCH_ENTRY_L0("boosted_tcp_server::handle_accept", void()); } - //--------------------------------------------------------------------------------- - template - bool boosted_tcp_server::connect(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_connection_context& conn_context, const std::string& bind_ip) - { - TRY_ENTRY(); + else { + LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); + if(e.value() != 2) { + LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); + shutdown(); + } + } + // If an error occurs then no new asynchronous operations are started. This + // means that all shared_ptr references to the connection object will + // disappear and the object will be destroyed automatically after this + // handler returns. The connection class's destructor closes the socket. + CATCH_ENTRY_L0("connection::handle_read", void()); +} +//--------------------------------------------------------------------------------- +template +bool connection::call_run_once_service_io() +{ + TRY_ENTRY(); + if(!m_is_multithreaded) { + //single thread model, we can wait in blocked call + size_t cnt = m_rio_service.run_one(); - connection_ptr new_connection_l(new connection(io_service_, m_config, m_sockets_count, m_pfilter) ); - boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); - - ////////////////////////////////////////////////////////////////////////// - boost::asio::ip::tcp::resolver resolver(io_service_); - boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); - boost::asio::ip::tcp::resolver::iterator end; - if(iterator == end) - { - LOG_ERROR("Failed to resolve " << adr); + if(!cnt) //service is going to quit + return false; + } + else { + //multi thread model, we can't(!) wait in blocked call + //so we make non blocking call and releasing CPU by calling sleep(0); + //if no handlers were called + //TODO: Maybe we need to have have critical section + event + callback to upper protocol to + //ask it inside(!) critical region if we still able to go in event wait... + size_t cnt = m_rio_service.poll_one(); + if(!cnt) + misc_utils::sleep_no_w(0); + } + + return true; + CATCH_ENTRY_L0("connection::call_run_once_service_io", false); +} +//--------------------------------------------------------------------------------- +template +bool connection::do_send(const void* ptr, size_t cb) +{ + TRY_ENTRY(); + // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted + auto self = safe_shared_from_this(); + if(!self) + return false; + if(m_was_shutdown) + return false; + + LOG_PRINT("[sock " << socket_.native_handle() << "] SEND " << cb, LOG_LEVEL_4); + context.m_last_send = time(NULL); + context.m_send_cnt += cb; + //some data should be wrote to stream + //request complete + + CRITICAL_REGION_LOCAL_VAR(m_send_que_lock, send_guard); + if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) { + send_guard.unlock(); //manual unlock + LOG_ERROR("send to [" << print_connection_context_short(context) << ", (" << (void*)this << ")] que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); + close(); + shutdown(); + return false; + } + + m_send_que.resize(m_send_que.size() + 1); + m_send_que.back().assign((const char*)ptr, cb); + + if(m_send_que.size() > 1) { + //active operation should be in progress, nothing to do, just wait last operation callback + } + else { + //no active operation + if(m_send_que.size() != 1) { + LOG_ERROR("Looks like no active operations, but send que size != 1!!"); return false; } - ////////////////////////////////////////////////////////////////////////// + boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), + //strand_.wrap( + boost::bind(&connection::handle_write, self, boost::placeholders::_1, boost::placeholders::_2) + //) + ); - //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); - boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); - - sock_.open(remote_endpoint.protocol()); - if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) - { - boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); - sock_.bind(local_endpoint); + LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Assync send requested " << m_send_que.front().size()); + } + + return true; + + CATCH_ENTRY_L0("connection::do_send", false); +} +//--------------------------------------------------------------------------------- +template +bool connection::shutdown() +{ + if(m_was_shutdown) + return true; + // Initiate graceful connection closure. + boost::system::error_code ignored_ec; + socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); + m_was_shutdown = true; + m_protocol_handler.release_protocol(); + return true; +} +//--------------------------------------------------------------------------------- +template +bool connection::close() +{ + TRY_ENTRY(); + LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Que Shutdown called."); + size_t send_que_size = 0; + CRITICAL_REGION_BEGIN(m_send_que_lock); + send_que_size = m_send_que.size(); + CRITICAL_REGION_END(); + boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); + if(!send_que_size) { + shutdown(); + } + + return true; + CATCH_ENTRY_L0("connection::close", false); +} +//--------------------------------------------------------------------------------- +template +bool connection::cancel() +{ + return close(); +} +//--------------------------------------------------------------------------------- +template +void connection::handle_write(const boost::system::error_code& e, size_t cb) +{ + TRY_ENTRY(); + LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Assync send calledback " << cb); + + if(e) { + LOG_PRINT_L0("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); + shutdown(); + return; + } + + bool do_shutdown = false; + CRITICAL_REGION_BEGIN(m_send_que_lock); + if(m_send_que.empty()) { + LOG_ERROR("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); + return; + } + + m_send_que.pop_front(); + if(m_send_que.empty()) { + if(boost::interprocess::ipcdetail::atomic_read32(&m_want_close_connection)) { + do_shutdown = true; + } + } + else { + //have more data to send + boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), + //strand_.wrap( + boost::bind(&connection::handle_write, connection::shared_from_this(), boost::placeholders::_1, boost::placeholders::_2)); + //); + } + CRITICAL_REGION_END(); + + if(do_shutdown) { + shutdown(); + } + CATCH_ENTRY_L0("connection::handle_write", void()); +} +/************************************************************************/ +/* */ +/************************************************************************/ +template +boosted_tcp_server::boosted_tcp_server() + : m_io_service_local_instance(new boost::asio::io_service()), + io_service_(*m_io_service_local_instance.get()), + acceptor_(io_service_), + new_connection_(new connection(io_service_, m_config, m_sockets_count, m_pfilter)), + m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) +{ + m_thread_name_prefix = "NET"; +} + +template +boosted_tcp_server::boosted_tcp_server(boost::asio::io_service& extarnal_io_service) + : io_service_(extarnal_io_service), + acceptor_(io_service_), + new_connection_(new connection(io_service_, m_config, m_sockets_count, m_pfilter)), + m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) +{ + m_thread_name_prefix = "NET"; +} +//--------------------------------------------------------------------------------- +template +boosted_tcp_server::~boosted_tcp_server() +{ + NESTED_TRY_ENTRY(); + + this->send_stop_signal(); + timed_wait_server_stop(10000); + + NESTED_CATCH_ENTRY(__func__); +} +//--------------------------------------------------------------------------------- +template +bool boosted_tcp_server::init_server(uint32_t port, const std::string address) +{ + TRY_ENTRY(); + m_stop_signal_sent = false; + m_port = port; + m_address = address; + // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast(port)); + boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); + acceptor_.open(endpoint.protocol()); + acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor_.bind(endpoint); + acceptor_.listen(); + boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); + m_port = binded_endpoint.port(); + acceptor_.async_accept(new_connection_->socket(), + boost::bind(&boosted_tcp_server::handle_accept, this, + boost::asio::placeholders::error)); + + return true; + CATCH_ENTRY_L0("boosted_tcp_server::init_server", false); +} +//----------------------------------------------------------------------------- +PUSH_GCC_WARNINGS +DISABLE_GCC_WARNING(maybe-uninitialized) +template +bool boosted_tcp_server::init_server(const std::string port, const std::string& address) +{ + uint32_t p = 0; + + if(port.size() && !string_tools::get_xtype_from_string(p, port)) { + LOG_ERROR("Failed to convert port no = " << port); + return false; + } + return this->init_server(p, address); +} +POP_GCC_WARNINGS +//--------------------------------------------------------------------------------- +template +bool boosted_tcp_server::worker_thread() +{ + TRY_ENTRY(); + uint32_t local_thr_index = boost::interprocess::ipcdetail::atomic_inc32(&m_thread_index); + std::string thread_name = std::string("[") + m_thread_name_prefix; + thread_name += boost::to_string(local_thr_index) + "]"; + log_space::log_singletone::set_thread_log_prefix(thread_name); + while(!m_stop_signal_sent) { + try { + io_service_.run(); + } + catch(const std::exception& ex) { + LOG_ERROR("Exception at server worker thread, what=" << ex.what()); + } + catch(...) { + LOG_ERROR("Exception at server worker thread, unknown execption"); + } + } + LOG_PRINT_L4("Worker thread finished"); + return true; + CATCH_ENTRY_L0("boosted_tcp_server::worker_thread", false); +} +//--------------------------------------------------------------------------------- +template +void boosted_tcp_server::set_threads_prefix(const std::string& prefix_name) +{ + m_thread_name_prefix = prefix_name; +} +//--------------------------------------------------------------------------------- +template +void boosted_tcp_server::set_connection_filter(i_connection_filter* pfilter) +{ + m_pfilter = pfilter; +} +//--------------------------------------------------------------------------------- +template +bool boosted_tcp_server::run_server(size_t threads_count, bool wait) +{ + TRY_ENTRY(); + m_threads_count = threads_count; + m_main_thread_id = boost::this_thread::get_id(); + log_space::log_singletone::set_thread_log_prefix("[SRV_MAIN]"); + while(!m_stop_signal_sent) { + // Create a pool of threads to run all of the io_services. + CRITICAL_REGION_BEGIN(m_threads_lock); + for(std::size_t i = 0; i < threads_count; ++i) { + boost::shared_ptr thread(new boost::thread( + boost::bind(&boosted_tcp_server::worker_thread, this))); + m_threads.push_back(thread); + } + CRITICAL_REGION_END(); + // Wait for all threads in the pool to exit. + if(wait) { + for(std::size_t i = 0; i < m_threads.size(); ++i) + m_threads[i]->join(); + m_threads.clear(); + } + else { + return true; } - /* + if(wait && !m_stop_signal_sent) { + //some problems with the listening socket ?.. + LOG_PRINT_L0("Net service stopped without stop request, restarting..."); + if(!this->init_server(m_port, m_address)) { + LOG_PRINT_L0("Reiniting service failed, exit."); + return false; + } + else { + LOG_PRINT_L0("Reiniting OK."); + } + } + } + return true; + CATCH_ENTRY_L0("boosted_tcp_server::run_server", false); +} +//--------------------------------------------------------------------------------- +template +bool boosted_tcp_server::is_thread_worker() +{ + TRY_ENTRY(); + CRITICAL_REGION_LOCAL(m_threads_lock); + BOOST_FOREACH(boost::shared_ptr& thp, m_threads) { + if(thp->get_id() == boost::this_thread::get_id()) + return true; + } + if(m_threads_count == 1 && boost::this_thread::get_id() == m_main_thread_id) + return true; + return false; + CATCH_ENTRY_L0("boosted_tcp_server::is_thread_worker", false); +} +//--------------------------------------------------------------------------------- +template +bool boosted_tcp_server::timed_wait_server_stop(uint64_t wait_mseconds) +{ + TRY_ENTRY(); + boost::chrono::milliseconds ms(wait_mseconds); + for(std::size_t i = 0; i < m_threads.size(); ++i) { +#ifdef ANDROID_BUILD + if (m_threads[i]->joinable()) + { + m_threads[i]->join(); + } +#else + if (m_threads[i]->joinable() && !m_threads[i]->try_join_for(ms)) + { + LOG_PRINT_L0("Interrupting thread " << m_threads[i]->native_handle()); + m_threads[i]->interrupt(); + } +#endif + } + return true; + CATCH_ENTRY_L0("boosted_tcp_server::timed_wait_server_stop", false); +} +//--------------------------------------------------------------------------------- +template +void boosted_tcp_server::send_stop_signal() +{ + m_stop_signal_sent = true; + TRY_ENTRY(); + m_config.on_send_stop_signal(); + + io_service_.stop(); + CATCH_ENTRY_L0("boosted_tcp_server::send_stop_signal()", void()); +} +//--------------------------------------------------------------------------------- +template +bool boosted_tcp_server::is_stop_signal_sent() +{ + return m_stop_signal_sent; +} +//--------------------------------------------------------------------------------- +template +void boosted_tcp_server::handle_accept(const boost::system::error_code& e) +{ + TRY_ENTRY(); + if(!e) { + connection_ptr conn(std::move(new_connection_)); + + new_connection_.reset(new connection(io_service_, m_config, m_sockets_count, m_pfilter)); + acceptor_.async_accept(new_connection_->socket(), + boost::bind(&boosted_tcp_server::handle_accept, this, + boost::asio::placeholders::error)); + + conn->start(true, 1 < m_threads_count); + } + else { + LOG_ERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_sockets_count); + } + CATCH_ENTRY_L0("boosted_tcp_server::handle_accept", void()); +} +//--------------------------------------------------------------------------------- +template +bool boosted_tcp_server::connect(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_connection_context& conn_context, const std::string& bind_ip) +{ + TRY_ENTRY(); + + connection_ptr new_connection_l(new connection(io_service_, m_config, m_sockets_count, m_pfilter)); + boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); + + ////////////////////////////////////////////////////////////////////////// + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port); + boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::asio::ip::tcp::resolver::iterator end; + if(iterator == end) { + LOG_ERROR("Failed to resolve " << adr); + return false; + } + ////////////////////////////////////////////////////////////////////////// + + //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); + boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); + + sock_.open(remote_endpoint.protocol()); + if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "") { + boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); + sock_.bind(local_endpoint); + } + + /* NOTICE: be careful to make sync connection from event handler: in case if all threads suddenly do sync connect, there will be no thread to dispatch events from io service. */ - boost::system::error_code ec = boost::asio::error::would_block; + boost::system::error_code ec = boost::asio::error::would_block; - //have another free thread(s), work in wait mode, without event handling - struct local_async_context - { - boost::system::error_code ec; - boost::mutex connect_mut; - boost::condition_variable cond; - }; + //have another free thread(s), work in wait mode, without event handling + struct local_async_context { + boost::system::error_code ec; + boost::mutex connect_mut; + boost::condition_variable cond; + }; - boost::shared_ptr local_shared_context(new local_async_context()); - local_shared_context->ec = boost::asio::error::would_block; - boost::unique_lock lock(local_shared_context->connect_mut); - auto connect_callback = [](boost::system::error_code ec_, boost::shared_ptr shared_context) - { - CRITICAL_SECTION_LOCK(shared_context->connect_mut); - shared_context->ec = ec_; - CRITICAL_SECTION_UNLOCK(shared_context->connect_mut); - shared_context->cond.notify_one(); - }; + boost::shared_ptr local_shared_context(new local_async_context()); + local_shared_context->ec = boost::asio::error::would_block; + boost::unique_lock lock(local_shared_context->connect_mut); + auto connect_callback = [](boost::system::error_code ec_, boost::shared_ptr shared_context) { + CRITICAL_SECTION_LOCK(shared_context->connect_mut); + shared_context->ec = ec_; + CRITICAL_SECTION_UNLOCK(shared_context->connect_mut); + shared_context->cond.notify_one(); + }; - sock_.async_connect(remote_endpoint, boost::bind(connect_callback, _1, local_shared_context)); - while(local_shared_context->ec == boost::asio::error::would_block) - { - bool r = false; - try{ - r = local_shared_context->cond.timed_wait(lock, boost::get_system_time() + boost::posix_time::milliseconds(conn_timeout)); - } - catch (...) - { - //timeout - sock_.close(); - LOG_PRINT_L3("timed_wait throwed, " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); - return false; - } - if(local_shared_context->ec == boost::asio::error::would_block && !r) - { - //timeout - sock_.close(); - LOG_PRINT_L3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); - return false; - } + sock_.async_connect(remote_endpoint, boost::bind(connect_callback, _1, local_shared_context)); + while(local_shared_context->ec == boost::asio::error::would_block) { + bool r = false; + try { + r = local_shared_context->cond.timed_wait(lock, boost::get_system_time() + boost::posix_time::milliseconds(conn_timeout)); } - ec = local_shared_context->ec; - - if (ec || !sock_.is_open()) - { - LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3); + catch(...) { + //timeout + sock_.close(); + LOG_PRINT_L3("timed_wait throwed, " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); return false; } - - LOG_PRINT_L3("Connected success to " << adr << ':' << port); - - bool r = new_connection_l->start(false, 1 < m_threads_count); - if (r) - { - new_connection_l->get_context(conn_context); - //new_connection_l.reset(new connection(io_service_, m_config, m_sockets_count, m_pfilter)); + if(local_shared_context->ec == boost::asio::error::would_block && !r) { + //timeout + sock_.close(); + LOG_PRINT_L3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); + return false; } - - return r; - - CATCH_ENTRY_L0("boosted_tcp_server::connect", false); } - //--------------------------------------------------------------------------------- - template template - bool boosted_tcp_server::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_callback cb, const std::string& bind_ip) - { - TRY_ENTRY(); - connection_ptr new_connection_l(new connection(io_service_, m_config, m_sockets_count, m_pfilter) ); - boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); + ec = local_shared_context->ec; - ////////////////////////////////////////////////////////////////////////// - boost::asio::ip::tcp::resolver resolver(io_service_); - boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); - boost::asio::ip::tcp::resolver::iterator end; - if(iterator == end) - { - LOG_ERROR("Failed to resolve " << adr); - return false; + if(ec || !sock_.is_open()) { + LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3); + return false; + } + + LOG_PRINT_L3("Connected success to " << adr << ':' << port); + + bool r = new_connection_l->start(false, 1 < m_threads_count); + if(r) { + new_connection_l->get_context(conn_context); + //new_connection_l.reset(new connection(io_service_, m_config, m_sockets_count, m_pfilter)); + } + + return r; + + CATCH_ENTRY_L0("boosted_tcp_server::connect", false); +} +//--------------------------------------------------------------------------------- +template +template +bool boosted_tcp_server::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, const t_callback& cb, const std::string& bind_ip) +{ + TRY_ENTRY(); + connection_ptr new_connection_l(new connection(io_service_, m_config, m_sockets_count, m_pfilter)); + boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); + + ////////////////////////////////////////////////////////////////////////// + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port); + boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::asio::ip::tcp::resolver::iterator end; + if(iterator == end) { + LOG_ERROR("Failed to resolve " << adr); + return false; + } + ////////////////////////////////////////////////////////////////////////// + boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); + + sock_.open(remote_endpoint.protocol()); + if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "") { + boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); + sock_.bind(local_endpoint); + } + + boost::shared_ptr sh_deadline(new boost::asio::deadline_timer(io_service_)); + //start deadline + sh_deadline->expires_from_now(boost::posix_time::milliseconds(conn_timeout)); + sh_deadline->async_wait([=](const boost::system::error_code& error) { + if(error != boost::asio::error::operation_aborted) { + LOG_PRINT_L3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")"); + new_connection_l->socket().close(); } - ////////////////////////////////////////////////////////////////////////// - boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); - - sock_.open(remote_endpoint.protocol()); - if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) - { - boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); - sock_.bind(local_endpoint); - } - - boost::shared_ptr sh_deadline(new boost::asio::deadline_timer(io_service_)); - //start deadline - sh_deadline->expires_from_now(boost::posix_time::milliseconds(conn_timeout)); - sh_deadline->async_wait([=](const boost::system::error_code& error) - { - if(error != boost::asio::error::operation_aborted) - { - LOG_PRINT_L3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")"); - new_connection_l->socket().close(); - } - }); - //start async connect - sock_.async_connect(remote_endpoint, [=](const boost::system::error_code& ec_) - { - t_connection_context conn_context = AUTO_VAL_INIT(conn_context); - boost::system::error_code ignored_ec; - boost::asio::ip::tcp::socket::endpoint_type lep = new_connection_l->socket().local_endpoint(ignored_ec); - if(!ec_) - {//success - if(!sh_deadline->cancel()) - { - cb(conn_context, boost::asio::error::operation_aborted);//this mean that deadline timer already queued callback with cancel operation, rare situation - }else if(new_connection_l->is_shutdown()) - { - //if deadline timer started and finished callback right after async_connect callback is started and before deadline timer sh_deadline->cancel() is called - cb(conn_context, boost::asio::error::operation_aborted); - } - else - { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port << - " from " << lep.address().to_string() << ':' << lep.port()); - bool r = new_connection_l->start(false, 1 < m_threads_count); - if (r) - { - new_connection_l->get_context(conn_context); - cb(conn_context, ec_); - } - else - { - cb(conn_context, boost::asio::error::fault); - } - } - }else - { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port << - " from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value()); + }); + //start async connect + sock_.async_connect(remote_endpoint, [=](const boost::system::error_code& ec_) { + t_connection_context conn_context = AUTO_VAL_INIT(conn_context); + boost::system::error_code ignored_ec; + boost::asio::ip::tcp::socket::endpoint_type lep = new_connection_l->socket().local_endpoint(ignored_ec); + if(!ec_) { //success + if(!sh_deadline->cancel()) { + cb(conn_context, boost::asio::error::operation_aborted); //this mean that deadline timer already queued callback with cancel operation, rare situation + } + else if(new_connection_l->is_shutdown()) { + //if deadline timer started and finished callback right after async_connect callback is started and before deadline timer sh_deadline->cancel() is called + cb(conn_context, boost::asio::error::operation_aborted); + } + else { + LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port << " from " << lep.address().to_string() << ':' << lep.port()); + bool r = new_connection_l->start(false, 1 < m_threads_count); + if(r) { + new_connection_l->get_context(conn_context); cb(conn_context, ec_); } - }); - return true; - CATCH_ENTRY_L0("boosted_tcp_server::connect_async", false); - } + else { + cb(conn_context, boost::asio::error::fault); + } + } + } + else { + LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port << " from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value()); + cb(conn_context, ec_); + } + }); + return true; + CATCH_ENTRY_L0("boosted_tcp_server::connect_async", false); } -} -POP_WARNINGS +} // namespace net_utils +} // namespace epee +POP_VS_WARNINGS diff --git a/src/contrib/epee/include/net/http_base.h b/src/contrib/epee/include/net/http_base.h index 49b0839..cca1add 100644 --- a/src/contrib/epee/include/net/http_base.h +++ b/src/contrib/epee/include/net/http_base.h @@ -1,3 +1,4 @@ +// Copyright (c) 2019, anonimal, // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -137,7 +138,8 @@ namespace net_utils http_request_info():m_http_method(http_method_unknown), m_http_ver_hi(0), m_http_ver_lo(0), - m_have_to_block(false) + m_have_to_block(false), + m_full_request_buf_size(0) {} http_method m_http_method; @@ -181,4 +183,4 @@ namespace net_utils }; } } -} \ No newline at end of file +} diff --git a/src/contrib/epee/include/net/http_client.h b/src/contrib/epee/include/net/http_client.h index 7de1f15..be81ba4 100644 --- a/src/contrib/epee/include/net/http_client.h +++ b/src/contrib/epee/include/net/http_client.h @@ -38,9 +38,9 @@ #include "net_helper.h" #include "http_client_base.h" -#ifdef HTTP_ENABLE_GZIP +//#ifdef HTTP_ENABLE_GZIP #include "gzip_encoding.h" -#endif +//#endif #include "string_tools.h" #include "reg_exp_definer.h" @@ -230,11 +230,11 @@ using namespace std; blocked_mode_client m_net_client; std::string m_host_buff; std::string m_port; - unsigned int m_timeout; + //unsigned int m_timeout; + unsigned int m_connection_timeout; + unsigned int m_recv_timeout; std::string m_header_cache; http_response_info m_response_info; - size_t m_len_in_summary; - size_t m_len_in_remain; //std::string* m_ptarget_buffer; boost::shared_ptr m_pcontent_encoding_handler; reciev_machine_state m_state; @@ -242,6 +242,10 @@ using namespace std; std::string m_chunked_cache; critical_section m_lock; + protected: + uint64_t m_len_in_summary; + uint64_t m_len_in_remain; + public: void set_host_name(const std::string& name) { @@ -259,14 +263,27 @@ using namespace std; { return connect(host, std::to_string(port), timeout); } - bool connect(const std::string& host, const std::string& port, unsigned int timeout) + + bool set_timeouts(unsigned int connection_timeout, unsigned int recv_timeout) + { + m_connection_timeout = connection_timeout; + m_recv_timeout = recv_timeout; + return true; + } + + bool connect(const std::string& host, std::string port) { CRITICAL_REGION_LOCAL(m_lock); m_host_buff = host; m_port = port; - m_timeout = timeout; - return m_net_client.connect(host, port, timeout, timeout); + return m_net_client.connect(host, port, m_connection_timeout, m_recv_timeout); + } + + bool connect(const std::string& host, const std::string& port, unsigned int timeout) + { + m_connection_timeout = m_recv_timeout = timeout; + return connect(host, port); } //--------------------------------------------------------------------------- bool disconnect() @@ -303,7 +320,7 @@ using namespace std; if(!is_connected()) { LOG_PRINT("Reconnecting...", LOG_LEVEL_3); - if(!connect(m_host_buff, m_port, m_timeout)) + if(!connect(m_host_buff, m_port)) { LOG_PRINT("Failed to connect to " << m_host_buff << ":" << m_port, LOG_LEVEL_3); return false; @@ -447,7 +464,14 @@ using namespace std; } CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()"); m_len_in_remain -= recv_buff.size(); - m_pcontent_encoding_handler->update_in(recv_buff); + bool r = m_pcontent_encoding_handler->update_in(recv_buff); + //CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_pcontent_encoding_handler->update_in returned false"); + if (!r) + { + m_state = reciev_machine_state_error; + disconnect(); + return false; + } if(m_len_in_remain == 0) m_state = reciev_machine_state_done; @@ -483,7 +507,7 @@ using namespace std; } //--------------------------------------------------------------------------- inline - bool get_len_from_chunk_head(const std::string &chunk_head, size_t& result_size) + bool get_len_from_chunk_head(const std::string &chunk_head, uint64_t& result_size) { std::stringstream str_stream; str_stream << std::hex; @@ -494,7 +518,7 @@ using namespace std; } //--------------------------------------------------------------------------- inline - bool get_chunk_head(std::string& buff, size_t& chunk_size, bool& is_matched) + bool get_chunk_head(std::string& buff, uint64_t& chunk_size, bool& is_matched) { is_matched = false; size_t offset = 0; @@ -850,13 +874,13 @@ using namespace std; return true; } }; + // class http_simple_client /************************************************************************/ /* */ /************************************************************************/ - //inline template bool invoke_request(const std::string& url, t_transport& tr, unsigned int timeout, const http_response_info** ppresponse_info, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list()) { @@ -870,13 +894,150 @@ using namespace std; if(!u_c.port) u_c.port = 80;//default for http - res = tr.connect(u_c.host, static_cast(u_c.port), timeout); - CHECK_AND_ASSERT_MES(res, false, "failed to connect " << u_c.host << ":" << u_c.port); + if (!tr.connect(u_c.host, static_cast(u_c.port), timeout)) + { + LOG_PRINT_L2("invoke_request: cannot connect to " << u_c.host << ":" << u_c.port); + return false; + } } return tr.invoke(u_c.uri, method, body, ppresponse_info, additional_params); } - } -} -} + struct idle_handler_base + { + virtual bool do_call(const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) = 0; + virtual ~idle_handler_base() {} + }; + + template + struct idle_handler : public idle_handler_base + { + callback_t m_cb; + + idle_handler(callback_t cb) : m_cb(cb) {} + virtual bool do_call(const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) + { + return m_cb(piece_of_data, total_bytes, received_bytes); + } + }; + + class interruptible_http_client : public http_simple_client + { + std::shared_ptr m_pcb; + bool m_permanent_error = false; + + virtual bool handle_target_data(std::string& piece_of_transfer) + { + bool r = m_pcb->do_call(piece_of_transfer, m_len_in_summary, m_len_in_summary - m_len_in_remain); + piece_of_transfer.clear(); + return r; + } + + public: + template + bool invoke_cb(callback_t cb, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list()) + { + m_pcb.reset(new idle_handler(cb)); + const http_response_info* p_hri = nullptr; + bool r = invoke_request(url, *this, timeout, &p_hri, method, body, additional_params); + if (p_hri && !(p_hri->m_response_code >= 200 && p_hri->m_response_code < 300)) + { + LOG_PRINT_L0("HTTP request to " << url << " failed with code: " << p_hri->m_response_code); + m_permanent_error = true; + return false; + } + return r; + } + + template + bool download(callback_t cb, const std::string& path_for_file, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), const fields_list& additional_params = fields_list()) + { + std::ofstream fs; + fs.open(path_for_file, std::ios::binary | std::ios::out | std::ios::trunc); + if (!fs.is_open()) + { + LOG_ERROR("Fsiled to open " << path_for_file); + return false; + } + auto local_cb = [&](const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) + { + fs.write(piece_of_data.data(), piece_of_data.size()); + return cb(total_bytes, received_bytes); + }; + bool r = this->invoke_cb(local_cb, url, timeout, method, body, additional_params); + fs.close(); + return r; + } + + // + template + bool download_and_unzip(callback_t cb, const std::string& path_for_file, const std::string& url, uint64_t timeout, const std::string& method = "GET", const std::string& body = std::string(), uint64_t fails_count = 1000, const fields_list& additional_params = fields_list()) + { + std::ofstream fs; + fs.open(path_for_file, std::ios::binary | std::ios::out | std::ios::trunc); + if (!fs.is_open()) + { + LOG_ERROR("Fsiled to open " << path_for_file); + return false; + } + std::string buff; + gzip_decoder_lambda zip_decoder; + uint64_t state_total_bytes = 0; + uint64_t state_received_bytes_base = 0; + uint64_t state_received_bytes_current = 0; + bool stopped = false; + auto local_cb = [&](const std::string& piece_of_data, uint64_t total_bytes, uint64_t received_bytes) + { + //remember total_bytes only for first attempt, where fetched full lenght of the file + if (!state_total_bytes) + state_total_bytes = total_bytes; + + buff += piece_of_data; + return zip_decoder.update_in(buff, [&](const std::string& unpacked_buff) + { + state_received_bytes_current = received_bytes; + fs.write(unpacked_buff.data(), unpacked_buff.size()); + stopped = !cb(unpacked_buff, state_total_bytes, state_received_bytes_base + received_bytes); + return !stopped; + }); + }; + uint64_t current_err_count = 0; + bool r = false; + m_permanent_error = false; + while (!r && current_err_count < fails_count) + { + LOG_PRINT_L0("Attempt " << current_err_count + 1 << "/" << fails_count << " to get " << url << " (offset:" << state_received_bytes_base << ")"); + fields_list additional_params_local = additional_params; + additional_params_local.push_back(std::make_pair("Range", std::string("bytes=") + std::to_string(state_received_bytes_base) + "-")); + r = this->invoke_cb(local_cb, url, timeout, method, body, additional_params_local); + if (!r) + { + if (stopped || m_permanent_error) + break; + current_err_count++; + state_received_bytes_base += state_received_bytes_current; + state_received_bytes_current = 0; + boost::this_thread::sleep_for(boost::chrono::milliseconds(2000)); + } + } + + if (current_err_count >= fails_count) + { + LOG_PRINT_YELLOW("Downloading from " << url << " FAILED as it's reached maximum (" << fails_count << ") number of attempts. Downloaded " << state_received_bytes_base << " bytes.", LOG_LEVEL_0); + } + else if (m_permanent_error) + { + LOG_PRINT_YELLOW("Downloading from " << url << " FAILED due to permanent HTTP error. Downloaded " << state_received_bytes_base << " bytes.", LOG_LEVEL_0); + } + + fs.close(); + return r; + } + }; + + + } // namespace http + +} // namespace net_utils +} // namespace epee diff --git a/src/contrib/epee/include/net/http_protocol_handler.inl b/src/contrib/epee/include/net/http_protocol_handler.inl index b84bf47..a5f42c4 100644 --- a/src/contrib/epee/include/net/http_protocol_handler.inl +++ b/src/contrib/epee/include/net/http_protocol_handler.inl @@ -37,7 +37,7 @@ #define HTTP_MAX_PRE_COMMAND_LINE_CHARS 20 #define HTTP_MAX_HEADER_LEN 100000 -PUSH_WARNINGS +PUSH_GCC_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) @@ -690,7 +690,7 @@ namespace net_utils } } -POP_WARNINGS +POP_GCC_WARNINGS //-------------------------------------------------------------------------------------------- //-------------------------------------------------------------------------------------------- //-------------------------------------------------------------------------------------------- diff --git a/src/contrib/epee/include/net/http_server_handlers_map2.h b/src/contrib/epee/include/net/http_server_handlers_map2.h index 0d439ca..6b45950 100644 --- a/src/contrib/epee/include/net/http_server_handlers_map2.h +++ b/src/contrib/epee/include/net/http_server_handlers_map2.h @@ -198,10 +198,12 @@ namespace epee struct response { std::string jsonrpc; + std::string method; t_error error; epee::serialization::storage_entry id; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(jsonrpc) + KV_SERIALIZE(method) KV_SERIALIZE(id) KV_SERIALIZE(error) END_KV_SERIALIZE_MAP() @@ -288,6 +290,7 @@ struct json_command_type_t PREPARE_OBJECTS_FROM_JSON(command_type) \ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ + fail_resp.method = req.method; \ fail_resp.id = req.id; \ if(!callback_f(req.params, resp.result, fail_resp.error, m_conn_context)) \ { \ @@ -304,6 +307,7 @@ struct json_command_type_t PREPARE_OBJECTS_FROM_JSON(command_type) \ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ + fail_resp.method = req.method; \ fail_resp.id = req.id; \ if(!callback_f(req.params, resp.result, fail_resp.error, m_conn_context, response_info)) \ { \ @@ -322,6 +326,7 @@ struct json_command_type_t { \ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ + fail_resp.method = req.method; \ fail_resp.id = req.id; \ fail_resp.error.code = -32603; \ fail_resp.error.message = "Internal error"; \ @@ -338,6 +343,7 @@ struct json_command_type_t epee::json_rpc::error_response rsp; \ rsp.id = id_; \ rsp.jsonrpc = "2.0"; \ + rsp.method = callback_name; \ rsp.error.code = -32601; \ rsp.error.message = "Method not found"; \ epee::serialization::store_t_to_json(static_cast(rsp), response_info.m_body); \ diff --git a/src/contrib/epee/include/net/levin_protocol_handler_async.h b/src/contrib/epee/include/net/levin_protocol_handler_async.h index 105af24..c3cdd02 100644 --- a/src/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/src/contrib/epee/include/net/levin_protocol_handler_async.h @@ -1,3 +1,4 @@ +// Copyright (c) 2019, anonimal // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -76,7 +77,7 @@ public: void on_send_stop_signal(); int invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id); template - int invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, callback_t cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); + int invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t& cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); int notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id); bool close(boost::uuids::uuid connection_id); @@ -86,14 +87,17 @@ public: bool foreach_connection(callback_t cb); size_t get_connections_count(); - async_protocol_handler_config() :m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_is_in_sendstop_loop(false) + async_protocol_handler_config() :m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_is_in_sendstop_loop(false), m_invoke_timeout{} {} ~async_protocol_handler_config() { + NESTED_TRY_ENTRY(); + CRITICAL_REGION_LOCAL(m_connects_lock); m_connects.clear(); + + NESTED_CATCH_ENTRY(__func__); } - }; @@ -213,9 +217,14 @@ public: std::list > m_invoke_response_handlers; template - bool add_invoke_response_handler(callback_t cb, uint64_t timeout, async_protocol_handler& con, int command) + bool add_invoke_response_handler(const callback_t& cb, uint64_t timeout, async_protocol_handler& con, int command) { CRITICAL_REGION_LOCAL(m_invoke_response_handlers_lock); + if (m_protocol_released) + { + LOG_PRINT_L0("ERROR: Adding response handler to a released object"); + return false; + } boost::shared_ptr handler(boost::make_shared>(cb, timeout, con, command)); m_invoke_response_handlers.push_back(handler); LOG_PRINT_L4("[LEVIN_PROTOCOL" << this << "] INVOKE_HANDLER_QUE: PUSH_BACK RESPONSE HANDLER"); @@ -226,6 +235,8 @@ public: async_protocol_handler(net_utils::i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context): + m_invoke_buf_ready{}, + m_invoke_result_code{}, m_current_head(bucket_head2()), m_pservice_endpoint(psnd_hndlr), m_config(config), @@ -243,6 +254,8 @@ public: virtual ~async_protocol_handler() { + NESTED_TRY_ENTRY(); + m_deletion_initiated = true; if(m_connection_initialized) { @@ -261,6 +274,8 @@ public: VALIDATE_MUTEX_IS_FREE(m_send_lock); VALIDATE_MUTEX_IS_FREE(m_call_lock); VALIDATE_MUTEX_IS_FREE(m_invoke_response_handlers_lock); + + NESTED_CATCH_ENTRY(__func__); } bool start_outer_call() @@ -529,7 +544,7 @@ public: } template - bool async_invoke(int command, const std::string& in_buff, callback_t cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke(int command, const std::string& in_buff, const callback_t& cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -784,7 +799,7 @@ int async_protocol_handler_config::invoke(int command, con } //------------------------------------------------------------------------------------------ template template -int async_protocol_handler_config::invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, callback_t cb, size_t timeout) +int async_protocol_handler_config::invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t& cb, size_t timeout) { async_protocol_handler* aph; int r = find_and_lock_connection(connection_id, aph); diff --git a/src/contrib/epee/include/net/net_helper.h b/src/contrib/epee/include/net/net_helper.h index 4f7ebfa..72ff4e7 100644 --- a/src/contrib/epee/include/net/net_helper.h +++ b/src/contrib/epee/include/net/net_helper.h @@ -1,3 +1,4 @@ +// Copyright (c) 2019, anonimal, // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -85,7 +86,9 @@ namespace net_utils m_initialized(false), m_connected(false), m_deadline(m_io_service), - m_shutdowned(0) + m_shutdowned(0), + m_connect_timeout{}, + m_reciev_timeout{} { @@ -104,8 +107,12 @@ namespace net_utils inline ~blocked_mode_client() { + NESTED_TRY_ENTRY(); + //profile_tools::local_coast lc("~blocked_mode_client()", 3); shutdown(); + + NESTED_CATCH_ENTRY(__func__); } inline void set_recv_timeout(int reciev_timeout) diff --git a/src/contrib/epee/include/net/net_utils_base.h b/src/contrib/epee/include/net/net_utils_base.h index 8858685..1ef0677 100644 --- a/src/contrib/epee/include/net/net_utils_base.h +++ b/src/contrib/epee/include/net/net_utils_base.h @@ -117,34 +117,34 @@ namespace net_utils //some helpers - inline - std::string print_connection_context(const connection_context_base& ctx) + inline std::string print_connection_context(const connection_context_base& ctx) { std::stringstream ss; ss << epee::string_tools::get_ip_string_from_int32(ctx.m_remote_ip) << ":" << ctx.m_remote_port << " " << epee::string_tools::get_str_from_guid_a(ctx.m_connection_id) << (ctx.m_is_income ? " INC":" OUT"); return ss.str(); } - inline - void print_connection_context_short(const connection_context_base& ctx, std::stringstream& ss) + inline std::ostream &operator <<(std::ostream &o, const connection_context_base& ctx) { - ss << epee::string_tools::get_ip_string_from_int32(ctx.m_remote_ip) << ":" << ctx.m_remote_port << (ctx.m_is_income ? " INC" : " OUT"); - } + o << epee::string_tools::get_ip_string_from_int32(ctx.m_remote_ip) << ":" << ctx.m_remote_port << (ctx.m_is_income ? " INC" : " OUT"); + return o; + } - inline - std::string print_connection_context_short(const connection_context_base& ctx) + inline std::string print_connection_context_short(const connection_context_base& ctx) { std::stringstream ss; - print_connection_context_short(ctx, ss); + ss << ctx; return ss.str(); } - inline std::string print_connection_context_list(const std::list& contexts) + inline std::string print_connection_context_list(const std::list& contexts, const std::string& delim = std::string("\n")) { std::stringstream ss; for (auto& c : contexts) { - ss << epee::string_tools::get_ip_string_from_int32(c.m_remote_ip) << ":" << c.m_remote_port << (c.m_is_income ? " INC" : " OUT") << ENDL; + if (ss.tellp()) + ss << delim; + ss << c; } return ss.str(); } @@ -179,7 +179,7 @@ namespace net_utils #define CHECK_AND_ASSERT_MES_CC(condition, return_val, err_message) CHECK_AND_ASSERT_MES(condition, return_val, "[" << epee::net_utils::print_connection_context_short(context) << "]" << err_message) -} -} +} // namespace net_utils +} // namespace epee #endif //_NET_UTILS_BASE_H_ diff --git a/src/contrib/epee/include/profile_tools.h b/src/contrib/epee/include/profile_tools.h index aec1806..b66adac 100644 --- a/src/contrib/epee/include/profile_tools.h +++ b/src/contrib/epee/include/profile_tools.h @@ -1,3 +1,4 @@ +// Copyright (c) 2019, anonimal // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -106,10 +107,14 @@ namespace profile_tools } ~local_call_account() { + NESTED_TRY_ENTRY(); + LOG_PRINT2("profile_details.log", "PROFILE "<< std::left << std::setw(50) << (m_name + ":") << "av_time:" << std::setw(15) << epee::string_tools::print_fixed_decimal_point (m_count_of_call ? (m_summary_time_used / m_count_of_call) : 0, 3) << "sum_time: " << std::setw(15) << epee::string_tools::print_fixed_decimal_point(m_summary_time_used, 3) << "call_count: " << std::setw(15) << m_count_of_call, LOG_LEVEL_0); + + NESTED_CATCH_ENTRY(__func__); } size_t m_count_of_call; @@ -124,17 +129,21 @@ namespace profile_tools { m_call_time = boost::posix_time::microsec_clock::local_time(); } - - ~call_frame() - { - boost::posix_time::ptime now_t(boost::posix_time::microsec_clock::local_time()); - boost::posix_time::time_duration delta_microsec = now_t - m_call_time; - uint64_t microseconds_used = delta_microsec.total_microseconds(); - m_cc.m_summary_time_used += microseconds_used; - m_cc.m_count_of_call++; - } - - private: + + ~call_frame() + { + NESTED_TRY_ENTRY(); + + boost::posix_time::ptime now_t(boost::posix_time::microsec_clock::local_time()); + boost::posix_time::time_duration delta_microsec = now_t - m_call_time; + uint64_t microseconds_used = delta_microsec.total_microseconds(); + m_cc.m_summary_time_used += microseconds_used; + m_cc.m_count_of_call++; + + NESTED_CATCH_ENTRY(__func__); + } + + private: local_call_account& m_cc; boost::posix_time::ptime m_call_time; }; diff --git a/src/contrib/epee/include/reg_exp_definer.h b/src/contrib/epee/include/reg_exp_definer.h index e2bed5c..06e476f 100644 --- a/src/contrib/epee/include/reg_exp_definer.h +++ b/src/contrib/epee/include/reg_exp_definer.h @@ -44,12 +44,17 @@ namespace epee const static global_regexp_critical_section gregexplock; + inline const boost::regex* build_regexp(const char* p, boost::regex::flag_type f) + { + return new boost::regex(p, f); + } + #define STATIC_REGEXP_EXPR_1(var_name, xpr_text, reg_exp_flags) \ static volatile uint32_t regexp_initialized_1 = 0;\ volatile uint32_t local_is_initialized_1 = regexp_initialized_1;\ if(!local_is_initialized_1)\ gregexplock.get_lock().lock();\ - static const boost::regex var_name(xpr_text , reg_exp_flags);\ + static const boost::regex& var_name = *build_regexp(xpr_text, reg_exp_flags);\ if(!local_is_initialized_1)\ {\ boost::interprocess::ipcdetail::atomic_write32(®exp_initialized_1, 1);\ @@ -61,7 +66,7 @@ namespace epee volatile uint32_t local_is_initialized_2 = regexp_initialized_2;\ if(!local_is_initialized_2)\ gregexplock.get_lock().lock().lock();\ - static const boost::regex var_name(xpr_text , reg_exp_flags);\ + static const boost::regex& var_name = *build_regexp(xpr_text, reg_exp_flags);\ if(!local_is_initialized_2)\ {\ boost::interprocess::ipcdetail::atomic_write32(®exp_initialized_2, 1);\ @@ -73,7 +78,7 @@ namespace epee volatile uint32_t local_is_initialized_3 = regexp_initialized_3;\ if(!local_is_initialized_3)\ gregexplock.get_lock().lock().lock();\ - static const boost::regex var_name(xpr_text , reg_exp_flags);\ + static const boost::regex& var_name = *build_regexp(xpr_text, reg_exp_flags);\ if(!local_is_initialized_3)\ {\ boost::interprocess::ipcdetail::atomic_write32(®exp_initialized_3, 1);\ diff --git a/src/contrib/epee/include/serialization/keyvalue_helpers.h b/src/contrib/epee/include/serialization/keyvalue_helpers.h index 924ab89..0ae911b 100644 --- a/src/contrib/epee/include/serialization/keyvalue_helpers.h +++ b/src/contrib/epee/include/serialization/keyvalue_helpers.h @@ -51,6 +51,7 @@ namespace epee } }; + //basic helpers for pod-to-hex serialization template std::string transform_t_pod_to_str(const t_pod_type & a) @@ -61,10 +62,34 @@ namespace epee t_pod_type transform_str_to_t_pod(const std::string& a) { t_pod_type res = AUTO_VAL_INIT(res); - epee::string_tools::hex_to_pod(a, res); + if (!epee::string_tools::hex_to_pod(a, res)) + throw std::runtime_error(std::string("Unable to transform \"") + a + "\" to pod type " + typeid(t_pod_type).name()); return res; } + + //basic helpers for blob-to-hex serialization + + inline std::string transform_binbuf_to_hexstr(const std::string& a) + { + return epee::string_tools::buff_to_hex_nodelimer(a); + } + + inline std::string transform_hexstr_to_binbuff(const std::string& a) + { + std::string res; + if (!epee::string_tools::parse_hexstr_to_binbuff(a, res)) + { + CHECK_AND_ASSERT_THROW_MES(false, "Failed to parse hex string:" << a); + } + return res; + } //------------------------------------------------------------------------------------------------------------------- - - +#pragma pack(push, 1) + template + struct pod_pair + { + first_t first; + second_t second; + }; +#pragma pack(pop) } \ No newline at end of file diff --git a/src/contrib/epee/include/serialization/keyvalue_hexemizer.h b/src/contrib/epee/include/serialization/keyvalue_hexemizer.h new file mode 100644 index 0000000..47e7e34 --- /dev/null +++ b/src/contrib/epee/include/serialization/keyvalue_hexemizer.h @@ -0,0 +1,41 @@ +// Copyright (c) 2006-2019, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#pragma once +#include "keyvalue_serialization.h" +namespace epee +{ + + struct hexemizer + { + std::string blob; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(blob) + END_KV_SERIALIZE_MAP() + }; + + +} \ No newline at end of file diff --git a/src/contrib/epee/include/serialization/keyvalue_serialization.h b/src/contrib/epee/include/serialization/keyvalue_serialization.h index a73c6b8..31a4fb8 100644 --- a/src/contrib/epee/include/serialization/keyvalue_serialization.h +++ b/src/contrib/epee/include/serialization/keyvalue_serialization.h @@ -78,7 +78,8 @@ public: \ #define KV_SERIALIZE_POD_AS_HEX_STRING_N(varialble, val_name) \ KV_SERIALIZE_CUSTOM_N(varialble, std::string, epee::transform_t_pod_to_str, epee::transform_str_to_t_pod, val_name) - +#define KV_SERIALIZE_BLOB_AS_HEX_STRING_N(varialble, val_name) \ + KV_SERIALIZE_CUSTOM_N(varialble, std::string, epee::transform_binbuf_to_hexstr, epee::transform_hexstr_to_binbuff, val_name) #define KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, val_name) \ @@ -99,6 +100,8 @@ public: \ #define KV_SERIALIZE_CONTAINER_POD_AS_BLOB(varialble) KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(varialble, #varialble) #define KV_SERIALIZE_CUSTOM(varialble, stored_type, from_v_to_stored, from_stored_to_v) KV_SERIALIZE_CUSTOM_N(varialble, stored_type, from_v_to_stored, from_stored_to_v, #varialble) #define KV_SERIALIZE_POD_AS_HEX_STRING(varialble) KV_SERIALIZE_POD_AS_HEX_STRING_N(varialble, #varialble) +#define KV_SERIALIZE_BLOB_AS_HEX_STRING(varialble) KV_SERIALIZE_BLOB_AS_HEX_STRING_N(varialble, #varialble) + #define KV_CHAIN_MAP(variable_obj) epee::namespace_accessor::template serialize_map(this_ref.variable_obj, stg, hparent_section); @@ -106,6 +109,20 @@ public: \ + template + struct uint_mask_selector + { + template + inline static bool get_value_of_flag_by_mask(const t_uint& given_flags) + { + return given_flags&mask == 0 ? false : true; + } + }; + + +#define KV_SERIALIZE_EPHEMERAL_BOOL_FROM_FLAG_N(var, mask, val_name) \ + KV_SERIALIZE_EPHEMERAL_N(bool, uint_mask_selector::get_value_of_flag_by_mask, val_name) + diff --git a/src/contrib/epee/include/singleton.h b/src/contrib/epee/include/singleton.h index f510c9a..a770ab2 100644 --- a/src/contrib/epee/include/singleton.h +++ b/src/contrib/epee/include/singleton.h @@ -31,7 +31,7 @@ #pragma once #include - +#include "static_helpers.h" template class abstract_singleton @@ -39,7 +39,7 @@ class abstract_singleton static std::shared_ptr get_set_instance_internal(bool is_need_set = false, owned_object_t* pnew_obj = nullptr) { - static std::shared_ptr val_pobj; + static epee::static_helpers::wrapper> val_pobj; if (is_need_set) val_pobj.reset(pnew_obj); diff --git a/src/contrib/epee/include/static_helpers.h b/src/contrib/epee/include/static_helpers.h new file mode 100644 index 0000000..d3c11c8 --- /dev/null +++ b/src/contrib/epee/include/static_helpers.h @@ -0,0 +1,101 @@ +// Copyright (c) 2006-2020, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + + + +#pragma once +#include +#include +#include +#include +#include "include_base_utils.h" +#include "auto_val_init.h" + +#define DEFINE_SECURE_STATIC_VAR(type, var) static epee::static_helpers::wrapper var##inst; \ + static type& var = var##inst; + +namespace epee +{ + namespace static_helpers + { + template + class initializer + { + public: + initializer() + { + to_initialize::init(); + //get_set_is_initialized(true, true); + } + ~initializer() + { + to_initialize::un_init(); + //get_set_is_uninitialized(true, true); + } + }; + + + typedef void(*static_destroy_handler_type)(); + + inline + bool set_or_call_on_destruct(bool set = false, static_destroy_handler_type destr_ptr = nullptr) + { + volatile static bool deinit_called = false; + volatile static static_destroy_handler_type static_destroy_handler = nullptr; + + if (set) + { + static_destroy_handler = destr_ptr; + return true; + } + if (!deinit_called) + { + + if (static_destroy_handler) + static_destroy_handler(); + + deinit_called = true; + } + + return true; + } + + template + struct wrapper : public t_base + { + ~wrapper() + { + set_or_call_on_destruct(); + } + + }; + + } + + + + +} diff --git a/src/contrib/epee/include/storages/levin_abstract_invoke2.h b/src/contrib/epee/include/storages/levin_abstract_invoke2.h index 4ec014b..1b2c74a 100644 --- a/src/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/src/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -1,3 +1,4 @@ +// Copyright (c) 2019, anonimal, // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -114,7 +115,7 @@ namespace epee } template - bool async_invoke_remote_command2(boost::uuids::uuid conn_id, int command, const t_arg& out_struct, t_transport& transport, callback_t cb, size_t inv_timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke_remote_command2(boost::uuids::uuid conn_id, int command, const t_arg& out_struct, t_transport& transport, const callback_t& cb, size_t inv_timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { typename serialization::portable_storage stg; const_cast(out_struct).store(stg);//TODO: add true const support to searilzation @@ -125,7 +126,8 @@ namespace epee t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) { - LOG_PRINT_L1("Failed to invoke command " << command << " return code " << code << "(" << epee::levin::get_err_descr(code) << ")"); + LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::get_callstack()); + LOG_PRINT_L1("Failed to invoke command " << command << " return code " << code << "(" << epee::levin::get_err_descr(code) << ")context:" << print_connection_context(context)); TRY_ENTRY() cb(code, result_struct, context); CATCH_ENTRY2(true) @@ -148,7 +150,8 @@ namespace epee }, inv_timeout); if( res <=0 ) { - LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res << "(" << epee::levin::get_err_descr(res)<< ")"); + LOG_PRINT_L2("BACKTRACE: " << ENDL << epee::misc_utils::get_callstack()); + LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res << "(" << epee::levin::get_err_descr(res) << ") conn_id=" << conn_id); return false; } return true; diff --git a/src/contrib/epee/include/storages/portable_storage.h b/src/contrib/epee/include/storages/portable_storage.h index b9b7aba..92fbeec 100644 --- a/src/contrib/epee/include/storages/portable_storage.h +++ b/src/contrib/epee/include/storages/portable_storage.h @@ -83,7 +83,7 @@ namespace epee bool load_from_binary(const binarybuffer& target); template bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); - bool dump_as_json(std::string& targetObj, size_t indent = 0); + bool dump_as_json(std::string& targetObj, size_t indent = 0, end_of_line_t eol = eol_crlf); bool load_from_json(const std::string& source); private: @@ -106,11 +106,11 @@ namespace epee #pragma pack(pop) }; inline - bool portable_storage::dump_as_json(std::string& buff, size_t indent) + bool portable_storage::dump_as_json(std::string& buff, size_t indent /* = 0 */, end_of_line_t eol /* = eol_crlf */) { TRY_ENTRY(); std::stringstream ss; - epee::serialization::dump_as_json(ss, m_root, indent); + epee::serialization::dump_as_json(ss, m_root, indent, eol); buff = ss.str(); return true; CATCH_ENTRY("portable_storage::dump_as_json", false) diff --git a/src/contrib/epee/include/storages/portable_storage_base.h b/src/contrib/epee/include/storages/portable_storage_base.h index 3f16375..99c792e 100644 --- a/src/contrib/epee/include/storages/portable_storage_base.h +++ b/src/contrib/epee/include/storages/portable_storage_base.h @@ -69,6 +69,8 @@ namespace epee { namespace serialization { + enum end_of_line_t { eol_crlf = 0, eol_lf = 1, eol_cr = 2, eol_space = 3 }; + struct section; /************************************************************************/ @@ -78,6 +80,7 @@ namespace epee struct array_entry_t { array_entry_t():m_it(m_array.end()){} + array_entry_t(const array_entry_t& other):m_array(other.m_array), m_it(m_array.end()){} const t_entry_type* get_first_val() const { diff --git a/src/contrib/epee/include/storages/portable_storage_template_helper.h b/src/contrib/epee/include/storages/portable_storage_template_helper.h index 4f91e08..896f9eb 100644 --- a/src/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/src/contrib/epee/include/storages/portable_storage_template_helper.h @@ -56,16 +56,16 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template - bool store_t_to_json(const t_struct& str_in, std::string& json_buff, size_t indent = 0) + bool store_t_to_json(const t_struct& str_in, std::string& json_buff, size_t indent = 0, end_of_line_t eol = eol_crlf) { portable_storage ps; str_in.store(ps); - ps.dump_as_json(json_buff, indent); + ps.dump_as_json(json_buff, indent, eol); return true; } //----------------------------------------------------------------------------------------------------------- template - std::string store_t_to_json(const t_struct& str_in, size_t indent = 0) + std::string store_t_to_json(const t_struct& str_in, size_t indent = 0, end_of_line_t eol = eol_crlf) { std::string json_buff; store_t_to_json(str_in, json_buff, indent); @@ -101,7 +101,7 @@ namespace epee return load_t_from_binary(out, f_buff); } //----------------------------------------------------------------------------------------------------------- -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4100) template bool store_t_to_binary(const t_struct& str_in, std::string& binary_buff, size_t indent = 0) @@ -110,7 +110,7 @@ DISABLE_VS_WARNINGS(4100) str_in.store(ps); return ps.store_to_binary(binary_buff); } -POP_WARNINGS +POP_VS_WARNINGS //----------------------------------------------------------------------------------------------------------- template std::string store_t_to_binary(const t_struct& str_in, size_t indent = 0) diff --git a/src/contrib/epee/include/storages/portable_storage_to_bin.h b/src/contrib/epee/include/storages/portable_storage_to_bin.h index cb74bc0..97c5cd5 100644 --- a/src/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/src/contrib/epee/include/storages/portable_storage_to_bin.h @@ -37,7 +37,7 @@ namespace epee { template - size_t pack_varint_t(t_stream& strm, uint8_t type_or, size_t& pv) + size_t pack_varint_t(t_stream& strm, uint8_t type_or, const uint64_t& pv) { pack_value v = (*((pack_value*)&pv)) << 2; v |= type_or; @@ -45,10 +45,10 @@ namespace epee return sizeof(pack_value); } - PUSH_WARNINGS + PUSH_GCC_WARNINGS DISABLE_GCC_WARNING(strict-aliasing) template - size_t pack_varint(t_stream& strm, size_t val) + size_t pack_varint(t_stream& strm, uint64_t val) { //the first two bits always reserved for size information if(val <= 63) @@ -67,7 +67,7 @@ namespace epee return pack_varint_t(strm, PORTABLE_RAW_SIZE_MARK_INT64, val); } } - POP_WARNINGS + POP_GCC_WARNINGS template bool put_string(t_stream& strm, const std::string& v) diff --git a/src/contrib/epee/include/storages/portable_storage_to_json.h b/src/contrib/epee/include/storages/portable_storage_to_json.h index 7c22f30..1e2d5d9 100644 --- a/src/contrib/epee/include/storages/portable_storage_to_json.h +++ b/src/contrib/epee/include/storages/portable_storage_to_json.h @@ -37,23 +37,23 @@ namespace epee { template - void dump_as_json(t_stream& strm, const array_entry& ae, size_t indent); + void dump_as_json(t_stream& strm, const array_entry& ae, size_t indent, end_of_line_t eol = eol_crlf); template - void dump_as_json(t_stream& strm, const storage_entry& se, size_t indent); + void dump_as_json(t_stream& strm, const storage_entry& se, size_t indent, end_of_line_t eol = eol_crlf); template - void dump_as_json(t_stream& strm, const std::string& v, size_t indent); + void dump_as_json(t_stream& strm, const std::string& v, size_t indent, end_of_line_t eol = eol_crlf); template - void dump_as_json(t_stream& strm, const int8_t& v, size_t indent); + void dump_as_json(t_stream& strm, const int8_t& v, size_t indent, end_of_line_t eol = eol_crlf); template - void dump_as_json(t_stream& strm, const uint8_t& v, size_t indent); + void dump_as_json(t_stream& strm, const uint8_t& v, size_t indent, end_of_line_t eol = eol_crlf); template - void dump_as_json(t_stream& strm, const bool& v, size_t indent); + void dump_as_json(t_stream& strm, const bool& v, size_t indent, end_of_line_t eol = eol_crlf); template - void dump_as_json(t_stream& strm, const double& v, size_t indent); + void dump_as_json(t_stream& strm, const double& v, size_t indent, end_of_line_t eol = eol_crlf); template - void dump_as_json(t_stream& strm, const t_type& v, size_t indent); + void dump_as_json(t_stream& strm, const t_type& v, size_t indent, end_of_line_t eol = eol_crlf); template - void dump_as_json(t_stream& strm, const section& sec, size_t indent); + void dump_as_json(t_stream& strm, const section& sec, size_t indent, end_of_line_t eol = eol_crlf); inline std::string make_indent(size_t indent) @@ -66,7 +66,13 @@ namespace epee { t_stream& m_strm; size_t m_indent; - array_entry_store_to_json_visitor(t_stream& strm, size_t indent):m_strm(strm), m_indent(indent){} + end_of_line_t m_eol; + + array_entry_store_to_json_visitor(t_stream& strm, size_t indent, end_of_line_t eol) + : m_strm(strm) + , m_indent(indent) + , m_eol(eol) + {} template void operator()(const array_entry_t& a) @@ -77,7 +83,7 @@ namespace epee auto last_it = --a.m_array.end(); for(auto it = a.m_array.begin(); it != a.m_array.end(); it++) { - dump_as_json(m_strm, *it, m_indent); + dump_as_json(m_strm, *it, m_indent, m_eol); if(it != last_it) m_strm << ","; } @@ -91,50 +97,56 @@ namespace epee { t_stream& m_strm; size_t m_indent; - storage_entry_store_to_json_visitor(t_stream& strm, size_t indent):m_strm(strm), m_indent(indent) + end_of_line_t m_eol; + + storage_entry_store_to_json_visitor(t_stream& strm, size_t indent, end_of_line_t eol) + : m_strm(strm) + , m_indent(indent) + , m_eol(eol) {} + //section, array_entry template void operator()(const visited_type& v) { - dump_as_json(m_strm, v, m_indent); + dump_as_json(m_strm, v, m_indent, m_eol); } }; template - void dump_as_json(t_stream& strm, const array_entry& ae, size_t indent) + void dump_as_json(t_stream& strm, const array_entry& ae, size_t indent, end_of_line_t eol) { - array_entry_store_to_json_visitor aesv(strm, indent); + array_entry_store_to_json_visitor aesv(strm, indent, eol); boost::apply_visitor(aesv, ae); } template - void dump_as_json(t_stream& strm, const storage_entry& se, size_t indent) + void dump_as_json(t_stream& strm, const storage_entry& se, size_t indent, end_of_line_t eol) { - storage_entry_store_to_json_visitor sv(strm, indent); + storage_entry_store_to_json_visitor sv(strm, indent, eol); boost::apply_visitor(sv, se); } template - void dump_as_json(t_stream& strm, const std::string& v, size_t indent) + void dump_as_json(t_stream& strm, const std::string& v, size_t indent, end_of_line_t eol) { strm << "\"" << misc_utils::parse::transform_to_json_escape_sequence(v) << "\""; } template - void dump_as_json(t_stream& strm, const int8_t& v, size_t indent) + void dump_as_json(t_stream& strm, const int8_t& v, size_t indent, end_of_line_t eol) { strm << static_cast(v); } template - void dump_as_json(t_stream& strm, const uint8_t& v, size_t indent) + void dump_as_json(t_stream& strm, const uint8_t& v, size_t indent, end_of_line_t eol) { strm << static_cast(v); } template - void dump_as_json(t_stream& strm, const bool& v, size_t indent) + void dump_as_json(t_stream& strm, const bool& v, size_t indent, end_of_line_t eol) { if(v) strm << "true"; @@ -143,23 +155,35 @@ namespace epee } template - void dump_as_json(t_stream& strm, const double& v, size_t indent) + void dump_as_json(t_stream& strm, const double& v, size_t indent, end_of_line_t eol) { + boost::io::ios_flags_saver ifs(strm); strm.precision(8); strm << std::fixed << v; } template - void dump_as_json(t_stream& strm, const t_type& v, size_t /*indent*/) + void dump_as_json(t_stream& strm, const t_type& v, size_t indent, end_of_line_t eol) { strm << v; } template - void dump_as_json(t_stream& strm, const section& sec, size_t indent) + void dump_as_json(t_stream& strm, const section& sec, size_t indent, end_of_line_t eol) { + auto put_eol = [&]() { + switch (eol) + { + case eol_lf: strm << "\n"; break; + case eol_cr: strm << "\r"; break; + case eol_space: strm << " "; break; + default: strm << "\r\n"; break; + } + }; + size_t local_indent = indent + 1; - strm << "{\r\n"; + strm << "{"; + put_eol(); std::string indent_str = make_indent(local_indent); if(sec.m_entries.size()) { @@ -167,10 +191,10 @@ namespace epee for(auto it = sec.m_entries.begin(); it!= sec.m_entries.end();it++) { strm << indent_str << "\"" << misc_utils::parse::transform_to_json_escape_sequence(it->first) << "\"" << ": "; - dump_as_json(strm, it->second, local_indent); + dump_as_json(strm, it->second, local_indent, eol); if(it_last != it) strm << ","; - strm << "\r\n"; + put_eol(); } } strm << make_indent(indent) << "}"; diff --git a/src/contrib/epee/include/storages/portable_storage_val_converters.h b/src/contrib/epee/include/storages/portable_storage_val_converters.h index e58ae34..caf7a32 100644 --- a/src/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/src/contrib/epee/include/storages/portable_storage_val_converters.h @@ -41,33 +41,37 @@ namespace epee template void convert_int_to_uint(const from_type& from, to_type& to) { -PUSH_WARNINGS +PUSH_VS_WARNINGS +PUSH_GCC_WARNINGS DISABLE_VS_WARNINGS(4018) CHECK_AND_ASSERT_THROW_MES(from >=0, "unexpected int value with signed storage value less than 0, and unsigned receiver value: " << from); DISABLE_GCC_AND_CLANG_WARNING(sign-compare) CHECK_AND_ASSERT_THROW_MES(from <= std::numeric_limits::max(), "int value overhead: try to set value " << from << " to type " << typeid(to_type).name() << " with max possible value = " << std::numeric_limits::max()); to = static_cast(from); -POP_WARNINGS +POP_GCC_WARNINGS +POP_VS_WARNINGS } template void convert_int_to_int(const from_type& from, to_type& to) { CHECK_AND_ASSERT_THROW_MES(from >= boost::numeric::bounds::lowest(), "int value overhead: try to set value " << from << " to type " << typeid(to_type).name() << " with lowest possible value = " << boost::numeric::bounds::lowest()); -PUSH_WARNINGS +PUSH_GCC_WARNINGS DISABLE_CLANG_WARNING(tautological-constant-out-of-range-compare) CHECK_AND_ASSERT_THROW_MES(from <= std::numeric_limits::max(), "int value overhead: try to set value " << from << " to type " << typeid(to_type).name() << " with max possible value = " << std::numeric_limits::max()); -POP_WARNINGS +POP_GCC_WARNINGS to = static_cast(from); } template void convert_uint_to_any_int(const from_type& from, to_type& to) { -PUSH_WARNINGS +PUSH_VS_WARNINGS +PUSH_GCC_WARNINGS DISABLE_VS_WARNINGS(4018) DISABLE_CLANG_WARNING(tautological-constant-out-of-range-compare) CHECK_AND_ASSERT_THROW_MES(from <= std::numeric_limits::max(), "uint value overhead: try to set value " << from << " to type " << typeid(to_type).name() << " with max possible value = " << std::numeric_limits::max()); to = static_cast(from); -POP_WARNINGS +POP_GCC_WARNINGS +POP_VS_WARNINGS } template //is from signed, is from to signed diff --git a/src/contrib/epee/include/string_coding.h b/src/contrib/epee/include/string_coding.h index a6d2acf..0b125bb 100644 --- a/src/contrib/epee/include/string_coding.h +++ b/src/contrib/epee/include/string_coding.h @@ -37,6 +37,8 @@ #endif #include #endif +#include "warnings.h" + namespace epee { @@ -59,8 +61,10 @@ namespace string_encoding inline std::string convert_to_ansii(const std::wstring& str_from) { - + PUSH_VS_WARNINGS + DISABLE_VS_WARNINGS(4244) std::string res(str_from.begin(), str_from.end()); + POP_VS_WARNINGS return res; } #ifdef WIN32 diff --git a/src/contrib/epee/include/string_tools.h b/src/contrib/epee/include/string_tools.h index f66abfc..a4e67f4 100644 --- a/src/contrib/epee/include/string_tools.h +++ b/src/contrib/epee/include/string_tools.h @@ -42,6 +42,7 @@ #include #include "warnings.h" #include "auto_val_init.h" +#include "string_coding.h" #ifndef OUT @@ -215,7 +216,7 @@ namespace string_tools return t_pod; } //---------------------------------------------------------------------------- -PUSH_WARNINGS +PUSH_GCC_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) template inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) @@ -246,7 +247,7 @@ DISABLE_GCC_WARNING(maybe-uninitialized) return true; } -POP_WARNINGS +POP_GCC_WARNINGS //--------------------------------------------------- template bool get_xnum_from_hex_string(const std::string str, int_t& res ) @@ -536,38 +537,15 @@ POP_WARNINGS return module_folder; } //---------------------------------------------------------------------------- -#ifdef _WIN32 - inline std::string get_current_module_path() + inline bool set_module_name_and_folder(const std::string& path_to_process_) { - char pname [5000] = {0}; - GetModuleFileNameA( NULL, pname, sizeof(pname)); - pname[sizeof(pname)-1] = 0; //be happy ;) - return pname; - } -#endif - //---------------------------------------------------------------------------- - inline bool set_module_name_and_folder(const std::string& path_to_process_) - { - std::string path_to_process = path_to_process_; - boost::system::error_code ec; - path_to_process = boost::filesystem::canonical(path_to_process, ec).string(); - #ifdef _WIN32 - path_to_process = get_current_module_path(); - #endif - std::string::size_type a = path_to_process.rfind( '\\' ); - if(a == std::string::npos ) - { - a = path_to_process.rfind( '/' ); - } - if ( a != std::string::npos ) - { - get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); - get_current_module_folder() = path_to_process.substr(0, a); - return true; - }else - return false; + boost::filesystem::path path(epee::string_encoding::utf8_to_wstring(path_to_process_)); - } + get_current_module_folder() = epee::string_encoding::wstring_to_utf8(path.parent_path().wstring()); + get_current_module_name() = epee::string_encoding::wstring_to_utf8(path.filename().wstring()); + + return true; + } //---------------------------------------------------------------------------- inline bool trim_left(std::string& str) { @@ -625,6 +603,15 @@ POP_WARNINGS s = *(t_pod_type*)bin_buff.data(); return true; } + //---------------------------------------------------------------------------- + template + t_pod_type hex_to_pod(const std::string& hex_str) + { + t_pod_type p = AUTO_VAL_INIT(p); + if (!hex_to_pod(hex_str, p)) + return AUTO_VAL_INIT_T(t_pod_type); + return p; + } //---------------------------------------------------------------------------- inline std::string get_extension(const std::string& str) { @@ -648,9 +635,6 @@ POP_WARNINGS return res; } //---------------------------------------------------------------------------- - - - inline std::string cut_off_extension(const std::string& str) { std::string res; @@ -661,7 +645,17 @@ POP_WARNINGS res = str.substr(0, pos); return res; } + //---------------------------------------------------------------------------- + inline std::wstring cut_off_extension(const std::wstring& str) + { + std::wstring res; + std::wstring::size_type pos = str.rfind('.'); + if (std::wstring::npos == pos) + return str; + res = str.substr(0, pos); + return res; + } //---------------------------------------------------------------------------- // replaces all non-ascii characters with mask_character inline std::string mask_non_ascii_chars(const std::string& str, const char mask_character = '?') diff --git a/src/contrib/epee/include/sync_locked_object.h b/src/contrib/epee/include/sync_locked_object.h index 09e8517..2980bbb 100644 --- a/src/contrib/epee/include/sync_locked_object.h +++ b/src/contrib/epee/include/sync_locked_object.h @@ -54,8 +54,10 @@ namespace epee {} ~locked_object_proxy() { + TRY_ENTRY(); uint64_t lock_time = epee::misc_utils::get_tick_count() - start_lock_time; lock_time_watching_policy::watch_lock_time(lock_time); + CATCH_ALL_DO_NOTHING(); } /* @@ -82,6 +84,13 @@ namespace epee template friend class locked_object_proxy; public: + std::shared_ptr> lock() + { + std::shared_ptr> res; + res.reset(new locked_object_proxy(t, m)); + return res; + } + std::shared_ptr> try_lock() { std::shared_ptr> res; diff --git a/src/contrib/epee/include/syncobj.h b/src/contrib/epee/include/syncobj.h index c31b5e0..07c0a48 100644 --- a/src/contrib/epee/include/syncobj.h +++ b/src/contrib/epee/include/syncobj.h @@ -1,4 +1,6 @@ -// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// Copyright (c) 2019, Zano Project +// Copyright (c) 2019, anonimal, +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // // Redistribution and use in source and binary forms, with or without @@ -23,12 +25,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - - - -#ifndef __WINH_OBJ_H__ -#define __WINH_OBJ_H__ +#pragma once #include #include #include @@ -38,27 +35,12 @@ #include #include "singleton.h" -#include "static_initializer.h" - +#include "static_helpers.h" +#include "misc_helpers.h" //#define DISABLE_DEADLOCK_GUARD -#define VALIDATE_MUTEX_IS_FREE(mutex_mame) \ - if (mutex_mame.try_lock()) \ - { \ - mutex_mame.unlock(); \ - return; \ - } \ - else \ - { \ - LOG_ERROR("MUTEX IS NOT FREE ON DESTRUCTOR: " << #mutex_mame); \ - } - - - - - namespace epee { @@ -525,9 +507,11 @@ namespace epee << prev_it->second.func_name << " @ " << prev_it->second.block_location << std::endl << " |" << std::endl << " V" << std::endl; prev_it = current_it; } - - ss << prev_it->second.thread_name << "(tid:" << prev_it->first << ") blocked by locker \"" << lock_name << "(owned by " << (*threads_chain.begin())->second.thread_name << " tid:" << (*threads_chain.begin())->first << ")] at " - << func_name << " @ " << location << std::endl; + if (prev_it != m_thread_owned_locks.end()) + { + ss << prev_it->second.thread_name << "(tid:" << prev_it->first << ") blocked by locker \"" << lock_name << "(owned by " << (*threads_chain.begin())->second.thread_name << " tid:" << (*threads_chain.begin())->first << ")] at " + << func_name << " @ " << location << std::endl; + } m_deadlock_journal.push_back(ss.str()); throw std::runtime_error(ss.str()); } @@ -536,7 +520,7 @@ namespace epee } }; - const static initializer > singleton_initializer; + //const static initializer > singleton_initializer; /************************************************************************/ /* */ @@ -604,8 +588,9 @@ namespace epee ~guarded_critical_region_t() { + TRY_ENTRY(); unlock(); - + CATCH_ALL_DO_NOTHING(); } void unlock() @@ -643,13 +628,27 @@ namespace epee #define DEADLOCK_LOCATION __FILE__ ":" DEADLOCK_STRINGIFY(__LINE__) -/* + /* + + We do DEADLOCK_LOCATION and DEADLOCK_FUNCTION_DEF as separate variables passed into deadlock_guard + because in GCC __PRETTY_FUNCTION__ is not a literal (like __FILE__ macro) but const variable, and + can't concatenate it with other macro like we did in DEADLOCK_LOCATION. + + */ + +#define VALIDATE_MUTEX_IS_FREE(mutex_mame) \ + if (mutex_mame.try_lock()) \ + { \ + mutex_mame.unlock(); \ + return; \ + } \ + else \ + { \ + auto state_str = epee::deadlock_guard_singleton::get_dlg_state(); \ + LOG_ERROR("MUTEX IS NOT FREE ON DESTRUCTOR: " << #mutex_mame << ", address:" << (void*)&mutex_mame << ENDL << "DEAD LOCK GUARD state:" << ENDL << state_str); \ + } - We do DEADLOCK_LOCATION and DEADLOCK_FUNCTION_DEF as separate variables passed into deadlock_guard - because in GCC __PRETTY_FUNCTION__ is not a literal (like __FILE__ macro) but const variable, and - can't concatenate it with other macro like we did in DEADLOCK_LOCATION. -*/ #define DLG_CRITICAL_REGION_LOCAL_VAR(lock, varname) epee::guarded_critical_region_t varname(lock, DEADLOCK_FUNCTION_DEF, DEADLOCK_LOCATION, #lock, epee::log_space::log_singletone::get_thread_log_prefix()) #define DLG_CRITICAL_REGION_BEGIN_VAR(lock, varname) { epee::guarded_critical_region_t varname(lock, DEADLOCK_FUNCTION_DEF, DEADLOCK_LOCATION, #lock, epee::log_space::log_singletone::get_thread_log_prefix()) @@ -701,14 +700,15 @@ namespace epee #define CRITICAL_REGION_BEGIN1(x) CRITICAL_REGION_BEGIN_VAR(x, critical_region_var1) #define CRITICAL_REGION_END() } + +#define CIRITCAL_OPERATION(obj,op) {obj##_lock.lock();obj . op;obj##_lock.unlock();} + #define SHARED_CRITICAL_REGION_LOCAL(x) boost::shared_lock< boost::shared_mutex > critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_LOCAL(x) boost::unique_lock< boost::shared_mutex > critical_region_var(x) #define SHARED_CRITICAL_REGION_BEGIN(x) { SHARED_CRITICAL_REGION_LOCAL(x) +#define SHARED_CRITICAL_REGION_END() } #define EXCLUSIVE_CRITICAL_REGION_BEGIN(x) { EXCLUSIVE_CRITICAL_REGION_LOCAL(x) - +#define EXCLUSIVE_CRITICAL_REGION_END() } } - -#endif - diff --git a/src/contrib/epee/include/time_helper.h b/src/contrib/epee/include/time_helper.h index 457f0b2..f11dae9 100644 --- a/src/contrib/epee/include/time_helper.h +++ b/src/contrib/epee/include/time_helper.h @@ -59,10 +59,10 @@ namespace misc_utils char tmpbuf[200] = {0}; tm* pt = NULL; -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4996) pt = localtime(&time_); -POP_WARNINGS +POP_VS_WARNINGS if(pt) strftime( tmpbuf, 199, "%d.%m.%Y %H:%M:%S", pt ); @@ -81,10 +81,10 @@ POP_WARNINGS char tmpbuf[200] = {0}; tm* pt = NULL; -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4996) pt = localtime(&time_); -POP_WARNINGS +POP_VS_WARNINGS if(pt) strftime( tmpbuf, 199, "%Y_%m_%d %H_%M_%S", pt ); @@ -109,10 +109,10 @@ POP_WARNINGS { char tmpbuf[200] = {0}; tm* pt = NULL; -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4996) pt = gmtime(&time_); -POP_WARNINGS +POP_VS_WARNINGS strftime( tmpbuf, 199, "%a, %d %b %Y %H:%M:%S GMT", pt ); return tmpbuf; } @@ -126,7 +126,7 @@ POP_WARNINGS tail = -tail; res = "-"; } -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4244) int days = tail/(60*60*24); tail = tail%(60*60*24); @@ -135,7 +135,7 @@ DISABLE_VS_WARNINGS(4244) int minutes = tail/(60); tail = tail%(60); int seconds = tail; -POP_WARNINGS +POP_VS_WARNINGS res += std::string("d") + boost::lexical_cast(days) + ".h" + boost::lexical_cast(hours) + ".m" + boost::lexical_cast(minutes) + ".s" + boost::lexical_cast(seconds); return res; } diff --git a/src/contrib/epee/include/warnings.h b/src/contrib/epee/include/warnings.h index 89f2ec7..bbdcd9f 100644 --- a/src/contrib/epee/include/warnings.h +++ b/src/contrib/epee/include/warnings.h @@ -2,8 +2,12 @@ #if defined(_MSC_VER) -#define PUSH_WARNINGS __pragma(warning(push)) -#define POP_WARNINGS __pragma(warning(pop)) +#define PUSH_VS_WARNINGS __pragma(warning(push)) +#define POP_VS_WARNINGS __pragma(warning(pop)) + +#define PUSH_GCC_WARNINGS +#define POP_GCC_WARNINGS + #define DISABLE_VS_WARNINGS(w) __pragma(warning(disable: w)) #define DISABLE_GCC_WARNING(w) #define DISABLE_CLANG_WARNING(w) @@ -13,8 +17,12 @@ #include -#define PUSH_WARNINGS _Pragma("GCC diagnostic push") -#define POP_WARNINGS _Pragma("GCC diagnostic pop") +#define PUSH_VS_WARNINGS +#define POP_VS_WARNINGS + +#define PUSH_GCC_WARNINGS _Pragma("GCC diagnostic push") +#define POP_GCC_WARNINGS _Pragma("GCC diagnostic pop") + #define DISABLE_VS_WARNINGS(w) #if defined(__clang__) diff --git a/src/contrib/epee/include/zlib_helper.h b/src/contrib/epee/include/zlib_helper.h index 86f5f2c..f63b589 100644 --- a/src/contrib/epee/include/zlib_helper.h +++ b/src/contrib/epee/include/zlib_helper.h @@ -42,9 +42,8 @@ namespace zlib_helper int ret = deflateInit(&zstream, Z_DEFAULT_COMPRESSION); if(target.size()) { - - - result_packed_buff.resize(target.size()*2, 'X'); + size_t estimated_output_size_max = deflateBound(&zstream, static_cast(target.size())); + result_packed_buff.resize(estimated_output_size_max, 'X'); zstream.next_in = (Bytef*)target.data(); zstream.avail_in = (uInt)target.size(); @@ -52,12 +51,10 @@ namespace zlib_helper zstream.avail_out = (uInt)result_packed_buff.size(); ret = deflate(&zstream, Z_FINISH); - CHECK_AND_ASSERT_MES(ret>=0, false, "Failed to deflate. err = " << ret); + // as we allocated enough room for a signel pass avail_out should not be zero + CHECK_AND_ASSERT_MES(ret == Z_STREAM_END && zstream.avail_out != 0, false, "Failed to deflate. err = " << ret); - if(result_packed_buff.size() != zstream.avail_out) - result_packed_buff.resize(result_packed_buff.size()-zstream.avail_out); - - + result_packed_buff.resize(result_packed_buff.size() - zstream.avail_out); result_packed_buff.erase(0, 2); } diff --git a/src/contrib/ethereum/libethash/bit_manipulation.h b/src/contrib/ethereum/libethash/bit_manipulation.h index 3fa2294..5c7baa7 100644 --- a/src/contrib/ethereum/libethash/bit_manipulation.h +++ b/src/contrib/ethereum/libethash/bit_manipulation.h @@ -9,6 +9,7 @@ #include "support/attributes.h" #include +#include #ifdef __cplusplus extern "C" { @@ -39,7 +40,7 @@ static inline uint32_t clz32(uint32_t x) static inline uint32_t popcount32(uint32_t x) { - return (uint32_t)__builtin_popcount(x); + return std::bitset<32>(x).count(); } static inline uint32_t mul_hi32(uint32_t x, uint32_t y) diff --git a/src/contrib/ethereum/libethash/ethash.cpp b/src/contrib/ethereum/libethash/ethash.cpp index f12faab..b631523 100644 --- a/src/contrib/ethereum/libethash/ethash.cpp +++ b/src/contrib/ethereum/libethash/ethash.cpp @@ -145,7 +145,11 @@ epoch_context_full* create_epoch_context( char* const alloc_data = static_cast(std::calloc(1, alloc_size)); if (!alloc_data) - return nullptr; // Signal out-of-memory by returning null pointer. + { + LOG_CUSTOM_WITH_CALLSTACK("CRITICAL: std::calloc(" << alloc_size << ") failed in create_epoch_context()", 0); + return nullptr; // Signal out-of-memory by returning null pointer. + } + LOG_CUSTOM("context for epoch " << epoch_number << " allocated, size: " << alloc_size << " bytes, full dataset size: " << full_dataset_size << " bytes", 0); hash512* const light_cache = reinterpret_cast(alloc_data + context_alloc_size); const hash256 epoch_seed = calculate_epoch_seed(epoch_number); @@ -373,6 +377,33 @@ search_result search(const epoch_context_full& context, const hash256& header_ha } return {}; } + +custom_log_level_function*& access_custom_log_level_function() +{ + static custom_log_level_function* p_custom_log_level_function = nullptr; + return p_custom_log_level_function; +} + +custom_log_function*& access_custom_log_function() +{ + static custom_log_function* p_custom_log_function = nullptr; + return p_custom_log_function; +} + +int get_custom_log_level() +{ + if (access_custom_log_level_function() != nullptr) + return access_custom_log_level_function()(); + return -1; +} + +void custom_log(const std::string& m, bool add_callstack) +{ + if (access_custom_log_function() != nullptr) + access_custom_log_function()(m, add_callstack); +} + + } // namespace ethash using namespace ethash; @@ -434,6 +465,8 @@ void ethash_destroy_epoch_context_full(epoch_context_full* context) noexcept void ethash_destroy_epoch_context(epoch_context* context) noexcept { + LOG_CUSTOM("context for epoch " << context->epoch_number << " is about to be freed", 0); + context->~epoch_context(); std::free(context); } diff --git a/src/contrib/ethereum/libethash/ethash/ethash.hpp b/src/contrib/ethereum/libethash/ethash/ethash.hpp index 03f29af..0f2962a 100644 --- a/src/contrib/ethereum/libethash/ethash/ethash.hpp +++ b/src/contrib/ethereum/libethash/ethash/ethash.hpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include namespace ethash { @@ -156,5 +158,34 @@ int find_epoch_number(const hash256& seed) noexcept; const epoch_context& get_global_epoch_context(int epoch_number); /// Get global shared epoch context with full dataset initialized. -const epoch_context_full& get_global_epoch_context_full(int epoch_number); +std::shared_ptr get_global_epoch_context_full(int epoch_number); + +typedef int (custom_log_level_function)(); +typedef void (custom_log_function)(const std::string& m, bool add_callstack); + +custom_log_level_function*& access_custom_log_level_function(); +custom_log_function*& access_custom_log_function(); +int get_custom_log_level(); +void custom_log(const std::string& m, bool add_callstack); + +#define LOG_CUSTOM(msg, level) \ +{ \ + if (level <= ethash::get_custom_log_level()) \ + { \ + std::stringstream ss; \ + ss << msg << std::endl; \ + ethash::custom_log(ss.str(), false); \ + } \ +} + +#define LOG_CUSTOM_WITH_CALLSTACK(msg, level) \ +{ \ + if (level <= ethash::get_custom_log_level()) \ + { \ + std::stringstream ss; \ + ss << msg << std::endl; \ + ethash::custom_log(ss.str(), true); \ + } \ +} + } // namespace ethash diff --git a/src/contrib/ethereum/libethash/managed.cpp b/src/contrib/ethereum/libethash/managed.cpp index 900da7e..82ff69b 100644 --- a/src/contrib/ethereum/libethash/managed.cpp +++ b/src/contrib/ethereum/libethash/managed.cpp @@ -89,12 +89,12 @@ const epoch_context& get_global_epoch_context(int epoch_number) return *thread_local_context; } -const epoch_context_full& get_global_epoch_context_full(int epoch_number) +std::shared_ptr get_global_epoch_context_full(int epoch_number) { // Check if local context matches epoch number. if (!thread_local_context_full || thread_local_context_full->epoch_number != epoch_number) update_local_context_full(epoch_number); - return *thread_local_context_full; + return thread_local_context_full; } } // namespace ethash diff --git a/src/contrib/miniupnp/.travis.yml b/src/contrib/miniupnp/.travis.yml new file mode 100644 index 0000000..56395eb --- /dev/null +++ b/src/contrib/miniupnp/.travis.yml @@ -0,0 +1,49 @@ +language: c + +os: + - linux + - osx + +addons: + apt: + packages: + # packages list: https://github.com/travis-ci/apt-package-whitelist/blob/master/ubuntu-precise + - iptables-dev + - libevent-dev + - libnfnetlink-dev + - uuid-dev + +# container-based builds +sudo: false + +env: + - 'PROJECT=minissdpd' + - 'PROJECT=miniupnpc' + - 'PROJECT=miniupnpc-async' + - 'PROJECT=miniupnpc-libevent' + - 'PROJECT=miniupnpd' + +matrix: + exclude: + - os: osx + env: PROJECT=miniupnpd + - os: osx + compiler: gcc + +compiler: + - gcc + - clang + +before_install: + - 'if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "$CC" == "gcc" ] ; then CC=gcc-4.9; fi' + +script: + - 'cd $TRAVIS_BUILD_DIR && cd $PROJECT' + - 'MAKEFILE=Makefile && if [ -f Makefile.linux -a "$TRAVIS_OS_NAME" = "linux" ]; then MAKEFILE=Makefile.linux; elif [ -f Makefile.macosx -a "$TRAVIS_OS_NAME" = "osx" ]; then MAKEFILE=Makefile.macosx; fi' + - 'if [ "$MAKEFILE" = "Makefile.macosx" ]; then make -f $MAKEFILE depend; fi' + - 'CONFIG_OPTIONS="--ipv6 --igd2" make -f $MAKEFILE -j3' + - 'if [ "$PROJECT" = "miniupnpc" -o "$PROJECT" = "minissdpd" -o "$PROJECT" = "miniupnpd" ]; then make -f $MAKEFILE check; fi' + - 'if [ "$PROJECT" = "miniupnpc" ]; then INSTALLPREFIX="$HOME/_pythonmodule" make -f $MAKEFILE pythonmodule; fi' + +after_success: + - 'INSTALLPREFIX="$HOME/$PROJECT" make -f $MAKEFILE install' diff --git a/src/contrib/miniupnp/CODE_OF_CONDUCT b/src/contrib/miniupnp/CODE_OF_CONDUCT new file mode 100644 index 0000000..9684f77 --- /dev/null +++ b/src/contrib/miniupnp/CODE_OF_CONDUCT @@ -0,0 +1 @@ +behave diff --git a/src/contrib/miniupnp/CONTRIBUTING b/src/contrib/miniupnp/CONTRIBUTING new file mode 100644 index 0000000..88372e8 --- /dev/null +++ b/src/contrib/miniupnp/CONTRIBUTING @@ -0,0 +1,11 @@ +MiniUPnP Project + +Before reporting a bug, please test using the latest development sources +available on github : https://github.com/miniupnp/miniupnp/ +Have a look at existing bug reports on github +https://github.com/miniupnp/miniupnp/issues +and on the forum https://miniupnp.tuxfamily.org/forum/ + +Please prefer the github repository to report bugs, send patches, etc. + +Thomas Bernard diff --git a/src/contrib/miniupnp/LICENSE b/src/contrib/miniupnp/LICENSE new file mode 100644 index 0000000..a37fdac --- /dev/null +++ b/src/contrib/miniupnp/LICENSE @@ -0,0 +1,26 @@ +MiniUPnP Project +Copyright (c) 2005-2017, Thomas BERNARD +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/src/contrib/miniupnp/README b/src/contrib/miniupnp/README new file mode 100644 index 0000000..301e742 --- /dev/null +++ b/src/contrib/miniupnp/README @@ -0,0 +1,61 @@ + -= MiniUPnP project =- + +Main author : Thomas BERNARD +Web site : http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ +Github : https://github.com/miniupnp/miniupnp/ +Travis CI : https://travis-ci.org/miniupnp/miniupnp +Appveyor : https://ci.appveyor.com/project/miniupnp/miniupnp + +miniupnpc/ : MiniUPnP client - an UPnP IGD control point +miniupnpd/ : MiniUPnP daemon - an implementation of a UPnP IGD + + NAT-PMP / PCP gateway +minissdpd/ : SSDP managing daemon. Designed to work with miniupnpc, + miniupnpd, ReadyMedia (formerly MiniDLNA), etc. +miniupnpc-async/ : Proof of concept for a UPnP IGD control point using + asynchronous (non blocking) sockets. +miniupnpc-libevent/ : UPnP IGD control point using libevent2 + http://libevent.org/ + +Thanks to : + * Ryan Wagoner + * João Paulo Barraca + * Craig Kadziolka + * Seth Mos + * Rick Richard + * Michael van Tellingen + * Julien Wajsberg + * Jeremy Collake + * Matthew Sporleder + * Greg Hazel + * Rico Huber + * Jelle Huitema + * Xavier Martin + * Thomas Goirand + * Darren Reed + * Robbie Hanson + * Nikos Mavrogiannopoulos + * Elsö András + * Justin Maggard + * David Wu + * Michael Trebilcock + * Soren Dreijer + * Colin McFarlane + * Daniel Dickinson + * Guillaume Habault + * Alexey Osipov + * Alexey Kuznetsov + * Chiaki Ishikawa + * David Kerr + * Jardel Weyrich + * Leah X. Schmidt + * Peter Tatrai + * Leo Moll + * Daniel Becker + * Yonetani Tomokazu + * Markus Stenberg + * Tomofumi Hayashi + * Konstantin Tokarev + * Mike Tzou + * Nevo Hed + * Salva Peiró + * Stephan Zeisberg diff --git a/src/contrib/miniupnp/appveyor.yml b/src/contrib/miniupnp/appveyor.yml new file mode 100644 index 0000000..b4fb414 --- /dev/null +++ b/src/contrib/miniupnp/appveyor.yml @@ -0,0 +1,21 @@ +version: '2.1.{build}' + +install: + - set PATH=%PATH%;C:\msys64\mingw32\bin + +build_script: + - cd miniupnpc + - appveyor AddCompilationMessage "Building miniupnpc" + - mingw32-make -f Makefile.mingw + - mingw32-make -f Makefile.mingw pythonmodule PYTHON=C:\Python27\python +# - upnpc-static.exe -l + +after_build: + - 7z a ..\miniupnpc-%APPVEYOR_BUILD_VERSION%.zip *.exe *.dll *.a *.lib + - 7z a ..\miniupnpc-python-%APPVEYOR_BUILD_VERSION%.zip build\ dist\ + +artifacts: + - path: miniupnpc-$(appveyor_build_version).zip + name: miniupnpc binaries + - path: miniupnpc-python-$(appveyor_build_version).zip + name: miniupnpc python module diff --git a/src/contrib/miniupnp/minissdpd/.gitignore b/src/contrib/miniupnp/minissdpd/.gitignore new file mode 100644 index 0000000..4ce115e --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/.gitignore @@ -0,0 +1,9 @@ +*.o +minissdpd +testcodelength +testminissdpd +listifaces +Makefile.bak +validateminissdpd +validatecodelength +showminissdpdnotif diff --git a/src/contrib/miniupnp/minissdpd/Changelog.txt b/src/contrib/miniupnp/minissdpd/Changelog.txt new file mode 100644 index 0000000..049c561 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/Changelog.txt @@ -0,0 +1,158 @@ +$Id: Changelog.txt,v 1.49 2018/02/23 13:01:23 nanard Exp $ + +2018/02/23: + Fix build with IPv6 disabled and SSDP_LISTEN_ON_SPECIFIC_ADDR enabled + +2018/02/03: + Properly parse several requests read() at once + Ignore the version of devices while answering to requests + +2016/11/11: + Fix for Solaris build + +2016/03/01: + Fix broken overflow test (p+l > buf+n) thanks to Salva Peiro + +VERSION 1.5: + +2016/01/13: + add "notification" mode (command 5) + +2015/08/06: + disable multicast loop + add -f command line option to filter for a specific device type + +VERSION 1.4: + +2015/08/06: + added command 0 (version) + +2015/07/21: + set multicast TTL to 2 by default and configurable + +2015/05/27: + support larger buffer size (useful for type 3 requests) + +VERSION 1.3: + +2014/12/05: + clean up select call() + fix non blocking write to sockets + +2014/12/04: + Fixes removing of devices on ssdp:byebye + handle ssdp:update messages + +2014/11/28: + revert "listen on only 1 IPv4 if only 1 interface is specified" + because it prevents broadcast messages to be received + Change the list of LAN addresses/interfaces (code taken from miniupnpd) + Check that the peer is from a LAN for each SSDP packet + +2014/11/06: + listen on only 1 IPv4 if only 1 interface is specified + also when ENABLE_IPV6 is not defined + +2014/09/06: + freebsd-glue for Debian/kFreeBSD + use LDFLAGS when linking binary + +2014/05/01: + listen on only 1 IPv4 if only 1 interface is specified + +2014/02/03: + silently ignore EAGAIN, EWOULDBLOCK, EINTR of recv calls + Discover devices on the network at startup + +2013/08/19: + Translate README in english + +2012/09/27: + Rename all include guards to not clash with C99 + (7.1.3 Reserved identifiers). + +VERSION 1.2: + +2012/05/21: + Clean signal handling + Set sockets non blocking + +2012/05/18: + Improve ProcessInterfaceWatch() under BSD. + +2012/05/15: + Improve ProcessInterfaceWatch() under linux. + +2012/05/02: + Clean CLFAGS in Makefile. + Remove a few signed/unsigned compares + +2012/04/09: + Added -ansi to compilation flags. + Handle ssdp:update messages and update logging + +2012/01/02: + Install manpage. Fix installation under Mac OS X. + +2011/10/07: + unlink unix socket before binding. + set SO_REUSEADDR on SSDP socket. + daemonize after init + +VERSION 1.1: + +2011/07/30: + fixes. More overflow checks + +2011/07/29: + added a lot of buffer overflow checks. Check malloc() failure, etc. + Better cleanup in case of crash at start. + network interface watch to add/drop multicast membership when the interface get live. + +2011/06/18: + Starting to add support for UPnP Device Architecture v1.1 + +2011/05/23: + Added IPv6 support. + -i option now understands interface names as well as addresses. + +VERSION 1.0: + +2008/10/07: + added codelength.h + Fixing response to M-SEARCH + Doc update + +2008/10/06: + UPnP server support (answering M-SEARCH) + +2008/10/04: + listening on several interfaces. + +2008/10/01: + use of daemon() instead of home made daemonize. + +2007/12/19: + added uuid in responses + 3 types of requests supported. + preventing buffer overflow + +2007/12/18: + It is now possible to change the location of both pid file and + unix socket. + +2007/10/08: + Added a man page + +2007/09/27: + Support for install in different location $ PREFIX=... make install + +2007/09/23: + added a script for use in /etc/init.d + improved Makefile + creating /var/run/minissdpd.pid + adding synthetic messages for new devices/removed devices + +2007/09/19: + Take SSDP announce packets lifetime into account. + diff --git a/src/contrib/miniupnp/minissdpd/LICENSE b/src/contrib/miniupnp/minissdpd/LICENSE new file mode 100644 index 0000000..d994f14 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2007-2016, Thomas BERNARD +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/contrib/miniupnp/minissdpd/Makefile b/src/contrib/miniupnp/minissdpd/Makefile new file mode 100644 index 0000000..2380e79 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/Makefile @@ -0,0 +1,127 @@ +# $Id: Makefile,v 1.30 2018/02/23 13:59:44 nanard Exp $ +# MiniUPnP project +# author: Thomas Bernard +# website: http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ +# for use with GNU Make (gmake) +# install with : +# $ PREFIX=/tmp/dummylocation make install +# or +# $ INSTALLPREFIX=/usr/local make install +# or +# make install (miniupnpd will be put in /usr/sbin) +# +# install target is made for linux... sorry BSD users... +#CFLAGS = -g -O0 +CFLAGS ?= -Os +CFLAGS += -Wall +CFLAGS += -W -Wstrict-prototypes +CFLAGS += -fno-strict-aliasing -fno-common +CFLAGS += -D_GNU_SOURCE +#CFLAGS += -ansi +CC ?= gcc +RM = rm -f +INSTALL = install +OS = $(shell $(CC) -dumpmachine) + +ifneq (, $(findstring linux, $(OS))) + LDLIBS += -lnfnetlink +endif +ifeq ($(DEB_HOST_ARCH_OS), kfreebsd) + LDLIBS += -lfreebsd-glue +else +ifneq (, $(findstring sun, $(OS))) + CFLAGS += -D_XOPEN_SOURCE + CFLAGS += -D_XOPEN_SOURCE_EXTENDED=1 + CFLAGS += -D__EXTENSIONS__ + LDFLAGS += -lsocket -lnsl -lresolv +endif +endif + +#EXECUTABLES = minissdpd testminissdpd listifaces +EXECUTABLES = minissdpd testminissdpd testcodelength \ + showminissdpdnotif +MINISSDPDOBJS = minissdpd.o openssdpsocket.o daemonize.o upnputils.o \ + ifacewatch.o getroute.o getifaddr.o asyncsendto.o +TESTMINISSDPDOBJS = testminissdpd.o printresponse.o +SHOWMINISSDPDNOTIFOBJS = showminissdpdnotif.o printresponse.o + +ALLOBJS = $(MINISSDPDOBJS) $(TESTMINISSDPDOBJS) \ + $(SHOWMINISSDPDNOTIFOBJS) \ + testcodelength.o + +INSTALLPREFIX ?= $(PREFIX)/usr +SBININSTALLDIR = $(INSTALLPREFIX)/sbin +MANINSTALLDIR = $(INSTALLPREFIX)/share/man + + +.PHONY: all clean install depend check test + +all: $(EXECUTABLES) + +test: check + +clean: + $(RM) $(ALLOBJS) $(EXECUTABLES) + +install: minissdpd + $(INSTALL) -d $(SBININSTALLDIR) + $(INSTALL) minissdpd $(SBININSTALLDIR) + $(INSTALL) -d $(MANINSTALLDIR)/man1 + $(INSTALL) minissdpd.1 $(MANINSTALLDIR)/man1/minissdpd.1 +ifeq (, $(findstring darwin, $(OS))) + $(INSTALL) -d $(PREFIX)/etc/init.d + $(INSTALL) minissdpd.init.d.script $(PREFIX)/etc/init.d/minissdpd +endif + +check: validateminissdpd validatecodelength + +validateminissdpd: testminissdpd minissdpd + ./testminissdpd.sh + touch $@ + +validatecodelength: testcodelength + ./testcodelength + touch $@ + +minissdpd: $(MINISSDPDOBJS) + +testminissdpd: $(TESTMINISSDPDOBJS) + +showminissdpdnotif: $(SHOWMINISSDPDNOTIFOBJS) + +testcodelength: testcodelength.o + +listifaces: listifaces.o upnputils.o + +config.h: VERSION + @tmp=`grep -n MINISSDPD_VERSION $@` ; \ + line=`echo $$tmp | cut -d: -f1` ; \ + old_version=`echo $$tmp | cut -d\\" -f2` ; \ + new_version=`cat VERSION` ; \ + if [ "$$new_version" != "$$old_version" ] ; then \ + echo "updating VERSION in $@ from $$old_version to $$new_version"; \ + sed "$${line}s/$${old_version}/$${new_version}/" $@ > $@.temp ; \ + mv $@.temp $@ ; \ + fi + +depend: + makedepend -f$(MAKEFILE_LIST) -Y \ + $(ALLOBJS:.o=.c) 2>/dev/null + +# DO NOT DELETE + +minissdpd.o: config.h getifaddr.h upnputils.h openssdpsocket.h +minissdpd.o: minissdpdtypes.h daemonize.h codelength.h ifacewatch.h +minissdpd.o: asyncsendto.h +openssdpsocket.o: config.h openssdpsocket.h minissdpdtypes.h upnputils.h +daemonize.o: daemonize.h config.h +upnputils.o: config.h upnputils.h getroute.h minissdpdtypes.h +ifacewatch.o: config.h openssdpsocket.h minissdpdtypes.h upnputils.h +getroute.o: getroute.h upnputils.h +getifaddr.o: config.h getifaddr.h +asyncsendto.o: asyncsendto.h upnputils.h +testminissdpd.o: codelength.h printresponse.h +printresponse.o: codelength.h +showminissdpdnotif.o: codelength.h printresponse.h +printresponse.o: codelength.h +testcodelength.o: codelength.h diff --git a/src/contrib/miniupnp/minissdpd/README b/src/contrib/miniupnp/minissdpd/README new file mode 100644 index 0000000..3712ff6 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/README @@ -0,0 +1,70 @@ + * MiniSSDPd - SSDP daemon + +(c) Thomas Bernard +http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ +https://github.com/miniupnp/miniupnp/ + +MiniSSDPd is a daemon that : +1 - keeps track of all UPnP devices announcing themselves on the network. +its database can be queried by local processes using a protocol based on +a unix socket. That enables local processes to quickly discover UPnP devices +without broadcasting SSDP message and waiting several seconds for a response. +2 - keeps a database of local UPnP devices hosted on the machine and +answering SSDP searchs on their behalf. It enables to run several UPnP devices, +like an IGD and a MediaServer, on the same machine. + +to build, use GNU Make. + + +* protocol : + +Connect to the unix socket. +Sent request, get response. +close unix socket connection. + +* Request format : +1st byte : request type + 0 - version + 1 - type + 2 - USN (unique id) + 3 - everything + 4 - submit service (see below) + 5 - switch connection to notification mode +n bytes : string length : 1 byte if < 128 else the upper bit indicate that +one additional byte should be read, etc. (see codelength.h) +n bytes = string + +Response format : + +request type 0 (version) : +n bytes string length +n bytes = version string + +request type 1 / 2 / 3 / 5 : +1st byte : number of services/devices, from 0 to 254. + 255 is a special value, see below +For each service/device : +URL : + n bytes string length + n bytes = Location string +ST: + n bytes string length + n bytes = type string +USN: + n bytes string length + n bytes = identifier string + +if the 1st byte is 255, the format is as follows : +1st byte = 255 +2nd byte = notification type (1=NEW, 2=UPDATE, 3=REMOVE) +3rd byte = number of services/devices, from 0 to 255. + + +request type 4 = submit service +1st byte = 4 +(k,n) bytes : length and string "ST" (service type) +(k,n) bytes : length and string "USN" +(k,n) bytes : length and string "Server" +(k,n) bytes : length and string "Location" +No answer + diff --git a/src/contrib/miniupnp/minissdpd/README.fr b/src/contrib/miniupnp/minissdpd/README.fr new file mode 100644 index 0000000..d94ee54 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/README.fr @@ -0,0 +1,49 @@ +protocole : + +connection à la socket unix. +envoie d'une requete, retour d'une reponse. +fermeture de la connexion. + +format de requete : +1 octet : type de la requete + 0 - version + 1 - type + 2 - USN (id unique) + 3 - tout +n octets longueur de la chaine : 1 octet si < 128 sinon le bit haut +indique s'il existe un octet suplementaire, etc... +n octets = chaine + +format reponse : +1 octet : nombre de reponses (de 0 à 254) +pour chaque rep : +URL : + n octets longueur de la chaine + n octets = chaine Location +ST: + n octets longueur de la chaine + n octets = chaine type +USN: + n octets longueur de la chaine + n octets = chaine identifiant + +si le 1er octet est 255, alors le format est le suivant : +1 octet : 255 +1 octet : type de notification + 1 = NOTIF_NEW, 2 = NOTIF_NEW, 3 = NOTIF_REMOVE +1 octet : nombre de reponses (0 à 255) +puis comme ci dessus pour chaque réponse + + + +* Type de requete 4 = submit service +1 octet = 4 +(k,n) octets : longueur et chaine "ST" (service type) +(k,n) octets : longueur et chaine "USN" +(k,n) octets : longueur et chaine "Server" +(k,n) octets : longueur et chaine "Location" +Pas de reponse + +* Type de requete 5 = mode notification +Reste connecté et reçoit au fur et à mesure les nouvelles connections +réponses au format normal diff --git a/src/contrib/miniupnp/minissdpd/VERSION b/src/contrib/miniupnp/minissdpd/VERSION new file mode 100644 index 0000000..c239c60 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/VERSION @@ -0,0 +1 @@ +1.5 diff --git a/src/contrib/miniupnp/minissdpd/asyncsendto.c b/src/contrib/miniupnp/minissdpd/asyncsendto.c new file mode 100644 index 0000000..5689f0b --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/asyncsendto.c @@ -0,0 +1,348 @@ +/* $Id: asyncsendto.c,v 1.8 2017/05/24 22:51:57 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "asyncsendto.h" +#include "upnputils.h" + +/* state diagram for a packet : + * + * | + * V + * -> ESCHEDULED -> ESENDNOW -> sent + * ^ | + * | V + * EWAITREADY -> sent + */ +struct scheduled_send { + LIST_ENTRY(scheduled_send) entries; + struct timeval ts; + enum {ESCHEDULED=1, EWAITREADY=2, ESENDNOW=3} state; + int sockfd; + const void * buf; + size_t len; + int flags; + const struct sockaddr *dest_addr; + socklen_t addrlen; + const struct sockaddr_in6 *src_addr; + char data[]; +}; + +static LIST_HEAD(listhead, scheduled_send) send_list = { NULL }; + +/* + * ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, + * const struct sockaddr *dest_addr, socklen_t addrlen); + */ +static ssize_t +send_from_to(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr_in6 *src_addr, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ +#ifdef IPV6_PKTINFO + if(src_addr) { + struct iovec iov; + struct in6_pktinfo ipi6; + uint8_t c[CMSG_SPACE(sizeof(ipi6))]; + struct msghdr msg; + struct cmsghdr* cmsg; + + iov.iov_base = (void *)buf; + iov.iov_len = len; + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + ipi6.ipi6_addr = src_addr->sin6_addr; + ipi6.ipi6_ifindex = src_addr->sin6_scope_id; + msg.msg_control = c; + msg.msg_controllen = sizeof(c); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = IPPROTO_IPV6; + cmsg->cmsg_type = IPV6_PKTINFO; + cmsg->cmsg_len = CMSG_LEN(sizeof(ipi6)); + memcpy(CMSG_DATA(cmsg), &ipi6, sizeof(ipi6)); + msg.msg_name = (void *)dest_addr; + msg.msg_namelen = addrlen; + return sendmsg(sockfd, &msg, flags); + } else { +#endif /* IPV6_PKTINFO */ + return sendto(sockfd, buf, len, flags, dest_addr, addrlen); +#ifdef IPV6_PKTINFO + } +#endif /* IPV6_PKTINFO */ +} + +/* delay = milli seconds */ +ssize_t +sendto_schedule2(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + const struct sockaddr_in6 *src_addr, + unsigned int delay) +{ + enum {ESCHEDULED, EWAITREADY, ESENDNOW} state; + ssize_t n; + size_t alloc_len; + struct timeval tv; + struct scheduled_send * elt; + + if(delay == 0) { + /* first try to send at once */ + n = send_from_to(sockfd, buf, len, flags, src_addr, dest_addr, addrlen); + if(n >= 0) + return n; + else if(errno == EAGAIN || errno == EWOULDBLOCK) { + /* use select() on this socket */ + state = EWAITREADY; + } else if(errno == EINTR) { + state = ESENDNOW; + } else { + /* uncatched error */ + return n; + } + } else { + state = ESCHEDULED; + } + + /* schedule */ + if(gettimeofday(&tv, 0) < 0) { + return -1; + } + /* allocate enough space for structure + buffers */ + alloc_len = sizeof(struct scheduled_send) + len + addrlen; + if(src_addr) + alloc_len += sizeof(struct sockaddr_in6); + elt = malloc(alloc_len); + if(elt == NULL) { + syslog(LOG_ERR, "malloc failed to allocate %u bytes", + (unsigned)alloc_len); + return -1; + } + elt->state = state; + /* time the packet should be sent */ + elt->ts.tv_sec = tv.tv_sec + (delay / 1000); + elt->ts.tv_usec = tv.tv_usec + (delay % 1000) * 1000; + if(elt->ts.tv_usec > 1000000) { + elt->ts.tv_sec++; + elt->ts.tv_usec -= 1000000; + } + elt->sockfd = sockfd; + elt->flags = flags; + memcpy(elt->data, dest_addr, addrlen); + elt->dest_addr = (struct sockaddr *)elt->data; + elt->addrlen = addrlen; + if(src_addr) { + elt->src_addr = (struct sockaddr_in6 *)(elt->data + addrlen); + memcpy((void *)elt->src_addr, src_addr, sizeof(struct sockaddr_in6)); + elt->buf = (void *)(elt->data + addrlen + sizeof(struct sockaddr_in6)); + } else { + elt->src_addr = NULL; + elt->buf = (void *)(elt->data + addrlen); + } + elt->len = len; + memcpy((void *)elt->buf, buf, len); + /* insert */ + LIST_INSERT_HEAD( &send_list, elt, entries); + return 0; +} + +/* try to send at once, and queue the packet if needed */ +ssize_t +sendto_or_schedule(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ + return sendto_schedule2(sockfd, buf, len, flags, dest_addr, addrlen, NULL, 0); +} + +ssize_t +sendto_or_schedule2(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + const struct sockaddr_in6 *src_addr) +{ + return sendto_schedule2(sockfd, buf, len, flags, dest_addr, addrlen, src_addr, 0); +} + +/* get_next_scheduled_send() return number of scheduled send in list */ +int get_next_scheduled_send(struct timeval * next_send) +{ + int n = 0; + struct scheduled_send * elt; + if(next_send == NULL) + return -1; + for(elt = send_list.lh_first; elt != NULL; elt = elt->entries.le_next) { + if(n == 0 || (elt->ts.tv_sec < next_send->tv_sec) || + (elt->ts.tv_sec == next_send->tv_sec && elt->ts.tv_usec < next_send->tv_usec)) { + next_send->tv_sec = elt->ts.tv_sec; + next_send->tv_usec = elt->ts.tv_usec; + } + n++; + } + return n; +} + +/* update writefds for select() call + * return the number of packets to try to send at once */ +int get_sendto_fds(fd_set * writefds, int * max_fd, const struct timeval * now) +{ + int n = 0; + struct scheduled_send * elt; + for(elt = send_list.lh_first; elt != NULL; elt = elt->entries.le_next) { + if(elt->state == EWAITREADY) { + /* last sendto() call returned EAGAIN/EWOULDBLOCK */ + FD_SET(elt->sockfd, writefds); + if(elt->sockfd > *max_fd) + *max_fd = elt->sockfd; + n++; + } else if((elt->ts.tv_sec < now->tv_sec) || + (elt->ts.tv_sec == now->tv_sec && elt->ts.tv_usec <= now->tv_usec)) { + /* we waited long enough, now send ! */ + elt->state = ESENDNOW; + n++; + } + } + return n; +} + +/* executed sendto() when needed */ +int try_sendto(fd_set * writefds) +{ + int ret = 0; + ssize_t n; + struct scheduled_send * elt; + struct scheduled_send * next; + for(elt = send_list.lh_first; elt != NULL; elt = next) { + next = elt->entries.le_next; + if((elt->state == ESENDNOW) || + (elt->state == EWAITREADY && FD_ISSET(elt->sockfd, writefds))) { +#ifdef DEBUG + syslog(LOG_DEBUG, "%s: %d bytes on socket %d", + "try_sendto", (int)elt->len, elt->sockfd); +#endif + n = send_from_to(elt->sockfd, elt->buf, elt->len, elt->flags, + elt->src_addr, elt->dest_addr, elt->addrlen); + /*n = sendto(elt->sockfd, elt->buf, elt->len, elt->flags, + elt->dest_addr, elt->addrlen);*/ + if(n < 0) { + if(errno == EINTR) { + /* retry at once */ + elt->state = ESENDNOW; + continue; + } else if(errno == EAGAIN || errno == EWOULDBLOCK) { + /* retry once the socket is ready for writing */ + elt->state = EWAITREADY; + continue; + } else { + char addr_str[64]; + /* uncatched error */ + if(sockaddr_to_string(elt->dest_addr, addr_str, sizeof(addr_str)) <= 0) + addr_str[0] = '\0'; + syslog(LOG_ERR, "%s(sock=%d, len=%u, dest=%s): sendto: %m", + "try_sendto", elt->sockfd, (unsigned)elt->len, + addr_str); + ret--; + } + } else if((int)n != (int)elt->len) { + syslog(LOG_WARNING, "%s: %d bytes sent out of %d", + "try_sendto", (int)n, (int)elt->len); + } + /* remove from the list */ + LIST_REMOVE(elt, entries); + free(elt); + } + } + return ret; +} + +/* maximum execution time for finalize_sendto() in milliseconds */ +#define FINALIZE_SENDTO_DELAY (500) + +/* empty the list */ +void finalize_sendto(void) +{ + ssize_t n; + struct scheduled_send * elt; + struct scheduled_send * next; + fd_set writefds; + struct timeval deadline; + struct timeval now; + struct timeval timeout; + int max_fd; + + if(gettimeofday(&deadline, NULL) < 0) { + syslog(LOG_ERR, "gettimeofday: %m"); + return; + } + deadline.tv_usec += FINALIZE_SENDTO_DELAY*1000; + if(deadline.tv_usec > 1000000) { + deadline.tv_sec++; + deadline.tv_usec -= 1000000; + } + while(send_list.lh_first) { + FD_ZERO(&writefds); + max_fd = -1; + for(elt = send_list.lh_first; elt != NULL; elt = next) { + next = elt->entries.le_next; + syslog(LOG_DEBUG, "finalize_sendto(): %d bytes on socket %d", + (int)elt->len, elt->sockfd); + n = send_from_to(elt->sockfd, elt->buf, elt->len, elt->flags, + elt->src_addr, elt->dest_addr, elt->addrlen); + /*n = sendto(elt->sockfd, elt->buf, elt->len, elt->flags, + elt->dest_addr, elt->addrlen);*/ + if(n < 0) { + if(errno==EAGAIN || errno==EWOULDBLOCK) { + FD_SET(elt->sockfd, &writefds); + if(elt->sockfd > max_fd) + max_fd = elt->sockfd; + continue; + } + syslog(LOG_WARNING, "finalize_sendto(): socket=%d sendto: %m", elt->sockfd); + } + /* remove from the list */ + LIST_REMOVE(elt, entries); + free(elt); + } + /* check deadline */ + if(gettimeofday(&now, NULL) < 0) { + syslog(LOG_ERR, "gettimeofday: %m"); + return; + } + if(now.tv_sec > deadline.tv_sec || + (now.tv_sec == deadline.tv_sec && now.tv_usec > deadline.tv_usec)) { + /* deadline ! */ + while((elt = send_list.lh_first) != NULL) { + LIST_REMOVE(elt, entries); + free(elt); + } + return; + } + /* compute timeout value */ + timeout.tv_sec = deadline.tv_sec - now.tv_sec; + timeout.tv_usec = deadline.tv_usec - now.tv_usec; + if(timeout.tv_usec < 0) { + timeout.tv_sec--; + timeout.tv_usec += 1000000; + } + if(max_fd >= 0) { + if(select(max_fd + 1, NULL, &writefds, NULL, &timeout) < 0) { + syslog(LOG_ERR, "select: %m"); + return; + } + } + } +} + diff --git a/src/contrib/miniupnp/minissdpd/asyncsendto.h b/src/contrib/miniupnp/minissdpd/asyncsendto.h new file mode 100644 index 0000000..ef670c2 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/asyncsendto.h @@ -0,0 +1,51 @@ +/* $Id: asyncsendto.h,v 1.3 2017/11/02 15:48:29 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef ASYNCSENDTO_H_INCLUDED +#define ASYNCSENDTO_H_INCLUDED +/* for fd_set */ +#include + +/* sendto_schedule() : see sendto(2) + * schedule sendto() call after delay (milliseconds) */ +ssize_t +sendto_schedule2(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + const struct sockaddr_in6 *src_addr, + unsigned int delay); + +#define sendto_schedule(sockfd, buf, len, flags, dest_addr, addrlen, delay) \ + sendto_schedule2(sockfd, buf, len, flags, dest_addr, addrlen, NULL, delay) + +/* sendto_schedule() : see sendto(2) + * try sendto() at once and schedule if EINTR/EAGAIN/EWOULDBLOCK */ +ssize_t +sendto_or_schedule(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen); + +/* same as sendto_schedule() except it will try to set source address + * (for IPV6 only) */ +ssize_t +sendto_or_schedule2(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + const struct sockaddr_in6 *src_addr); + +/* get_next_scheduled_send() + * return number of scheduled sendto + * set next_send to timestamp to send next packet */ +int get_next_scheduled_send(struct timeval * next_send); + +/* execute sendto() for needed packets */ +int try_sendto(fd_set * writefds); + +/* set writefds before select() */ +int get_sendto_fds(fd_set * writefds, int * max_fd, const struct timeval * now); + +/* empty the list */ +void finalize_sendto(void); + +#endif diff --git a/src/contrib/miniupnp/minissdpd/codelength.h b/src/contrib/miniupnp/minissdpd/codelength.h new file mode 100644 index 0000000..f5f8e30 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/codelength.h @@ -0,0 +1,54 @@ +/* $Id: codelength.h,v 1.5 2015/07/09 12:40:18 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas BERNARD + * copyright (c) 2005-2015 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ +#ifndef CODELENGTH_H_INCLUDED +#define CODELENGTH_H_INCLUDED + +/* Encode length by using 7bit per Byte : + * Most significant bit of each byte specifies that the + * following byte is part of the code */ + +/* n : unsigned + * p : unsigned char * + */ +#define DECODELENGTH(n, p) n = 0; \ + do { n = (n << 7) | (*p & 0x7f); } \ + while((*(p++)&0x80) && (n<(1<<25))); + +/* n : unsigned + * READ : function/macro to read one byte (unsigned char) + */ +#define DECODELENGTH_READ(n, READ) \ + n = 0; \ + do { \ + unsigned char c; \ + READ(c); \ + n = (n << 7) | (c & 0x07f); \ + if(!(c&0x80)) break; \ + } while(n<(1<<25)); + +/* n : unsigned + * p : unsigned char * + * p_limit : unsigned char * + */ +#define DECODELENGTH_CHECKLIMIT(n, p, p_limit) \ + n = 0; \ + do { \ + if((p) >= (p_limit)) break; \ + n = (n << 7) | (*(p) & 0x7f); \ + } while((*((p)++)&0x80) && (n<(1<<25))); + + +/* n : unsigned + * p : unsigned char * + */ +#define CODELENGTH(n, p) if(n>=268435456) *(p++) = (n >> 28) | 0x80; \ + if(n>=2097152) *(p++) = (n >> 21) | 0x80; \ + if(n>=16384) *(p++) = (n >> 14) | 0x80; \ + if(n>=128) *(p++) = (n >> 7) | 0x80; \ + *(p++) = n & 0x7f; + +#endif /* CODELENGTH_H_INCLUDED */ diff --git a/src/contrib/miniupnp/minissdpd/config.h b/src/contrib/miniupnp/minissdpd/config.h new file mode 100644 index 0000000..5288426 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/config.h @@ -0,0 +1,33 @@ +/* $Id: config.h,v 1.10 2018/02/23 13:58:14 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef CONFIG_H_INCLUDED +#define CONFIG_H_INCLUDED + +#define MINISSDPD_VERSION "1.5" + +/* use BSD daemon() ? */ +#define USE_DAEMON + +/* set the syslog facility to use. See man syslog(3) and syslog.conf(5). */ +#define LOG_MINISSDPD LOG_DAEMON + +/* enable IPv6 */ +#define ENABLE_IPV6 + +/* The size of unix socket response buffer */ +#define RESPONSE_BUFFER_SIZE (1024 * 4) + +/* Uncomment the following line in order to make minissdpd + * listen on 1.2.3.4:1900 instead of *:1900 + * FOR TESTING PURPOSE ONLY + * Note : it prevents multicast packets to be received, + * at least with linux + * As miniSSDPd needs to receive SSDP packets both multicasted + * and unicasted, we cannot bind to 239.255.255.250 neither */ +/*#define SSDP_LISTEN_ON_SPECIFIC_ADDR*/ + +#endif diff --git a/src/contrib/miniupnp/minissdpd/daemonize.c b/src/contrib/miniupnp/minissdpd/daemonize.c new file mode 100644 index 0000000..4c5909d --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/daemonize.c @@ -0,0 +1,129 @@ +/* $Id: daemonize.c,v 1.12 2011/05/27 09:35:02 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "daemonize.h" +#include "config.h" + +#ifndef USE_DAEMON + +int +daemonize(void) +{ + int pid, i; + + switch(fork()) + { + /* fork error */ + case -1: + perror("fork()"); + exit(1); + + /* child process */ + case 0: + /* obtain a new process group */ + if( (pid = setsid()) < 0) + { + perror("setsid()"); + exit(1); + } + + /* close all descriptors */ + for (i=getdtablesize();i>=0;--i) close(i); + + i = open("/dev/null", O_RDWR); /* open stdin */ + dup(i); /* stdout */ + dup(i); /* stderr */ + + umask(027); + chdir("/"); /* chdir to /tmp ? */ + + return pid; + + /* parent process */ + default: + exit(0); + } +} +#endif + +int +writepidfile(const char * fname, int pid) +{ + char pidstring[16]; + int pidstringlen; + int pidfile; + + if(!fname || fname[0] == '\0') + return -1; + + if( (pidfile = open(fname, O_WRONLY|O_CREAT, 0644)) < 0) + { + syslog(LOG_ERR, "Unable to open pidfile for writing %s: %m", fname); + return -1; + } + + pidstringlen = snprintf(pidstring, sizeof(pidstring), "%d\n", pid); + if(pidstringlen <= 0) + { + syslog(LOG_ERR, + "Unable to write to pidfile %s: snprintf(): FAILED", fname); + close(pidfile); + return -1; + } + else + { + if(write(pidfile, pidstring, pidstringlen) < 0) + syslog(LOG_ERR, "Unable to write to pidfile %s: %m", fname); + } + + close(pidfile); + + return 0; +} + +int +checkforrunning(const char * fname) +{ + char buffer[64]; + int pidfile; + pid_t pid; + + if(!fname || fname[0] == '\0') + return -1; + + if( (pidfile = open(fname, O_RDONLY)) < 0) + return 0; + + memset(buffer, 0, 64); + + if(read(pidfile, buffer, 63)) + { + if( (pid = atol(buffer)) > 0) + { + if(!kill(pid, 0)) + { + close(pidfile); + return -2; + } + } + } + + close(pidfile); + + return 0; +} + diff --git a/src/contrib/miniupnp/minissdpd/daemonize.h b/src/contrib/miniupnp/minissdpd/daemonize.h new file mode 100644 index 0000000..818ce73 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/daemonize.h @@ -0,0 +1,35 @@ +/* $Id: daemonize.h,v 1.6 2008/01/29 13:04:46 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef DAEMONIZE_H_INCLUDED +#define DAEMONIZE_H_INCLUDED + +#include "config.h" + +#ifndef USE_DAEMON +/* daemonize() + * "fork" to background, detach from terminal, etc... + * returns: pid of the daemon, exits upon failure */ +int +daemonize(void); +#endif + +/* writepidfile() + * write the pid to a file */ +int +writepidfile(const char * fname, int pid); + +/* checkforrunning() + * check for another instance running + * returns: 0 only instance + * -1 invalid filename + * -2 another instance running */ +int +checkforrunning(const char * fname); + +#endif + diff --git a/src/contrib/miniupnp/minissdpd/getifaddr.c b/src/contrib/miniupnp/minissdpd/getifaddr.c new file mode 100644 index 0000000..3febe64 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/getifaddr.c @@ -0,0 +1,261 @@ +/* $Id: getifaddr.c,v 1.24 2015/07/09 12:27:26 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(sun) +#include +#endif + +#include "config.h" +#include "getifaddr.h" +#if defined(USE_GETIFADDRS) || defined(ENABLE_IPV6) || defined(ENABLE_PCP) +#include +#endif + +int +getifaddr(const char * ifname, char * buf, int len, + struct in_addr * addr, struct in_addr * mask) +{ +#ifndef USE_GETIFADDRS + /* use ioctl SIOCGIFADDR. Works only for ip v4 */ + /* SIOCGIFADDR struct ifreq * */ + int s; + struct ifreq ifr; + int ifrlen; + struct sockaddr_in * ifaddr; + ifrlen = sizeof(ifr); + + if(!ifname || ifname[0]=='\0') + return -1; + s = socket(PF_INET, SOCK_DGRAM, 0); + if(s < 0) + { + syslog(LOG_ERR, "socket(PF_INET, SOCK_DGRAM): %m"); + return -1; + } + strncpy(ifr.ifr_name, ifname, IFNAMSIZ); + ifr.ifr_name[IFNAMSIZ-1] = '\0'; + if(ioctl(s, SIOCGIFFLAGS, &ifr, &ifrlen) < 0) + { + syslog(LOG_DEBUG, "ioctl(s, SIOCGIFFLAGS, ...): %m"); + close(s); + return -1; + } + if ((ifr.ifr_flags & IFF_UP) == 0) + { + syslog(LOG_DEBUG, "network interface %s is down", ifname); + close(s); + return -1; + } + strncpy(ifr.ifr_name, ifname, IFNAMSIZ); + if(ioctl(s, SIOCGIFADDR, &ifr, &ifrlen) < 0) + { + syslog(LOG_ERR, "ioctl(s, SIOCGIFADDR, ...): %m"); + close(s); + return -1; + } + ifaddr = (struct sockaddr_in *)&ifr.ifr_addr; + if(addr) *addr = ifaddr->sin_addr; + if(buf) + { + if(!inet_ntop(AF_INET, &ifaddr->sin_addr, buf, len)) + { + syslog(LOG_ERR, "inet_ntop(): %m"); + close(s); + return -1; + } + } + if(mask) + { + strncpy(ifr.ifr_name, ifname, IFNAMSIZ); + if(ioctl(s, SIOCGIFNETMASK, &ifr, &ifrlen) < 0) + { + syslog(LOG_ERR, "ioctl(s, SIOCGIFNETMASK, ...): %m"); + close(s); + return -1; + } +#ifdef ifr_netmask + *mask = ((struct sockaddr_in *)&ifr.ifr_netmask)->sin_addr; +#else + *mask = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr; +#endif + } + close(s); +#else /* ifndef USE_GETIFADDRS */ + /* Works for all address families (both ip v4 and ip v6) */ + struct ifaddrs * ifap; + struct ifaddrs * ife; + + if(!ifname || ifname[0]=='\0') + return -1; + if(getifaddrs(&ifap)<0) + { + syslog(LOG_ERR, "getifaddrs: %m"); + return -1; + } + for(ife = ifap; ife; ife = ife->ifa_next) + { + /* skip other interfaces if one was specified */ + if(ifname && (0 != strcmp(ifname, ife->ifa_name))) + continue; + if(ife->ifa_addr == NULL) + continue; + switch(ife->ifa_addr->sa_family) + { + case AF_INET: + if(buf) + { + inet_ntop(ife->ifa_addr->sa_family, + &((struct sockaddr_in *)ife->ifa_addr)->sin_addr, + buf, len); + } + if(addr) *addr = ((struct sockaddr_in *)ife->ifa_addr)->sin_addr; + if(mask) *mask = ((struct sockaddr_in *)ife->ifa_netmask)->sin_addr; + break; +/* + case AF_INET6: + inet_ntop(ife->ifa_addr->sa_family, + &((struct sockaddr_in6 *)ife->ifa_addr)->sin6_addr, + buf, len); +*/ + } + } + freeifaddrs(ifap); +#endif + return 0; +} + +#ifdef ENABLE_PCP + +int getifaddr_in6(const char * ifname, int af, struct in6_addr * addr) +{ +#if defined(ENABLE_IPV6) || defined(USE_GETIFADDRS) + struct ifaddrs * ifap; + struct ifaddrs * ife; +#ifdef ENABLE_IPV6 + const struct sockaddr_in6 * tmpaddr; +#endif /* ENABLE_IPV6 */ + int found = 0; + + if(!ifname || ifname[0]=='\0') + return -1; + if(getifaddrs(&ifap)<0) + { + syslog(LOG_ERR, "getifaddrs: %m"); + return -1; + } + for(ife = ifap; ife && !found; ife = ife->ifa_next) + { + /* skip other interfaces if one was specified */ + if(ifname && (0 != strcmp(ifname, ife->ifa_name))) + continue; + if(ife->ifa_addr == NULL) + continue; + if (ife->ifa_addr->sa_family != af) + continue; + switch(ife->ifa_addr->sa_family) + { + case AF_INET: + /* IPv4-mapped IPv6 address ::ffff:1.2.3.4 */ + memset(addr->s6_addr, 0, 10); + addr->s6_addr[10] = 0xff; + addr->s6_addr[11] = 0xff; + memcpy(addr->s6_addr + 12, + &(((struct sockaddr_in *)ife->ifa_addr)->sin_addr.s_addr), + 4); + found = 1; + break; + +#ifdef ENABLE_IPV6 + case AF_INET6: + tmpaddr = (const struct sockaddr_in6 *)ife->ifa_addr; + if(!IN6_IS_ADDR_LOOPBACK(&tmpaddr->sin6_addr) + && !IN6_IS_ADDR_LINKLOCAL(&tmpaddr->sin6_addr)) + { + memcpy(addr->s6_addr, + &tmpaddr->sin6_addr, + 16); + found = 1; + } + break; +#endif /* ENABLE_IPV6 */ + } + } + freeifaddrs(ifap); + return (found ? 0 : -1); +#else /* defined(ENABLE_IPV6) || defined(USE_GETIFADDRS) */ + /* IPv4 only */ + struct in_addr addr4; + if(af != AF_INET) + return -1; + if(getifaddr(ifname, NULL, 0, &addr4, NULL) < 0) + return -1; + /* IPv4-mapped IPv6 address ::ffff:1.2.3.4 */ + memset(addr->s6_addr, 0, 10); + addr->s6_addr[10] = 0xff; + addr->s6_addr[11] = 0xff; + memcpy(addr->s6_addr + 12, &addr4.s_addr, 4); + return 0; +#endif +} +#endif /* ENABLE_PCP */ + +#ifdef ENABLE_IPV6 +int +find_ipv6_addr(const char * ifname, + char * dst, int n) +{ + struct ifaddrs * ifap; + struct ifaddrs * ife; + const struct sockaddr_in6 * addr; + char buf[64]; + int r = 0; + + if(!dst) + return -1; + + if(getifaddrs(&ifap)<0) + { + syslog(LOG_ERR, "getifaddrs: %m"); + return -1; + } + for(ife = ifap; ife; ife = ife->ifa_next) + { + /* skip other interfaces if one was specified */ + if(ifname && (0 != strcmp(ifname, ife->ifa_name))) + continue; + if(ife->ifa_addr == NULL) + continue; + if(ife->ifa_addr->sa_family == AF_INET6) + { + addr = (const struct sockaddr_in6 *)ife->ifa_addr; + if(!IN6_IS_ADDR_LOOPBACK(&addr->sin6_addr) + && !IN6_IS_ADDR_LINKLOCAL(&addr->sin6_addr)) + { + inet_ntop(ife->ifa_addr->sa_family, + &addr->sin6_addr, + buf, sizeof(buf)); + /* add brackets */ + snprintf(dst, n, "[%s]", buf); + r = 1; + } + } + } + freeifaddrs(ifap); + return r; +} +#endif + diff --git a/src/contrib/miniupnp/minissdpd/getifaddr.h b/src/contrib/miniupnp/minissdpd/getifaddr.h new file mode 100644 index 0000000..9dcfb02 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/getifaddr.h @@ -0,0 +1,32 @@ +/* $Id: getifaddr.h,v 1.10 2014/05/06 14:40:53 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef GETIFADDR_H_INCLUDED +#define GETIFADDR_H_INCLUDED + +struct in_addr; +struct in6_addr; + +/* getifaddr() + * take a network interface name and write the + * ip v4 address as text in the buffer + * returns: 0 success, -1 failure */ +int +getifaddr(const char * ifname, char * buf, int len, + struct in_addr * addr, struct in_addr * mask); + +int +getifaddr_in6(const char * ifname, int af, struct in6_addr* addr); + +/* find a non link local IP v6 address for the interface. + * if ifname is NULL, look for all interfaces */ +int +find_ipv6_addr(const char * ifname, + char * dst, int n); + +#endif + diff --git a/src/contrib/miniupnp/minissdpd/getroute.c b/src/contrib/miniupnp/minissdpd/getroute.c new file mode 100644 index 0000000..90b9933 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/getroute.c @@ -0,0 +1,282 @@ +/* $Id: getroute.c,v 1.4 2014/12/01 09:07:17 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +/*#include */ +#include +#include +#include +#else /* __linux__ */ +#include +#include +#include +#ifdef AF_LINK +#include +#endif /* AF_LINK */ +#endif /* __linux__ */ + +#include "getroute.h" +#include "upnputils.h" +#include "config.h" + +/* get_src_for_route_to() function is only called in code + * enabled with ENABLE_IPV6 */ +#ifdef ENABLE_IPV6 + +int +get_src_for_route_to(const struct sockaddr * dst, + void * src, size_t * src_len, + int * index) +{ +#if __linux__ + int fd = -1; + struct nlmsghdr *h; + int status; + struct { + struct nlmsghdr n; + struct rtmsg r; + char buf[1024]; + } req; + struct sockaddr_nl nladdr; + struct iovec iov = { + .iov_base = (void*) &req.n, + }; + struct msghdr msg = { + .msg_name = &nladdr, + .msg_namelen = sizeof(nladdr), + .msg_iov = &iov, + .msg_iovlen = 1, + }; + const struct sockaddr_in * dst4; + const struct sockaddr_in6 * dst6; + + memset(&req, 0, sizeof(req)); + req.n.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + req.n.nlmsg_flags = NLM_F_REQUEST; + req.n.nlmsg_type = RTM_GETROUTE; + req.r.rtm_family = dst->sa_family; + req.r.rtm_table = 0; + req.r.rtm_protocol = 0; + req.r.rtm_scope = 0; + req.r.rtm_type = 0; + req.r.rtm_src_len = 0; + req.r.rtm_dst_len = 0; + req.r.rtm_tos = 0; + + { + char dst_str[128]; + sockaddr_to_string(dst, dst_str, sizeof(dst_str)); + syslog(LOG_DEBUG, "get_src_for_route_to (%s)", dst_str); + } + /* add address */ + if(dst->sa_family == AF_INET) { + dst4 = (const struct sockaddr_in *)dst; + nfnl_addattr_l(&req.n, sizeof(req), RTA_DST, &dst4->sin_addr, 4); + req.r.rtm_dst_len = 32; + } else { + dst6 = (const struct sockaddr_in6 *)dst; + nfnl_addattr_l(&req.n, sizeof(req), RTA_DST, &dst6->sin6_addr, 16); + req.r.rtm_dst_len = 128; + } + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd < 0) { + syslog(LOG_ERR, "socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE) : %m"); + return -1; + } + + memset(&nladdr, 0, sizeof(nladdr)); + nladdr.nl_family = AF_NETLINK; + + req.n.nlmsg_seq = 1; + iov.iov_len = req.n.nlmsg_len; + + status = sendmsg(fd, &msg, 0); + + if (status < 0) { + syslog(LOG_ERR, "sendmsg(rtnetlink) : %m"); + goto error; + } + + memset(&req, 0, sizeof(req)); + + for(;;) { + iov.iov_len = sizeof(req); + status = recvmsg(fd, &msg, 0); + if(status < 0) { + if (errno == EINTR || errno == EAGAIN) + continue; + syslog(LOG_ERR, "recvmsg(rtnetlink) %m"); + goto error; + } + if(status == 0) { + syslog(LOG_ERR, "recvmsg(rtnetlink) EOF"); + goto error; + } + for (h = (struct nlmsghdr*)&req.n; status >= (int)sizeof(*h); ) { + int len = h->nlmsg_len; + int l = len - sizeof(*h); + + if (l<0 || len>status) { + if (msg.msg_flags & MSG_TRUNC) { + syslog(LOG_ERR, "Truncated message"); + } + syslog(LOG_ERR, "malformed message: len=%d", len); + goto error; + } + + if(nladdr.nl_pid != 0 || h->nlmsg_seq != 1/*seq*/) { + syslog(LOG_ERR, "wrong seq = %d\n", h->nlmsg_seq); + /* Don't forget to skip that message. */ + status -= NLMSG_ALIGN(len); + h = (struct nlmsghdr*)((char*)h + NLMSG_ALIGN(len)); + continue; + } + + if(h->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = (struct nlmsgerr*)NLMSG_DATA(h); + syslog(LOG_ERR, "NLMSG_ERROR %d : %s", err->error, strerror(-err->error)); + goto error; + } + if(h->nlmsg_type == RTM_NEWROUTE) { + struct rtattr * rta; + int len = h->nlmsg_len; + len -= NLMSG_LENGTH(sizeof(struct rtmsg)); + for(rta = RTM_RTA(NLMSG_DATA((h))); RTA_OK(rta, len); rta = RTA_NEXT(rta,len)) { + unsigned char * data = RTA_DATA(rta); + if(rta->rta_type == RTA_PREFSRC) { + if(src_len && src) { + if(*src_len < RTA_PAYLOAD(rta)) { + syslog(LOG_WARNING, "cannot copy src: %u<%lu", + (unsigned)*src_len, (unsigned long)RTA_PAYLOAD(rta)); + goto error; + } + *src_len = RTA_PAYLOAD(rta); + memcpy(src, data, RTA_PAYLOAD(rta)); + } + } else if(rta->rta_type == RTA_OIF) { + if(index) + memcpy(index, data, sizeof(int)); + } + } + close(fd); + return 0; + } + status -= NLMSG_ALIGN(len); + h = (struct nlmsghdr*)((char*)h + NLMSG_ALIGN(len)); + } + } + syslog(LOG_WARNING, "get_src_for_route_to() : src not found"); +error: + if(fd >= 0) + close(fd); + return -1; +#else /* __linux__ */ + int found = 0; + int s; + int l, i; + char * p; + struct sockaddr * sa; + struct { + struct rt_msghdr m_rtm; + char m_space[512]; + } m_rtmsg; +#define rtm m_rtmsg.m_rtm + + if(dst == NULL) + return -1; +#ifdef __APPLE__ + if(dst->sa_family == AF_INET6) { + syslog(LOG_ERR, "Sorry, get_src_for_route_to() is known to fail with IPV6 on OS X..."); + return -1; + } +#endif + s = socket(PF_ROUTE, SOCK_RAW, dst->sa_family); + if(s < 0) { + syslog(LOG_ERR, "socket(PF_ROUTE) failed : %m"); + return -1; + } + memset(&rtm, 0, sizeof(rtm)); + rtm.rtm_type = RTM_GET; + rtm.rtm_flags = RTF_UP; + rtm.rtm_version = RTM_VERSION; + rtm.rtm_seq = 1; + rtm.rtm_addrs = RTA_DST; /* destination address */ + memcpy(m_rtmsg.m_space, dst, sizeof(struct sockaddr)); + rtm.rtm_msglen = sizeof(struct rt_msghdr) + sizeof(struct sockaddr); + if(write(s, &m_rtmsg, rtm.rtm_msglen) < 0) { + syslog(LOG_ERR, "write: %m"); + close(s); + return -1; + } + + do { + l = read(s, &m_rtmsg, sizeof(m_rtmsg)); + if(l<0) { + syslog(LOG_ERR, "read: %m"); + close(s); + return -1; + } + syslog(LOG_DEBUG, "read l=%d seq=%d pid=%d", + l, rtm.rtm_seq, rtm.rtm_pid); + } while(l > 0 && (rtm.rtm_pid != getpid() || rtm.rtm_seq != 1)); + close(s); + p = m_rtmsg.m_space; + if(rtm.rtm_addrs) { + for(i=1; i<0x8000; i <<= 1) { + if(i & rtm.rtm_addrs) { + char tmp[256] = { 0 }; + sa = (struct sockaddr *)p; + sockaddr_to_string(sa, tmp, sizeof(tmp)); + syslog(LOG_DEBUG, "type=%d sa_len=%d sa_family=%d %s", + i, SA_LEN(sa), sa->sa_family, tmp); + if((i == RTA_DST || i == RTA_GATEWAY) && + (src_len && src)) { + size_t len = 0; + void * paddr = NULL; + if(sa->sa_family == AF_INET) { + paddr = &((struct sockaddr_in *)sa)->sin_addr; + len = sizeof(struct in_addr); + } else if(sa->sa_family == AF_INET6) { + paddr = &((struct sockaddr_in6 *)sa)->sin6_addr; + len = sizeof(struct in6_addr); + } + if(paddr) { + if(*src_len < len) { + syslog(LOG_WARNING, "cannot copy src. %u<%u", + (unsigned)*src_len, (unsigned)len); + return -1; + } + memcpy(src, paddr, len); + *src_len = len; + found = 1; + } + } +#ifdef AF_LINK + if(sa->sa_family == AF_LINK) { + struct sockaddr_dl * sdl = (struct sockaddr_dl *)sa; + if(index) + *index = sdl->sdl_index; + } +#endif + p += SA_LEN(sa); + } + } + } + return found ? 0 : -1; +#endif /* __linux__ */ +} + +#endif /* ENABLE_IPV6 */ diff --git a/src/contrib/miniupnp/minissdpd/getroute.h b/src/contrib/miniupnp/minissdpd/getroute.h new file mode 100644 index 0000000..86d0496 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/getroute.h @@ -0,0 +1,17 @@ +/* $Id: getroute.h,v 1.3 2013/02/06 10:50:04 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef GETROUTE_H_INCLUDED +#define GETROUTE_H_INCLUDED + +int +get_src_for_route_to(const struct sockaddr * dst, + void * src, size_t * src_len, + int * index); + +#endif + diff --git a/src/contrib/miniupnp/minissdpd/ifacewatch.c b/src/contrib/miniupnp/minissdpd/ifacewatch.c new file mode 100644 index 0000000..c73450a --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/ifacewatch.c @@ -0,0 +1,345 @@ +/* $Id: ifacewatch.c,v 1.16 2015/09/03 18:31:25 nanard Exp $ */ +/* MiniUPnP project + * (c) 2011-2018 Thomas Bernard + * website : http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#include +#else /* __linux__ */ +#include +#ifdef AF_LINK +#include +#endif +#endif /* __linux__ */ +#include +#include + +#include "config.h" +#include "openssdpsocket.h" +#include "upnputils.h" +#include "minissdpdtypes.h" + +extern struct lan_addr_list lan_addrs; + +#ifndef __linux__ +#if defined(__OpenBSD__) || defined(__FreeBSD__) +#define SALIGN (sizeof(long) - 1) +#else +#define SALIGN (sizeof(int32_t) - 1) +#endif +#define SA_RLEN(sa) (SA_LEN(sa) ? ((SA_LEN(sa) + SALIGN) & ~SALIGN) : (SALIGN + 1)) +#endif + +int +OpenAndConfInterfaceWatchSocket(void) +{ + int s; +#ifdef __linux__ + struct sockaddr_nl addr; + + s = socket(PF_NETLINK, SOCK_RAW, NETLINK_ROUTE); +#else /* __linux__*/ + /*s = socket(PF_ROUTE, SOCK_RAW, AF_INET);*/ + s = socket(PF_ROUTE, SOCK_RAW, AF_UNSPEC); +/* The family parameter may be AF_UNSPEC which will provide routing informa- + * tion for all address families, or can be restricted to a specific address + * family by specifying which one is desired. There can be more than one + * routing socket open per system. */ +#endif + if(s < 0) { + syslog(LOG_ERR, "%s socket: %m", + "OpenAndConfInterfaceWatchSocket"); + return -1; + } + if(!set_non_blocking(s)) { + syslog(LOG_WARNING, "%s failed to set socket non blocking : %m", + "OpenAndConfInterfaceWatchSocket"); + } +#ifdef __linux__ + memset(&addr, 0, sizeof(addr)); + addr.nl_family = AF_NETLINK; + addr.nl_groups = RTMGRP_LINK | RTMGRP_IPV4_IFADDR | RTMGRP_IPV6_IFADDR; + + if (bind(s, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + syslog(LOG_ERR, "bind(netlink): %m"); + close(s); + return -1; + } +#endif + return s; +} + +/** + * Process the message and add/drop multicast membership if needed + */ +int +ProcessInterfaceWatch(int s, int s_ssdp, int s_ssdp6) +{ + struct lan_addr_s * lan_addr; + ssize_t len; + char buffer[4096]; +#ifdef __linux__ + struct iovec iov; + struct msghdr hdr; + struct nlmsghdr *nlhdr; + struct ifaddrmsg *ifa; + struct rtattr *rta; + int ifa_len; + +#ifndef ENABLE_IPV6 + (void)s_ssdp6; +#endif + iov.iov_base = buffer; + iov.iov_len = sizeof(buffer); + + memset(&hdr, 0, sizeof(hdr)); + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + + len = recvmsg(s, &hdr, 0); + if(len < 0) { + syslog(LOG_ERR, "recvmsg(s, &hdr, 0): %m"); + return -1; + } + + for(nlhdr = (struct nlmsghdr *)buffer; + NLMSG_OK(nlhdr, len); + nlhdr = NLMSG_NEXT(nlhdr, len)) { + int is_del = 0; + char address[48]; + char ifname[IFNAMSIZ]; + address[0] = '\0'; + ifname[0] = '\0'; + if(nlhdr->nlmsg_type == NLMSG_DONE) + break; + switch(nlhdr->nlmsg_type) { + /* case RTM_NEWLINK: */ + /* case RTM_DELLINK: */ + case RTM_DELADDR: + is_del = 1; + case RTM_NEWADDR: + /* http://linux-hacks.blogspot.fr/2009/01/sample-code-to-learn-netlink.html */ + ifa = (struct ifaddrmsg *)NLMSG_DATA(nlhdr); + rta = (struct rtattr *)IFA_RTA(ifa); + ifa_len = IFA_PAYLOAD(nlhdr); + syslog(LOG_DEBUG, "%s %s index=%d fam=%d prefixlen=%d flags=%d scope=%d", + "ProcessInterfaceWatchNotify", is_del ? "RTM_DELADDR" : "RTM_NEWADDR", + ifa->ifa_index, ifa->ifa_family, ifa->ifa_prefixlen, + ifa->ifa_flags, ifa->ifa_scope); + for(;RTA_OK(rta, ifa_len); rta = RTA_NEXT(rta, ifa_len)) { + /*RTA_DATA(rta)*/ + /*rta_type : IFA_ADDRESS, IFA_LOCAL, etc. */ + char tmp[128]; + memset(tmp, 0, sizeof(tmp)); + switch(rta->rta_type) { + case IFA_ADDRESS: + case IFA_LOCAL: + case IFA_BROADCAST: + case IFA_ANYCAST: + inet_ntop(ifa->ifa_family, RTA_DATA(rta), tmp, sizeof(tmp)); + if(rta->rta_type == IFA_ADDRESS) + strncpy(address, tmp, sizeof(address)); + break; + case IFA_LABEL: + strncpy(tmp, RTA_DATA(rta), sizeof(tmp)); + strncpy(ifname, tmp, sizeof(ifname)); + break; + case IFA_CACHEINFO: + { + struct ifa_cacheinfo *cache_info; + cache_info = RTA_DATA(rta); + snprintf(tmp, sizeof(tmp), "valid=%u preferred=%u", + cache_info->ifa_valid, cache_info->ifa_prefered); + } + break; + default: + strncpy(tmp, "*unknown*", sizeof(tmp)); + } + syslog(LOG_DEBUG, " rta_len=%d rta_type=%d '%s'", rta->rta_len, rta->rta_type, tmp); + } + syslog(LOG_INFO, "%s: %s/%d %s", + is_del ? "RTM_DELADDR" : "RTM_NEWADDR", + address, ifa->ifa_prefixlen, ifname); + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) { +#ifdef ENABLE_IPV6 + if((0 == strcmp(address, lan_addr->str)) || + (0 == strcmp(ifname, lan_addr->ifname)) || + (ifa->ifa_index == lan_addr->index)) { +#else + if((0 == strcmp(address, lan_addr->str)) || + (0 == strcmp(ifname, lan_addr->ifname))) { +#endif + if(ifa->ifa_family == AF_INET) + AddDropMulticastMembership(s_ssdp, lan_addr, 0, is_del); +#ifdef ENABLE_IPV6 + else if(ifa->ifa_family == AF_INET6) + AddDropMulticastMembership(s_ssdp6, lan_addr, 1, is_del); +#endif + break; + } + } + break; + default: + syslog(LOG_DEBUG, "unknown nlmsg_type=%d", nlhdr->nlmsg_type); + } + } +#else /* __linux__ */ + struct rt_msghdr * rtm; + struct ifa_msghdr * ifam; + int is_del = 0; + char tmp[64]; + char * p; + struct sockaddr * sa; + int addr; + char address[48]; + char ifname[IFNAMSIZ]; + int family = AF_UNSPEC; + int prefixlen = 0; + +#ifndef ENABLE_IPV6 + (void)s_ssdp6; +#endif + address[0] = '\0'; + ifname[0] = '\0'; + + len = recv(s, buffer, sizeof(buffer), 0); + if(len < 0) { + syslog(LOG_ERR, "%s recv: %m", "ProcessInterfaceWatchNotify"); + return -1; + } + rtm = (struct rt_msghdr *)buffer; + switch(rtm->rtm_type) { + case RTM_DELADDR: + is_del = 1; + case RTM_NEWADDR: + ifam = (struct ifa_msghdr *)buffer; + syslog(LOG_DEBUG, "%s %s len=%d/%hu index=%hu addrs=%x flags=%x", + "ProcessInterfaceWatchNotify", is_del?"RTM_DELADDR":"RTM_NEWADDR", + (int)len, ifam->ifam_msglen, + ifam->ifam_index, ifam->ifam_addrs, ifam->ifam_flags); + p = buffer + sizeof(struct ifa_msghdr); + addr = 1; + while(p < buffer + len) { + sa = (struct sockaddr *)p; + while(!(addr & ifam->ifam_addrs) && (addr <= ifam->ifam_addrs)) + addr = addr << 1; + sockaddr_to_string(sa, tmp, sizeof(tmp)); + syslog(LOG_DEBUG, " %s", tmp); + switch(addr) { + case RTA_DST: + case RTA_GATEWAY: + break; + case RTA_NETMASK: + if(sa->sa_family == AF_INET +#if defined(__OpenBSD__) + || (sa->sa_family == 0 && + sa->sa_len <= sizeof(struct sockaddr_in)) +#endif + ) { + uint32_t sin_addr = ntohl(((struct sockaddr_in *)sa)->sin_addr.s_addr); + while((prefixlen < 32) && + ((sin_addr & (1 << (31 - prefixlen))) != 0)) + prefixlen++; + } else if(sa->sa_family == AF_INET6 +#if defined(__OpenBSD__) + || (sa->sa_family == 0 && + sa->sa_len == sizeof(struct sockaddr_in6)) +#endif + ) { + int i = 0; + uint8_t * q = ((struct sockaddr_in6 *)sa)->sin6_addr.s6_addr; + while((*q == 0xff) && (i < 16)) { + prefixlen += 8; + q++; i++; + } + if(i < 16) { + i = 0; + while((i < 8) && + ((*q & (1 << (7 - i))) != 0)) + i++; + prefixlen += i; + } + } + break; + case RTA_GENMASK: + break; + case RTA_IFP: +#ifdef AF_LINK + if(sa->sa_family == AF_LINK) { + struct sockaddr_dl * sdl = (struct sockaddr_dl *)sa; + memset(ifname, 0, sizeof(ifname)); + memcpy(ifname, sdl->sdl_data, sdl->sdl_nlen); + } +#endif + break; + case RTA_IFA: + family = sa->sa_family; + if(sa->sa_family == AF_INET) { + inet_ntop(sa->sa_family, + &((struct sockaddr_in *)sa)->sin_addr, + address, sizeof(address)); + } else if(sa->sa_family == AF_INET6) { + inet_ntop(sa->sa_family, + &((struct sockaddr_in6 *)sa)->sin6_addr, + address, sizeof(address)); + } + break; + case RTA_AUTHOR: + break; + case RTA_BRD: + break; + } +#if 0 + syslog(LOG_DEBUG, " %d.%d.%d.%d %02x%02x%02x%02x", + (uint8_t)p[0], (uint8_t)p[1], (uint8_t)p[2], (uint8_t)p[3], + (uint8_t)p[0], (uint8_t)p[1], (uint8_t)p[2], (uint8_t)p[3]); + syslog(LOG_DEBUG, " %d.%d.%d.%d %02x%02x%02x%02x", + (uint8_t)p[4], (uint8_t)p[5], (uint8_t)p[6], (uint8_t)p[7], + (uint8_t)p[4], (uint8_t)p[5], (uint8_t)p[6], (uint8_t)p[7]); +#endif + p += SA_RLEN(sa); + addr = addr << 1; + } + syslog(LOG_INFO, "%s: %s/%d %s", + is_del ? "RTM_DELADDR" : "RTM_NEWADDR", + address, prefixlen, ifname); + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) { +#ifdef ENABLE_IPV6 + if((0 == strcmp(address, lan_addr->str)) || + (0 == strcmp(ifname, lan_addr->ifname)) || + (ifam->ifam_index == lan_addr->index)) { +#else + if((0 == strcmp(address, lan_addr->str)) || + (0 == strcmp(ifname, lan_addr->ifname))) { +#endif + if(family == AF_INET) + AddDropMulticastMembership(s_ssdp, lan_addr, 0, is_del); +#ifdef ENABLE_IPV6 + else if(family == AF_INET6) + AddDropMulticastMembership(s_ssdp6, lan_addr, 1, is_del); +#endif + break; + } + } + break; + default: + syslog(LOG_DEBUG, "Unknown RTM message : rtm->rtm_type=%d len=%d", + rtm->rtm_type, (int)len); + } +#endif + return 0; +} + diff --git a/src/contrib/miniupnp/minissdpd/ifacewatch.h b/src/contrib/miniupnp/minissdpd/ifacewatch.h new file mode 100644 index 0000000..4b4f28f --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/ifacewatch.h @@ -0,0 +1,17 @@ +/* $Id: ifacewatch.h,v 1.4 2014/11/28 16:20:57 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef IFACEWATCH_H_INCLUDED +#define IFACEWATCH_H_INCLUDED + +int +OpenAndConfInterfaceWatchSocket(void); + +int +ProcessInterfaceWatch(int s, int s_ssdp, int s_ssdp6); + +#endif diff --git a/src/contrib/miniupnp/minissdpd/listifaces.c b/src/contrib/miniupnp/minissdpd/listifaces.c new file mode 100644 index 0000000..190ee21 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/listifaces.c @@ -0,0 +1,120 @@ +/* $Id: listifaces.c,v 1.7 2015/02/08 08:51:54 nanard Exp $ */ +/* (c) 2006-2015 Thomas BERNARD + * http://miniupnp.free.fr/ http://miniupnp.tuxfamily.org/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "upnputils.h" + +/* hexdump */ +void printhex(const unsigned char * p, int n) +{ + int i; + while(n>0) + { + for(i=0; i<16; i++) + printf("%02x ", p[i]); + printf("| "); + for(i=0; i<16; i++) + { + putchar((p[i]>=32 && p[i]<127)?p[i]:'.'); + } + printf("\n"); + p+=16; + n -= 16; + } +} + +/* List network interfaces */ +void listifaces(void) +{ + struct ifconf ifc; + char * buf = NULL; + int buflen; + int s, i; + int j; + char saddr[256/*INET_ADDRSTRLEN*/]; +#ifdef __linux__ + buflen = sizeof(struct ifreq)*10; +#else + buflen = 0; +#endif + /*s = socket(PF_INET, SOCK_DGRAM, 0);*/ + s = socket(AF_INET, SOCK_DGRAM, 0); + do { + char * tmp; +#ifdef __linux__ + buflen += buflen; +#endif + if(buflen > 0) { + tmp = realloc(buf, buflen); + if(!tmp) { + fprintf(stderr, "error allocating %d bytes.\n", buflen); + close(f); + free(buf); + return; + } + buf = tmp; + } + ifc.ifc_len = buflen; + ifc.ifc_buf = (caddr_t)buf; + if(ioctl(s, SIOCGIFCONF, &ifc) < 0) + { + perror("ioctl"); + close(s); + free(buf); + return; + } + printf("buffer length=%d - buffer used=%d - sizeof(struct ifreq)=%d\n", + buflen, ifc.ifc_len, (int)sizeof(struct ifreq)); + printf("IFNAMSIZ=%d ", IFNAMSIZ); + printf("sizeof(struct sockaddr)=%d sizeof(struct sockaddr_in)=%d\n", + (int)sizeof(struct sockaddr), (int)sizeof(struct sockaddr_in) ); +#ifndef __linux__ + if(buflen == 0) + buflen = ifc.ifc_len; + else + break; + } while(1); +#else + } while(buflen <= ifc.ifc_len); +#endif + printhex((const unsigned char *)ifc.ifc_buf, ifc.ifc_len); + printf("off index fam name address\n"); + for(i = 0, j = 0; iifr_addr))->sin_addr), saddr, sizeof(saddr));*/ + saddr[0] = '\0'; + /* inet_ntop(ifrp->ifr_addr.sa_family, &(ifrp->ifr_addr.sa_data[2]), saddr, sizeof(saddr)); */ + sockaddr_to_string(&ifrp->ifr_addr, saddr, sizeof(saddr)); + printf("0x%03x %2d %2d %-16s %s\n", i, j, ifrp->ifr_addr.sa_family, ifrp->ifr_name, saddr); + /*ifrp->ifr_addr.sa_len is only available on BSD */ +#ifdef __linux__ + i += sizeof(struct ifreq); +#else + if(ifrp->ifr_addr.sa_len == 0) + break; + i += IFNAMSIZ + ifrp->ifr_addr.sa_len; +#endif + } + free(buf); + close(s); +} + +int main(int argc, char * * argv) +{ + (void)argc; + (void)argv; + listifaces(); + return 0; +} + diff --git a/src/contrib/miniupnp/minissdpd/minissdpd.1 b/src/contrib/miniupnp/minissdpd/minissdpd.1 new file mode 100644 index 0000000..f898b9e --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/minissdpd.1 @@ -0,0 +1,43 @@ +.TH "minissdpd" 1 +.SH NAME +minissdpd \- daemon keeping track of UPnP devices up +.SH SYNOPSIS +.B minissdpd +.RB [ -d "] [" -6 "] [" "-s \fIsocket" "] [" "-p \fIpidfile" "] [" "-t \fITTL" "] [" "-f \fIdevice" ] " -i \fR<\fIinterface\fR> " [ "-i \fR<\fIinterface2\fR>" "] ..." +.SH DESCRIPTION +minissdpd listen for SSDP traffic and keeps track +of what are the UPnP devices up on the network. +The list of the UPnP devices is accessed by programs +looking for devices, skipping the UPnP discovery process. +.SH OPTIONS +.TP +.B \-d +debug : do not go to background, output messages to console +and do not filter out low priority messages. +.TP +.B \-6 +IPv6 : Enable IPv6 in addition to IPv4. +.TP +.BI \-s " socket" +path of the unix socket open for communicating with other processes. +By default /var/run/minissdpd.sock is used. +.TP +.BI \-p " pidfile" +path of the file where pid is written at startup. +By default /var/run/minissdpd.pid is used. +.TP +.BI \-t " TTL" +TTL of the package. +By default 2 is used according to UDA. +.TP +.BI \-f " device" +search/filter a specific device type. +.TP +.BI \-i " interface" +name or IP address of the interface used to listen to SSDP packets +coming on port 1900, multicast address 239.255.255.250. +.SH "SEE ALSO" +miniupnpd(1) miniupnpc(3) +.SH BUGS +No known bugs. + diff --git a/src/contrib/miniupnp/minissdpd/minissdpd.c b/src/contrib/miniupnp/minissdpd/minissdpd.c new file mode 100644 index 0000000..8a815ca --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/minissdpd.c @@ -0,0 +1,1646 @@ +/* $Id: minissdpd.c,v 1.53 2016/03/01 18:06:46 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * (c) 2007-2018 Thomas Bernard + * website : http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* for chmod : */ +#include +/* unix sockets */ +#include +/* for getpwnam() and getgrnam() */ +#if 0 +#include +#include +#endif + +/* LOG_PERROR does not exist on Solaris */ +#ifndef LOG_PERROR +#define LOG_PERROR 0 +#endif /* LOG_PERROR */ + +#include "getifaddr.h" +#include "upnputils.h" +#include "openssdpsocket.h" +#include "daemonize.h" +#include "codelength.h" +#include "ifacewatch.h" +#include "minissdpdtypes.h" +#include "asyncsendto.h" + +#define SET_MAX(max, x) if((x) > (max)) (max) = (x) +#ifndef MIN +#define MIN(x,y) (((x)<(y))?(x):(y)) +#endif + +/* current request management structure */ +struct reqelem { + int socket; + int is_notify; /* has subscribed to notifications */ + LIST_ENTRY(reqelem) entries; + unsigned char * output_buffer; + int output_buffer_offset; + int output_buffer_len; +}; + +/* device data structures */ +struct header { + const char * p; /* string pointer */ + int l; /* string length */ +}; + +#define HEADER_NT 0 +#define HEADER_USN 1 +#define HEADER_LOCATION 2 + +struct device { + struct device * next; + time_t t; /* validity time */ + struct header headers[3]; /* NT, USN and LOCATION headers */ + char data[]; +}; + +/* Services stored for answering to M-SEARCH */ +struct service { + char * st; /* Service type */ + char * usn; /* Unique identifier */ + char * server; /* Server string */ + char * location; /* URL */ + LIST_ENTRY(service) entries; +}; +LIST_HEAD(servicehead, service) servicelisthead; + +#define NTS_SSDP_ALIVE 1 +#define NTS_SSDP_BYEBYE 2 +#define NTS_SSDP_UPDATE 3 + +/* request types */ +enum request_type { + MINISSDPD_GET_VERSION = 0, + MINISSDPD_SEARCH_TYPE = 1, + MINISSDPD_SEARCH_USN = 2, + MINISSDPD_SEARCH_ALL = 3, + MINISSDPD_SUBMIT = 4, + MINISSDPD_NOTIF = 5 +}; + +/* discovered device list kept in memory */ +struct device * devlist = 0; + +/* bootid and configid */ +unsigned int upnp_bootid = 1; +unsigned int upnp_configid = 1337; + +/* LAN interfaces/addresses */ +struct lan_addr_list lan_addrs; + +/* connected clients */ +LIST_HEAD(reqstructhead, reqelem) reqlisthead; + +/* functions prototypes */ + +#define NOTIF_NEW 1 +#define NOTIF_UPDATE 2 +#define NOTIF_REMOVE 3 +static void +sendNotifications(int notif_type, const struct device * dev, const struct service * serv); + +/* functions */ + +/* parselanaddr() + * parse address with mask + * ex: 192.168.1.1/24 or 192.168.1.1/255.255.255.0 + * + * Can also use the interface name (ie eth0) + * + * return value : + * 0 : ok + * -1 : error */ +static int +parselanaddr(struct lan_addr_s * lan_addr, const char * str) +{ + const char * p; + int n; + char tmp[16]; + + memset(lan_addr, 0, sizeof(struct lan_addr_s)); + p = str; + while(*p && *p != '/' && !isspace(*p)) + p++; + n = p - str; + if(!isdigit(str[0]) && n < (int)sizeof(lan_addr->ifname)) { + /* not starting with a digit : suppose it is an interface name */ + memcpy(lan_addr->ifname, str, n); + lan_addr->ifname[n] = '\0'; + if(getifaddr(lan_addr->ifname, lan_addr->str, sizeof(lan_addr->str), + &lan_addr->addr, &lan_addr->mask) < 0) + goto parselan_error; + /*printf("%s => %s\n", lan_addr->ifname, lan_addr->str);*/ + } else { + if(n>15) + goto parselan_error; + memcpy(lan_addr->str, str, n); + lan_addr->str[n] = '\0'; + if(!inet_aton(lan_addr->str, &lan_addr->addr)) + goto parselan_error; + } + if(*p == '/') { + const char * q = ++p; + while(*p && isdigit(*p)) + p++; + if(*p=='.') { + /* parse mask in /255.255.255.0 format */ + while(*p && (*p=='.' || isdigit(*p))) + p++; + n = p - q; + if(n>15) + goto parselan_error; + memcpy(tmp, q, n); + tmp[n] = '\0'; + if(!inet_aton(tmp, &lan_addr->mask)) + goto parselan_error; + } else { + /* it is a /24 format */ + int nbits = atoi(q); + if(nbits > 32 || nbits < 0) + goto parselan_error; + lan_addr->mask.s_addr = htonl(nbits ? (0xffffffffu << (32 - nbits)) : 0); + } + } else if(lan_addr->mask.s_addr == 0) { + /* by default, networks are /24 */ + lan_addr->mask.s_addr = htonl(0xffffff00u); + } +#ifdef ENABLE_IPV6 + if(lan_addr->ifname[0] != '\0') { + lan_addr->index = if_nametoindex(lan_addr->ifname); + if(lan_addr->index == 0) + fprintf(stderr, "Cannot get index for network interface %s", + lan_addr->ifname); + } else { + fprintf(stderr, + "Error: please specify LAN network interface by name instead of IPv4 address : %s\n", + str); + return -1; + } +#endif /* ENABLE_IPV6 */ + return 0; +parselan_error: + fprintf(stderr, "Error parsing address/mask (or interface name) : %s\n", + str); + return -1; +} + +static int +write_buffer(struct reqelem * req) +{ + if(req->output_buffer && req->output_buffer_len > 0) { + int n = write(req->socket, + req->output_buffer + req->output_buffer_offset, + req->output_buffer_len); + if(n >= 0) { + req->output_buffer_offset += n; + req->output_buffer_len -= n; + } else if(errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) { + return 0; + } + return n; + } else { + return 0; + } +} + +static int +add_to_buffer(struct reqelem * req, const unsigned char * data, int len) +{ + unsigned char * tmp; + if(req->output_buffer_offset > 0) { + memmove(req->output_buffer, req->output_buffer + req->output_buffer_offset, req->output_buffer_len); + req->output_buffer_offset = 0; + } + tmp = realloc(req->output_buffer, req->output_buffer_len + len); + if(tmp == NULL) { + syslog(LOG_ERR, "%s: failed to allocate %d bytes", + __func__, req->output_buffer_len + len); + return -1; + } + req->output_buffer = tmp; + memcpy(req->output_buffer + req->output_buffer_len, data, len); + req->output_buffer_len += len; + return len; +} + +static int +write_or_buffer(struct reqelem * req, const unsigned char * data, int len) +{ + if(write_buffer(req) < 0) + return -1; + if(req->output_buffer && req->output_buffer_len > 0) { + return add_to_buffer(req, data, len); + } else { + int n = write(req->socket, data, len); + if(n == len) + return len; + if(n < 0) { + if(errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) { + n = add_to_buffer(req, data, len); + if(n < 0) return n; + } else { + return n; + } + } else { + n = add_to_buffer(req, data + n, len - n); + if(n < 0) return n; + } + } + return len; +} + +static const char * +nts_to_str(int nts) +{ + switch(nts) + { + case NTS_SSDP_ALIVE: + return "ssdp:alive"; + case NTS_SSDP_BYEBYE: + return "ssdp:byebye"; + case NTS_SSDP_UPDATE: + return "ssdp:update"; + } + return "unknown"; +} + +/* updateDevice() : + * adds or updates the device to the list. + * return value : + * 0 : the device was updated (or nothing done) + * 1 : the device was new */ +static int +updateDevice(const struct header * headers, time_t t) +{ + struct device ** pp = &devlist; + struct device * p = *pp; /* = devlist; */ + while(p) + { + if( p->headers[HEADER_NT].l == headers[HEADER_NT].l + && (0==memcmp(p->headers[HEADER_NT].p, headers[HEADER_NT].p, headers[HEADER_NT].l)) + && p->headers[HEADER_USN].l == headers[HEADER_USN].l + && (0==memcmp(p->headers[HEADER_USN].p, headers[HEADER_USN].p, headers[HEADER_USN].l)) ) + { + /*printf("found! %d\n", (int)(t - p->t));*/ + syslog(LOG_DEBUG, "device updated : %.*s", headers[HEADER_USN].l, headers[HEADER_USN].p); + p->t = t; + /* update Location ! */ + if(headers[HEADER_LOCATION].l > p->headers[HEADER_LOCATION].l) + { + struct device * tmp; + tmp = realloc(p, sizeof(struct device) + + headers[0].l+headers[1].l+headers[2].l); + if(!tmp) /* allocation error */ + { + syslog(LOG_ERR, "updateDevice() : memory allocation error"); + free(p); + return 0; + } + p = tmp; + *pp = p; + } + memcpy(p->data + p->headers[0].l + p->headers[1].l, + headers[2].p, headers[2].l); + /* TODO : check p->headers[HEADER_LOCATION].l */ + return 0; + } + pp = &p->next; + p = *pp; /* p = p->next; */ + } + syslog(LOG_INFO, "new device discovered : %.*s", + headers[HEADER_USN].l, headers[HEADER_USN].p); + /* add */ + { + char * pc; + int i; + p = malloc( sizeof(struct device) + + headers[0].l+headers[1].l+headers[2].l ); + if(!p) { + syslog(LOG_ERR, "updateDevice(): cannot allocate memory"); + return -1; + } + p->next = devlist; + p->t = t; + pc = p->data; + for(i = 0; i < 3; i++) + { + p->headers[i].p = pc; + p->headers[i].l = headers[i].l; + memcpy(pc, headers[i].p, headers[i].l); + pc += headers[i].l; + } + devlist = p; + sendNotifications(NOTIF_NEW, p, NULL); + } + return 1; +} + +/* removeDevice() : + * remove a device from the list + * return value : + * 0 : no device removed + * -1 : device removed */ +static int +removeDevice(const struct header * headers) +{ + struct device ** pp = &devlist; + struct device * p = *pp; /* = devlist */ + while(p) + { + if( p->headers[HEADER_NT].l == headers[HEADER_NT].l + && (0==memcmp(p->headers[HEADER_NT].p, headers[HEADER_NT].p, headers[HEADER_NT].l)) + && p->headers[HEADER_USN].l == headers[HEADER_USN].l + && (0==memcmp(p->headers[HEADER_USN].p, headers[HEADER_USN].p, headers[HEADER_USN].l)) ) + { + syslog(LOG_INFO, "remove device : %.*s", headers[HEADER_USN].l, headers[HEADER_USN].p); + sendNotifications(NOTIF_REMOVE, p, NULL); + *pp = p->next; + free(p); + return -1; + } + pp = &p->next; + p = *pp; /* p = p->next; */ + } + syslog(LOG_WARNING, "device not found for removing : %.*s", headers[HEADER_USN].l, headers[HEADER_USN].p); + return 0; +} + +/* sent notifications to client having subscribed */ +static void +sendNotifications(int notif_type, const struct device * dev, const struct service * serv) +{ + struct reqelem * req; + unsigned int m; + unsigned char rbuf[RESPONSE_BUFFER_SIZE]; + unsigned char * rp; + + for(req = reqlisthead.lh_first; req; req = req->entries.le_next) { + if(!req->is_notify) continue; + rbuf[0] = '\xff'; /* special code for notifications */ + rbuf[1] = (unsigned char)notif_type; + rbuf[2] = 0; + rp = rbuf + 3; + if(dev) { + /* response : + * 1 - Location + * 2 - NT (device/service type) + * 3 - usn */ + m = dev->headers[HEADER_LOCATION].l; + CODELENGTH(m, rp); + memcpy(rp, dev->headers[HEADER_LOCATION].p, dev->headers[HEADER_LOCATION].l); + rp += dev->headers[HEADER_LOCATION].l; + m = dev->headers[HEADER_NT].l; + CODELENGTH(m, rp); + memcpy(rp, dev->headers[HEADER_NT].p, dev->headers[HEADER_NT].l); + rp += dev->headers[HEADER_NT].l; + m = dev->headers[HEADER_USN].l; + CODELENGTH(m, rp); + memcpy(rp, dev->headers[HEADER_USN].p, dev->headers[HEADER_USN].l); + rp += dev->headers[HEADER_USN].l; + rbuf[2]++; + } + if(serv) { + /* response : + * 1 - Location + * 2 - NT (device/service type) + * 3 - usn */ + m = strlen(serv->location); + CODELENGTH(m, rp); + memcpy(rp, serv->location, m); + rp += m; + m = strlen(serv->st); + CODELENGTH(m, rp); + memcpy(rp, serv->st, m); + rp += m; + m = strlen(serv->usn); + CODELENGTH(m, rp); + memcpy(rp, serv->usn, m); + rp += m; + rbuf[2]++; + } + if(rbuf[2] > 0) { + if(write_or_buffer(req, rbuf, rp - rbuf) < 0) { + syslog(LOG_ERR, "(s=%d) write: %m", req->socket); + /*goto error;*/ + } + } + } +} + +/* SendSSDPMSEARCHResponse() : + * build and send response to M-SEARCH SSDP packets. */ +static void +SendSSDPMSEARCHResponse(int s, const struct sockaddr * sockname, + const char * st, size_t st_len, const char * usn, + const char * server, const char * location) +{ + int l, n; + char buf[1024]; + socklen_t sockname_len; + /* + * follow guideline from document "UPnP Device Architecture 1.0" + * uppercase is recommended. + * DATE: is recommended + * SERVER: OS/ver UPnP/1.0 miniupnpd/1.0 + * - check what to put in the 'Cache-Control' header + * + * have a look at the document "UPnP Device Architecture v1.1 */ + l = snprintf(buf, sizeof(buf), "HTTP/1.1 200 OK\r\n" + "CACHE-CONTROL: max-age=120\r\n" + /*"DATE: ...\r\n"*/ + "ST: %.*s\r\n" + "USN: %s\r\n" + "EXT:\r\n" + "SERVER: %s\r\n" + "LOCATION: %s\r\n" + "OPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n" /* UDA v1.1 */ + "01-NLS: %u\r\n" /* same as BOOTID. UDA v1.1 */ + "BOOTID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "CONFIGID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "\r\n", + (int)st_len, st, usn, + server, location, + upnp_bootid, upnp_bootid, upnp_configid); +#ifdef ENABLE_IPV6 + sockname_len = (sockname->sa_family == PF_INET6) + ? sizeof(struct sockaddr_in6) + : sizeof(struct sockaddr_in); +#else /* ENABLE_IPV6 */ + sockname_len = sizeof(struct sockaddr_in); +#endif /* ENABLE_IPV6 */ + n = sendto_or_schedule(s, buf, l, 0, sockname, sockname_len); + if(n < 0) { + syslog(LOG_ERR, "%s: sendto(udp): %m", __func__); + } +} + +/* Process M-SEARCH requests */ +static void +processMSEARCH(int s, const char * st, size_t st_len, + const struct sockaddr * addr) +{ + struct service * serv; +#ifdef ENABLE_IPV6 + char buf[64]; +#endif /* ENABLE_IPV6 */ + + if(!st || st_len==0) + return; +#ifdef ENABLE_IPV6 + sockaddr_to_string(addr, buf, sizeof(buf)); + syslog(LOG_INFO, "SSDP M-SEARCH from %s ST:%.*s", + buf, (int)st_len, st); +#else /* ENABLE_IPV6 */ + syslog(LOG_INFO, "SSDP M-SEARCH from %s:%d ST: %.*s", + inet_ntoa(((const struct sockaddr_in *)addr)->sin_addr), + ntohs(((const struct sockaddr_in *)addr)->sin_port), + (int)st_len, st); +#endif /* ENABLE_IPV6 */ + if(st_len==8 && (0==memcmp(st, "ssdp:all", 8))) { + /* send a response for all services */ + for(serv = servicelisthead.lh_first; + serv; + serv = serv->entries.le_next) { + SendSSDPMSEARCHResponse(s, addr, + serv->st, strlen(serv->st), serv->usn, + serv->server, serv->location); + } + } else if(st_len > 5 && (0==memcmp(st, "uuid:", 5))) { + /* find a matching UUID value */ + for(serv = servicelisthead.lh_first; + serv; + serv = serv->entries.le_next) { + if(0 == strncmp(serv->usn, st, st_len)) { + SendSSDPMSEARCHResponse(s, addr, + serv->st, strlen(serv->st), serv->usn, + serv->server, serv->location); + } + } + } else { + size_t l; + int st_ver = 0; + char atoi_buffer[8]; + + /* remove version at the end of the ST string */ + for (l = st_len; l > 0; l--) { + if (st[l-1] == ':') { + memset(atoi_buffer, 0, sizeof(atoi_buffer)); + memcpy(atoi_buffer, st + l, MIN((sizeof(atoi_buffer) - 1), st_len - l)); + st_ver = atoi(atoi_buffer); + break; + } + } + if (l == 0) + l = st_len; + /* answer for each matching service */ + /* From UPnP Device Architecture v1.1 : + * 1.3.2 [...] Updated versions of device and service types + * are REQUIRED to be full backward compatible with + * previous versions. Devices MUST respond to M-SEARCH + * requests for any supported version. For example, if a + * device implements “urn:schemas-upnporg:service:xyz:2â€, + * it MUST respond to search requests for both that type + * and “urn:schemas-upnp-org:service:xyz:1â€. The response + * MUST specify the same version as was contained in the + * search request. [...] */ + for(serv = servicelisthead.lh_first; + serv; + serv = serv->entries.le_next) { + if(0 == strncmp(serv->st, st, l)) { + syslog(LOG_DEBUG, "Found matching service : %s %s", serv->st, serv->location); + SendSSDPMSEARCHResponse(s, addr, + st, st_len, serv->usn, + serv->server, serv->location); + } + } + } +} + +/** + * helper function. + * reject any non ASCII or non printable character. + */ +static int +containsForbiddenChars(const unsigned char * p, int len) +{ + while(len > 0) { + if(*p < ' ' || *p >= '\x7f') + return 1; + p++; + len--; + } + return 0; +} + +#define METHOD_MSEARCH 1 +#define METHOD_NOTIFY 2 + +/* ParseSSDPPacket() : + * parse a received SSDP Packet and call + * updateDevice() or removeDevice() as needed + * return value : + * -1 : a device was removed + * 0 : no device removed nor added + * 1 : a device was added. */ +static int +ParseSSDPPacket(int s, const char * p, ssize_t n, + const struct sockaddr * addr, + const char * searched_device) +{ + const char * linestart; + const char * lineend; + const char * nameend; + const char * valuestart; + struct header headers[3]; + int i, r = 0; + int methodlen; + int nts = -1; + int method = -1; + unsigned int lifetime = 180; /* 3 minutes by default */ + const char * st = NULL; + int st_len = 0; + + /* first check from what subnet is the sender */ + if(get_lan_for_peer(addr) == NULL) { + char addr_str[64]; + sockaddr_to_string(addr, addr_str, sizeof(addr_str)); + syslog(LOG_WARNING, "peer %s is not from a LAN", + addr_str); + return 0; + } + + /* do the parsing */ + memset(headers, 0, sizeof(headers)); + for(methodlen = 0; + methodlen < n && (isalpha(p[methodlen]) || p[methodlen]=='-'); + methodlen++); + if(methodlen==8 && 0==memcmp(p, "M-SEARCH", 8)) + method = METHOD_MSEARCH; + else if(methodlen==6 && 0==memcmp(p, "NOTIFY", 6)) + method = METHOD_NOTIFY; + else if(methodlen==4 && 0==memcmp(p, "HTTP", 4)) { + /* answer to a M-SEARCH => process it as a NOTIFY + * with NTS: ssdp:alive */ + method = METHOD_NOTIFY; + nts = NTS_SSDP_ALIVE; + } + linestart = p; + while(linestart < p + n - 2) { + /* start parsing the line : detect line end */ + lineend = linestart; + while(lineend < p + n && *lineend != '\n' && *lineend != '\r') + lineend++; + /*printf("line: '%.*s'\n", lineend - linestart, linestart);*/ + /* detect name end : ':' character */ + nameend = linestart; + while(nameend < lineend && *nameend != ':') + nameend++; + /* detect value */ + if(nameend < lineend) + valuestart = nameend + 1; + else + valuestart = nameend; + /* trim spaces */ + while(valuestart < lineend && isspace(*valuestart)) + valuestart++; + /* suppress leading " if needed */ + if(valuestart < lineend && *valuestart=='\"') + valuestart++; + if(nameend > linestart && valuestart < lineend) { + int l = nameend - linestart; /* header name length */ + int m = lineend - valuestart; /* header value length */ + /* suppress tailing spaces */ + while(m>0 && isspace(valuestart[m-1])) + m--; + /* suppress tailing ' if needed */ + if(m>0 && valuestart[m-1] == '\"') + m--; + i = -1; + /*printf("--%.*s: (%d)%.*s--\n", l, linestart, + m, m, valuestart);*/ + if(l==2 && 0==strncasecmp(linestart, "nt", 2)) + i = HEADER_NT; + else if(l==3 && 0==strncasecmp(linestart, "usn", 3)) + i = HEADER_USN; + else if(l==3 && 0==strncasecmp(linestart, "nts", 3)) { + if(m==10 && 0==strncasecmp(valuestart, "ssdp:alive", 10)) + nts = NTS_SSDP_ALIVE; + else if(m==11 && 0==strncasecmp(valuestart, "ssdp:byebye", 11)) + nts = NTS_SSDP_BYEBYE; + else if(m==11 && 0==strncasecmp(valuestart, "ssdp:update", 11)) + nts = NTS_SSDP_UPDATE; + } + else if(l==8 && 0==strncasecmp(linestart, "location", 8)) + i = HEADER_LOCATION; + else if(l==13 && 0==strncasecmp(linestart, "cache-control", 13)) { + /* parse "name1=value1, name_alone, name2=value2" string */ + const char * name = valuestart; /* name */ + const char * val; /* value */ + int rem = m; /* remaining bytes to process */ + while(rem > 0) { + val = name; + while(val < name + rem && *val != '=' && *val != ',') + val++; + if(val >= name + rem) + break; + if(*val == '=') { + while(val < name + rem && (*val == '=' || isspace(*val))) + val++; + if(val >= name + rem) + break; + if(0==strncasecmp(name, "max-age", 7)) + lifetime = (unsigned int)strtoul(val, 0, 0); + /* move to the next name=value pair */ + while(rem > 0 && *name != ',') { + rem--; + name++; + } + /* skip spaces */ + while(rem > 0 && (*name == ',' || isspace(*name))) { + rem--; + name++; + } + } else { + rem -= (val - name); + name = val; + while(rem > 0 && (*name == ',' || isspace(*name))) { + rem--; + name++; + } + } + } + /*syslog(LOG_DEBUG, "**%.*s**%u", m, valuestart, lifetime);*/ + } else if(l==2 && 0==strncasecmp(linestart, "st", 2)) { + st = valuestart; + st_len = m; + if(method == METHOD_NOTIFY) + i = HEADER_NT; /* it was a M-SEARCH response */ + } + if(i>=0) { + headers[i].p = valuestart; + headers[i].l = m; + } + } + linestart = lineend; + while((*linestart == '\n' || *linestart == '\r') && linestart < p + n) + linestart++; + } +#if 0 + printf("NTS=%d\n", nts); + for(i=0; i<3; i++) { + if(headers[i].p) + printf("%d-'%.*s'\n", i, headers[i].l, headers[i].p); + } +#endif + syslog(LOG_DEBUG,"SSDP request: '%.*s' (%d) %s %s=%.*s", + methodlen, p, method, nts_to_str(nts), + (method==METHOD_NOTIFY)?"nt":"st", + (method==METHOD_NOTIFY)?headers[HEADER_NT].l:st_len, + (method==METHOD_NOTIFY)?headers[HEADER_NT].p:st); + switch(method) { + case METHOD_NOTIFY: + if(nts==NTS_SSDP_ALIVE || nts==NTS_SSDP_UPDATE) { + if(headers[HEADER_NT].p && headers[HEADER_USN].p && headers[HEADER_LOCATION].p) { + /* filter if needed */ + if(searched_device && + 0 != memcmp(headers[HEADER_NT].p, searched_device, headers[HEADER_NT].l)) + break; + r = updateDevice(headers, time(NULL) + lifetime); + } else { + syslog(LOG_WARNING, "missing header nt=%p usn=%p location=%p", + headers[HEADER_NT].p, headers[HEADER_USN].p, + headers[HEADER_LOCATION].p); + } + } else if(nts==NTS_SSDP_BYEBYE) { + if(headers[HEADER_NT].p && headers[HEADER_USN].p) { + r = removeDevice(headers); + } else { + syslog(LOG_WARNING, "missing header nt=%p usn=%p", + headers[HEADER_NT].p, headers[HEADER_USN].p); + } + } + break; + case METHOD_MSEARCH: + processMSEARCH(s, st, st_len, addr); + break; + default: + { + char addr_str[64]; + sockaddr_to_string(addr, addr_str, sizeof(addr_str)); + syslog(LOG_WARNING, "method %.*s, don't know what to do (from %s)", + methodlen, p, addr_str); + } + } + return r; +} + +/* OpenUnixSocket() + * open the unix socket and call bind() and listen() + * return -1 in case of error */ +static int +OpenUnixSocket(const char * path) +{ + struct sockaddr_un addr; + int s; + int rv; + s = socket(AF_UNIX, SOCK_STREAM, 0); + if(s < 0) + { + syslog(LOG_ERR, "socket(AF_UNIX): %m"); + return -1; + } + /* unlink the socket pseudo file before binding */ + rv = unlink(path); + if(rv < 0 && errno != ENOENT) + { + syslog(LOG_ERR, "unlink(unixsocket, \"%s\"): %m", path); + close(s); + return -1; + } + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, path, sizeof(addr.sun_path)); + if(bind(s, (struct sockaddr *)&addr, + sizeof(struct sockaddr_un)) < 0) + { + syslog(LOG_ERR, "bind(unixsocket, \"%s\"): %m", path); + close(s); + return -1; + } + else if(listen(s, 5) < 0) + { + syslog(LOG_ERR, "listen(unixsocket): %m"); + close(s); + return -1; + } + /* Change rights so everyone can communicate with us */ + if(chmod(path, 0666) < 0) + { + syslog(LOG_WARNING, "chmod(\"%s\"): %m", path); + } + return s; +} + +static ssize_t processRequestSub(struct reqelem * req, const unsigned char * buf, ssize_t n); + +/* processRequest() : + * process the request coming from a unix socket */ +void processRequest(struct reqelem * req) +{ + ssize_t n, r; + unsigned char buf[2048]; + const unsigned char * p; + + n = read(req->socket, buf, sizeof(buf)); + if(n<0) { + if(errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) + return; /* try again later */ + syslog(LOG_ERR, "(s=%d) processRequest(): read(): %m", req->socket); + goto error; + } + if(n==0) { + syslog(LOG_INFO, "(s=%d) request connection closed", req->socket); + goto error; + } + p = buf; + while (n > 0) + { + r = processRequestSub(req, p, n); + if (r < 0) + goto error; + p += r; + n -= r; + } + return; +error: + close(req->socket); + req->socket = -1; +} + +static ssize_t processRequestSub(struct reqelem * req, const unsigned char * buf, ssize_t n) +{ + unsigned int l, m; + unsigned int baselen; /* without the version */ + const unsigned char * p; + enum request_type type; + struct device * d = devlist; + unsigned char rbuf[RESPONSE_BUFFER_SIZE]; + unsigned char * rp; + unsigned char nrep = 0; + time_t t; + struct service * newserv = NULL; + struct service * serv; + + t = time(NULL); + type = buf[0]; + p = buf + 1; + DECODELENGTH_CHECKLIMIT(l, p, buf + n); + if(l > (unsigned)(buf+n-p)) { + syslog(LOG_WARNING, "bad request (length encoding l=%u n=%u)", + l, (unsigned)n); + goto error; + } + if(l == 0 && type != MINISSDPD_SEARCH_ALL + && type != MINISSDPD_GET_VERSION && type != MINISSDPD_NOTIF) { + syslog(LOG_WARNING, "bad request (length=0, type=%d)", type); + goto error; + } + syslog(LOG_INFO, "(s=%d) request type=%d str='%.*s'", + req->socket, type, l, p); + switch(type) { + case MINISSDPD_GET_VERSION: + rp = rbuf; + CODELENGTH((sizeof(MINISSDPD_VERSION) - 1), rp); + memcpy(rp, MINISSDPD_VERSION, sizeof(MINISSDPD_VERSION) - 1); + rp += (sizeof(MINISSDPD_VERSION) - 1); + if(write_or_buffer(req, rbuf, rp - rbuf) < 0) { + syslog(LOG_ERR, "(s=%d) write: %m", req->socket); + goto error; + } + p += l; + break; + case MINISSDPD_SEARCH_TYPE: /* request by type */ + case MINISSDPD_SEARCH_USN: /* request by USN (unique id) */ + case MINISSDPD_SEARCH_ALL: /* everything */ + rp = rbuf+1; + /* From UPnP Device Architecture v1.1 : + * 1.3.2 [...] Updated versions of device and service types + * are REQUIRED to be full backward compatible with + * previous versions. Devices MUST respond to M-SEARCH + * requests for any supported version. For example, if a + * device implements “urn:schemas-upnporg:service:xyz:2â€, + * it MUST respond to search requests for both that type + * and “urn:schemas-upnp-org:service:xyz:1â€. The response + * MUST specify the same version as was contained in the + * search request. [...] */ + baselen = l; /* remove the version */ + while(baselen > 0) { + if(p[baselen-1] == ':') + break; + if(!(p[baselen-1] >= '0' && p[baselen-1] <= '9')) + break; + baselen--; + } + while(d && (nrep < 255)) { + if(d->t < t) { + syslog(LOG_INFO, "outdated device"); + } else { + /* test if we can put more responses in the buffer */ + if(d->headers[HEADER_LOCATION].l + d->headers[HEADER_NT].l + + d->headers[HEADER_USN].l + 6 + + (rp - rbuf) >= (int)sizeof(rbuf)) + break; + if( (type==MINISSDPD_SEARCH_TYPE && 0==memcmp(d->headers[HEADER_NT].p, p, baselen)) + ||(type==MINISSDPD_SEARCH_USN && 0==memcmp(d->headers[HEADER_USN].p, p, l)) + ||(type==MINISSDPD_SEARCH_ALL) ) { + /* response : + * 1 - Location + * 2 - NT (device/service type) + * 3 - usn */ + m = d->headers[HEADER_LOCATION].l; + CODELENGTH(m, rp); + memcpy(rp, d->headers[HEADER_LOCATION].p, d->headers[HEADER_LOCATION].l); + rp += d->headers[HEADER_LOCATION].l; + m = d->headers[HEADER_NT].l; + CODELENGTH(m, rp); + memcpy(rp, d->headers[HEADER_NT].p, d->headers[HEADER_NT].l); + rp += d->headers[HEADER_NT].l; + m = d->headers[HEADER_USN].l; + CODELENGTH(m, rp); + memcpy(rp, d->headers[HEADER_USN].p, d->headers[HEADER_USN].l); + rp += d->headers[HEADER_USN].l; + nrep++; + } + } + d = d->next; + } + /* Also look in service list */ + for(serv = servicelisthead.lh_first; + serv && (nrep < 255); + serv = serv->entries.le_next) { + /* test if we can put more responses in the buffer */ + if(strlen(serv->location) + strlen(serv->st) + + strlen(serv->usn) + 6 + (rp - rbuf) >= sizeof(rbuf)) + break; + if( (type==MINISSDPD_SEARCH_TYPE && 0==strncmp(serv->st, (const char *)p, l)) + ||(type==MINISSDPD_SEARCH_USN && 0==strncmp(serv->usn, (const char *)p, l)) + ||(type==MINISSDPD_SEARCH_ALL) ) { + /* response : + * 1 - Location + * 2 - NT (device/service type) + * 3 - usn */ + m = strlen(serv->location); + CODELENGTH(m, rp); + memcpy(rp, serv->location, m); + rp += m; + m = strlen(serv->st); + CODELENGTH(m, rp); + memcpy(rp, serv->st, m); + rp += m; + m = strlen(serv->usn); + CODELENGTH(m, rp); + memcpy(rp, serv->usn, m); + rp += m; + nrep++; + } + } + rbuf[0] = nrep; + syslog(LOG_DEBUG, "(s=%d) response : %d device%s", + req->socket, nrep, (nrep > 1) ? "s" : ""); + if(write_or_buffer(req, rbuf, rp - rbuf) < 0) { + syslog(LOG_ERR, "(s=%d) write: %m", req->socket); + goto error; + } + p += l; + break; + case MINISSDPD_SUBMIT: /* submit service */ + newserv = malloc(sizeof(struct service)); + if(!newserv) { + syslog(LOG_ERR, "cannot allocate memory"); + goto error; + } + memset(newserv, 0, sizeof(struct service)); /* set pointers to NULL */ + if(containsForbiddenChars(p, l)) { + syslog(LOG_ERR, "bad request (st contains forbidden chars)"); + goto error; + } + newserv->st = malloc(l + 1); + if(!newserv->st) { + syslog(LOG_ERR, "cannot allocate memory"); + goto error; + } + memcpy(newserv->st, p, l); + newserv->st[l] = '\0'; + p += l; + if(p >= buf + n) { + syslog(LOG_WARNING, "bad request (missing usn)"); + goto error; + } + DECODELENGTH_CHECKLIMIT(l, p, buf + n); + if(l > (unsigned)(buf+n-p)) { + syslog(LOG_WARNING, "bad request (length encoding)"); + goto error; + } + if(containsForbiddenChars(p, l)) { + syslog(LOG_ERR, "bad request (usn contains forbidden chars)"); + goto error; + } + syslog(LOG_INFO, "usn='%.*s'", l, p); + newserv->usn = malloc(l + 1); + if(!newserv->usn) { + syslog(LOG_ERR, "cannot allocate memory"); + goto error; + } + memcpy(newserv->usn, p, l); + newserv->usn[l] = '\0'; + p += l; + DECODELENGTH_CHECKLIMIT(l, p, buf + n); + if(l > (unsigned)(buf+n-p)) { + syslog(LOG_WARNING, "bad request (length encoding)"); + goto error; + } + if(containsForbiddenChars(p, l)) { + syslog(LOG_ERR, "bad request (server contains forbidden chars)"); + goto error; + } + syslog(LOG_INFO, "server='%.*s'", l, p); + newserv->server = malloc(l + 1); + if(!newserv->server) { + syslog(LOG_ERR, "cannot allocate memory"); + goto error; + } + memcpy(newserv->server, p, l); + newserv->server[l] = '\0'; + p += l; + DECODELENGTH_CHECKLIMIT(l, p, buf + n); + if(l > (unsigned)(buf+n-p)) { + syslog(LOG_WARNING, "bad request (length encoding)"); + goto error; + } + if(containsForbiddenChars(p, l)) { + syslog(LOG_ERR, "bad request (location contains forbidden chars)"); + goto error; + } + syslog(LOG_INFO, "location='%.*s'", l, p); + newserv->location = malloc(l + 1); + if(!newserv->location) { + syslog(LOG_ERR, "cannot allocate memory"); + goto error; + } + memcpy(newserv->location, p, l); + newserv->location[l] = '\0'; + p += l; + /* look in service list for duplicate */ + for(serv = servicelisthead.lh_first; + serv; + serv = serv->entries.le_next) { + if(0 == strcmp(newserv->usn, serv->usn) + && 0 == strcmp(newserv->st, serv->st)) { + syslog(LOG_INFO, "Service already in the list. Updating..."); + free(newserv->st); + free(newserv->usn); + free(serv->server); + serv->server = newserv->server; + free(serv->location); + serv->location = newserv->location; + free(newserv); + newserv = NULL; + return (p - buf); + } + } + /* Inserting new service */ + LIST_INSERT_HEAD(&servicelisthead, newserv, entries); + sendNotifications(NOTIF_NEW, NULL, newserv); + newserv = NULL; + break; + case MINISSDPD_NOTIF: /* switch socket to notify */ + rbuf[0] = '\0'; + if(write_or_buffer(req, rbuf, 1) < 0) { + syslog(LOG_ERR, "(s=%d) write: %m", req->socket); + goto error; + } + req->is_notify = 1; + p += l; + break; + default: + syslog(LOG_WARNING, "Unknown request type %d", type); + rbuf[0] = '\0'; + if(write_or_buffer(req, rbuf, 1) < 0) { + syslog(LOG_ERR, "(s=%d) write: %m", req->socket); + goto error; + } + } + return (p - buf); +error: + if(newserv) { + free(newserv->st); + free(newserv->usn); + free(newserv->server); + free(newserv->location); + free(newserv); + newserv = NULL; + } + return -1; +} + +static volatile sig_atomic_t quitting = 0; +/* SIGTERM signal handler */ +static void +sigterm(int sig) +{ + (void)sig; + /*int save_errno = errno;*/ + /*signal(sig, SIG_IGN);*/ +#if 0 + /* calling syslog() is forbidden in a signal handler according to + * signal(3) */ + syslog(LOG_NOTICE, "received signal %d, good-bye", sig); +#endif + quitting = 1; + /*errno = save_errno;*/ +} + +#define PORT 1900 +#define XSTR(s) STR(s) +#define STR(s) #s +#define UPNP_MCAST_ADDR "239.255.255.250" +/* for IPv6 */ +#define UPNP_MCAST_LL_ADDR "FF02::C" /* link-local */ +#define UPNP_MCAST_SL_ADDR "FF05::C" /* site-local */ + +/* send the M-SEARCH request for devices + * either all devices (third argument is NULL or "*") or a specific one */ +static void ssdpDiscover(int s, int ipv6, const char * search) +{ + static const char MSearchMsgFmt[] = + "M-SEARCH * HTTP/1.1\r\n" + "HOST: %s:" XSTR(PORT) "\r\n" + "ST: %s\r\n" + "MAN: \"ssdp:discover\"\r\n" + "MX: %u\r\n" + "\r\n"; + char bufr[512]; + int n; + int mx = 3; + int linklocal = 1; + struct sockaddr_storage sockudp_w; + + { + n = snprintf(bufr, sizeof(bufr), + MSearchMsgFmt, + ipv6 ? + (linklocal ? "[" UPNP_MCAST_LL_ADDR "]" : "[" UPNP_MCAST_SL_ADDR "]") + : UPNP_MCAST_ADDR, + (search ? search : "ssdp:all"), mx); + memset(&sockudp_w, 0, sizeof(struct sockaddr_storage)); + if(ipv6) { + struct sockaddr_in6 * p = (struct sockaddr_in6 *)&sockudp_w; + p->sin6_family = AF_INET6; + p->sin6_port = htons(PORT); + inet_pton(AF_INET6, + linklocal ? UPNP_MCAST_LL_ADDR : UPNP_MCAST_SL_ADDR, + &(p->sin6_addr)); + } else { + struct sockaddr_in * p = (struct sockaddr_in *)&sockudp_w; + p->sin_family = AF_INET; + p->sin_port = htons(PORT); + p->sin_addr.s_addr = inet_addr(UPNP_MCAST_ADDR); + } + + n = sendto_or_schedule(s, bufr, n, 0, (const struct sockaddr *)&sockudp_w, + ipv6 ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)); + if (n < 0) { + syslog(LOG_ERR, "%s: sendto: %m", __func__); + } + } +} + +/* main(): program entry point */ +int main(int argc, char * * argv) +{ + int ret = 0; + int pid; + struct sigaction sa; + char buf[1500]; + ssize_t n; + int s_ssdp = -1; /* udp socket receiving ssdp packets */ +#ifdef ENABLE_IPV6 + int s_ssdp6 = -1; /* udp socket receiving ssdp packets IPv6*/ +#else /* ENABLE_IPV6 */ +#define s_ssdp6 (-1) +#endif /* ENABLE_IPV6 */ + int s_unix = -1; /* unix socket communicating with clients */ + int s_ifacewatch = -1; /* socket to receive Route / network interface config changes */ + struct reqelem * req; + struct reqelem * reqnext; + fd_set readfds; + fd_set writefds; + struct timeval now; + int max_fd; + struct lan_addr_s * lan_addr; + int i; + const char * sockpath = "/var/run/minissdpd.sock"; + const char * pidfilename = "/var/run/minissdpd.pid"; + int debug_flag = 0; +#ifdef ENABLE_IPV6 + int ipv6 = 0; +#endif /* ENABLE_IPV6 */ + int deltadev = 0; + struct sockaddr_in sendername; + socklen_t sendername_len; +#ifdef ENABLE_IPV6 + struct sockaddr_in6 sendername6; + socklen_t sendername6_len; +#endif /* ENABLE_IPV6 */ + unsigned char ttl = 2; /* UDA says it should default to 2 */ + const char * searched_device = NULL; /* if not NULL, search/filter a specific device type */ + + LIST_INIT(&reqlisthead); + LIST_INIT(&servicelisthead); + LIST_INIT(&lan_addrs); + /* process command line */ + for(i=1; i= argc) { + fprintf(stderr, "option %s needs an argument.\n", argv[i]); + break; + } + if(0==strcmp(argv[i], "-i")) { + lan_addr = malloc(sizeof(struct lan_addr_s)); + if(lan_addr == NULL) { + fprintf(stderr, "malloc(%d) FAILED\n", (int)sizeof(struct lan_addr_s)); + break; + } + if(parselanaddr(lan_addr, argv[++i]) != 0) { + fprintf(stderr, "can't parse \"%s\" as a valid address or interface name\n", argv[i]); + free(lan_addr); + } else { + LIST_INSERT_HEAD(&lan_addrs, lan_addr, list); + } + } else if(0==strcmp(argv[i], "-s")) + sockpath = argv[++i]; + else if(0==strcmp(argv[i], "-p")) + pidfilename = argv[++i]; + else if(0==strcmp(argv[i], "-t")) + ttl = (unsigned char)atoi(argv[++i]); + else if(0==strcmp(argv[i], "-f")) + searched_device = argv[++i]; + else + fprintf(stderr, "unknown commandline option %s.\n", argv[i]); + } + } + if(lan_addrs.lh_first == NULL) + { + fprintf(stderr, + "Usage: %s [-d] " +#ifdef ENABLE_IPV6 + "[-6] " +#endif /* ENABLE_IPV6 */ + "[-s socket] [-p pidfile] [-t TTL] " + "[-f device] " + "-i [-i ] ...\n", + argv[0]); + fprintf(stderr, + "\n is either an IPv4 address with mask such as\n" + " 192.168.1.42/255.255.255.0, or an interface name such as eth0.\n"); + fprintf(stderr, + "\n By default, socket will be open as %s\n" + " and pid written to file %s\n", + sockpath, pidfilename); + return 1; + } + + /* open log */ + openlog("minissdpd", + LOG_CONS|LOG_PID|(debug_flag?LOG_PERROR:0), + LOG_MINISSDPD); + if(!debug_flag) /* speed things up and ignore LOG_INFO and LOG_DEBUG */ + setlogmask(LOG_UPTO(LOG_NOTICE)); + + if(checkforrunning(pidfilename) < 0) + { + syslog(LOG_ERR, "MiniSSDPd is already running. EXITING"); + return 1; + } + + upnp_bootid = (unsigned int)time(NULL); + + /* set signal handlers */ + memset(&sa, 0, sizeof(struct sigaction)); + sa.sa_handler = sigterm; + if(sigaction(SIGTERM, &sa, NULL)) + { + syslog(LOG_ERR, "Failed to set SIGTERM handler. EXITING"); + ret = 1; + goto quit; + } + if(sigaction(SIGINT, &sa, NULL)) + { + syslog(LOG_ERR, "Failed to set SIGINT handler. EXITING"); + ret = 1; + goto quit; + } + /* open route/interface config changes socket */ + s_ifacewatch = OpenAndConfInterfaceWatchSocket(); + /* open UDP socket(s) for receiving SSDP packets */ + s_ssdp = OpenAndConfSSDPReceiveSocket(0, ttl); + if(s_ssdp < 0) + { + syslog(LOG_ERR, "Cannot open socket for receiving SSDP messages, exiting"); + ret = 1; + goto quit; + } +#ifdef ENABLE_IPV6 + if(ipv6) { + s_ssdp6 = OpenAndConfSSDPReceiveSocket(1, ttl); + if(s_ssdp6 < 0) + { + syslog(LOG_ERR, "Cannot open socket for receiving SSDP messages (IPv6), exiting"); + ret = 1; + goto quit; + } + } +#endif /* ENABLE_IPV6 */ + /* Open Unix socket to communicate with other programs on + * the same machine */ + s_unix = OpenUnixSocket(sockpath); + if(s_unix < 0) + { + syslog(LOG_ERR, "Cannot open unix socket for communicating with clients. Exiting"); + ret = 1; + goto quit; + } + + /* drop privileges */ +#if 0 + /* if we drop privileges, how to unlink(/var/run/minissdpd.sock) ? */ + if(getuid() == 0) { + struct passwd * user; + struct group * group; + user = getpwnam("nobody"); + if(!user) { + syslog(LOG_ERR, "getpwnam(\"%s\") : %m", "nobody"); + ret = 1; + goto quit; + } + group = getgrnam("nogroup"); + if(!group) { + syslog(LOG_ERR, "getgrnam(\"%s\") : %m", "nogroup"); + ret = 1; + goto quit; + } + if(setgid(group->gr_gid) < 0) { + syslog(LOG_ERR, "setgit(%d) : %m", group->gr_gid); + ret = 1; + goto quit; + } + if(setuid(user->pw_uid) < 0) { + syslog(LOG_ERR, "setuid(%d) : %m", user->pw_uid); + ret = 1; + goto quit; + } + } +#endif + + /* daemonize or in any case get pid ! */ + if(debug_flag) + pid = getpid(); + else { +#ifdef USE_DAEMON + if(daemon(0, 0) < 0) + perror("daemon()"); + pid = getpid(); +#else /* USE_DAEMON */ + pid = daemonize(); +#endif /* USE_DAEMON */ + } + + writepidfile(pidfilename, pid); + + /* send M-SEARCH ssdp:all Requests */ + if(s_ssdp >= 0) + ssdpDiscover(s_ssdp, 0, searched_device); + if(s_ssdp6 >= 0) + ssdpDiscover(s_ssdp6, 1, searched_device); + + /* Main loop */ + while(!quitting) { + /* fill readfds fd_set */ + FD_ZERO(&readfds); + FD_ZERO(&writefds); + + FD_SET(s_unix, &readfds); + max_fd = s_unix; + if(s_ssdp >= 0) { + FD_SET(s_ssdp, &readfds); + SET_MAX(max_fd, s_ssdp); + } +#ifdef ENABLE_IPV6 + if(s_ssdp6 >= 0) { + FD_SET(s_ssdp6, &readfds); + SET_MAX(max_fd, s_ssdp6); + } +#endif /* ENABLE_IPV6 */ + if(s_ifacewatch >= 0) { + FD_SET(s_ifacewatch, &readfds); + SET_MAX(max_fd, s_ifacewatch); + } + for(req = reqlisthead.lh_first; req; req = req->entries.le_next) { + if(req->socket >= 0) { + FD_SET(req->socket, &readfds); + SET_MAX(max_fd, req->socket); + } + if(req->output_buffer_len > 0) { + FD_SET(req->socket, &writefds); + SET_MAX(max_fd, req->socket); + } + } + gettimeofday(&now, NULL); + i = get_sendto_fds(&writefds, &max_fd, &now); + /* select call */ + if(select(max_fd + 1, &readfds, &writefds, 0, 0) < 0) { + if(errno != EINTR) { + syslog(LOG_ERR, "select: %m"); + break; /* quit */ + } + continue; /* try again */ + } + if(try_sendto(&writefds) < 0) { + syslog(LOG_ERR, "try_sendto: %m"); + break; + } +#ifdef ENABLE_IPV6 + if((s_ssdp6 >= 0) && FD_ISSET(s_ssdp6, &readfds)) + { + sendername6_len = sizeof(struct sockaddr_in6); + n = recvfrom(s_ssdp6, buf, sizeof(buf), 0, + (struct sockaddr *)&sendername6, &sendername6_len); + if(n<0) + { + /* EAGAIN, EWOULDBLOCK, EINTR : silently ignore (try again next time) + * other errors : log to LOG_ERR */ + if(errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) + syslog(LOG_ERR, "recvfrom: %m"); + } + else + { + /* Parse and process the packet received */ + /*printf("%.*s", n, buf);*/ + i = ParseSSDPPacket(s_ssdp6, buf, n, + (struct sockaddr *)&sendername6, searched_device); + syslog(LOG_DEBUG, "** i=%d deltadev=%d **", i, deltadev); + if(i==0 || (i*deltadev < 0)) + { + if(deltadev > 0) + syslog(LOG_NOTICE, "%d new devices added", deltadev); + else if(deltadev < 0) + syslog(LOG_NOTICE, "%d devices removed (good-bye!)", -deltadev); + deltadev = i; + } + else if((i*deltadev) >= 0) + { + deltadev += i; + } + } + } +#endif /* ENABLE_IPV6 */ + if((s_ssdp >= 0) && FD_ISSET(s_ssdp, &readfds)) + { + sendername_len = sizeof(struct sockaddr_in); + n = recvfrom(s_ssdp, buf, sizeof(buf), 0, + (struct sockaddr *)&sendername, &sendername_len); + if(n<0) + { + /* EAGAIN, EWOULDBLOCK, EINTR : silently ignore (try again next time) + * other errors : log to LOG_ERR */ + if(errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) + syslog(LOG_ERR, "recvfrom: %m"); + } + else + { + /* Parse and process the packet received */ + /*printf("%.*s", n, buf);*/ + i = ParseSSDPPacket(s_ssdp, buf, n, + (struct sockaddr *)&sendername, searched_device); + syslog(LOG_DEBUG, "** i=%d deltadev=%d **", i, deltadev); + if(i==0 || (i*deltadev < 0)) + { + if(deltadev > 0) + syslog(LOG_NOTICE, "%d new devices added", deltadev); + else if(deltadev < 0) + syslog(LOG_NOTICE, "%d devices removed (good-bye!)", -deltadev); + deltadev = i; + } + else if((i*deltadev) >= 0) + { + deltadev += i; + } + } + } + /* processing unix socket requests */ + for(req = reqlisthead.lh_first; req;) { + reqnext = req->entries.le_next; + if((req->socket >= 0) && FD_ISSET(req->socket, &readfds)) { + processRequest(req); + } + if((req->socket >= 0) && FD_ISSET(req->socket, &writefds)) { + write_buffer(req); + } + if(req->socket < 0) { + LIST_REMOVE(req, entries); + free(req->output_buffer); + free(req); + } + req = reqnext; + } + /* processing new requests */ + if(FD_ISSET(s_unix, &readfds)) + { + struct reqelem * tmp; + int s = accept(s_unix, NULL, NULL); + if(s < 0) { + syslog(LOG_ERR, "accept(s_unix): %m"); + } else { + syslog(LOG_INFO, "(s=%d) new request connection", s); + if(!set_non_blocking(s)) + syslog(LOG_WARNING, "Failed to set new socket non blocking : %m"); + tmp = malloc(sizeof(struct reqelem)); + if(!tmp) { + syslog(LOG_ERR, "cannot allocate memory for request"); + close(s); + } else { + memset(tmp, 0, sizeof(struct reqelem)); + tmp->socket = s; + LIST_INSERT_HEAD(&reqlisthead, tmp, entries); + } + } + } + /* processing route/network interface config changes */ + if((s_ifacewatch >= 0) && FD_ISSET(s_ifacewatch, &readfds)) { + ProcessInterfaceWatch(s_ifacewatch, s_ssdp, s_ssdp6); + } + } + syslog(LOG_DEBUG, "quitting..."); + finalize_sendto(); + + /* closing and cleaning everything */ +quit: + if(s_ssdp >= 0) { + close(s_ssdp); + s_ssdp = -1; + } +#ifdef ENABLE_IPV6 + if(s_ssdp6 >= 0) { + close(s_ssdp6); + s_ssdp6 = -1; + } +#endif /* ENABLE_IPV6 */ + if(s_unix >= 0) { + close(s_unix); + s_unix = -1; + if(unlink(sockpath) < 0) + syslog(LOG_ERR, "unlink(%s): %m", sockpath); + } + if(s_ifacewatch >= 0) { + close(s_ifacewatch); + s_ifacewatch = -1; + } + /* empty LAN interface/address list */ + while(lan_addrs.lh_first != NULL) { + lan_addr = lan_addrs.lh_first; + LIST_REMOVE(lan_addrs.lh_first, list); + free(lan_addr); + } + /* empty device list */ + while(devlist != NULL) { + struct device * next = devlist->next; + free(devlist); + devlist = next; + } + /* empty service list */ + while(servicelisthead.lh_first != NULL) { + struct service * serv = servicelisthead.lh_first; + LIST_REMOVE(servicelisthead.lh_first, entries); + free(serv->st); + free(serv->usn); + free(serv->server); + free(serv->location); + free(serv); + } + if(unlink(pidfilename) < 0) + syslog(LOG_ERR, "unlink(%s): %m", pidfilename); + closelog(); + return ret; +} + diff --git a/src/contrib/miniupnp/minissdpd/minissdpd.init.d.script b/src/contrib/miniupnp/minissdpd/minissdpd.init.d.script new file mode 100644 index 0000000..8b0631b --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/minissdpd.init.d.script @@ -0,0 +1,39 @@ +#!/bin/sh +# $Id: minissdpd.init.d.script,v 1.2 2007/09/23 17:46:57 nanard Exp $ +# MiniUPnP project +# author: Thomas Bernard +# website: http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + +MINISSDPD=/usr/sbin/minissdpd +PIDFILE=/var/run/minissdpd.pid +# get default interface +IF=`route | grep default |awk -- '{ print $8 }'` +ARGS="-i $IF" + +test -f $MINISSDPD || exit 0 + +. /lib/lsb/init-functions + +case "$1" in +start) log_daemon_msg "Starting minissdpd" "minissdpd" + start-stop-daemon --start --quiet --pidfile $PIDFILE \ + --exec $MINISSDPD -- $ARGS $LSBNAMES + log_end_msg $? + ;; +stop) log_daemon_msg "Stopping minissdpd" "minissdpd" + start-stop-daemon --stop --quiet --pidfile $PIDFILE + log_end_msg $? + ;; +restart|reload|force-reload) + log_daemon_msg "Restarting minissdpd" "minissdpd" + start-stop-daemon --stop --retry 5 --quiet --pidfile $PIDFILE + start-stop-daemon --start --quiet --pidfile $PIDFILE \ + --exec $MINISSDPD -- $ARGS $LSBNAMES + log_end_msg $? + ;; +*) log_action_msg "Usage: /etc/init.d/minissdpd {start|stop|restart|reload|force-reload}" + exit 2 + ;; +esac +exit 0 + diff --git a/src/contrib/miniupnp/minissdpd/minissdpdtypes.h b/src/contrib/miniupnp/minissdpd/minissdpdtypes.h new file mode 100644 index 0000000..daf18e3 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/minissdpdtypes.h @@ -0,0 +1,28 @@ +/* $Id: minissdpdtypes.h,v 1.1 2014/11/28 16:20:58 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef MINISSDPDTYPES_H_INCLUDED +#define MINISSDPDTYPES_H_INCLUDED + +#include "config.h" +#include +#include +#include + +/* structure and list for storing lan addresses + * with ascii representation and mask */ +struct lan_addr_s { + char ifname[IFNAMSIZ]; /* example: eth0 */ +#ifdef ENABLE_IPV6 + unsigned int index; /* use if_nametoindex() */ +#endif /* ENABLE_IPV6 */ + char str[16]; /* example: 192.168.0.1 */ + struct in_addr addr, mask; /* ip/mask */ + LIST_ENTRY(lan_addr_s) list; +}; +LIST_HEAD(lan_addr_list, lan_addr_s); + +#endif /* MINISSDPDTYPES_H_INCLUDED */ diff --git a/src/contrib/miniupnp/minissdpd/openssdpsocket.c b/src/contrib/miniupnp/minissdpd/openssdpsocket.c new file mode 100644 index 0000000..702856d --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/openssdpsocket.c @@ -0,0 +1,233 @@ +/* $Id: openssdpsocket.c,v 1.17 2015/08/06 14:05:37 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "openssdpsocket.h" +#include "upnputils.h" +#include "minissdpdtypes.h" + +extern struct lan_addr_list lan_addrs; + +/* SSDP ip/port */ +#define SSDP_PORT (1900) +#define SSDP_MCAST_ADDR ("239.255.255.250") +/* Link Local and Site Local SSDP IPv6 multicast addresses */ +#define LL_SSDP_MCAST_ADDR ("FF02::C") +#define SL_SSDP_MCAST_ADDR ("FF05::C") + +/** + * Add the multicast membership for SSDP on the interface + * @param s the socket + * @param ifaddr the IPv4 address or interface name + * @param ipv6 IPv6 or IPv4 + * return -1 on error, 0 on success */ +int +AddDropMulticastMembership(int s, struct lan_addr_s * lan_addr, int ipv6, int drop) +{ + struct ip_mreq imr; /* Ip multicast membership */ +#ifdef ENABLE_IPV6 + struct ipv6_mreq mr; +#else /* ENABLE_IPV6 */ + (void)ipv6; +#endif /* ENABLE_IPV6 */ + + if(s <= 0) + return -1; /* nothing to do */ +#ifdef ENABLE_IPV6 + if(ipv6) + { + memset(&mr, 0, sizeof(mr)); + inet_pton(AF_INET6, LL_SSDP_MCAST_ADDR, &mr.ipv6mr_multiaddr); + mr.ipv6mr_interface = lan_addr->index; + if(setsockopt(s, IPPROTO_IPV6, drop ? IPV6_LEAVE_GROUP : IPV6_JOIN_GROUP, + &mr, sizeof(struct ipv6_mreq)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp, %s)(%s, %s): %m", + drop ? "IPV6_LEAVE_GROUP" : "IPV6_JOIN_GROUP", + LL_SSDP_MCAST_ADDR, + lan_addr->ifname); + return -1; + } + inet_pton(AF_INET6, SL_SSDP_MCAST_ADDR, &mr.ipv6mr_multiaddr); + if(setsockopt(s, IPPROTO_IPV6, drop ? IPV6_LEAVE_GROUP : IPV6_JOIN_GROUP, + &mr, sizeof(struct ipv6_mreq)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp, %s)(%s, %s): %m", + drop ? "IPV6_LEAVE_GROUP" : "IPV6_JOIN_GROUP", + SL_SSDP_MCAST_ADDR, + lan_addr->ifname); + return -1; + } + } + else + { +#endif /* ENABLE_IPV6 */ + /* setting up imr structure */ + imr.imr_multiaddr.s_addr = inet_addr(SSDP_MCAST_ADDR); + imr.imr_interface.s_addr = lan_addr->addr.s_addr; + if(imr.imr_interface.s_addr == INADDR_NONE) + { + syslog(LOG_ERR, "no IPv4 address for interface %s", + lan_addr->ifname); + return -1; + } + + if (setsockopt(s, IPPROTO_IP, drop ? IP_DROP_MEMBERSHIP : IP_ADD_MEMBERSHIP, + (void *)&imr, sizeof(struct ip_mreq)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp, %s)(%s): %m", + drop ? "IP_DROP_MEMBERSHIP" : "IP_ADD_MEMBERSHIP", + lan_addr->ifname); + return -1; + } +#ifdef ENABLE_IPV6 + } +#endif /* ENABLE_IPV6 */ + + return 0; +} + +int +OpenAndConfSSDPReceiveSocket(int ipv6, unsigned char ttl) +{ + int s; + int opt = 1; + unsigned char loopchar = 0; +#ifdef ENABLE_IPV6 + struct sockaddr_storage sockname; +#else /* ENABLE_IPV6 */ + struct sockaddr_in sockname; +#endif /* ENABLE_IPV6 */ + socklen_t sockname_len; + struct lan_addr_s * lan_addr; + +#ifndef ENABLE_IPV6 + if(ipv6) { + syslog(LOG_ERR, "%s: please compile with ENABLE_IPV6 to allow ipv6=1", __func__); + return -1; + } +#endif /* ENABLE_IPV6 */ + +#ifdef ENABLE_IPV6 + if( (s = socket(ipv6 ? PF_INET6 : PF_INET, SOCK_DGRAM, 0)) < 0) +#else /* ENABLE_IPV6 */ + if( (s = socket(PF_INET, SOCK_DGRAM, 0)) < 0) +#endif /* ENABLE_IPV6 */ + { + syslog(LOG_ERR, "socket(udp): %m"); + return -1; + } + + if(!set_non_blocking(s)) { + syslog(LOG_WARNING, "Failed to set SSDP socket non blocking : %m"); + } + +#ifdef ENABLE_IPV6 + memset(&sockname, 0, sizeof(struct sockaddr_storage)); + if(ipv6) + { +#ifdef IPV6_V6ONLY + if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, + (char *)&opt, sizeof(opt)) < 0) + { + syslog(LOG_WARNING, "setsockopt(IPV6_V6ONLY): %m"); + } +#endif /* IPV6_V6ONLY */ + struct sockaddr_in6 * sa = (struct sockaddr_in6 *)&sockname; + sa->sin6_family = AF_INET6; + sa->sin6_port = htons(SSDP_PORT); + sa->sin6_addr = in6addr_any; + sockname_len = sizeof(struct sockaddr_in6); + } + else + { + struct sockaddr_in * sa = (struct sockaddr_in *)&sockname; + sa->sin_family = AF_INET; + sa->sin_port = htons(SSDP_PORT); +#ifdef SSDP_LISTEN_ON_SPECIFIC_ADDR + if(lan_addrs.lh_first != NULL && lan_addrs.lh_first->list.le_next == NULL) + { + sa->sin_addr.s_addr = lan_addrs.lh_first->addr.s_addr; + if(sa->sin_addr.s_addr == INADDR_NONE) + { + syslog(LOG_ERR, "no IPv4 address for interface %s", + lan_addrs.lh_first->ifname); + close(s); + return -1; + } + } + else +#endif /* SSDP_LISTEN_ON_SPECIFIC_ADDR */ + sa->sin_addr.s_addr = htonl(INADDR_ANY); + sockname_len = sizeof(struct sockaddr_in); + } +#else /* ENABLE_IPV6 */ + memset(&sockname, 0, sizeof(struct sockaddr_in)); + sockname.sin_family = AF_INET; + sockname.sin_port = htons(SSDP_PORT); +#ifdef SSDP_LISTEN_ON_SPECIFIC_ADDR + if(lan_addrs.lh_first != NULL && lan_addrs.lh_first->list.le_next == NULL) + { + sockname.sin_addr.s_addr = lan_addrs.lh_first->addr.s_addr; + if(sockname.sin_addr.s_addr == INADDR_NONE) + { + syslog(LOG_ERR, "no IPv4 address for interface %s", + lan_addrs.lh_first->ifname); + close(s); + return -1; + } + } + else +#endif /* SSDP_LISTEN_ON_SPECIFIC_ADDR */ + sockname.sin_addr.s_addr = htonl(INADDR_ANY); + sockname_len = sizeof(struct sockaddr_in); +#endif /* ENABLE_IPV6 */ + + if(setsockopt(s, IPPROTO_IP, IP_MULTICAST_LOOP, (char *)&loopchar, sizeof(loopchar)) < 0) + { + syslog(LOG_WARNING, "setsockopt(IP_MULTICAST_LOOP): %m"); + } + + if(setsockopt(s, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) < 0) + { + syslog(LOG_WARNING, "setsockopt(IP_MULTICAST_TTL): %m"); + } + + if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)) < 0) + { + syslog(LOG_WARNING, "setsockopt(SO_REUSEADDR): %m"); + } + + if(bind(s, (struct sockaddr *)&sockname, sockname_len) < 0) + { + syslog(LOG_ERR, "bind(udp%s): %m", ipv6 ? "6" : ""); + close(s); + return -1; + } + + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) + { + if(AddDropMulticastMembership(s, lan_addr, ipv6, 0) < 0) + { + syslog(LOG_WARNING, "Failed to add IPv%d multicast membership for interface %s.", + ipv6 ? 6 : 4, + lan_addr->ifname); + } + } + + return s; +} diff --git a/src/contrib/miniupnp/minissdpd/openssdpsocket.h b/src/contrib/miniupnp/minissdpd/openssdpsocket.h new file mode 100644 index 0000000..957a928 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/openssdpsocket.h @@ -0,0 +1,33 @@ +/* $Id: openssdpsocket.h,v 1.7 2015/07/21 15:39:38 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2015 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef OPENSSDPSOCKET_H_INCLUDED +#define OPENSSDPSOCKET_H_INCLUDED + +#include "minissdpdtypes.h" + +/** + * Open a socket and configure it for receiving SSDP packets + * + * @param ipv6 open INET6 or INET socket + * @param ttl multicast TTL + * @return socket + */ +int +OpenAndConfSSDPReceiveSocket(int ipv6, unsigned char ttl); + +/** + * Add or Drop the multicast membership for SSDP on the interface + * @param s the socket + * @param lan_addr the LAN address or interface name + * @param ipv6 IPv6 or IPv4 + * @param drop 0 to add, 1 to drop + * return -1 on error, 0 on success */ +int +AddDropMulticastMembership(int s, struct lan_addr_s * lan_addr, int ipv6, int drop); + +#endif + diff --git a/src/contrib/miniupnp/minissdpd/printresponse.c b/src/contrib/miniupnp/minissdpd/printresponse.c new file mode 100644 index 0000000..270c91f --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/printresponse.c @@ -0,0 +1,90 @@ +/* $Id: $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp + * website : http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author : Thomas BERNARD + * copyright (c) 2005-2016 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ + +#include +#include "codelength.h" + +#define NOTIF_NEW 1 +#define NOTIF_UPDATE 2 +#define NOTIF_REMOVE 3 + +void printresponse(const unsigned char * resp, int n) +{ + int i, l; + int notif_type; + unsigned int nresp; + const unsigned char * p; + + if(n == 0) + return; + /* first, hexdump the response : */ + for(i = 0; i < n; i += 16) { + printf("%06x | ", i); + for(l = i; l < n && l < (i + 16); l++) + printf("%02x ", resp[l]); + while(l < (i + 16)) { + printf(" "); + l++; + } + printf("| "); + for(l = i; l < n && l < (i + 16); l++) + putchar((resp[l] >= ' ' && resp[l] < 128) ? resp[l] : '.'); + putchar('\n'); + } + for(p = resp; p < resp + n; ) { + /* now parse and display all devices of response */ + nresp = p[0]; /* 1st byte : number of devices in response */ + if(nresp == 0xff) { + /* notification */ + notif_type = p[1]; + nresp = p[2]; + printf("Notification : "); + switch(notif_type) { + case NOTIF_NEW: printf("new\n"); break; + case NOTIF_UPDATE: printf("update\n"); break; + case NOTIF_REMOVE: printf("remove\n"); break; + default: printf("**UNKNOWN**\n"); + } + p += 3; + } else { + p++; + } + for(i = 0; i < (int)nresp; i++) { + if(p >= resp + n) + goto error; + /*l = *(p++);*/ + DECODELENGTH(l, p); + if(p + l > resp + n) + goto error; + printf("%d - %.*s\n", i, l, p); /* URL */ + p += l; + if(p >= resp + n) + goto error; + /*l = *(p++);*/ + DECODELENGTH(l, p); + if(p + l > resp + n) + goto error; + printf(" %.*s\n", l, p); /* ST */ + p += l; + if(p >= resp + n) + goto error; + /*l = *(p++);*/ + DECODELENGTH(l, p); + if(p + l > resp + n) + goto error; + printf(" %.*s\n", l, p); /* USN */ + p += l; + } + } + return; +error: + printf("*** WARNING : TRUNCATED RESPONSE ***\n"); +} + + diff --git a/src/contrib/miniupnp/minissdpd/printresponse.h b/src/contrib/miniupnp/minissdpd/printresponse.h new file mode 100644 index 0000000..f0cf613 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/printresponse.h @@ -0,0 +1,12 @@ +/* $Id: $ */ +/* Project : miniupnp + * Author : Thomas BERNARD + * copyright (c) 2016 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ +#ifndef PRINTRESPONSE_H_INCLUDED +#define PRINTRESPONSE_H_INCLUDED + +void printresponse(const unsigned char * resp, int n); + +#endif /* PRINTRESPONSE_H_INCLUDED */ diff --git a/src/contrib/miniupnp/minissdpd/showminissdpdnotif.c b/src/contrib/miniupnp/minissdpd/showminissdpdnotif.c new file mode 100644 index 0000000..c64c830 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/showminissdpdnotif.c @@ -0,0 +1,85 @@ +/* $Id: $ */ +/* vim: shiftwidth=4 tabstop=4 noexpandtab + * MiniUPnP project + * (c) 2016 Thomas Bernard + * website : http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "codelength.h" +#include "printresponse.h" + +static volatile sig_atomic_t quitting = 0; + +static void sighandler(int sig) +{ + (void)sig; + quitting = 1; +} + +int main(int argc, char * * argv) +{ + int i; + int s; + struct sockaddr_un addr; + const char * sockpath = "/var/run/minissdpd.sock"; + unsigned char buffer[4096]; + ssize_t n; + const char command5[] = { 0x05, 0x00 }; + struct sigaction sa; + + for(i=0; i +#include "codelength.h" + +int main(int argc, char * * argv) +{ + unsigned char buf[256]; + unsigned char * p; + long i, j; + (void)argc; (void)argv; + + for(i = 1; i < 1000000000; i *= 2) { + /* encode i, decode to j */ + printf("%ld ", i); + p = buf; + CODELENGTH(i, p); + p = buf; + DECODELENGTH(j, p); + if(i != j) { + fprintf(stderr, "Error ! encoded %ld, decoded %ld.\n", i, j); + return 1; + } + } + printf("Test successful\n"); + return 0; +} diff --git a/src/contrib/miniupnp/minissdpd/testminissdpd.c b/src/contrib/miniupnp/minissdpd/testminissdpd.c new file mode 100644 index 0000000..05ddfd6 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/testminissdpd.c @@ -0,0 +1,196 @@ +/* $Id: testminissdpd.c,v 1.14 2016/03/01 17:49:51 nanard Exp $ */ +/* Project : miniupnp + * website : http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * Author : Thomas BERNARD + * copyright (c) 2005-2018 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ + +#include +#include +#include +#include +#include +#include +#include + +#include "codelength.h" +#include "printresponse.h" + +void printversion(const unsigned char * resp, int n) +{ + int l; + const unsigned char * p; + + p = resp; + DECODELENGTH(l, p); + if(resp + n < p + l) { + printf("get version error\n"); + } + printf("MiniSSDPd version : %.*s\n", l, p); +} + +#define SENDCOMMAND(command, size) write(s, command, size); \ + printf("Command written type=%u\n", (unsigned char)command[0]); + +int connect_unix_socket(const char * sockpath) +{ + int s; + struct sockaddr_un addr; + + s = socket(AF_UNIX, SOCK_STREAM, 0); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, sockpath, sizeof(addr.sun_path)); + if(connect(s, (struct sockaddr *)&addr, sizeof(struct sockaddr_un)) < 0) { + fprintf(stderr, "connecting to %s : ", addr.sun_path); + perror("connect"); + exit(1); + } + printf("Connected to %s\n", addr.sun_path); + return s; +} + +/* test program for minissdpd */ +int +main(int argc, char * * argv) +{ + const unsigned char command0[] = { 0x00, 0x00 }; + char command1[] = "\x01\x00urn:schemas-upnp-org:device:InternetGatewayDevice"; + char command2[] = "\x02\x00uuid:fc4ec57e-b051-11db-88f8-0060085db3f6::upnp:rootdevice"; + const unsigned char command3[] = { 0x03, 0x00 }; + /* old versions of minissdpd would reject a command with + * a zero length string argument */ + char command3compat[] = "\x03\x00ssdp:all"; + char command4[] = "\x04\x00test:test:test"; + const unsigned char bad_command[] = { 0xff, 0xff }; + const unsigned char overflow[] = { 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + const unsigned char command5[] = { 0x05, 0x00 }; + const unsigned char bad_command4[] = { 0x04, 0x01, 0x60, 0x8f, 0xff, 0xff, 0xff, 0x7f}; + int s; + int i; + void * tmp; + unsigned char * resp = NULL; + size_t respsize = 0; + unsigned char buf[4096]; + ssize_t n; + int total = 0; + const char * sockpath = "/var/run/minissdpd.sock"; + + for(i=0; i 0) { + printversion(buf, n); + } else { + printf("Command 0 (get version) not supported\n"); + close(s); + s = connect_unix_socket(sockpath); + } + + n = SENDCOMMAND(command1, sizeof(command1) - 1); + n = read(s, buf, sizeof(buf)); + printf("Response received %d bytes\n", (int)n); + printresponse(buf, n); + if(n == 0) { + close(s); + s = connect_unix_socket(sockpath); + } + + n = SENDCOMMAND(command2, sizeof(command2) - 1); + n = read(s, buf, sizeof(buf)); + printf("Response received %d bytes\n", (int)n); + printresponse(buf, n); + if(n == 0) { + close(s); + s = connect_unix_socket(sockpath); + } + + buf[0] = 0; /* Slight hack for printing num devices when 0 */ + n = SENDCOMMAND(command3, sizeof(command3)); + n = read(s, buf, sizeof(buf)); + if(n == 0) { + printf("command3 failed, testing compatible one\n"); + close(s); + s = connect_unix_socket(sockpath); + n = SENDCOMMAND(command3compat, sizeof(command3compat) - 1); + n = read(s, buf, sizeof(buf)); + } + printf("Response received %d bytes\n", (int)n); + printf("Number of devices %d\n", (int)buf[0]); + while(n > 0) { + tmp = realloc(resp, respsize + n); + if(tmp == NULL) { + fprintf(stderr, "memory allocation error\n"); + break; + } + resp = tmp; + respsize += n; + if (n > 0) { + memcpy(resp + total, buf, n); + total += n; + } + if (n < (ssize_t)sizeof(buf)) { + break; + } + + n = read(s, buf, sizeof(buf)); + printf("response received %d bytes\n", (int)n); + } + if(resp != NULL) { + printresponse(resp, total); + free(resp); + resp = NULL; + } + if(n == 0) { + close(s); + s = connect_unix_socket(sockpath); + } + + n = SENDCOMMAND(command4, sizeof(command4)); + /* no response for request type 4 */ + + n = SENDCOMMAND(bad_command, sizeof(bad_command)); + n = read(s, buf, sizeof(buf)); + printf("Response received %d bytes\n", (int)n); + printresponse(buf, n); + if(n == 0) { + close(s); + s = connect_unix_socket(sockpath); + } + + n = SENDCOMMAND(overflow, sizeof(overflow)); + n = read(s, buf, sizeof(buf)); + printf("Response received %d bytes\n", (int)n); + printresponse(buf, n); + if(n == 0) { + close(s); + s = connect_unix_socket(sockpath); + } + + n = SENDCOMMAND(command5, sizeof(command5)); + n = read(s, buf, sizeof(buf)); + printf("Response received %d bytes\n", (int)n); + printresponse(buf, n); + if(n == 0) { + close(s); + s = connect_unix_socket(sockpath); + } + + n = SENDCOMMAND(bad_command4, sizeof(bad_command4)); + n = read(s, buf, sizeof(buf)); + printf("Response received %d bytes\n", (int)n); + printresponse(buf, n); + + close(s); + return 0; +} diff --git a/src/contrib/miniupnp/minissdpd/testminissdpd.sh b/src/contrib/miniupnp/minissdpd/testminissdpd.sh new file mode 100644 index 0000000..66b41fc --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/testminissdpd.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# $Id: testminissdpd.sh,v 1.8 2017/04/21 11:57:59 nanard Exp $ +# (c) 2017 Thomas Bernard + +OS=`uname -s` +IF=lo +if [ "$OS" = "Darwin" ] || [ "$OS" = "OpenBSD" ] || [ "$OS" = "SunOS" ] || [ "$OS" = "FreeBSD" ] ; then + IF=lo0 +fi +# if set, 1st argument is network interface +if [ -n "$1" ] ; then + IF=$1 +fi +SOCKET=`mktemp -t minissdpdsocketXXXXXX` +PID="${SOCKET}.pid" +./minissdpd -s $SOCKET -p $PID -i $IF || exit 1 +./testminissdpd -s $SOCKET || exit 2 +kill `cat $PID` diff --git a/src/contrib/miniupnp/minissdpd/testminissdpdnotif.sh b/src/contrib/miniupnp/minissdpd/testminissdpdnotif.sh new file mode 100644 index 0000000..2099ea6 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/testminissdpdnotif.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# $Id: $ +# (c) 2016 Thomas Bernard + +OS=`uname -s` +IF=lo +if [ "$OS" = "Darwin" ] || [ "$OS" = "SunOS" ] ; then + IF=lo0 +fi +# if set, 1st argument is network interface +if [ -n "$1" ] ; then + IF=$1 +fi + +# trap sigint in the script so CTRL-C interrupts the running program, +# not the script +trap 'echo SIGINT' INT + +SOCKET=`mktemp -t minissdpdsocketXXXXXX` +PID="${SOCKET}.pid" +./minissdpd -s $SOCKET -p $PID -i $IF || exit 1 +sleep .5 +echo "minissdpd process id `cat $PID`" +./showminissdpdnotif -s $SOCKET +echo "showminissdpdnotif returned $?" +kill `cat $PID` diff --git a/src/contrib/miniupnp/minissdpd/upnputils.c b/src/contrib/miniupnp/minissdpd/upnputils.c new file mode 100644 index 0000000..a207cc2 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/upnputils.c @@ -0,0 +1,172 @@ +/* $Id: upnputils.c,v 1.2 2014/11/28 16:20:58 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef AF_LINK +#include +#endif + +#include "upnputils.h" +#include "getroute.h" +#include "minissdpdtypes.h" + +extern struct lan_addr_list lan_addrs; + +int +sockaddr_to_string(const struct sockaddr * addr, char * str, size_t size) +{ + char buffer[64]; + unsigned short port = 0; + int n = -1; + + switch(addr->sa_family) + { + case AF_INET6: + inet_ntop(addr->sa_family, + &((struct sockaddr_in6 *)addr)->sin6_addr, + buffer, sizeof(buffer)); + port = ntohs(((struct sockaddr_in6 *)addr)->sin6_port); + n = snprintf(str, size, "[%s]:%hu", buffer, port); + break; + case AF_INET: + inet_ntop(addr->sa_family, + &((struct sockaddr_in *)addr)->sin_addr, + buffer, sizeof(buffer)); + port = ntohs(((struct sockaddr_in *)addr)->sin_port); + n = snprintf(str, size, "%s:%hu", buffer, port); + break; +#ifdef AF_LINK +#if defined(__sun) + /* solaris does not seem to have link_ntoa */ + /* #define link_ntoa _link_ntoa */ +#define link_ntoa(x) "dummy-link_ntoa" +#endif + case AF_LINK: + { + struct sockaddr_dl * sdl = (struct sockaddr_dl *)addr; + n = snprintf(str, size, "index=%hu type=%d %s", + sdl->sdl_index, sdl->sdl_type, + link_ntoa(sdl)); + } + break; +#endif + default: + n = snprintf(str, size, "unknown address family %d", addr->sa_family); +#if 0 + n = snprintf(str, size, "unknown address family %d " + "%02x %02x %02x %02x %02x %02x %02x %02x", + addr->sa_family, + addr->sa_data[0], addr->sa_data[1], (unsigned)addr->sa_data[2], addr->sa_data[3], + addr->sa_data[4], addr->sa_data[5], (unsigned)addr->sa_data[6], addr->sa_data[7]); +#endif + } + return n; +} + + +int +set_non_blocking(int fd) +{ + int flags = fcntl(fd, F_GETFL); + if(flags < 0) + return 0; + if(fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) + return 0; + return 1; +} + +struct lan_addr_s * +get_lan_for_peer(const struct sockaddr * peer) +{ + struct lan_addr_s * lan_addr = NULL; +#ifdef DEBUG + char dbg_str[64]; +#endif /* DEBUG */ + +#ifdef ENABLE_IPV6 + if(peer->sa_family == AF_INET6) + { + struct sockaddr_in6 * peer6 = (struct sockaddr_in6 *)peer; + if(IN6_IS_ADDR_V4MAPPED(&peer6->sin6_addr)) + { + struct in_addr peer_addr; + memcpy(&peer_addr, &peer6->sin6_addr.s6_addr[12], 4); + for(lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + if( (peer_addr.s_addr & lan_addr->mask.s_addr) + == (lan_addr->addr.s_addr & lan_addr->mask.s_addr)) + break; + } + } + else + { + int index = -1; + if(peer6->sin6_scope_id > 0) + index = (int)peer6->sin6_scope_id; + else + { + if(get_src_for_route_to(peer, NULL, NULL, &index) < 0) + return NULL; + } + syslog(LOG_DEBUG, "%s looking for LAN interface index=%d", + "get_lan_for_peer()", index); + for(lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + syslog(LOG_DEBUG, + "ifname=%s index=%u str=%s addr=%08x mask=%08x", + lan_addr->ifname, lan_addr->index, + lan_addr->str, + ntohl(lan_addr->addr.s_addr), + ntohl(lan_addr->mask.s_addr)); + if(index == (int)lan_addr->index) + break; + } + } + } + else if(peer->sa_family == AF_INET) + { +#endif /* ENABLE_IPV6 */ + for(lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + if( (((const struct sockaddr_in *)peer)->sin_addr.s_addr & lan_addr->mask.s_addr) + == (lan_addr->addr.s_addr & lan_addr->mask.s_addr)) + break; + } +#ifdef ENABLE_IPV6 + } +#endif /* ENABLE_IPV6 */ + +#ifdef DEBUG + sockaddr_to_string(peer, dbg_str, sizeof(dbg_str)); + if(lan_addr) { + syslog(LOG_DEBUG, "%s: %s found in LAN %s %s", + "get_lan_for_peer()", dbg_str, + lan_addr->ifname, lan_addr->str); + } else { + syslog(LOG_DEBUG, "%s: %s not found !", "get_lan_for_peer()", + dbg_str); + } +#endif /* DEBUG */ + return lan_addr; +} + diff --git a/src/contrib/miniupnp/minissdpd/upnputils.h b/src/contrib/miniupnp/minissdpd/upnputils.h new file mode 100644 index 0000000..4072eb0 --- /dev/null +++ b/src/contrib/miniupnp/minissdpd/upnputils.h @@ -0,0 +1,53 @@ +/* $Id: upnputils.h,v 1.2 2014/11/28 16:20:07 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011-2016 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPUTILS_H_INCLUDED +#define UPNPUTILS_H_INCLUDED + +/** + * convert a struct sockaddr to a human readable string. + * [ipv6]:port or ipv4:port + * return the number of characters used (as snprintf) + */ +int +sockaddr_to_string(const struct sockaddr * addr, char * str, size_t size); + +/** + * set the file description as non blocking + * return 0 in case of failure, 1 in case of success + */ +int +set_non_blocking(int fd); + +/** + * get the LAN which the peer belongs to + */ +struct lan_addr_s * +get_lan_for_peer(const struct sockaddr * peer); + +/** + * define portability macros + */ +#if defined(__sun) +static __inline size_t _sa_len(const struct sockaddr *addr) +{ + if (addr->sa_family == AF_INET) + return (sizeof(struct sockaddr_in)); + else if (addr->sa_family == AF_INET6) + return (sizeof(struct sockaddr_in6)); + else + return (sizeof(struct sockaddr)); +} +# define SA_LEN(sa) (_sa_len(sa)) +#else +#if !defined(SA_LEN) +# define SA_LEN(sa) ((sa)->sa_len) +#endif +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnp.podspec b/src/contrib/miniupnp/miniupnp.podspec new file mode 100644 index 0000000..1a8bd71 --- /dev/null +++ b/src/contrib/miniupnp/miniupnp.podspec @@ -0,0 +1,54 @@ +Pod::Spec.new do |spec| + spec.name = "miniupnp" + spec.summary = "Mini UPnP client" + spec.homepage = 'http://miniupnp.free.fr/' + spec.authors = "The MiniUPnP Authors" + spec.license = { type: "BSD", file: "miniupnpc/LICENSE" } + + spec.version = "2.0.0.2" + spec.source = { + git: 'https://github.com/cpp-ethereum-ios/miniupnp.git', + tag: "v#{spec.version}" + } + + spec.platform = :ios + spec.ios.deployment_target = '8.0' + + spec.prepare_command = <<-CMD + build_for_ios() { + build_for_architecture iphoneos armv7 arm-apple-darwin + build_for_architecture iphonesimulator i386 i386-apple-darwin + build_for_architecture iphoneos arm64 arm-apple-darwin + build_for_architecture iphonesimulator x86_64 x86_64-apple-darwin + create_universal_library + } + + build_for_architecture() { + PLATFORM=$1 + ARCH=$2 + HOST=$3 + SDKPATH=`xcrun -sdk $PLATFORM --show-sdk-path` + PREFIX="build-ios/$ARCH" + mkdir -p "$PREFIX" + xcrun -sdk $PLATFORM make clean + xcrun -sdk $PLATFORM make -j 16 install \ + PREFIX="$PREFIX" \ + CC=`xcrun -sdk $PLATFORM -find cc` \ + CFLAGS="-arch $ARCH -isysroot $SDKPATH" \ + LIBTOOL=`xcrun -sdk $PLATFORM -find libtool` \ + LDFLAGS="-arch $ARCH -headerpad_max_install_names" + } + + create_universal_library() { + lipo -create -output libminiupnpc.dylib \ + build-ios/{armv7,arm64,i386,x86_64}/usr/lib/libminiupnpc.dylib + install_name_tool -id "@rpath/libminiupnpc.dylib" libminiupnpc.dylib + } + + cd miniupnpc + build_for_ios + CMD + + spec.source_files = "miniupnpc/build-ios/armv7/usr/include/**/*.h" + spec.ios.vendored_libraries = "miniupnpc/libminiupnpc.dylib" +end diff --git a/src/contrib/miniupnp/miniupnpc-async/.gitignore b/src/contrib/miniupnp/miniupnpc-async/.gitignore new file mode 100644 index 0000000..7a67161 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/.gitignore @@ -0,0 +1,3 @@ +*.a +*.o +testasync diff --git a/src/contrib/miniupnp/miniupnpc-async/Changelog.txt b/src/contrib/miniupnp/miniupnpc-async/Changelog.txt new file mode 100644 index 0000000..649a120 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/Changelog.txt @@ -0,0 +1,4 @@ +$Id: Changelog.txt,v 1.1 2009/11/12 14:04:33 nanard Exp $ + +initial version... + diff --git a/src/contrib/miniupnp/miniupnpc-async/Makefile b/src/contrib/miniupnp/miniupnpc-async/Makefile new file mode 100644 index 0000000..ee75a58 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/Makefile @@ -0,0 +1,115 @@ +# $Id: Makefile,v 1.12 2017/05/26 11:01:37 nanard Exp $ +# MiniUPnP Project +# http://miniupnp.free.fr/ +# (c) 2005-2017 Thomas Bernard +# to install use : +# $ PREFIX=/tmp/dummylocation make install +# or +# $ INSTALLPREFIX=/usr/local make install +# or +# make install (will go to /usr/bin, /usr/lib, etc...) +OS = $(shell $(CC) -dumpmachine) + +CC ?= gcc +#AR = gar +CFLAGS = -O0 -g -DDEBUG +#CFLAGS = -O +CFLAGS += -fPIC +CFLAGS += -ansi +CFLAGS += -Wall -W +CFLAGS += -D_BSD_SOURCE +ifeq (, $(findstring darwin, $(OS))$(findstring freebsd, $(OS))) +#CFLAGS += -D_POSIX_C_SOURCE=200112L +CFLAGS += -D_XOPEN_SOURCE=600 +endif +CFLAGS += -DUPNPC_USE_SELECT +INSTALL = install + +#following libs are needed on Solaris +ifneq (, $(findstring sun, $(OS))) + LDLIBS += -lsocket -lnsl -lresolv + CFLAGS += -D__EXTENSIONS__ +endif + +# APIVERSION is used to build SONAME +APIVERSION = 0 + +SRCS = miniupnpc-async.c parsessdpreply.c \ + upnputils.c igd_desc_parse.c minixml.c \ + upnpreplyparse.c \ + testasync.c + +LIBOBJS = miniupnpc-async.o parsessdpreply.o \ + upnputils.o igd_desc_parse.o minixml.o \ + upnpreplyparse.o + +OBJS = $(patsubst %.c,%.o,$(SRCS)) + +# HEADERS to install +HEADERS = miniupnpc-async.h +LIBRARY = libminiupnpc-async.a +SHAREDLIBRARY = libminiupnpc-async.so +SONAME = $(SHAREDLIBRARY).$(APIVERSION) +EXECUTABLES = testasync + +INSTALLPREFIX ?= $(PREFIX)/usr +INSTALLDIRINC = $(INSTALLPREFIX)/include/miniupnpc +INSTALLDIRLIB = $(INSTALLPREFIX)/lib +INSTALLDIRBIN = $(INSTALLPREFIX)/bin + +.PHONY: install clean depend all installpythonmodule + +all: $(LIBRARY) $(EXECUTABLES) + +pythonmodule: $(LIBRARY) miniupnpcmodule.c setup.py + python setup.py build + touch $@ + +installpythonmodule: pythonmodule + python setup.py install + +clean: + $(RM) $(LIBRARY) $(SHAREDLIBRARY) $(EXECUTABLES) $(OBJS) + # clean python stuff + $(RM) pythonmodule + $(RM) -r build/ dist/ + #python setup.py clean + +install: $(LIBRARY) $(SHAREDLIBRARY) + $(INSTALL) -d $(INSTALLDIRINC) + $(INSTALL) -m 644 $(HEADERS) $(INSTALLDIRINC) + $(INSTALL) -d $(INSTALLDIRLIB) + $(INSTALL) -m 644 $(LIBRARY) $(INSTALLDIRLIB) + $(INSTALL) -m 644 $(SHAREDLIBRARY) $(INSTALLDIRLIB)/$(SONAME) + $(INSTALL) -d $(INSTALLDIRBIN) + $(INSTALL) -m 755 $(EXECUTABLES) $(INSTALLDIRBIN) + ln -fs $(SONAME) $(INSTALLDIRLIB)/$(SHAREDLIBRARY) + +cleaninstall: + $(RM) -r $(INSTALLDIRINC) + $(RM) $(INSTALLDIRLIB)/$(LIBRARY) + $(RM) $(INSTALLDIRLIB)/$(SHAREDLIBRARY) + +depend: + makedepend -Y -- $(CFLAGS) -- $(SRCS) 2>/dev/null + +$(LIBRARY): $(LIBOBJS) + $(AR) crs $@ $? + +$(SHAREDLIBRARY): $(LIBOBJS) + $(CC) -shared -Wl,-soname,$(SONAME) -o $@ $^ + +upnpc-static: upnpc.o $(LIBRARY) + $(CC) -o $@ $^ + +upnpc-shared: upnpc.o $(SHAREDLIBRARY) + $(CC) -o $@ $^ + +testasync: testasync.o libminiupnpc-async.a +#testasync: testasync.o -lminiupnpc-async + +# DO NOT DELETE THIS LINE -- make depend depends on it. + +miniupnpc-async.o: miniupnpc-async.h declspec.h parsessdpreply.h upnputils.h +parsessdpreply.o: parsessdpreply.h +testasync.o: miniupnpc-async.h declspec.h diff --git a/src/contrib/miniupnp/miniupnpc-async/README b/src/contrib/miniupnp/miniupnpc-async/README new file mode 100644 index 0000000..6b097e2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/README @@ -0,0 +1,10 @@ +(c) 2014 Thomas BERNARD +http://miniupnp.free.fr/ +https://github.com/miniupnp/miniupnp + +miniupnpc-async : + proof of concept of a UPnP IGD client using asynchronous socket calls + (ie non blocking sockets) + + To be reimplemented using libevent2 (http://libevent.org/) + diff --git a/src/contrib/miniupnp/miniupnpc-async/config.h b/src/contrib/miniupnp/miniupnpc-async/config.h new file mode 100644 index 0000000..020ac6e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/config.h @@ -0,0 +1,5 @@ +/* $Id: config.h,v 1.1 2012/05/20 14:58:50 nanard Exp $ */ +#ifndef __CONFIG_H__ +#define __CONFIG_H__ + +#endif diff --git a/src/contrib/miniupnp/miniupnpc-async/declspec.h b/src/contrib/miniupnp/miniupnpc-async/declspec.h new file mode 100644 index 0000000..b804247 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/declspec.h @@ -0,0 +1,15 @@ +#ifndef __DECLSPEC_H__ +#define __DECLSPEC_H__ + +#if defined(WIN32) && !defined(STATICLIB) + #ifdef MINIUPNP_EXPORTS + #define LIBSPEC __declspec(dllexport) + #else + #define LIBSPEC __declspec(dllimport) + #endif +#else + #define LIBSPEC +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc-async/igd_desc_parse.c b/src/contrib/miniupnp/miniupnpc-async/igd_desc_parse.c new file mode 100644 index 0000000..d2999ad --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/igd_desc_parse.c @@ -0,0 +1,123 @@ +/* $Id: igd_desc_parse.c,v 1.17 2015/09/15 13:30:04 nanard Exp $ */ +/* Project : miniupnp + * http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005-2015 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ + +#include "igd_desc_parse.h" +#include +#include + +/* Start element handler : + * update nesting level counter and copy element name */ +void IGDstartelt(void * d, const char * name, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + if(l >= MINIUPNPC_URL_MAXSIZE) + l = MINIUPNPC_URL_MAXSIZE-1; + memcpy(datas->cureltname, name, l); + datas->cureltname[l] = '\0'; + datas->level++; + if( (l==7) && !memcmp(name, "service", l) ) { + datas->tmp.controlurl[0] = '\0'; + datas->tmp.eventsuburl[0] = '\0'; + datas->tmp.scpdurl[0] = '\0'; + datas->tmp.servicetype[0] = '\0'; + } +} + +#define COMPARE(str, cstr) (0==memcmp(str, cstr, sizeof(cstr) - 1)) + +/* End element handler : + * update nesting level counter and update parser state if + * service element is parsed */ +void IGDendelt(void * d, const char * name, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + datas->level--; + /*printf("endelt %2d %.*s\n", datas->level, l, name);*/ + if( (l==7) && !memcmp(name, "service", l) ) + { + if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:")) { + memcpy(&datas->CIF, &datas->tmp, sizeof(struct IGDdatas_service)); + } else if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANIPv6FirewallControl:")) { + memcpy(&datas->IPv6FC, &datas->tmp, sizeof(struct IGDdatas_service)); + } else if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANIPConnection:") + || COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANPPPConnection:") ) { + if(datas->first.servicetype[0] == '\0') { + memcpy(&datas->first, &datas->tmp, sizeof(struct IGDdatas_service)); + } else { + memcpy(&datas->second, &datas->tmp, sizeof(struct IGDdatas_service)); + } + } + } +} + +/* Data handler : + * copy data depending on the current element name and state */ +void IGDdata(void * d, const char * data, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + char * dstmember = 0; + /*printf("%2d %s : %.*s\n", + datas->level, datas->cureltname, l, data); */ + if( !strcmp(datas->cureltname, "URLBase") ) + dstmember = datas->urlbase; + else if( !strcmp(datas->cureltname, "presentationURL") ) + dstmember = datas->presentationurl; + else if( !strcmp(datas->cureltname, "serviceType") ) + dstmember = datas->tmp.servicetype; + else if( !strcmp(datas->cureltname, "controlURL") ) + dstmember = datas->tmp.controlurl; + else if( !strcmp(datas->cureltname, "eventSubURL") ) + dstmember = datas->tmp.eventsuburl; + else if( !strcmp(datas->cureltname, "SCPDURL") ) + dstmember = datas->tmp.scpdurl; +/* else if( !strcmp(datas->cureltname, "deviceType") ) + dstmember = datas->devicetype_tmp;*/ + if(dstmember) + { + if(l>=MINIUPNPC_URL_MAXSIZE) + l = MINIUPNPC_URL_MAXSIZE-1; + memcpy(dstmember, data, l); + dstmember[l] = '\0'; + } +} + +#ifdef DEBUG +void printIGD(struct IGDdatas * d) +{ + printf("urlbase = '%s'\n", d->urlbase); + printf("WAN Device (Common interface config) :\n"); + /*printf(" deviceType = '%s'\n", d->CIF.devicetype);*/ + printf(" serviceType = '%s'\n", d->CIF.servicetype); + printf(" controlURL = '%s'\n", d->CIF.controlurl); + printf(" eventSubURL = '%s'\n", d->CIF.eventsuburl); + printf(" SCPDURL = '%s'\n", d->CIF.scpdurl); + printf("primary WAN Connection Device (IP or PPP Connection):\n"); + /*printf(" deviceType = '%s'\n", d->first.devicetype);*/ + printf(" servicetype = '%s'\n", d->first.servicetype); + printf(" controlURL = '%s'\n", d->first.controlurl); + printf(" eventSubURL = '%s'\n", d->first.eventsuburl); + printf(" SCPDURL = '%s'\n", d->first.scpdurl); + printf("secondary WAN Connection Device (IP or PPP Connection):\n"); + /*printf(" deviceType = '%s'\n", d->second.devicetype);*/ + printf(" servicetype = '%s'\n", d->second.servicetype); + printf(" controlURL = '%s'\n", d->second.controlurl); + printf(" eventSubURL = '%s'\n", d->second.eventsuburl); + printf(" SCPDURL = '%s'\n", d->second.scpdurl); + printf("WAN IPv6 Firewall Control :\n"); + /*printf(" deviceType = '%s'\n", d->IPv6FC.devicetype);*/ + printf(" servicetype = '%s'\n", d->IPv6FC.servicetype); + printf(" controlURL = '%s'\n", d->IPv6FC.controlurl); + printf(" eventSubURL = '%s'\n", d->IPv6FC.eventsuburl); + printf(" SCPDURL = '%s'\n", d->IPv6FC.scpdurl); +} +#endif /* DEBUG */ + diff --git a/src/contrib/miniupnp/miniupnpc-async/igd_desc_parse.h b/src/contrib/miniupnp/miniupnpc-async/igd_desc_parse.h new file mode 100644 index 0000000..0de546b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/igd_desc_parse.h @@ -0,0 +1,49 @@ +/* $Id: igd_desc_parse.h,v 1.12 2014/11/17 17:19:13 nanard Exp $ */ +/* Project : miniupnp + * http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005-2014 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef IGD_DESC_PARSE_H_INCLUDED +#define IGD_DESC_PARSE_H_INCLUDED + +/* Structure to store the result of the parsing of UPnP + * descriptions of Internet Gateway Devices */ +#define MINIUPNPC_URL_MAXSIZE (128) +struct IGDdatas_service { + char controlurl[MINIUPNPC_URL_MAXSIZE]; + char eventsuburl[MINIUPNPC_URL_MAXSIZE]; + char scpdurl[MINIUPNPC_URL_MAXSIZE]; + char servicetype[MINIUPNPC_URL_MAXSIZE]; + /*char devicetype[MINIUPNPC_URL_MAXSIZE];*/ +}; + +struct IGDdatas { + char cureltname[MINIUPNPC_URL_MAXSIZE]; + char urlbase[MINIUPNPC_URL_MAXSIZE]; + char presentationurl[MINIUPNPC_URL_MAXSIZE]; + int level; + /*int state;*/ + /* "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1" */ + struct IGDdatas_service CIF; + /* "urn:schemas-upnp-org:service:WANIPConnection:1" + * "urn:schemas-upnp-org:service:WANPPPConnection:1" */ + struct IGDdatas_service first; + /* if both WANIPConnection and WANPPPConnection are present */ + struct IGDdatas_service second; + /* "urn:schemas-upnp-org:service:WANIPv6FirewallControl:1" */ + struct IGDdatas_service IPv6FC; + /* tmp */ + struct IGDdatas_service tmp; +}; + +void IGDstartelt(void *, const char *, int); +void IGDendelt(void *, const char *, int); +void IGDdata(void *, const char *, int); +#ifdef DEBUG +void printIGD(struct IGDdatas *); +#endif /* DEBUG */ + +#endif /* IGD_DESC_PARSE_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpc-async/miniupnpc-async.c b/src/contrib/miniupnp/miniupnpc-async/miniupnpc-async.c new file mode 100644 index 0000000..3d9af3c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/miniupnpc-async.c @@ -0,0 +1,1044 @@ +/* $Id: miniupnpc-async.c,v 1.19 2014/11/07 12:05:40 nanard Exp $ */ +/* miniupnpc-async + * Copyright (c) 2008-2017, Thomas BERNARD + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef WIN32 +#include +#include +#include +#define PRINT_SOCKET_ERROR printf +#define SOCKET_ERROR GetWSALastError() +#define WOULDBLOCK(err) (err == WSAEWOULDBLOCK) +#else +#include +#include +#define closesocket close +#define PRINT_SOCKET_ERROR perror +#define SOCKET_ERROR errno +#define WOULDBLOCK(err) (err == EAGAIN || err == EWOULDBLOCK) +#endif +#include "miniupnpc-async.h" +#include "parsessdpreply.h" +#include "upnputils.h" +#include "minixml.h" +#include "igd_desc_parse.h" +#include "upnpreplyparse.h" + +#ifndef MIN +#define MIN(x,y) (((x)<(y))?(x):(y)) +#endif /* MIN */ + +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN 64 +#endif /* MAXHOSTNAMELEN */ + +#define SSDP_PORT 1900 +#define SSDP_MCAST_ADDR "239.255.255.250" +#define XSTR(s) STR(s) +#define STR(s) #s + +#ifdef DEBUG +#define debug_printf(...) fprintf(stderr, __VA_ARGS__) +#else +#define debug_printf(...) +#endif + +/* stuctures */ + +struct upnp_args { + const char * elt; + const char * val; +}; + +/* private functions */ + +static int upnpc_connect(upnpc_device_t * p, const char * url); +static int upnpc_send_request(upnpc_device_t * p); + + +/* parse_msearch_reply() + * the last 4 arguments are filled during the parsing : + * - location/locationsize : "location:" field of the SSDP reply packet + * - st/stsize : "st:" field of the SSDP reply packet. + * The strings are NOT null terminated */ +static void +parse_msearch_reply(const char * reply, int size, + const char * * location, unsigned int * locationsize, + const char * * st, unsigned int * stsize) +{ + int a, b, i; + i = 0; /* current character index */ + a = i; /* start of the line */ + b = 0; /* end of the "header" (position of the colon) */ + while(issdp_socket, bufr, n, 0, + (struct sockaddr *)&addr, sizeof(struct sockaddr_in)); + if (n < 0) { + int err = SOCKET_ERROR; + if(err == EINTR || WOULDBLOCK(err)) { + debug_printf("upnpc_send_ssdp_msearch: should try again"); + p->state = EUPnPSendSSDP; + return 0; + } + PRINT_SOCKET_ERROR("sendto"); + return -1; + } + p->state = EUPnPReceiveSSDP; + return 0; +} + +static int upnpc_set_root_desc_location(upnpc_device_t * d, const char * location, int locationsize) +{ + char * tmp; + tmp = realloc(d->root_desc_location, locationsize + 1); + if(tmp == 0) { + return -1; + } + memcpy(tmp, location, locationsize); + tmp[locationsize] = '\0'; + d->root_desc_location = tmp; + return 0; +} + +static int upnpc_receive_and_parse_ssdp(upnpc_t * p) +{ + int n; + char bufr[1024]; + n = recv(p->ssdp_socket, bufr, sizeof(bufr), 0); + if (n<0) { + PRINT_SOCKET_ERROR("recv"); + } else if (n==0) { + debug_printf("empty packet received\n"); + } else { + const char * location = NULL; + unsigned int locationsize; + const char * st = NULL; + unsigned int stsize; + debug_printf("%.*s", n, bufr); + parse_msearch_reply(bufr, n, &location, &locationsize, &st, &stsize); + debug_printf("location = '%.*s'\n", locationsize, location); + debug_printf("st = '%.*s'\n", stsize, st); + if(location != NULL) { + upnpc_device_t * dev = p->device_list; + while(dev != NULL) { + if(dev->root_desc_location != NULL + && strlen(dev->root_desc_location) == locationsize + && memcmp(dev->root_desc_location, location, locationsize) == 0) { + debug_printf("device already in list (location='%s')\n", dev->root_desc_location); + return -1; + } + dev = dev->next; + } + dev = calloc(1, sizeof(upnpc_device_t)); + if(dev == NULL) { + p->state = EUPnPError; + return -1; + } + if(upnpc_set_root_desc_location(dev, location, locationsize) < 0) { + free(dev); + p->state = EUPnPError; + return -1; + } + dev->next = p->device_list; + p->device_list = dev; + dev->state = EDevGetDescConnect; + upnpc_connect(dev, dev->root_desc_location); + } else { + /* or do nothing ? */ + p->state = EUPnPError; + } + } + return 0; +} + +static int +parseURL(const char * url, + char * hostname, unsigned short * port, + char * * path, unsigned int * scope_id) +{ + char * p1, *p2, *p3; + if(!url) + return 0; + p1 = strstr(url, "://"); + if(!p1) + return 0; + p1 += 3; + if( (url[0]!='h') || (url[1]!='t') + ||(url[2]!='t') || (url[3]!='p')) + return 0; + memset(hostname, 0, MAXHOSTNAMELEN + 1); + if(*p1 == '[') { + /* IP v6 : http://[2a00:1450:8002::6a]/path/abc */ + char * scope; + scope = strchr(p1, '%'); + p2 = strchr(p1, ']'); + if(p2 && scope && scope < p2 && scope_id) { + /* parse scope */ +#ifdef IF_NAMESIZE + char tmp[IF_NAMESIZE]; + int l; + scope++; + /* "%25" is just '%' in URL encoding */ + if(scope[0] == '2' && scope[1] == '5') + scope += 2; /* skip "25" */ + l = p2 - scope; + if(l >= IF_NAMESIZE) + l = IF_NAMESIZE - 1; + memcpy(tmp, scope, l); + tmp[l] = '\0'; + *scope_id = if_nametoindex(tmp); + if(*scope_id == 0) { + *scope_id = (unsigned int)strtoul(tmp, NULL, 10); + } +#else + /* under windows, scope is numerical */ + char tmp[8]; + int l; + scope++; + /* "%25" is just '%' in URL encoding */ + if(scope[0] == '2' && scope[1] == '5') + scope += 2; /* skip "25" */ + l = p2 - scope; + if(l >= (int)sizeof(tmp)) + l = sizeof(tmp) - 1; + memcpy(tmp, scope, l); + tmp[l] = '\0'; + *scope_id = (unsigned int)strtoul(tmp, NULL, 10); +#endif + } + p3 = strchr(p1, '/'); + if(p2 && p3) { + p2++; + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p2-p1))); + if(*p2 == ':') { + *port = 0; + p2++; + while( (*p2 >= '0') && (*p2 <= '9')) { + *port *= 10; + *port += (unsigned short)(*p2 - '0'); + p2++; + } + } else { + *port = 80; + } + *path = p3; + return 1; + } + } + p2 = strchr(p1, ':'); + p3 = strchr(p1, '/'); + if(!p3) + return 0; + if(!p2 || (p2>p3)) { + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p3-p1))); + *port = 80; + } else { + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p2-p1))); + *port = 0; + p2++; + while( (*p2 >= '0') && (*p2 <= '9')) { + *port *= 10; + *port += (unsigned short)(*p2 - '0'); + p2++; + } + } + *path = p3; + return 1; +} + +static int upnpc_connect(upnpc_device_t * p, const char * url) +{ + int r; + char hostname[MAXHOSTNAMELEN+1]; + unsigned short port; + char * path; + unsigned int scope_id; + struct sockaddr_in addr; + socklen_t addrlen; + + /*if(p->root_desc_location == 0) { + p->state = EError; + return -1; + }*/ + if(!parseURL(url/*p->root_desc_location*/, hostname, &port, + &path, &scope_id)) { + p->state = EDevError; + return -1; + } + p->http_socket = socket(PF_INET, SOCK_STREAM, 0); + if(p->http_socket < 0) { + PRINT_SOCKET_ERROR("socket"); + p->state = EDevError; + return -1; + } + if(!set_non_blocking(p->http_socket)) { + /* TODO : ERROR */ + } + memset(&addr, 0, sizeof(struct sockaddr_in)); + addr.sin_family = AF_INET; + inet_pton(AF_INET, hostname, &(addr.sin_addr)); + addr.sin_port = htons(port); + addrlen = sizeof(struct sockaddr_in); + do { + r = connect(p->http_socket, (struct sockaddr *)&addr, addrlen); + if(r < 0) { + if(errno == EINPROGRESS) { + /*p->state = EGetDescConnect;*/ + return 0; + } else if(errno != EINTR) { + PRINT_SOCKET_ERROR("connect"); + p->state = EDevError; + return -1; + } + } + } while(r < 0 && errno == EINTR); + if(p->state == EDevGetDescConnect) { + p->state = EDevGetDescRequest; + } else { + p->state = EDevSoapRequest; + } + upnpc_send_request(p); + return 0; +} + +static int upnpc_complete_connect(upnpc_device_t * p) +{ + socklen_t len; + int err; + len = sizeof(err); + if(getsockopt(p->http_socket, SOL_SOCKET, SO_ERROR, &err, &len) < 0) { + PRINT_SOCKET_ERROR("getsockopt"); + p->state = EDevError; + return -1; + } + if(err != 0) { + debug_printf("connect failed %d\n", err); + p->state = EDevError; + return -1; + } + if(p->state == EDevGetDescConnect) + p->state = EDevGetDescRequest; + else + p->state = EDevSoapRequest; + upnpc_send_request(p); + return 0; +} + +static int upnpc_send_request(upnpc_device_t * p) +{ + ssize_t n; + static const char reqfmt[] = "GET %s HTTP/1.1\r\n" + "Host: %s:%hu\r\n" + "Connection: Close\r\n" + "User-Agent: MiniUPnPc-async\r\n" + "\r\n"; + + /* retrieve "our" IP address used to connect to the UPnP device */ + p->selfaddrlen = sizeof(struct sockaddr_storage); + if(getsockname(p->http_socket, (struct sockaddr *)&p->selfaddr, &p->selfaddrlen) < 0) { + PRINT_SOCKET_ERROR("getsockname()"); + } + + if(p->http_request == NULL) { + char hostname[MAXHOSTNAMELEN+1]; + unsigned short port; + char * path; + unsigned int scope_id; + int len; + if(!parseURL(p->root_desc_location, hostname, &port, + &path, &scope_id)) { + p->state = EDevError; + return -1; + } + len = snprintf(NULL, 0, reqfmt, path, hostname, port); + p->http_request = malloc(len + 1); + if(p->http_request == NULL) { + p->state = EDevError; + return -1; + } + p->http_request_len = snprintf(p->http_request, len + 1, + reqfmt, path, hostname, port); + p->http_request_sent = 0; + } + n = send(p->http_socket, p->http_request + p->http_request_sent, + p->http_request_len - p->http_request_sent, 0/* flags */); + if(n < 0) { + PRINT_SOCKET_ERROR("send"); + p->state = EDevError; + return -1; + } else { + debug_printf("sent %d bytes\n", (int)n); + /*if(n == 0) { + p->state = EError; + return -1; + }*/ + p->http_request_sent += n; + if(p->http_request_sent >= p->http_request_len) { + /* all bytes sent */ +#if 0 + shutdown(p->http_socket, SHUT_WR); /* some routers don't like that */ +#endif + free(p->http_request); + p->http_request = NULL; + p->http_request_len = 0; + if(p->state == EDevGetDescRequest) + p->state = EDevGetDescResponse; + else + p->state = EDevSoapResponse; + free(p->http_response); + p->http_response = NULL; + p->http_response_received = 0; + p->http_response_end_of_headers = 0; + /* get response */ + } + } + return 0; +} + +static int upnpc_parse_headers(upnpc_device_t * p) +{ + /* search for CR LF CR LF (end of headers) + * recognize also LF LF */ + int i = 0; + while(i < (p->http_response_received-1) && + p->http_response_end_of_headers == 0) { + if(p->http_response[i] == '\r') { + i++; + if(p->http_response[i] == '\n') { + i++; + if(i < p->http_response_received && p->http_response[i] == '\r') { + i++; + if(i < p->http_response_received && p->http_response[i] == '\n') { + p->http_response_end_of_headers = i + 1; + } + } + } + } else if(p->http_response[i] == '\n') { + i++; + if(p->http_response[i] == '\n') { + p->http_response_end_of_headers = i + 1; + } + } + i++; + } + if(p->http_response_end_of_headers != 0) { + int colon = 0; + int linestart = 0; + int valuestart = 0; + p->http_response_code = -1; + for(i = 0; i < p->http_response_end_of_headers - 1; i++) { + if(linestart == 0) { + /* reading HTTP response code on the 1st line */ + if(p->http_response[i] == ' ' && p->http_response_code < 0) + p->http_response_code = 0; + else if(p->http_response[i] >= '0' && p->http_response[i] <= '9') { + p->http_response_code = p->http_response_code * 10 + (p->http_response[i] - '0'); + } else if(p->http_response[i] == ' ') + linestart = 1; + } + if(colon <= linestart && p->http_response[i] == ':') { + colon = i; + while(i < p->http_response_end_of_headers - 1 && + (p->http_response[i+1] == ' ' || p->http_response[i+1] == '\t')) + i++; + valuestart = i + 1; + } else if(p->http_response[i + 1] == '\r' || + p->http_response[i + 1] == '\n') { + if(colon > linestart && valuestart > colon) { + debug_printf("header='%.*s', value='%.*s'\n", + colon-linestart, p->http_response+linestart, + i+1-valuestart, p->http_response+valuestart); + if(0==strncasecmp(p->http_response+linestart, "content-length", colon-linestart)) { + p->http_response_content_length = atoi(p->http_response + valuestart); + debug_printf("Content-Length: %d\n", p->http_response_content_length); + if(p->http_response_content_length < 0) { + debug_printf("Content-Length overflow ? setting to 0\n"); + p->http_response_content_length = 0; + } + } else if(0==strncasecmp(p->http_response+linestart, "transfer-encoding", colon-linestart) + && 0==strncasecmp(p->http_response+valuestart, "chunked", 7)) { + debug_printf("Chunked transfer-encoding !\n"); + p->http_response_chunked = 1; + } + } + /* find next line */ + while((i < p->http_response_received) && + (p->http_response[i]=='\r' || p->http_response[i] == '\n')) + i++; + linestart = i; + colon = linestart; + valuestart = 0; + } + } + } + return 0; +} + +static char * build_url_string(const char * urlbase, const char * root_desc_url, const char * controlurl) +{ + int l, n; + char * s; + const char * base; + char * p; + /* if controlurl is an absolute url, return it */ + if(0 == memcmp("http://", controlurl, 7)) + return strdup(controlurl); + base = (urlbase[0] == '\0') ? root_desc_url : urlbase; + n = strlen(base); + if(n > 7) { + p = strchr(base + 7, '/'); + if(p) + n = p - base; + } + l = n + strlen(controlurl) + 1; + if(controlurl[0] != '/') + l++; + s = malloc(l); + if(s == NULL) return NULL; + memcpy(s, base, n); + if(controlurl[0] != '/') + s[n++] = '/'; + memcpy(s + n, controlurl, l - n); + return s; +} + +static int upnpc_get_response(upnpc_device_t * p) +{ + ssize_t n; + ssize_t count; + char buffer[2048]; + if(p->http_response_content_length > 0) { + count = p->http_response_content_length + + p->http_response_end_of_headers + - p->http_response_received; + if(count > (ssize_t)sizeof(buffer)) count = sizeof(buffer); + } else { + count = sizeof(buffer); + } + debug_printf("recv(..., %d)\n", (int)count); + n = recv(p->http_socket, buffer, count, 0/* flags */); + if(n < 0) { + if(errno == EINTR || WOULDBLOCK(errno)) + return 0; /* try again later */ + PRINT_SOCKET_ERROR("read"); + p->state = EDevError; + return -1; + } else if(n == 0) { + /* receiving finished */ + debug_printf("%.*s\n", p->http_response_received, p->http_response); + close(p->http_socket); + p->http_socket = -1; + /* parse */ + if(p->http_response_end_of_headers == 0) { + upnpc_parse_headers(p); + } + /* TODO : decode chunked transfer-encoding */ + /* parse xml */ + if(p->state == EDevGetDescResponse) { + struct IGDdatas igd; + struct xmlparser parser; + memset(&igd, 0, sizeof(struct IGDdatas)); + memset(&parser, 0, sizeof(struct xmlparser)); + parser.xmlstart = p->http_response + p->http_response_end_of_headers; + parser.xmlsize = p->http_response_received - p->http_response_end_of_headers; + parser.data = &igd; + parser.starteltfunc = IGDstartelt; + parser.endeltfunc = IGDendelt; + parser.datafunc = IGDdata; + parsexml(&parser); +#ifdef DEBUG + printIGD(&igd); +#endif /* DEBUG */ + p->control_conn_url = build_url_string(igd.urlbase, p->root_desc_location, igd.first.controlurl); + p->control_cif_url = build_url_string(igd.urlbase, p->root_desc_location, igd.CIF.controlurl); + debug_printf("control_conn_url='%s'\n", p->control_conn_url); + debug_printf("control_cif_url='%s'\n", p->control_cif_url); + } else { + ClearNameValueList(&p->soap_response_data); + ParseNameValue(p->http_response + p->http_response_end_of_headers, + p->http_response_received - p->http_response_end_of_headers, + &p->soap_response_data); + } + free(p->http_response); + p->http_response = NULL; + p->http_response_received = 0; + p->http_response_end_of_headers = 0; + p->state = EDevReady; + } else { + /* receiving in progress */ + debug_printf("received %d bytes:\n%.*s\n", (int)n, (int)n, buffer); + if(p->http_response == NULL) { + p->http_response = malloc(n); + if(p->http_response == NULL) { + debug_printf("failed to malloc %d bytes\n", (int)n); + p->state = EDevError; + return -1; + } + p->http_response_received = n; + memcpy(p->http_response, buffer, n); + } else { + char * tmp = realloc(p->http_response, p->http_response_received + n); + if(tmp == NULL) { + debug_printf("failed to realloc %d bytes\n", (int)(p->http_response_received + n)); + p->state = EDevError; + return -1; + } + p->http_response = tmp; + memcpy(p->http_response + p->http_response_received, buffer, n); + p->http_response_received += n; + } + if(p->http_response_end_of_headers == 0) { + upnpc_parse_headers(p); + } + } + return 0; +} + +#define SOAPPREFIX "s" +#define SERVICEPREFIX "u" +#define SERVICEPREFIX2 'u' + +static int upnpc_build_soap_request(upnpc_device_t * p, const char * url, + const char * service, + const char * action, + const struct upnp_args * args, int arg_count) +{ + char * body; + const char fmt_soap[] = + "\r\n" + "<" SOAPPREFIX ":Envelope " + "xmlns:" SOAPPREFIX "=\"http://schemas.xmlsoap.org/soap/envelope/\" " + SOAPPREFIX ":encodingStyle=\"http://schemas.xmlsoap.org/soap/encoding/\">" + "<" SOAPPREFIX ":Body>" + "<" SERVICEPREFIX ":%s xmlns:" SERVICEPREFIX "=\"%s\">" + "%s" + "" + "" + "\r\n"; + int body_len; + const char fmt_http[] = + "POST %s HTTP/1.1\r\n" + "Host: %s%s\r\n" + "User-Agent: MiniUPnPc-async\r\n" + "Content-Length: %d\r\n" + "Content-Type: text/xml\r\n" + "SOAPAction: \"%s#%s\"\r\n" + "Connection: Close\r\n" + "Cache-Control: no-cache\r\n" /* ??? */ + "Pragma: no-cache\r\n" + "\r\n" + "%s"; + char hostname[MAXHOSTNAMELEN+1]; + unsigned short port; + char * path; + unsigned int scope_id; + char portstr[8]; + char * args_xml = NULL; + + if(arg_count > 0) { + int i; + size_t l, n; + for(i = 0, l = 0; i < arg_count; i++) { + /* VAL */ + l += strlen(args[i].elt) * 2 + strlen(args[i].val) + 5; + } + args_xml = malloc(++l); + if(args_xml == NULL) { + p->state = EDevError; + return -1; + } + for(i = 0, n = 0; i < arg_count && n < l; i++) { + /* VAL */ + n += snprintf(args_xml + n, l - n, "<%s>%s", + args[i].elt, args[i].val, args[i].elt); + } + } + + body_len = snprintf(NULL, 0, fmt_soap, action, service, args_xml?args_xml:"", action); + body = malloc(body_len + 1); + if(body == NULL) { + p->state = EDevError; + free(args_xml); + return -1; + } + if(snprintf(body, body_len + 1, fmt_soap, action, service, args_xml?args_xml:"", action) != body_len) { + debug_printf("snprintf() returned strange value...\n"); + } + free(args_xml); + args_xml = NULL; + if(!parseURL(url, hostname, &port, &path, &scope_id)) { + p->state = EDevError; + free(body); + return -1; + } + if(port != 80) + snprintf(portstr, sizeof(portstr), ":%hu", port); + else + portstr[0] = '\0'; + p->http_request_len = snprintf(NULL, 0, fmt_http, + path/*url*/, hostname, portstr, body_len, service, action, body); + free(p->http_request); + p->http_request = malloc(p->http_request_len + 1); + if(snprintf(p->http_request, p->http_request_len + 1, fmt_http, + path/*url*/, hostname, portstr, body_len, service, action, body) != p->http_request_len) { + debug_printf("snprintf() returned strange value...\n"); + } + free(body); + debug_printf("%s", p->http_request); + p->http_request_sent = 0; + return 0; +} + +/* public functions */ +int upnpc_init(upnpc_t * p, const char * multicastif) +{ + int opt = 1; + struct sockaddr_in addr; + if(!p) + return UPNPC_ERR_INVALID_ARGS; + p->state = EUPnPError; + memset(p, 0, sizeof(upnpc_t)); /* clean everything */ + /* open the socket for SSDP */ + p->ssdp_socket = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP); + if(p->ssdp_socket < 0) { + return UPNPC_ERR_SOCKET_FAILED; + } + /* set REUSEADDR */ +#ifdef WIN32 + if(setsockopt(p->ssdp_socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&opt, sizeof(opt)) < 0) { +#else + if(setsockopt(p->ssdp_socket, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)) < 0) { +#endif + /* non fatal error ! */ + } + if(!set_non_blocking(p->ssdp_socket)) { + /* TODO log error */ + } + + /* receive address */ + memset(&addr, 0, sizeof(struct sockaddr_in)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + /*addr.sin_port = htons(SSDP_PORT);*/ + + if(multicastif) { + struct in_addr mc_if; + mc_if.s_addr = inet_addr(multicastif); + addr.sin_addr.s_addr = mc_if.s_addr; + if(setsockopt(p->ssdp_socket, IPPROTO_IP, IP_MULTICAST_IF, (const char *)&mc_if, sizeof(mc_if)) < 0) { + PRINT_SOCKET_ERROR("setsockopt"); + /* non fatal error ! */ + } + } + + /* bind the socket to the ssdp address in order to receive responses */ + if(bind(p->ssdp_socket, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) != 0) { + close(p->ssdp_socket); + return UPNPC_ERR_BIND_FAILED; + } + + p->state = EUPnPInit; + return UPNPC_OK; +} + +int upnpc_finalize(upnpc_t * p) +{ + if(!p) return UPNPC_ERR_INVALID_ARGS; + if(p->ssdp_socket >= 0) { + close(p->ssdp_socket); + p->ssdp_socket = -1; + } + while(p->device_list) { + upnpc_device_t * next = p->device_list->next; + free(p->device_list->root_desc_location); + p->device_list->root_desc_location = NULL; + free(p->device_list->http_request); + p->device_list->http_request = NULL; + free(p->device_list->http_response); + p->device_list->http_response = NULL; + free(p->device_list->control_cif_url); + p->device_list->control_cif_url = NULL; + free(p->device_list->control_conn_url); + p->device_list->control_conn_url = NULL; + if(p->device_list->http_socket >= 0) { + close(p->device_list->http_socket); + p->device_list->http_socket = -1; + } + ClearNameValueList(&p->device_list->soap_response_data); + free(p->device_list); + p->device_list = next; + } + p->state = EUPnPFinalized; + return UPNPC_OK; +} + +int upnpc_get_external_ip_address(upnpc_device_t * p) +{ + upnpc_build_soap_request(p, p->control_conn_url, + "urn:schemas-upnp-org:service:WANIPConnection:1", + "GetExternalIPAddress", NULL, 0); + p->state = EDevSoapConnect; + upnpc_connect(p, p->control_conn_url); + return 0; +} + +int upnpc_get_link_layer_max_rate(upnpc_device_t * p) +{ + upnpc_build_soap_request(p, p->control_cif_url, + "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1", + "GetCommonLinkProperties", NULL, 0); + p->state = EDevSoapConnect; + upnpc_connect(p, p->control_conn_url); + return 0; +} + +int upnpc_add_port_mapping(upnpc_device_t * p, + const char * remote_host, unsigned short ext_port, + unsigned short int_port, const char * int_client, + const char * proto, const char * description, + unsigned int lease_duration) +{ + struct upnp_args args[8]; + char lease_duration_str[16]; + char int_port_str[8]; + char ext_port_str[8]; + + if(int_client == NULL || int_port == 0 || ext_port == 0 || proto == NULL) + return UPNPC_ERR_INVALID_ARGS; + snprintf(lease_duration_str, sizeof(lease_duration_str), "%u", lease_duration); + snprintf(int_port_str, sizeof(int_port_str), "%hu", int_port); + snprintf(ext_port_str, sizeof(ext_port_str), "%hu", ext_port); + args[0].elt = "NewRemoteHost"; + args[0].val = remote_host?remote_host:""; + args[1].elt = "NewExternalPort"; + args[1].val = ext_port_str; + args[2].elt = "NewProtocol"; + args[2].val = proto; + args[3].elt = "NewInternalPort"; + args[3].val = int_port_str; + args[4].elt = "NewInternalClient"; + args[4].val = int_client; + args[5].elt = "NewEnabled"; + args[5].val = "1"; + args[6].elt = "NewPortMappingDescription"; + args[6].val = description?description:"miniupnpc-async"; + args[7].elt = "NewLeaseDuration"; + args[7].val = lease_duration_str; + upnpc_build_soap_request(p, p->control_conn_url, + "urn:schemas-upnp-org:service:WANIPConnection:1", + "AddPortMapping", + args, 8); + p->state = EDevSoapConnect; + upnpc_connect(p, p->control_conn_url); + return 0; +} + +#ifdef UPNPC_USE_SELECT +int upnpc_select_fds(upnpc_t * p, int * nfds, fd_set * readfds, fd_set * writefds) +{ + upnpc_device_t * d; + int n = 0; + if(!p) return UPNPC_ERR_INVALID_ARGS; + for(d = p->device_list; d != NULL; d = d->next) { + switch(d->state) { + case EDevGetDescConnect: + case EDevGetDescRequest: + case EDevSoapConnect: + case EDevSoapRequest: + FD_SET(d->http_socket, writefds); + if(*nfds < d->http_socket) + *nfds = d->http_socket; + n++; + break; + case EDevGetDescResponse: + case EDevSoapResponse: + FD_SET(d->http_socket, readfds); + if(*nfds < d->http_socket) + *nfds = d->http_socket; + n++; + break; + default: + break; + } + } + + switch(p->state) { + case EUPnPSendSSDP: + FD_SET(p->ssdp_socket, writefds); + if(*nfds < p->ssdp_socket) + *nfds = p->ssdp_socket; + n++; + break; + case EUPnPReceiveSSDP: + default: + /* still receive SSDP responses when processing Description, etc. */ + FD_SET(p->ssdp_socket, readfds); + if(*nfds < p->ssdp_socket) + *nfds = p->ssdp_socket; + n++; + break; + } + return n; +} + +void upnpc_check_select_fds(upnpc_t * p, const fd_set * readfds, const fd_set * writefds) +{ + upnpc_device_t * d; + + p->socket_flags = 0; + if(FD_ISSET(p->ssdp_socket, readfds)) + p->socket_flags = UPNPC_SSDP_READABLE; + if(FD_ISSET(p->ssdp_socket, writefds)) + p->socket_flags = UPNPC_SSDP_WRITEABLE; + + for(d = p->device_list; d != NULL; d = d->next) { + d->socket_flags = 0; + if(FD_ISSET(d->http_socket, readfds)) + d->socket_flags = UPNPC_HTTP_READABLE; + if(FD_ISSET(d->http_socket, writefds)) + d->socket_flags = UPNPC_HTTP_WRITEABLE; + } +} +#endif + +static const char * devices_to_search[] = { + "urn:schemas-upnp-org:device:InternetGatewayDevice:1", + "urn:schemas-upnp-org:service:WANIPConnection:1", + "urn:schemas-upnp-org:service:WANPPPConnection:1", + "upnp:rootdevice", + 0 +}; + +int upnpc_process(upnpc_t * p) +{ + upnpc_device_t * d; +/* +1) Envoyer les paquets de discovery SSDP +2) Recevoir et traiter les reponses +3) recup les descriptions +4) tester les etats +TODO : translate comments to English +*/ + if(!p) return UPNPC_ERR_INVALID_ARGS; + debug_printf("state=%d socket_flags=0x%04x\n", (int)p->state, p->socket_flags); + + for(d = p->device_list; d != NULL; d = d->next) { + switch(d->state) { + case EDevGetDescConnect: + case EDevSoapConnect: + upnpc_complete_connect(d); + break; + case EDevGetDescRequest: + case EDevSoapRequest: + upnpc_send_request(d); + break; + case EDevGetDescResponse: + case EDevSoapResponse: + upnpc_get_response(d); + break; + default: + break; + } + } + /* all devices ready => ready */ + if(p->device_list != NULL) { + d = p->device_list; + while(d && d->state == EDevReady) d = d->next; + p->state = (d == NULL) ? EUPnPReady : EUPnPProcessing; + } + + if(p->socket_flags & UPNPC_SSDP_READABLE) { + upnpc_receive_and_parse_ssdp(p); + } + switch(p->state) { + case EUPnPInit: + upnpc_send_ssdp_msearch(p, devices_to_search[0], 2); + break; + case EUPnPSendSSDP: + upnpc_send_ssdp_msearch(p, devices_to_search[0], 2); + break; + case EUPnPReceiveSSDP: + /*upnpc_receive_and_parse_ssdp(p);*/ + break; + /*case EGetDesc: + upnpc_connect(p); + break;*/ + case EUPnPReady: + case EUPnPProcessing: + break; + default: + return UPNPC_ERR_UNKNOWN_STATE; + } + return UPNPC_OK; +} + diff --git a/src/contrib/miniupnp/miniupnpc-async/miniupnpc-async.h b/src/contrib/miniupnp/miniupnpc-async/miniupnpc-async.h new file mode 100644 index 0000000..53b433c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/miniupnpc-async.h @@ -0,0 +1,118 @@ +/* $Id: miniupnpc-async.h,v 1.13 2014/11/07 11:25:52 nanard Exp $ */ +/* miniupnpc-async + * Copyright (c) 2008-2017, Thomas BERNARD + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#ifndef MINIUPNPC_ASYNC_H_INCLUDED +#define MINIUPNPC_ASYNC_H_INCLUDED + +/* for struct sockaddr_storage */ +#include +/* for fd_set */ +#include + +#include "declspec.h" +#include "upnpreplyparse.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define UPNPC_OK 0 +#define UPNPC_ERR_INVALID_ARGS (-1) +#define UPNPC_ERR_SOCKET_FAILED (-2) +#define UPNPC_ERR_BIND_FAILED (-3) +#define UPNPC_ERR_UNKNOWN_STATE (-4) + +#define UPNPC_SSDP_READABLE 0x0001 +#define UPNPC_SSDP_WRITEABLE 0x0100 +#define UPNPC_HTTP_READABLE 0x0002 +#define UPNPC_HTTP_WRITEABLE 0x0200 + +typedef struct upnpc_device { + struct upnpc_device * next; + enum { + EDevInit = 1, + EDevGetDescConnect, + EDevGetDescRequest, + EDevGetDescResponse, + EDevReady, + EDevSoapConnect, + EDevSoapRequest, + EDevSoapResponse, + EDevFinalized = 99, + EDevError = 1000 + } state; + char * root_desc_location; + char * control_cif_url; + char * control_conn_url; + int http_socket; + int socket_flags; /* see UPNPC_*_READABLE, etc. */ + char * http_request; + int http_request_len; + int http_request_sent; + char * http_response; + int http_response_received; + int http_response_end_of_headers; + int http_response_content_length; + int http_response_chunked; + int http_response_code; + struct NameValueParserData soap_response_data; + struct sockaddr_storage selfaddr; + socklen_t selfaddrlen; +} upnpc_device_t; + +typedef struct { + enum { + EUPnPInit = 1, + EUPnPSendSSDP, + EUPnPReceiveSSDP, + EUPnPGetDesc, + EUPnPReady, + EUPnPProcessing, + EUPnPFinalized = 99, + EUPnPError = 1000 + } state; + int socket_flags; /* see UPNPC_*_READABLE, etc. */ + int ssdp_socket; + upnpc_device_t * device_list; +} upnpc_t; + +int upnpc_init(upnpc_t * p, const char * multicastif); + +int upnpc_finalize(upnpc_t * p); + +int upnpc_get_external_ip_address(upnpc_device_t * p); + +int upnpc_get_link_layer_max_rate(upnpc_device_t * p); + +int upnpc_add_port_mapping(upnpc_device_t * p, + const char * remote_host, unsigned short ext_port, + unsigned short int_port, const char * int_client, + const char * proto, const char * description, + unsigned int lease_duration); + +#ifdef UPNPC_USE_SELECT +int upnpc_select_fds(upnpc_t * p, int * nfds, fd_set * readfds, fd_set * writefds); +void upnpc_check_select_fds(upnpc_t * p, const fd_set * readfds, const fd_set * writefds); +#endif /* UPNPC_USE_SELECT */ + +int upnpc_process(upnpc_t * p); + +#ifdef __cplusplus +} +#endif + +#endif /* MINIUPNPC_ASYNC_H_INCLUDED */ + diff --git a/src/contrib/miniupnp/miniupnpc-async/minixml.c b/src/contrib/miniupnp/miniupnpc-async/minixml.c new file mode 100644 index 0000000..3e201ec --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/minixml.c @@ -0,0 +1,229 @@ +/* $Id: minixml.c,v 1.11 2014/02/03 15:54:12 nanard Exp $ */ +/* minixml.c : the minimum size a xml parser can be ! */ +/* Project : miniupnp + * webpage: http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author : Thomas Bernard + +Copyright (c) 2005-2014, Thomas BERNARD +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include "minixml.h" + +/* parseatt : used to parse the argument list + * return 0 (false) in case of success and -1 (true) if the end + * of the xmlbuffer is reached. */ +static int parseatt(struct xmlparser * p) +{ + const char * attname; + int attnamelen; + const char * attvalue; + int attvaluelen; + while(p->xml < p->xmlend) + { + if(*p->xml=='/' || *p->xml=='>') + return 0; + if( !IS_WHITE_SPACE(*p->xml) ) + { + char sep; + attname = p->xml; + attnamelen = 0; + while(*p->xml!='=' && !IS_WHITE_SPACE(*p->xml) ) + { + attnamelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + while(*(p->xml++) != '=') + { + if(p->xml >= p->xmlend) + return -1; + } + while(IS_WHITE_SPACE(*p->xml)) + { + p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + sep = *p->xml; + if(sep=='\'' || sep=='\"') + { + p->xml++; + if(p->xml >= p->xmlend) + return -1; + attvalue = p->xml; + attvaluelen = 0; + while(*p->xml != sep) + { + attvaluelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + } + else + { + attvalue = p->xml; + attvaluelen = 0; + while( !IS_WHITE_SPACE(*p->xml) + && *p->xml != '>' && *p->xml != '/') + { + attvaluelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + } + /*printf("%.*s='%.*s'\n", + attnamelen, attname, attvaluelen, attvalue);*/ + if(p->attfunc) + p->attfunc(p->data, attname, attnamelen, attvalue, attvaluelen); + } + p->xml++; + } + return -1; +} + +/* parseelt parse the xml stream and + * call the callback functions when needed... */ +static void parseelt(struct xmlparser * p) +{ + int i; + const char * elementname; + while(p->xml < (p->xmlend - 1)) + { + if((p->xml + 4) <= p->xmlend && (0 == memcmp(p->xml, "", 3) != 0); + p->xml += 3; + } + else if((p->xml)[0]=='<' && (p->xml)[1]!='?') + { + i = 0; elementname = ++p->xml; + while( !IS_WHITE_SPACE(*p->xml) + && (*p->xml!='>') && (*p->xml!='/') + ) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + /* to ignore namespace : */ + if(*p->xml==':') + { + i = 0; + elementname = ++p->xml; + } + } + if(i>0) + { + if(p->starteltfunc) + p->starteltfunc(p->data, elementname, i); + if(parseatt(p)) + return; + if(*p->xml!='/') + { + const char * data; + i = 0; data = ++p->xml; + if (p->xml >= p->xmlend) + return; + while( IS_WHITE_SPACE(*p->xml) ) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + } + if(memcmp(p->xml, "xml += 9; + data = p->xml; + i = 0; + while(memcmp(p->xml, "]]>", 3) != 0) + { + i++; p->xml++; + if ((p->xml + 3) >= p->xmlend) + return; + } + if(i>0 && p->datafunc) + p->datafunc(p->data, data, i); + while(*p->xml!='<') + { + p->xml++; + if (p->xml >= p->xmlend) + return; + } + } + else + { + while(*p->xml!='<') + { + i++; p->xml++; + if ((p->xml + 1) >= p->xmlend) + return; + } + if(i>0 && p->datafunc && *(p->xml + 1) == '/') + p->datafunc(p->data, data, i); + } + } + } + else if(*p->xml == '/') + { + i = 0; elementname = ++p->xml; + if (p->xml >= p->xmlend) + return; + while((*p->xml != '>')) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + } + if(p->endeltfunc) + p->endeltfunc(p->data, elementname, i); + p->xml++; + } + } + else + { + p->xml++; + } + } +} + +/* the parser must be initialized before calling this function */ +void parsexml(struct xmlparser * parser) +{ + parser->xml = parser->xmlstart; + parser->xmlend = parser->xmlstart + parser->xmlsize; + parseelt(parser); +} + + diff --git a/src/contrib/miniupnp/miniupnpc-async/minixml.h b/src/contrib/miniupnp/miniupnpc-async/minixml.h new file mode 100644 index 0000000..9f43aa4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/minixml.h @@ -0,0 +1,37 @@ +/* $Id: minixml.h,v 1.7 2012/09/27 15:42:10 nanard Exp $ */ +/* minimal xml parser + * + * Project : miniupnp + * Website : http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef MINIXML_H_INCLUDED +#define MINIXML_H_INCLUDED +#define IS_WHITE_SPACE(c) ((c==' ') || (c=='\t') || (c=='\r') || (c=='\n')) + +/* if a callback function pointer is set to NULL, + * the function is not called */ +struct xmlparser { + const char *xmlstart; + const char *xmlend; + const char *xml; /* pointer to current character */ + int xmlsize; + void * data; + void (*starteltfunc) (void *, const char *, int); + void (*endeltfunc) (void *, const char *, int); + void (*datafunc) (void *, const char *, int); + void (*attfunc) (void *, const char *, int, const char *, int); +}; + +/* parsexml() + * the xmlparser structure must be initialized before the call + * the following structure members have to be initialized : + * xmlstart, xmlsize, data, *func + * xml is for internal usage, xmlend is computed automatically */ +void parsexml(struct xmlparser *); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc-async/parsessdpreply.c b/src/contrib/miniupnp/miniupnpc-async/parsessdpreply.c new file mode 100644 index 0000000..05fda36 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/parsessdpreply.c @@ -0,0 +1,80 @@ +/* $Id: parsessdpreply.c,v 1.2 2009/11/14 10:37:55 nanard Exp $ */ +/* Project : miniupnp + * website : http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author : Thomas Bernard + * copyright (c) 2005-2009 Thomas Bernard + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include +#include "parsessdpreply.h" + +/* parseMSEARCHReply() + * the last 4 arguments are filled during the parsing : + * - location/locationsize : "location:" field of the SSDP reply packet + * - st/stsize : "st:" field of the SSDP reply packet. + * The strings are NOT null terminated */ +void +parseMSEARCHReply(const char * reply, int size, + const char * * location, int * locationsize, + const char * * st, int * stsize) +{ + int a, b, i; + i = 0; + a = i; /* start of the line */ + b = 0; + while(i + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include +#include +#include +/* for getnameinfo() : */ +#include +#include +#include +/* compile with -DUPNPC_USE_SELECT to enable upnpc_select_fds() function */ +#include "miniupnpc-async.h" +#include "upnpreplyparse.h" + +enum methods { + EGetExternalIP, + EGetRates, + EAddPortMapping, + ENothing +}; + +int main(int argc, char * * argv) +{ + char ip_address[64]; + int r, n; + upnpc_t upnp; + upnpc_device_t * device = NULL; + const char * multicastif = NULL; + enum methods next_method_to_call = EGetExternalIP; + enum methods last_method = ENothing; + if(argc>1) + multicastif = argv[1]; + if((r = upnpc_init(&upnp, multicastif)) < 0) { + fprintf(stderr, "upnpc_init failed : %d", r); + return 1; + } + r = upnpc_process(&upnp); + printf("upnpc_process returned %d\n", r); + while(upnp.state != EUPnPError) { + int nfds; + fd_set readfds; + fd_set writefds; + /*struct timeval timeout;*/ + + FD_ZERO(&readfds); + FD_ZERO(&writefds); + nfds = 0; + n = upnpc_select_fds(&upnp, &nfds, &readfds, &writefds); + if(n <= 0) { + printf("nothing to select()...\n"); + break; + } +#if 0 + timeout.tv_sec = 0; + timeout.tv_usec = 0; +#endif +#if DEBUG + printf("select(%d, ...);\n", nfds+1); +#endif /* DEBUG */ + if(select(nfds+1, &readfds, &writefds, NULL, NULL/*&timeout*/) < 0) { + perror("select"); + return 1; + } + upnpc_check_select_fds(&upnp, &readfds, &writefds); + r = upnpc_process(&upnp); +#if DEBUG + printf("upnpc_process returned %d\n", r); +#endif /* DEBUG */ + if(r < 0) + break; + if(upnp.state == EUPnPReady) { + char * p; + if(device == NULL) { + /* select one device */ + device = upnp.device_list; /* pick up the first one */ + } + printf("Process UPnP IGD Method results : HTTP %d\n", device->http_response_code); + if(device->http_response_code == 200) { + switch(last_method) { + case EGetExternalIP: + p = GetValueFromNameValueList(&device->soap_response_data, "NewExternalIPAddress"); + printf("ExternalIPAddress = %s\n", p); + /* p = GetValueFromNameValueList(&pdata, "errorCode");*/ + break; + case EGetRates: + p = GetValueFromNameValueList(&device->soap_response_data, "NewLayer1DownstreamMaxBitRate"); + printf("DownStream MaxBitRate = %s\t", p); + p = GetValueFromNameValueList(&device->soap_response_data, "NewLayer1UpstreamMaxBitRate"); + printf("UpStream MaxBitRate = %s\n", p); + break; + case EAddPortMapping: + printf("OK\n"); + break; + case ENothing: + break; + } + } else { + printf("SOAP error :\n"); + printf(" faultcode='%s'\n", GetValueFromNameValueList(&device->soap_response_data, "faultcode")); + printf(" faultstring='%s'\n", GetValueFromNameValueList(&device->soap_response_data, "faultstring")); + printf(" errorCode=%s\n", GetValueFromNameValueList(&device->soap_response_data, "errorCode")); + printf(" errorDescription='%s'\n", GetValueFromNameValueList(&device->soap_response_data, "errorDescription")); + } + if(next_method_to_call == ENothing) + break; + printf("Ready to call UPnP IGD methods\n"); + last_method = next_method_to_call; + switch(next_method_to_call) { + case EGetExternalIP: + printf("GetExternalIPAddress\n"); + upnpc_get_external_ip_address(device); + next_method_to_call = EGetRates; + break; + case EGetRates: + printf("GetCommonLinkProperties\n"); + upnpc_get_link_layer_max_rate(device); + next_method_to_call = EAddPortMapping; + break; + case EAddPortMapping: + if(getnameinfo((struct sockaddr *)&device->selfaddr, device->selfaddrlen, + ip_address, sizeof(ip_address), NULL, 0, NI_NUMERICHOST | NI_NUMERICSERV) < 0) { + fprintf(stderr, "getnameinfo() failed\n"); + } + printf("our IP address is %s\n", ip_address); + printf("AddPortMapping\n"); + upnpc_add_port_mapping(device, + NULL /* remote_host */, 40002 /* ext_port */, + 42042 /* int_port */, ip_address /* int_client */, + "TCP" /* proto */, "this is a test" /* description */, + 0 /* lease duration */); + next_method_to_call = ENothing; + case ENothing: + break; + } + } + } + upnpc_finalize(&upnp); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpc-async/upnpreplyparse.c b/src/contrib/miniupnp/miniupnpc-async/upnpreplyparse.c new file mode 100644 index 0000000..5de5796 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/upnpreplyparse.c @@ -0,0 +1,197 @@ +/* $Id: upnpreplyparse.c,v 1.19 2015/07/15 10:29:11 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2015 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include + +#include "upnpreplyparse.h" +#include "minixml.h" + +static void +NameValueParserStartElt(void * d, const char * name, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + data->topelt = 1; + if(l>63) + l = 63; + memcpy(data->curelt, name, l); + data->curelt[l] = '\0'; + data->cdata = NULL; + data->cdatalen = 0; +} + +static void +NameValueParserEndElt(void * d, const char * name, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + struct NameValue * nv; + (void)name; + (void)l; + if(!data->topelt) + return; + if(strcmp(data->curelt, "NewPortListing") != 0) + { + int l; + /* standard case. Limited to n chars strings */ + l = data->cdatalen; + nv = malloc(sizeof(struct NameValue)); + if(nv == NULL) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "NameValueParserEndElt"); +#endif /* DEBUG */ + return; + } + if(l>=(int)sizeof(nv->value)) + l = sizeof(nv->value) - 1; + strncpy(nv->name, data->curelt, 64); + nv->name[63] = '\0'; + if(data->cdata != NULL) + { + memcpy(nv->value, data->cdata, l); + nv->value[l] = '\0'; + } + else + { + nv->value[0] = '\0'; + } + nv->l_next = data->l_head; /* insert in list */ + data->l_head = nv; + } + data->cdata = NULL; + data->cdatalen = 0; + data->topelt = 0; +} + +static void +NameValueParserGetData(void * d, const char * datas, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + if(strcmp(data->curelt, "NewPortListing") == 0) + { + /* specific case for NewPortListing which is a XML Document */ + data->portListing = malloc(l + 1); + if(!data->portListing) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "NameValueParserGetData"); +#endif /* DEBUG */ + return; + } + memcpy(data->portListing, datas, l); + data->portListing[l] = '\0'; + data->portListingLength = l; + } + else + { + /* standard case. */ + data->cdata = datas; + data->cdatalen = l; + } +} + +void +ParseNameValue(const char * buffer, int bufsize, + struct NameValueParserData * data) +{ + struct xmlparser parser; + data->l_head = NULL; + data->portListing = NULL; + data->portListingLength = 0; + /* init xmlparser object */ + parser.xmlstart = buffer; + parser.xmlsize = bufsize; + parser.data = data; + parser.starteltfunc = NameValueParserStartElt; + parser.endeltfunc = NameValueParserEndElt; + parser.datafunc = NameValueParserGetData; + parser.attfunc = 0; + parsexml(&parser); +} + +void +ClearNameValueList(struct NameValueParserData * pdata) +{ + struct NameValue * nv; + if(pdata->portListing) + { + free(pdata->portListing); + pdata->portListing = NULL; + pdata->portListingLength = 0; + } + while((nv = pdata->l_head) != NULL) + { + pdata->l_head = nv->l_next; + free(nv); + } +} + +char * +GetValueFromNameValueList(struct NameValueParserData * pdata, + const char * Name) +{ + struct NameValue * nv; + char * p = NULL; + for(nv = pdata->l_head; + (nv != NULL) && (p == NULL); + nv = nv->l_next) + { + if(strcmp(nv->name, Name) == 0) + p = nv->value; + } + return p; +} + +#if 0 +/* useless now that minixml ignores namespaces by itself */ +char * +GetValueFromNameValueListIgnoreNS(struct NameValueParserData * pdata, + const char * Name) +{ + struct NameValue * nv; + char * p = NULL; + char * pname; + for(nv = pdata->head.lh_first; + (nv != NULL) && (p == NULL); + nv = nv->entries.le_next) + { + pname = strrchr(nv->name, ':'); + if(pname) + pname++; + else + pname = nv->name; + if(strcmp(pname, Name)==0) + p = nv->value; + } + return p; +} +#endif + +/* debug all-in-one function + * do parsing then display to stdout */ +#ifdef DEBUG +void +DisplayNameValueList(char * buffer, int bufsize) +{ + struct NameValueParserData pdata; + struct NameValue * nv; + ParseNameValue(buffer, bufsize, &pdata); + for(nv = pdata.l_head; + nv != NULL; + nv = nv->l_next) + { + printf("%s = %s\n", nv->name, nv->value); + } + ClearNameValueList(&pdata); +} +#endif /* DEBUG */ + diff --git a/src/contrib/miniupnp/miniupnpc-async/upnpreplyparse.h b/src/contrib/miniupnp/miniupnpc-async/upnpreplyparse.h new file mode 100644 index 0000000..6badd15 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/upnpreplyparse.h @@ -0,0 +1,63 @@ +/* $Id: upnpreplyparse.h,v 1.19 2014/10/27 16:33:19 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPREPLYPARSE_H_INCLUDED +#define UPNPREPLYPARSE_H_INCLUDED + +#ifdef __cplusplus +extern "C" { +#endif + +struct NameValue { + struct NameValue * l_next; + char name[64]; + char value[128]; +}; + +struct NameValueParserData { + struct NameValue * l_head; + char curelt[64]; + char * portListing; + int portListingLength; + int topelt; + const char * cdata; + int cdatalen; +}; + +/* ParseNameValue() */ +void +ParseNameValue(const char * buffer, int bufsize, + struct NameValueParserData * data); + +/* ClearNameValueList() */ +void +ClearNameValueList(struct NameValueParserData * pdata); + +/* GetValueFromNameValueList() */ +char * +GetValueFromNameValueList(struct NameValueParserData * pdata, + const char * Name); + +#if 0 +/* GetValueFromNameValueListIgnoreNS() */ +char * +GetValueFromNameValueListIgnoreNS(struct NameValueParserData * pdata, + const char * Name); +#endif + +/* DisplayNameValueList() */ +#ifdef DEBUG +void +DisplayNameValueList(char * buffer, int bufsize); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc-async/upnputils.c b/src/contrib/miniupnp/miniupnpc-async/upnputils.c new file mode 100644 index 0000000..8519da9 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/upnputils.c @@ -0,0 +1,87 @@ +/* $Id: upnputils.c,v 1.1 2013/09/07 06:45:39 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef AF_LINK +#include +#endif + +#include "upnputils.h" + +int +sockaddr_to_string(const struct sockaddr * addr, char * str, size_t size) +{ + char buffer[64]; + unsigned short port = 0; + int n = -1; + + switch(addr->sa_family) + { + case AF_INET6: + inet_ntop(addr->sa_family, + &((struct sockaddr_in6 *)addr)->sin6_addr, + buffer, sizeof(buffer)); + port = ntohs(((struct sockaddr_in6 *)addr)->sin6_port); + n = snprintf(str, size, "[%s]:%hu", buffer, port); + break; + case AF_INET: + inet_ntop(addr->sa_family, + &((struct sockaddr_in *)addr)->sin_addr, + buffer, sizeof(buffer)); + port = ntohs(((struct sockaddr_in *)addr)->sin_port); + n = snprintf(str, size, "%s:%hu", buffer, port); + break; +#ifdef AF_LINK +#if defined(__sun) + /* solaris does not seem to have link_ntoa */ + /* #define link_ntoa _link_ntoa */ +#define link_ntoa(x) "dummy-link_ntoa" +#endif + case AF_LINK: + { + struct sockaddr_dl * sdl = (struct sockaddr_dl *)addr; + n = snprintf(str, size, "index=%hu type=%d %s", + sdl->sdl_index, sdl->sdl_type, + link_ntoa(sdl)); + } + break; +#endif + default: + n = snprintf(str, size, "unknown address family %d", addr->sa_family); +#if 0 + n = snprintf(str, size, "unknown address family %d " + "%02x %02x %02x %02x %02x %02x %02x %02x", + addr->sa_family, + addr->sa_data[0], addr->sa_data[1], (unsigned)addr->sa_data[2], addr->sa_data[3], + addr->sa_data[4], addr->sa_data[5], (unsigned)addr->sa_data[6], addr->sa_data[7]); +#endif + } + return n; +} + + +int +set_non_blocking(int fd) +{ + int flags = fcntl(fd, F_GETFL); + if(flags < 0) + return 0; + if(fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) + return 0; + return 1; +} + diff --git a/src/contrib/miniupnp/miniupnpc-async/upnputils.h b/src/contrib/miniupnp/miniupnpc-async/upnputils.h new file mode 100644 index 0000000..adf387f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-async/upnputils.h @@ -0,0 +1,27 @@ +/* $Id: upnputils.h,v 1.1 2013/09/07 06:45:39 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPUTILS_H_INCLUDED +#define UPNPUTILS_H_INCLUDED + +/** + * convert a struct sockaddr to a human readable string. + * [ipv6]:port or ipv4:port + * return the number of characters used (as snprintf) + */ +int +sockaddr_to_string(const struct sockaddr * addr, char * str, size_t size); + +/** + * set the file description as non blocking + * return 0 in case of failure, 1 in case of success + */ +int +set_non_blocking(int fd); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc-libevent/.gitignore b/src/contrib/miniupnp/miniupnpc-libevent/.gitignore new file mode 100644 index 0000000..b845178 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/.gitignore @@ -0,0 +1,3 @@ +*.o +*.a +upnpc-libevent diff --git a/src/contrib/miniupnp/miniupnpc-libevent/Makefile b/src/contrib/miniupnp/miniupnpc-libevent/Makefile new file mode 100644 index 0000000..29abc4a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/Makefile @@ -0,0 +1,69 @@ +# $Id: Makefile,v 1.7 2014/11/28 13:14:19 nanard Exp $ + +OS = $(shell $(CC) -dumpmachine) +PKG_CONFIG ?= pkg-config + +CFLAGS = -O0 -g -DDEBUG +# libevent debug +CFLAGS += -DUSE_DEBUG + +CFLAGS += -fPIC +CFLAGS += -ansi +CFLAGS += -Wall -W +CFLAGS += -D_BSD_SOURCE +ifeq (, $(findstring darwin, $(OS))$(findstring freebsd, $(OS))) +CFLAGS += -D_POSIX_C_SOURCE=200112L +endif +#CFLAGS += -I/usr/local/include +CFLAGS += $(shell $(PKG_CONFIG) --cflags libevent) + +#CFLAGS += -DENABLE_UPNP_EVENTS + +#LDLIBS = -levent +LDLIBS = $(shell $(PKG_CONFIG) --libs-only-l libevent) +#LDFLAGS += -L/usr/local/lib +LDFLAGS += $(shell $(PKG_CONFIG) --libs-only-L libevent) + +ifneq (, $(findstring darwin, $(OS))) +CFLAGS += -D_DARWIN_C_SOURCE +#CFLAGS += -I/opt/local/include +#LDFLAGS += -L/opt/local/lib +endif + +LIB = libminiupnpc-ev.a + +LIB_SRCS = miniupnpc-libevent.c minixml.c igd_desc_parse.c upnpreplyparse.c + +SRCS = $(LIB_SRCS) upnpc-libevent.c + +LIB_OBJS = $(patsubst %.c,%.o,$(LIB_SRCS)) + +OBJS = $(patsubst %.c,%.o,$(SRCS)) + +EXECUTABLE = upnpc-libevent + +.PHONY: all clean depend + +all: $(EXECUTABLE) + +clean: + $(RM) $(OBJS) + $(RM) $(EXECUTABLE) + $(RM) $(LIB) + +upnpc-libevent: upnpc-libevent.o $(LIB) + +$(LIB): $(LIB_OBJS) + $(AR) crs $@ $? + +depend: + makedepend -Y -- $(CFLAGS) -- $(SRCS) 2>/dev/null + +# DO NOT DELETE THIS LINE -- make depend depends on it. + +miniupnpc-libevent.o: miniupnpc-libevent.h declspec.h upnpreplyparse.h +miniupnpc-libevent.o: minixml.h igd_desc_parse.h +minixml.o: minixml.h +igd_desc_parse.o: igd_desc_parse.h +upnpreplyparse.o: upnpreplyparse.h minixml.h +upnpc-libevent.o: miniupnpc-libevent.h declspec.h upnpreplyparse.h diff --git a/src/contrib/miniupnp/miniupnpc-libevent/README b/src/contrib/miniupnp/miniupnpc-libevent/README new file mode 100644 index 0000000..b102fcb --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/README @@ -0,0 +1,13 @@ +miniupnpc-libevent : + +UPnP IGD control point (ie client) using libevent + +http://libevent.org +https://github.com/libevent/libevent + +The UPnP Event code needs SUBSCRIBE / UNSUBSCRIBE / NOTIFY HTTP methods. +Implementation is in progress in libevent HTTP code : +https://github.com/libevent/libevent/pull/327 + + +TODO : improve error handling / reporting diff --git a/src/contrib/miniupnp/miniupnpc-libevent/declspec.h b/src/contrib/miniupnp/miniupnpc-libevent/declspec.h new file mode 100644 index 0000000..16be781 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/declspec.h @@ -0,0 +1,15 @@ +#ifndef DECLSPEC_H_DEFINED +#define DECLSPEC_H_DEFINED + +#if defined(_WIN32) && !defined(STATICLIB) + #ifdef MINIUPNP_EXPORTS + #define LIBSPEC __declspec(dllexport) + #else + #define LIBSPEC __declspec(dllimport) + #endif +#else + #define LIBSPEC +#endif + +#endif /* DECLSPEC_H_DEFINED */ + diff --git a/src/contrib/miniupnp/miniupnpc-libevent/igd_desc_parse.c b/src/contrib/miniupnp/miniupnpc-libevent/igd_desc_parse.c new file mode 100644 index 0000000..d2999ad --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/igd_desc_parse.c @@ -0,0 +1,123 @@ +/* $Id: igd_desc_parse.c,v 1.17 2015/09/15 13:30:04 nanard Exp $ */ +/* Project : miniupnp + * http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005-2015 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ + +#include "igd_desc_parse.h" +#include +#include + +/* Start element handler : + * update nesting level counter and copy element name */ +void IGDstartelt(void * d, const char * name, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + if(l >= MINIUPNPC_URL_MAXSIZE) + l = MINIUPNPC_URL_MAXSIZE-1; + memcpy(datas->cureltname, name, l); + datas->cureltname[l] = '\0'; + datas->level++; + if( (l==7) && !memcmp(name, "service", l) ) { + datas->tmp.controlurl[0] = '\0'; + datas->tmp.eventsuburl[0] = '\0'; + datas->tmp.scpdurl[0] = '\0'; + datas->tmp.servicetype[0] = '\0'; + } +} + +#define COMPARE(str, cstr) (0==memcmp(str, cstr, sizeof(cstr) - 1)) + +/* End element handler : + * update nesting level counter and update parser state if + * service element is parsed */ +void IGDendelt(void * d, const char * name, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + datas->level--; + /*printf("endelt %2d %.*s\n", datas->level, l, name);*/ + if( (l==7) && !memcmp(name, "service", l) ) + { + if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:")) { + memcpy(&datas->CIF, &datas->tmp, sizeof(struct IGDdatas_service)); + } else if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANIPv6FirewallControl:")) { + memcpy(&datas->IPv6FC, &datas->tmp, sizeof(struct IGDdatas_service)); + } else if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANIPConnection:") + || COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANPPPConnection:") ) { + if(datas->first.servicetype[0] == '\0') { + memcpy(&datas->first, &datas->tmp, sizeof(struct IGDdatas_service)); + } else { + memcpy(&datas->second, &datas->tmp, sizeof(struct IGDdatas_service)); + } + } + } +} + +/* Data handler : + * copy data depending on the current element name and state */ +void IGDdata(void * d, const char * data, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + char * dstmember = 0; + /*printf("%2d %s : %.*s\n", + datas->level, datas->cureltname, l, data); */ + if( !strcmp(datas->cureltname, "URLBase") ) + dstmember = datas->urlbase; + else if( !strcmp(datas->cureltname, "presentationURL") ) + dstmember = datas->presentationurl; + else if( !strcmp(datas->cureltname, "serviceType") ) + dstmember = datas->tmp.servicetype; + else if( !strcmp(datas->cureltname, "controlURL") ) + dstmember = datas->tmp.controlurl; + else if( !strcmp(datas->cureltname, "eventSubURL") ) + dstmember = datas->tmp.eventsuburl; + else if( !strcmp(datas->cureltname, "SCPDURL") ) + dstmember = datas->tmp.scpdurl; +/* else if( !strcmp(datas->cureltname, "deviceType") ) + dstmember = datas->devicetype_tmp;*/ + if(dstmember) + { + if(l>=MINIUPNPC_URL_MAXSIZE) + l = MINIUPNPC_URL_MAXSIZE-1; + memcpy(dstmember, data, l); + dstmember[l] = '\0'; + } +} + +#ifdef DEBUG +void printIGD(struct IGDdatas * d) +{ + printf("urlbase = '%s'\n", d->urlbase); + printf("WAN Device (Common interface config) :\n"); + /*printf(" deviceType = '%s'\n", d->CIF.devicetype);*/ + printf(" serviceType = '%s'\n", d->CIF.servicetype); + printf(" controlURL = '%s'\n", d->CIF.controlurl); + printf(" eventSubURL = '%s'\n", d->CIF.eventsuburl); + printf(" SCPDURL = '%s'\n", d->CIF.scpdurl); + printf("primary WAN Connection Device (IP or PPP Connection):\n"); + /*printf(" deviceType = '%s'\n", d->first.devicetype);*/ + printf(" servicetype = '%s'\n", d->first.servicetype); + printf(" controlURL = '%s'\n", d->first.controlurl); + printf(" eventSubURL = '%s'\n", d->first.eventsuburl); + printf(" SCPDURL = '%s'\n", d->first.scpdurl); + printf("secondary WAN Connection Device (IP or PPP Connection):\n"); + /*printf(" deviceType = '%s'\n", d->second.devicetype);*/ + printf(" servicetype = '%s'\n", d->second.servicetype); + printf(" controlURL = '%s'\n", d->second.controlurl); + printf(" eventSubURL = '%s'\n", d->second.eventsuburl); + printf(" SCPDURL = '%s'\n", d->second.scpdurl); + printf("WAN IPv6 Firewall Control :\n"); + /*printf(" deviceType = '%s'\n", d->IPv6FC.devicetype);*/ + printf(" servicetype = '%s'\n", d->IPv6FC.servicetype); + printf(" controlURL = '%s'\n", d->IPv6FC.controlurl); + printf(" eventSubURL = '%s'\n", d->IPv6FC.eventsuburl); + printf(" SCPDURL = '%s'\n", d->IPv6FC.scpdurl); +} +#endif /* DEBUG */ + diff --git a/src/contrib/miniupnp/miniupnpc-libevent/igd_desc_parse.h b/src/contrib/miniupnp/miniupnpc-libevent/igd_desc_parse.h new file mode 100644 index 0000000..0de546b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/igd_desc_parse.h @@ -0,0 +1,49 @@ +/* $Id: igd_desc_parse.h,v 1.12 2014/11/17 17:19:13 nanard Exp $ */ +/* Project : miniupnp + * http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005-2014 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef IGD_DESC_PARSE_H_INCLUDED +#define IGD_DESC_PARSE_H_INCLUDED + +/* Structure to store the result of the parsing of UPnP + * descriptions of Internet Gateway Devices */ +#define MINIUPNPC_URL_MAXSIZE (128) +struct IGDdatas_service { + char controlurl[MINIUPNPC_URL_MAXSIZE]; + char eventsuburl[MINIUPNPC_URL_MAXSIZE]; + char scpdurl[MINIUPNPC_URL_MAXSIZE]; + char servicetype[MINIUPNPC_URL_MAXSIZE]; + /*char devicetype[MINIUPNPC_URL_MAXSIZE];*/ +}; + +struct IGDdatas { + char cureltname[MINIUPNPC_URL_MAXSIZE]; + char urlbase[MINIUPNPC_URL_MAXSIZE]; + char presentationurl[MINIUPNPC_URL_MAXSIZE]; + int level; + /*int state;*/ + /* "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1" */ + struct IGDdatas_service CIF; + /* "urn:schemas-upnp-org:service:WANIPConnection:1" + * "urn:schemas-upnp-org:service:WANPPPConnection:1" */ + struct IGDdatas_service first; + /* if both WANIPConnection and WANPPPConnection are present */ + struct IGDdatas_service second; + /* "urn:schemas-upnp-org:service:WANIPv6FirewallControl:1" */ + struct IGDdatas_service IPv6FC; + /* tmp */ + struct IGDdatas_service tmp; +}; + +void IGDstartelt(void *, const char *, int); +void IGDendelt(void *, const char *, int); +void IGDdata(void *, const char *, int); +#ifdef DEBUG +void printIGD(struct IGDdatas *); +#endif /* DEBUG */ + +#endif /* IGD_DESC_PARSE_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpc-libevent/miniupnpc-libevent.c b/src/contrib/miniupnp/miniupnpc-libevent/miniupnpc-libevent.c new file mode 100644 index 0000000..a454cb4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/miniupnpc-libevent.c @@ -0,0 +1,1026 @@ +/* $Id: miniupnpc-libevent.c,v 1.27 2015/07/22 13:51:09 nanard Exp $ */ +/* miniupnpc-libevent + * Copyright (c) 2008-2016, Thomas BERNARD + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +/*#include */ +#include +#ifdef _WIN32 +#include +#include +#include +#define PRINT_SOCKET_ERROR printf +#define SOCKET_ERROR GetWSALastError() +#define WOULDBLOCK(err) (err == WSAEWOULDBLOCK) +#else /* _WIN32 */ +#include +#include +#define closesocket close +#define PRINT_SOCKET_ERROR perror +#define SOCKET_ERROR errno +#define WOULDBLOCK(err) (err == EAGAIN || err == EWOULDBLOCK) +#endif /* _WIN32 */ +#include "miniupnpc-libevent.h" +#include "minixml.h" +#include "igd_desc_parse.h" +#include "upnpreplyparse.h" + +#ifndef MIN +#define MIN(x,y) (((x)<(y))?(x):(y)) +#endif /* MIN */ + +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN 64 +#endif /* MAXHOSTNAMELEN */ + +#define SSDP_PORT 1900 +#define SSDP_MCAST_ADDR "239.255.255.250" +#define XSTR(s) STR(s) +#define STR(s) #s + +#ifdef DEBUG +#define debug_printf(...) fprintf(stderr, __VA_ARGS__) +#else +#define debug_printf(...) (void)0 +#endif + +/* compare the beginning of a string with a constant string */ +#define COMPARE(str, cstr) (0==memcmp(str, cstr, sizeof(cstr) - 1)) + +/* stuctures */ + +struct upnp_args { + const char * elt; + const char * val; +}; + +/* private functions */ + +static int upnpc_get_desc(upnpc_device_t * p, const char * url); +static char * build_url_string(const char * urlbase, const char * root_desc_url, const char * controlurl); + +/* data */ +static const char * devices_to_search[] = { + "urn:schemas-upnp-org:device:InternetGatewayDevice:1", + "urn:schemas-upnp-org:service:WANIPConnection:1", + "urn:schemas-upnp-org:service:WANPPPConnection:1", + "upnp:rootdevice", + 0 +}; + +#ifdef DEBUG +static void upnpc_conn_close_cb(struct evhttp_connection * conn, void * data) +{ + upnpc_device_t * d = (upnpc_device_t *)data; + debug_printf("%s %p %p\n", __func__, conn, d); +} +#endif /* DEBUG */ + +/* parse_msearch_reply() + * the last 4 arguments are filled during the parsing : + * - location/locationsize : "location:" field of the SSDP reply packet + * - st/stsize : "st:" field of the SSDP reply packet. + * The strings are NOT null terminated */ +static void +parse_msearch_reply(const char * reply, int size, + const char * * location, int * locationsize, + const char * * st, int * stsize) +{ + int a, b, i; + i = 0; /* current character index */ + a = i; /* start of the line */ + b = 0; /* end of the "header" (position of the colon) */ + while(idiscover_device_index++], mx); + debug_printf("%s: %s", __func__, bufr); + n = sendto(s, bufr, n, 0, + (struct sockaddr *)&addr, sizeof(struct sockaddr_in)); + if (n < 0) { + PRINT_SOCKET_ERROR("sendto"); + } +} + +static int upnpc_set_root_desc_location(upnpc_device_t * d, const char * location, int locationsize) +{ + char * tmp; + tmp = realloc(d->root_desc_location, locationsize + 1); + if(tmp == NULL) { + return -1; + } + memcpy(tmp, location, locationsize); + tmp[locationsize] = '\0'; + d->root_desc_location = tmp; + return 0; +} + +static upnpc_device_t * upnpc_find_device_with_location(upnpc_t * p, const char * location, int locationsize) +{ + upnpc_device_t * d; + for(d = p->devices; d != NULL; d = d->next) { + if(d->root_desc_location + && ((int)strlen(d->root_desc_location) == locationsize) + && (0 == memcmp(location, d->root_desc_location, locationsize))) + return d; + } + return NULL; +} + +static void upnpc_receive_and_parse_ssdp(evutil_socket_t s, short events, upnpc_t * p) +{ + char bufr[2048]; + ssize_t len; + + if(events == EV_TIMEOUT) { + /* nothing received ... */ + debug_printf("%s() TIMEOUT\n", __func__); + if(!devices_to_search[p->discover_device_index]) { + debug_printf("*** NO MORE DEVICES TO SEARCH ***\n"); + event_del(p->ev_ssdp_recv); + /* no device found : report error */ + p->ready_cb(UPNPC_ERR_NO_DEVICE_FOUND, p, NULL, p->cb_data); + } else { + /* send another SSDP M-SEARCH packet */ + if(event_add(p->ev_ssdp_writable, NULL)) { + debug_printf("event_add FAILED\n"); + } + } + return; + } + len = recv(s, bufr, sizeof(bufr), 0); + debug_printf("input %d bytes\n", (int)len); + if(len < 0) { + PRINT_SOCKET_ERROR("recv"); + } else if(len == 0) { + debug_printf("SSDP socket closed ?\n"); + } else { + const char * location = NULL; + int locationsize = 0; + const char * st = NULL; + int stsize = 0; + debug_printf("%.*s", (int)len, bufr); + parse_msearch_reply(bufr, len, &location, &locationsize, &st, &stsize); + debug_printf("location = '%.*s'\n", locationsize, location); + debug_printf("st = '%.*s'\n", stsize, st); + if(location != NULL) { + upnpc_device_t * device; + device = upnpc_find_device_with_location(p, location, locationsize); + if(device) { + debug_printf("device already known\n"); + } else { + device = malloc(sizeof(upnpc_device_t)); + if(device == NULL) { + debug_printf("Memory allocation error\n"); + return; + } + memset(device, 0, sizeof(upnpc_device_t)); + device->parent = p; + device->next = p->devices; + p->devices = device; + if(upnpc_set_root_desc_location(device, location, locationsize) < 0) { + return; + } + if(upnpc_get_desc(device, device->root_desc_location)) { + debug_printf("FAILED to request device root description\n"); + } + } +#if 0 + event_del(p->ev_ssdp_recv); /* stop receiving SSDP responses */ +#endif + } else { + /* or do nothing ? */ + debug_printf("no location\n"); + } + } +} + +static int +parseURL(const char * url, + char * hostname, unsigned short * port, + char * * path, unsigned int * scope_id) +{ + char * p1, *p2, *p3; + if(!url) + return 0; + p1 = strstr(url, "://"); + if(!p1) + return 0; + p1 += 3; + if( (url[0]!='h') || (url[1]!='t') + ||(url[2]!='t') || (url[3]!='p')) + return 0; + memset(hostname, 0, MAXHOSTNAMELEN + 1); + if(*p1 == '[') { + /* IP v6 : http://[2a00:1450:8002::6a]/path/abc */ + char * scope; + scope = strchr(p1, '%'); + p2 = strchr(p1, ']'); + if(p2 && scope && scope < p2 && scope_id) { + /* parse scope */ +#ifdef IF_NAMESIZE + char tmp[IF_NAMESIZE]; + int l; + scope++; + /* "%25" is just '%' in URL encoding */ + if(scope[0] == '2' && scope[1] == '5') + scope += 2; /* skip "25" */ + l = p2 - scope; + if(l >= IF_NAMESIZE) + l = IF_NAMESIZE - 1; + memcpy(tmp, scope, l); + tmp[l] = '\0'; + *scope_id = if_nametoindex(tmp); + if(*scope_id == 0) { + *scope_id = (unsigned int)strtoul(tmp, NULL, 10); + } +#else /* IF_NAMESIZE */ + /* under windows, scope is numerical */ + char tmp[8]; + int l; + scope++; + /* "%25" is just '%' in URL encoding */ + if(scope[0] == '2' && scope[1] == '5') + scope += 2; /* skip "25" */ + l = p2 - scope; + if(l >= (int)sizeof(tmp)) + l = sizeof(tmp) - 1; + memcpy(tmp, scope, l); + tmp[l] = '\0'; + *scope_id = (unsigned int)strtoul(tmp, NULL, 10); +#endif /* IF_NAMESIZE */ + } + p3 = strchr(p1, '/'); + if(p2 && p3) { + p2++; + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p2-p1))); + if(*p2 == ':') { + *port = 0; + p2++; + while( (*p2 >= '0') && (*p2 <= '9')) { + *port *= 10; + *port += (unsigned short)(*p2 - '0'); + p2++; + } + } else { + *port = 80; + } + *path = p3; + return 1; + } + } + p2 = strchr(p1, ':'); + p3 = strchr(p1, '/'); + if(!p3) + return 0; + if(!p2 || (p2>p3)) { + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p3-p1))); + *port = 80; + } else { + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p2-p1))); + *port = 0; + p2++; + while( (*p2 >= '0') && (*p2 <= '9')) { + *port *= 10; + *port += (unsigned short)(*p2 - '0'); + p2++; + } + } + *path = p3; + return 1; +} + +static void upnpc_desc_received(struct evhttp_request * req, void * pvoid) +{ + size_t len; + unsigned char * data; + struct evbuffer * input_buffer; + struct IGDdatas igd; + struct xmlparser parser; + upnpc_device_t * d = (upnpc_device_t *)pvoid; + + if(req == NULL) { + debug_printf("%s(%p, %p) NULL argument !\n", __func__, req, pvoid); + return; + } + input_buffer = evhttp_request_get_input_buffer(req); + len = evbuffer_get_length(input_buffer); + data = evbuffer_pullup(input_buffer, len); + debug_printf("%s %d (%d bytes)\n", __func__, evhttp_request_get_response_code(req), (int)len); + if(evhttp_request_get_response_code(req) != HTTP_OK) { + d->parent->ready_cb(evhttp_request_get_response_code(req), d->parent, d, d->parent->cb_data); + return; + } + if(data == NULL) { + d->parent->ready_cb(UPNPC_ERR_ROOT_DESC_ERROR, d->parent, d, d->parent->cb_data); + return; + } + debug_printf("%.*s\n", (int)len, (char *)data); + + memset(&igd, 0, sizeof(struct IGDdatas)); + memset(&parser, 0, sizeof(struct xmlparser)); + parser.xmlstart = (char *)data; + parser.xmlsize = len; + parser.data = &igd; + parser.starteltfunc = IGDstartelt; + parser.endeltfunc = IGDendelt; + parser.datafunc = IGDdata; + parsexml(&parser); +#ifdef DEBUG + printIGD(&igd); +#endif /* DEBUG */ + d->control_conn_url = build_url_string(igd.urlbase, d->root_desc_location, igd.first.controlurl); + d->event_conn_url = build_url_string(igd.urlbase, d->root_desc_location, igd.first.eventsuburl); + d->conn_service_type = strdup(igd.first.servicetype); + d->control_cif_url = build_url_string(igd.urlbase, d->root_desc_location, igd.CIF.controlurl); + d->event_cif_url = build_url_string(igd.urlbase, d->root_desc_location, igd.CIF.eventsuburl); + d->cif_service_type = strdup(igd.CIF.servicetype); + debug_printf("control_conn_url='%s'\n (service_type='%s')\n", + d->control_conn_url, d->conn_service_type); + debug_printf("event_conn_url='%s'\n", d->event_conn_url); + debug_printf("control_cif_url='%s'\n (service_type='%s')\n", + d->control_cif_url, d->cif_service_type); + + if((d->cif_service_type == NULL) + || (d->cif_service_type[0] == '\0') + || (!COMPARE(d->cif_service_type, "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:"))) { + d->parent->ready_cb(UPNPC_ERR_NOT_IGD, d->parent, d, d->parent->cb_data); + } else { + d->state |= UPNPC_DEVICE_GETSTATUS; + upnpc_get_status_info(d); + } +} + +#ifdef ENABLE_UPNP_EVENTS +static void upnpc_subscribe_response(struct evhttp_request * req, void * pvoid) +{ + size_t len; + unsigned char * data; + struct evbuffer * input_buffer; + upnpc_device_t * d = (upnpc_device_t *)pvoid; + + if(req == NULL) { + debug_printf("%s(%p, %p) NULL argument !\n", __func__, req, pvoid); + return; + } + input_buffer = evhttp_request_get_input_buffer(req); + len = evbuffer_get_length(input_buffer); + data = evbuffer_pullup(input_buffer, len); + debug_printf("%s %d (%d bytes)\n", __func__, evhttp_request_get_response_code(req), (int)len); + d->state &= ~UPNPC_DEVICE_SOAP_REQ; + if(evhttp_request_get_response_code(req) != HTTP_OK) { + /* TODO ERROR */ + } else { + const char * sid; + struct evkeyvalq * headers = evhttp_request_get_input_headers(req); + sid = evhttp_find_header(headers, "sid"); + debug_printf("SID=%s\n", sid); + if(sid) { + if(d->event_conn_sid) + free(d->event_conn_sid); + d->event_conn_sid = strdup(sid); + } + } +} +#endif /* ENABLE_UPNP_EVENTS */ + +static void upnpc_soap_response(struct evhttp_request * req, void * pvoid) +{ + size_t len; + unsigned char * data; + struct evbuffer * input_buffer; + upnpc_device_t * d = (upnpc_device_t *)pvoid; + int code; + + if(req == NULL) { + debug_printf("%s(%p, %p) NULL argument !\n", __func__, req, pvoid); + return; + } + code = evhttp_request_get_response_code(req); + input_buffer = evhttp_request_get_input_buffer(req); + len = evbuffer_get_length(input_buffer); + data = evbuffer_pullup(input_buffer, len); + debug_printf("%s %d (%d bytes)\n", __func__, code, (int)len); + debug_printf("%.*s\n", (int)len, (char *)data); + if(data == NULL) + return; + + ClearNameValueList(&d->soap_response_data); + ParseNameValue((char *)data, (int)len, + &d->soap_response_data); + d->state &= ~UPNPC_DEVICE_SOAP_REQ; + if(d->state & UPNPC_DEVICE_READY) { + d->parent->soap_cb(code, d->parent, d, d->parent->cb_data); + } else if(d->state & UPNPC_DEVICE_GETSTATUS) { + const char * connection_status; + d->state &= ~UPNPC_DEVICE_GETSTATUS; + connection_status = GetValueFromNameValueList(&d->soap_response_data, "NewConnectionStatus"); + d->state |= UPNPC_DEVICE_READY; + if((code == 200) && connection_status && (0 == strcmp("Connected", connection_status))) { + d->parent->ready_cb(code, d->parent, d, d->parent->cb_data); + d->state |= UPNPC_DEVICE_CONNECTED; + event_del(d->parent->ev_ssdp_recv); + } else { + d->parent->ready_cb(UPNPC_ERR_NOT_CONNECTED, d->parent, d, d->parent->cb_data); + } + } +} + +static int upnpc_get_desc(upnpc_device_t * d, const char * url) +{ + char hostname[MAXHOSTNAMELEN+1]; + char hostname_port[MAXHOSTNAMELEN+1+6]; + unsigned short port; + char * path; + unsigned int scope_id; + struct evhttp_request * req; + struct evkeyvalq * headers; + + /* if(d->root_desc_location == NULL) { + return -1; + } */ + if(!parseURL(url/*d->root_desc_location*/, hostname, &port, + &path, &scope_id)) { + return -1; + } + if(port != 80) + snprintf(hostname_port, sizeof(hostname_port), "%s:%hu", hostname, port); + else + strncpy(hostname_port, hostname, sizeof(hostname_port)); + if(d->desc_conn == NULL) { + d->desc_conn = evhttp_connection_base_new(d->parent->base, NULL, hostname, port); + } +#ifdef DEBUG + evhttp_connection_set_closecb(d->desc_conn, upnpc_conn_close_cb, d); +#endif /* DEBUG */ + /*evhttp_connection_set_timeout(p->desc_conn, 600);*/ + req = evhttp_request_new(upnpc_desc_received/*callback*/, d); + headers = evhttp_request_get_output_headers(req); + evhttp_add_header(headers, "Host", hostname_port); + evhttp_add_header(headers, "Connection", "close"); + /*evhttp_add_header(headers, "User-Agent", "***");*/ + return evhttp_make_request(d->desc_conn, req, EVHTTP_REQ_GET, path); +} + +static char * build_url_string(const char * urlbase, const char * root_desc_url, const char * controlurl) +{ + int l, n; + char * s; + const char * base; + char * p; + /* if controlurl is an absolute url, return it */ + if(0 == memcmp("http://", controlurl, 7)) + return strdup(controlurl); + base = (urlbase[0] == '\0') ? root_desc_url : urlbase; + n = strlen(base); + if(n > 7) { + p = strchr(base + 7, '/'); + if(p) + n = p - base; + } + l = n + strlen(controlurl) + 1; + if(controlurl[0] != '/') + l++; + s = malloc(l); + if(s == NULL) return NULL; + memcpy(s, base, n); + if(controlurl[0] != '/') + s[n++] = '/'; + memcpy(s + n, controlurl, l - n); + return s; +} + +#define SOAPPREFIX "s" +#define SERVICEPREFIX "u" +#define SERVICEPREFIX2 'u' + +static int upnpc_send_soap_request(upnpc_device_t * p, const char * url, + const char * service, + const char * method, + const struct upnp_args * args, int arg_count) +{ + char action[128]; + char * body; + const char fmt_soap[] = + "\r\n" + "<" SOAPPREFIX ":Envelope " + "xmlns:" SOAPPREFIX "=\"http://schemas.xmlsoap.org/soap/envelope/\" " + SOAPPREFIX ":encodingStyle=\"http://schemas.xmlsoap.org/soap/encoding/\">" + "<" SOAPPREFIX ":Body>" + "<" SERVICEPREFIX ":%s xmlns:" SERVICEPREFIX "=\"%s\">" + "%s" + "" + "" + "\r\n"; + int body_len; + char hostname[MAXHOSTNAMELEN+1]; + char hostname_port[MAXHOSTNAMELEN+1+6]; + unsigned short port; + char * path; + unsigned int scope_id; + char * args_xml = NULL; + struct evhttp_request * req; + struct evkeyvalq * headers; + struct evbuffer * buffer; + + if(p->state & UPNPC_DEVICE_SOAP_REQ) { + debug_printf("%s: another SOAP request in progress\n", __func__); + return UPNPC_ERR_REQ_IN_PROGRESS; + } + + if(arg_count > 0) { + int i; + size_t l, n; + for(i = 0, l = 0; i < arg_count; i++) { + /* VAL */ + l += strlen(args[i].elt) * 2 + strlen(args[i].val) + 5; + } + args_xml = malloc(++l); + if(args_xml == NULL) { + return -1; + } + for(i = 0, n = 0; i < arg_count && n < l; i++) { + /* VAL */ + n += snprintf(args_xml + n, l - n, "<%s>%s", + args[i].elt, args[i].val, args[i].elt); + } + } + + body_len = snprintf(NULL, 0, fmt_soap, method, service, args_xml?args_xml:"", method); + body = malloc(body_len + 1); + if(body == NULL) { + free(args_xml); + return -1; + } + if(snprintf(body, body_len + 1, fmt_soap, method, service, args_xml?args_xml:"", method) != body_len) { + debug_printf("%s: snprintf() returned strange value...\n", __func__); + } + free(args_xml); + args_xml = NULL; + if(!parseURL(url, hostname, &port, &path, &scope_id)) { + free(body); + return -1; + } + if(port != 80) + snprintf(hostname_port, sizeof(hostname_port), "%s:%hu", hostname, port); + else + strncpy(hostname_port, hostname, sizeof(hostname_port)); + snprintf(action, sizeof(action), "\"%s#%s\"", service, method); + if(p->soap_conn == NULL) { + p->soap_conn = evhttp_connection_base_new(p->parent->base, NULL, hostname, port); + } + req = evhttp_request_new(upnpc_soap_response, p); + headers = evhttp_request_get_output_headers(req); + buffer = evhttp_request_get_output_buffer(req); + evhttp_add_header(headers, "Host", hostname_port); + evhttp_add_header(headers, "SOAPAction", action); + evhttp_add_header(headers, "Content-Type", "text/xml"); + /*evhttp_add_header(headers, "User-Agent", "***");*/ + /*evhttp_add_header(headers, "Cache-Control", "no-cache");*/ + /*evhttp_add_header(headers, "Pragma", "no-cache");*/ + evbuffer_add(buffer, body, body_len); + evhttp_make_request(p->soap_conn, req, EVHTTP_REQ_POST, path); + free(body); + p->state |= UPNPC_DEVICE_SOAP_REQ; + return 0; +} + +#ifdef ENABLE_UPNP_EVENTS +#define EVHTTP_REQ_NOTIFY ((EVHTTP_REQ_MAX) << 1) +#define EVHTTP_REQ_SUBSCRIBE ((EVHTTP_REQ_NOTIFY) << 1) +static const struct evhttp_extended_method ext_methods[] = { + {"NOTIFY", EVHTTP_REQ_NOTIFY, EVHTTP_METHOD_HAS_BODY}, + {"SUBSCRIBE", EVHTTP_REQ_SUBSCRIBE, EVHTTP_METHOD_HAS_BODY}, + {NULL, 0, 0} +}; + +void upnpc_event_conn_req(struct evhttp_request * req, void * data) +{ + size_t len; + char * xml_data; + struct evbuffer * input_buffer; + struct evkeyvalq * headers; + const char * sid; + const char * nts; + const char * nt; + const char * seq; + struct NameValueParserData parsed_data; + struct NameValue * nv; + upnpc_device_t * d = (upnpc_device_t *)data; + + debug_printf("%s(%p, %p)\n", __func__, req, d); + headers = evhttp_request_get_input_headers(req); + input_buffer = evhttp_request_get_input_buffer(req); + len = evbuffer_get_length(input_buffer); + sid = evhttp_find_header(headers, "sid"); + nts = evhttp_find_header(headers, "nts"); + nt = evhttp_find_header(headers, "nt"); + seq = evhttp_find_header(headers, "seq"); + if(len == 0 || nts == NULL || nt == NULL) { + /* 400 Bad request : + * The NT or NTS header field is missing + * or the request is malformed. */ + evhttp_send_reply(req, 400, "Bad Request", NULL); + return; + } + debug_printf("SID=%s NTS=%s SEQ=%s\n", sid, nts, seq); + if(sid == NULL || 0 != strcmp(sid, d->event_conn_sid) + || 0 != strcmp(nt, "upnp:event") || 0 != strcmp(nts, "upnp:propchange")) { + /* 412 Precondition Failed : + * An SID does not correspond to a known, un-expired subscription + * or the NT header field does not equal upnp:event + * or the NTS header field does not equal upnp:propchange + * or the SID header field is missing or empty. */ + evhttp_send_reply(req, 412, "Precondition Failed", NULL); + return; + } + xml_data = (char *)evbuffer_pullup(input_buffer, len); + /*debug_printf("%.*s\n", len, xml_data);*/ + ParseNameValue(xml_data, len, &parsed_data); + for(nv = parsed_data.l_head; nv != NULL; nv = nv->l_next) { + if(d->parent->value_changed_cb) { + d->parent->value_changed_cb(d->parent, d, d->parent->cb_data, d->conn_service_type, nv->name, nv->value); + } else { + debug_printf("%s=%s\n", nv->name, nv->value); + } + } + ClearNameValueList(&parsed_data); + /* response : 200 OK */ + evhttp_send_reply(req, 200, "OK", NULL); +} +#endif /* ENABLE_UPNP_EVENTS */ + +/* public functions */ +int upnpc_init(upnpc_t * p, struct event_base * base, const char * multicastif, + upnpc_callback_fn ready_cb, upnpc_callback_fn soap_cb, void * cb_data) +{ + int opt = 1; + struct sockaddr_in addr; + + if(p == NULL || base == NULL) + return UPNPC_ERR_INVALID_ARGS; + memset(p, 0, sizeof(upnpc_t)); /* clean everything */ + p->base = base; + p->ready_cb = ready_cb; + p->soap_cb = soap_cb; + p->cb_data = cb_data; + p->ttl = 2; + /* open the socket for SSDP */ + p->ssdp_socket = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP); + if(p->ssdp_socket < 0) { + return UPNPC_ERR_SOCKET_FAILED; + } + /* set multicast TTL */ + if(setsockopt(p->ssdp_socket, IPPROTO_IP, IP_MULTICAST_TTL, &p->ttl, sizeof(p->ttl)) < 0) + { + /* not a fatal error */ + debug_printf("setsockopt(%d, ..., IP_MULTICAST_TTL, ...) FAILED\n", p->ssdp_socket); + } + /* set REUSEADDR */ +#ifdef _WIN32 + if(setsockopt(p->ssdp_socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&opt, sizeof(opt)) < 0) { +#else /* _WIN32 */ + if(setsockopt(p->ssdp_socket, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)) < 0) { +#endif /* _WIN32 */ + /* non fatal error ! */ + debug_printf("setsockopt(%d, SOL_SOCKET, SO_REUSEADDR, ...) FAILED\n", p->ssdp_socket); + } + if(evutil_make_socket_nonblocking(p->ssdp_socket) < 0) { + debug_printf("evutil_make_socket_nonblocking FAILED\n"); + } + + /* receive address */ + memset(&addr, 0, sizeof(struct sockaddr_in)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + /*addr.sin_port = htons(SSDP_PORT);*/ + + if(multicastif) { + struct in_addr mc_if; + mc_if.s_addr = inet_addr(multicastif); + addr.sin_addr.s_addr = mc_if.s_addr; + if(setsockopt(p->ssdp_socket, IPPROTO_IP, IP_MULTICAST_IF, (const char *)&mc_if, sizeof(mc_if)) < 0) { + PRINT_SOCKET_ERROR("setsockopt"); + /* non fatal error ! */ + } + } + + /* bind the socket to the ssdp address in order to receive responses */ + if(bind(p->ssdp_socket, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) != 0) { + close(p->ssdp_socket); + return UPNPC_ERR_BIND_FAILED; + } + return UPNPC_OK; +} + +int upnpc_start(upnpc_t * p) +{ + struct timeval timeout; + if(p == NULL || p->base == NULL) + return UPNPC_ERR_INVALID_ARGS; + /* event on SSDP */ + p->ev_ssdp_recv = event_new(p->base, p->ssdp_socket, + EV_READ|EV_PERSIST, + (event_callback_fn)upnpc_receive_and_parse_ssdp, p); + timeout.tv_sec = 3; + timeout.tv_usec = 0; + if(event_add(p->ev_ssdp_recv, &timeout)) { + debug_printf("event_add FAILED\n"); + } + p->ev_ssdp_writable = event_new(p->base, p->ssdp_socket, + EV_WRITE, + (event_callback_fn)upnpc_send_ssdp_msearch, p); + if(event_add(p->ev_ssdp_writable, NULL)) { + debug_printf("event_add FAILED\n"); + } + return UPNPC_OK; +} + +int upnpc_set_local_address(upnpc_t * p, const char * address, uint16_t port) +{ + if(!p || !address) return UPNPC_ERR_INVALID_ARGS; + p->local_address = strdup(address); /* TODO check error */ + p->local_port = port; + return UPNPC_OK; +} + +#ifdef ENABLE_UPNP_EVENTS +int upnpc_set_event_callback(upnpc_t * p, upnpc_event_callback_fn cb) +{ + if(!p || !cb) return UPNPC_ERR_INVALID_ARGS; + p->value_changed_cb = cb; + return UPNPC_OK; +} +#endif /* ENABLE_UPNP_EVENTS */ + +static void upnpc_device_finalize(upnpc_device_t * d) +{ + d->state = 0; + free(d->root_desc_location); + d->root_desc_location = NULL; + free(d->control_cif_url); + d->control_cif_url = NULL; + free(d->event_cif_url); + d->event_cif_url = NULL; + free(d->cif_service_type); + d->cif_service_type = NULL; + free(d->control_conn_url); + d->control_conn_url = NULL; + free(d->event_conn_url); + d->event_conn_url = NULL; + free(d->conn_service_type); + d->conn_service_type = NULL; + if(d->desc_conn) { + evhttp_connection_free(d->desc_conn); + d->desc_conn = NULL; + } + if(d->soap_conn) { + evhttp_connection_free(d->soap_conn); + d->soap_conn = NULL; + } + ClearNameValueList(&d->soap_response_data); +#ifdef ENABLE_UPNP_EVENTS + free(d->event_conn_sid); + d->event_conn_sid = NULL; +#endif /* ENABLE_UPNP_EVENTS */ +} + +int upnpc_finalize(upnpc_t * p) +{ + if(!p) return UPNPC_ERR_INVALID_ARGS; + p->discover_device_index = 0; + if(p->ssdp_socket >= 0) { + close(p->ssdp_socket); + p->ssdp_socket = -1; + } + if(p->ev_ssdp_recv) { + event_free(p->ev_ssdp_recv); + p->ev_ssdp_recv = NULL; + } + if(p->ev_ssdp_writable) { + event_free(p->ev_ssdp_writable); + p->ev_ssdp_writable = NULL; + } + while(p->devices != NULL) { + upnpc_device_t * d = p->devices; + upnpc_device_finalize(d); + p->devices = d->next; + free(d); + } + free(p->local_address); + p->local_address = NULL; +#ifdef ENABLE_UPNP_EVENTS + if(p->http_server) { + evhttp_free(p->http_server); + p->http_server = NULL; + } +#endif /* ENABLE_UPNP_EVENTS */ + return UPNPC_OK; +} + +#ifdef ENABLE_UPNP_EVENTS +int upnpc_event_subscribe(upnpc_device_t * p) +{ + char hostname[MAXHOSTNAMELEN+1]; + char hostname_port[MAXHOSTNAMELEN+1+6]; + unsigned short port; + char * path; + unsigned int scope_id; + struct evhttp_request * req; + struct evkeyvalq * headers; + char callback_header[7+15+1+5+9+2+1]; + + if(p->parent->http_server == NULL) { + /* HTTP server to receive event notifications */ + p->parent->http_server = evhttp_new(p->parent->base); + if(p->parent->http_server == NULL) { + debug_printf("evhttp_new() FAILED\n"); + return -1; + } + evhttp_set_extended_methods(p->parent->http_server, ext_methods); + evhttp_set_allowed_methods(p->parent->http_server, EVHTTP_REQ_NOTIFY); + evhttp_set_cb(p->parent->http_server, "/evt_conn", upnpc_event_conn_req, p); + if(evhttp_bind_socket(p->parent->http_server, p->parent->local_address, p->parent->local_port) < 0) { + debug_printf("evhttp_bind_socket() FAILED\n"); + return -1; + } + } + /*if(!parseURL(p->event_cif_url, hostname, &port, &path, &scope_id)) {*/ + if(!parseURL(p->event_conn_url, hostname, &port, &path, &scope_id)) { + return -1; + } + if(port != 80) + snprintf(hostname_port, sizeof(hostname_port), "%s:%hu", hostname, port); + else + strncpy(hostname_port, hostname, sizeof(hostname_port)); + if(p->soap_conn == NULL) { + p->soap_conn = evhttp_connection_base_new(p->parent->base, NULL, hostname, port); + } + evhttp_connection_set_extended_methods(p->soap_conn, ext_methods); + req = evhttp_request_new(upnpc_subscribe_response, p); + headers = evhttp_request_get_output_headers(req); + /*buffer = evhttp_request_get_output_buffer(req);*/ + evhttp_add_header(headers, "Host", hostname_port); + /*evhttp_add_header(headers, "User-Agent", "***");*/ + snprintf(callback_header, sizeof(callback_header), "", p->parent->local_address, p->parent->local_port); + evhttp_add_header(headers, "Callback", callback_header); + evhttp_add_header(headers, "NT", "upnp:event"); + /*evhttp_add_header(headers, "NTS", "");*/ + evhttp_add_header(headers, "Timeout", "3600"); + /*evbuffer_add(buffer, body, body_len);*/ + evhttp_make_request(p->soap_conn, req, EVHTTP_REQ_SUBSCRIBE, path); + p->state |= UPNPC_DEVICE_SOAP_REQ; + return 0; +} +#endif /* ENABLE_UPNP_EVENTS */ + +int upnpc_get_external_ip_address(upnpc_device_t * p) +{ + return upnpc_send_soap_request(p, p->control_conn_url, + p->conn_service_type/*"urn:schemas-upnp-org:service:WANIPConnection:1"*/, + "GetExternalIPAddress", NULL, 0); +} + +int upnpc_get_link_layer_max_rate(upnpc_device_t * p) +{ + return upnpc_send_soap_request(p, p->control_cif_url, + p->cif_service_type/*"urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1"*/, + "GetCommonLinkProperties", NULL, 0); +} + +int upnpc_delete_port_mapping(upnpc_device_t * p, + const char * remote_host, unsigned short ext_port, + const char * proto) +{ + struct upnp_args args[3]; + char ext_port_str[8]; + + if(proto == NULL || ext_port == 0) + return UPNPC_ERR_INVALID_ARGS; + snprintf(ext_port_str, sizeof(ext_port_str), "%hu", ext_port); + args[0].elt = "NewRemoteHost"; + args[0].val = remote_host?remote_host:""; + args[1].elt = "NewExternalPort"; + args[1].val = ext_port_str; + args[2].elt = "NewProtocol"; + args[2].val = proto; + return upnpc_send_soap_request(p, p->control_conn_url, + p->conn_service_type,/*"urn:schemas-upnp-org:service:WANIPConnection:1",*/ + "DeletePortMapping", + args, 3); +} + +int upnpc_add_port_mapping(upnpc_device_t * p, + const char * remote_host, unsigned short ext_port, + unsigned short int_port, const char * int_client, + const char * proto, const char * description, + unsigned int lease_duration) +{ + struct upnp_args args[8]; + char lease_duration_str[16]; + char int_port_str[8]; + char ext_port_str[8]; + + if(int_client == NULL || int_port == 0 || ext_port == 0 || proto == NULL) + return UPNPC_ERR_INVALID_ARGS; + snprintf(lease_duration_str, sizeof(lease_duration_str), "%u", lease_duration); + snprintf(int_port_str, sizeof(int_port_str), "%hu", int_port); + snprintf(ext_port_str, sizeof(ext_port_str), "%hu", ext_port); + args[0].elt = "NewRemoteHost"; + args[0].val = remote_host?remote_host:""; + args[1].elt = "NewExternalPort"; + args[1].val = ext_port_str; + args[2].elt = "NewProtocol"; + args[2].val = proto; + args[3].elt = "NewInternalPort"; + args[3].val = int_port_str; + args[4].elt = "NewInternalClient"; + args[4].val = int_client; + args[5].elt = "NewEnabled"; + args[5].val = "1"; + args[6].elt = "NewPortMappingDescription"; + args[6].val = description?description:"miniupnpc-libevent"; + args[7].elt = "NewLeaseDuration"; + args[7].val = lease_duration_str; + return upnpc_send_soap_request(p, p->control_conn_url, + p->conn_service_type/*"urn:schemas-upnp-org:service:WANIPConnection:1"*/, + "AddPortMapping", + args, 8); +} + +int upnpc_get_status_info(upnpc_device_t * p) +{ + return upnpc_send_soap_request(p, p->control_conn_url, + p->conn_service_type/*"urn:schemas-upnp-org:service:WANIPConnection:1"*/, + "GetStatusInfo", NULL, 0); +} diff --git a/src/contrib/miniupnp/miniupnpc-libevent/miniupnpc-libevent.h b/src/contrib/miniupnp/miniupnpc-libevent/miniupnpc-libevent.h new file mode 100644 index 0000000..47bc896 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/miniupnpc-libevent.h @@ -0,0 +1,132 @@ +/* $Id: miniupnpc-libevent.h,v 1.13 2015/07/22 13:48:37 nanard Exp $ */ +/* miniupnpc-libevent + * Copyright (c) 2008-2015, Thomas BERNARD + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#ifndef MINIUPNPC_LIBEVENT_H_INCLUDED +#define MINIUPNPC_LIBEVENT_H_INCLUDED + +#include + +#include "declspec.h" +#include "upnpreplyparse.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MINIUPNPC_LIBEVENT_API_VERSION 1 + +#define UPNPC_OK 0 +#define UPNPC_ERR_INVALID_ARGS (-1) +#define UPNPC_ERR_SOCKET_FAILED (-2) +#define UPNPC_ERR_BIND_FAILED (-3) +#define UPNPC_ERR_REQ_IN_PROGRESS (-4) + +#define UPNPC_ERR_NO_DEVICE_FOUND (-100) +#define UPNPC_ERR_ROOT_DESC_ERROR (-101) +#define UPNPC_ERR_NOT_IGD (-102) +#define UPNPC_ERR_NOT_CONNECTED (-103) + +/* device->state masks */ +#define UPNPC_DEVICE_SOAP_REQ (0x0001) +#define UPNPC_DEVICE_GETSTATUS (0x0002) +#define UPNPC_DEVICE_CONNECTED (0x4000) +#define UPNPC_DEVICE_READY (0x8000) + +typedef struct upnpc_device upnpc_device_t; +typedef struct upnpc upnpc_t; + +typedef void(* upnpc_callback_fn)(int, upnpc_t *, upnpc_device_t *, void *); +#ifdef ENABLE_UPNP_EVENTS +typedef void(* upnpc_event_callback_fn)(upnpc_t *, upnpc_device_t *, void *, const char *, const char *, const char *); +#endif /* ENABLE_UPNP_EVENTS */ + +struct upnpc_device { + upnpc_t * parent; + upnpc_device_t * next; + char * root_desc_location; + struct evhttp_connection * desc_conn; + char * control_cif_url; + char * event_cif_url; + char * cif_service_type; + char * control_conn_url; + char * event_conn_url; + char * conn_service_type; + struct evhttp_connection * soap_conn; + struct NameValueParserData soap_response_data; + unsigned int state; +#ifdef ENABLE_UPNP_EVENTS + char * event_conn_sid; +#endif /* ENABLE_UPNP_EVENTS */ +}; + +struct upnpc { + struct event_base * base; + evutil_socket_t ssdp_socket; + struct event * ev_ssdp_recv; + struct event * ev_ssdp_writable; + int discover_device_index; + upnpc_device_t * devices; + upnpc_callback_fn ready_cb; + upnpc_callback_fn soap_cb; + void * cb_data; +#ifdef ENABLE_UPNP_EVENTS + struct evhttp * http_server; + upnpc_event_callback_fn value_changed_cb; +#endif /* ENABLE_UPNP_EVENTS */ + char * local_address; + uint16_t local_port; + unsigned char ttl; +}; + +int upnpc_init(upnpc_t * p, struct event_base * base, const char * multicastif, + upnpc_callback_fn ready_cb, upnpc_callback_fn soap_cb, void * cb_data); + +int upnpc_set_local_address(upnpc_t * p, const char * address, uint16_t port); + +#ifdef ENABLE_UPNP_EVENTS +int upnpc_set_event_callback(upnpc_t * p, upnpc_event_callback_fn cb); +#endif /* ENABLE_UPNP_EVENTS */ + +int upnpc_start(upnpc_t * p); + +int upnpc_finalize(upnpc_t * p); + +#ifdef ENABLE_UPNP_EVENTS +int upnpc_event_subscribe(upnpc_device_t * p); +#endif /* ENABLE_UPNP_EVENTS */ + +int upnpc_get_external_ip_address(upnpc_device_t * p); + +int upnpc_get_link_layer_max_rate(upnpc_device_t * p); + +int upnpc_add_port_mapping(upnpc_device_t * p, + const char * remote_host, unsigned short ext_port, + unsigned short int_port, const char * int_client, + const char * proto, const char * description, + unsigned int lease_duration); + +int upnpc_delete_port_mapping(upnpc_device_t * p, + const char * remote_host, unsigned short ext_port, + const char * proto); + +int upnpc_get_status_info(upnpc_device_t * p); + +#ifdef __cplusplus +} +#endif + +#endif /* MINIUPNPC_LIBEVENT_H_INCLUDED */ + diff --git a/src/contrib/miniupnp/miniupnpc-libevent/minixml.c b/src/contrib/miniupnp/miniupnpc-libevent/minixml.c new file mode 100644 index 0000000..3e201ec --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/minixml.c @@ -0,0 +1,229 @@ +/* $Id: minixml.c,v 1.11 2014/02/03 15:54:12 nanard Exp $ */ +/* minixml.c : the minimum size a xml parser can be ! */ +/* Project : miniupnp + * webpage: http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author : Thomas Bernard + +Copyright (c) 2005-2014, Thomas BERNARD +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include "minixml.h" + +/* parseatt : used to parse the argument list + * return 0 (false) in case of success and -1 (true) if the end + * of the xmlbuffer is reached. */ +static int parseatt(struct xmlparser * p) +{ + const char * attname; + int attnamelen; + const char * attvalue; + int attvaluelen; + while(p->xml < p->xmlend) + { + if(*p->xml=='/' || *p->xml=='>') + return 0; + if( !IS_WHITE_SPACE(*p->xml) ) + { + char sep; + attname = p->xml; + attnamelen = 0; + while(*p->xml!='=' && !IS_WHITE_SPACE(*p->xml) ) + { + attnamelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + while(*(p->xml++) != '=') + { + if(p->xml >= p->xmlend) + return -1; + } + while(IS_WHITE_SPACE(*p->xml)) + { + p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + sep = *p->xml; + if(sep=='\'' || sep=='\"') + { + p->xml++; + if(p->xml >= p->xmlend) + return -1; + attvalue = p->xml; + attvaluelen = 0; + while(*p->xml != sep) + { + attvaluelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + } + else + { + attvalue = p->xml; + attvaluelen = 0; + while( !IS_WHITE_SPACE(*p->xml) + && *p->xml != '>' && *p->xml != '/') + { + attvaluelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + } + /*printf("%.*s='%.*s'\n", + attnamelen, attname, attvaluelen, attvalue);*/ + if(p->attfunc) + p->attfunc(p->data, attname, attnamelen, attvalue, attvaluelen); + } + p->xml++; + } + return -1; +} + +/* parseelt parse the xml stream and + * call the callback functions when needed... */ +static void parseelt(struct xmlparser * p) +{ + int i; + const char * elementname; + while(p->xml < (p->xmlend - 1)) + { + if((p->xml + 4) <= p->xmlend && (0 == memcmp(p->xml, "", 3) != 0); + p->xml += 3; + } + else if((p->xml)[0]=='<' && (p->xml)[1]!='?') + { + i = 0; elementname = ++p->xml; + while( !IS_WHITE_SPACE(*p->xml) + && (*p->xml!='>') && (*p->xml!='/') + ) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + /* to ignore namespace : */ + if(*p->xml==':') + { + i = 0; + elementname = ++p->xml; + } + } + if(i>0) + { + if(p->starteltfunc) + p->starteltfunc(p->data, elementname, i); + if(parseatt(p)) + return; + if(*p->xml!='/') + { + const char * data; + i = 0; data = ++p->xml; + if (p->xml >= p->xmlend) + return; + while( IS_WHITE_SPACE(*p->xml) ) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + } + if(memcmp(p->xml, "xml += 9; + data = p->xml; + i = 0; + while(memcmp(p->xml, "]]>", 3) != 0) + { + i++; p->xml++; + if ((p->xml + 3) >= p->xmlend) + return; + } + if(i>0 && p->datafunc) + p->datafunc(p->data, data, i); + while(*p->xml!='<') + { + p->xml++; + if (p->xml >= p->xmlend) + return; + } + } + else + { + while(*p->xml!='<') + { + i++; p->xml++; + if ((p->xml + 1) >= p->xmlend) + return; + } + if(i>0 && p->datafunc && *(p->xml + 1) == '/') + p->datafunc(p->data, data, i); + } + } + } + else if(*p->xml == '/') + { + i = 0; elementname = ++p->xml; + if (p->xml >= p->xmlend) + return; + while((*p->xml != '>')) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + } + if(p->endeltfunc) + p->endeltfunc(p->data, elementname, i); + p->xml++; + } + } + else + { + p->xml++; + } + } +} + +/* the parser must be initialized before calling this function */ +void parsexml(struct xmlparser * parser) +{ + parser->xml = parser->xmlstart; + parser->xmlend = parser->xmlstart + parser->xmlsize; + parseelt(parser); +} + + diff --git a/src/contrib/miniupnp/miniupnpc-libevent/minixml.h b/src/contrib/miniupnp/miniupnpc-libevent/minixml.h new file mode 100644 index 0000000..9f43aa4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/minixml.h @@ -0,0 +1,37 @@ +/* $Id: minixml.h,v 1.7 2012/09/27 15:42:10 nanard Exp $ */ +/* minimal xml parser + * + * Project : miniupnp + * Website : http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef MINIXML_H_INCLUDED +#define MINIXML_H_INCLUDED +#define IS_WHITE_SPACE(c) ((c==' ') || (c=='\t') || (c=='\r') || (c=='\n')) + +/* if a callback function pointer is set to NULL, + * the function is not called */ +struct xmlparser { + const char *xmlstart; + const char *xmlend; + const char *xml; /* pointer to current character */ + int xmlsize; + void * data; + void (*starteltfunc) (void *, const char *, int); + void (*endeltfunc) (void *, const char *, int); + void (*datafunc) (void *, const char *, int); + void (*attfunc) (void *, const char *, int, const char *, int); +}; + +/* parsexml() + * the xmlparser structure must be initialized before the call + * the following structure members have to be initialized : + * xmlstart, xmlsize, data, *func + * xml is for internal usage, xmlend is computed automatically */ +void parsexml(struct xmlparser *); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc-libevent/upnpc-libevent.c b/src/contrib/miniupnp/miniupnpc-libevent/upnpc-libevent.c new file mode 100644 index 0000000..ce6ec4a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/upnpc-libevent.c @@ -0,0 +1,252 @@ +/* $Id: upnpc-libevent.c,v 1.11 2014/12/02 13:33:42 nanard Exp $ */ +/* miniupnpc-libevent + * Copyright (c) 2008-2014, Thomas BERNARD + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + +#include +#include +#include +#include +#include +#include +#include + +#include "miniupnpc-libevent.h" + +static struct event_base *base = NULL; +static char local_address[32]; + +static void sighandler(int signal) +{ + (void)signal; + /*printf("signal %d\n", signal);*/ + if(base != NULL) + event_base_loopbreak(base); +} + +/* ready callback */ +static void ready(int code, upnpc_t * p, upnpc_device_t * d, void * data) +{ + (void)data; (void)p; + + if(code == 200) { + printf("READY ! %d\n", code); + printf(" root_desc_location='%s'\n", d->root_desc_location); + /* 1st request */ +#ifdef ENABLE_UPNP_EVENTS + upnpc_event_subscribe(d); +#else + upnpc_get_status_info(d); +#endif /* ENABLE_UPNP_EVENTS */ + } else { + printf("DISCOVER ERROR : %d\n", code); + switch(code) { + case UPNPC_ERR_NO_DEVICE_FOUND: + printf("UPNPC_ERR_NO_DEVICE_FOUND\n"); + break; + case UPNPC_ERR_ROOT_DESC_ERROR: + printf("UPNPC_ERR_ROOT_DESC_ERROR\n"); + break; + case 404: + printf("Root desc not found (404)\n"); + break; + default: + printf("unknown error\n"); + } + } +} + +static enum { + EGetStatusInfo = 0, + EGetExtIp, + EGetMaxRate, + EAddPortMapping, + EDeletePortMapping, + EFinished + } state = EGetStatusInfo; + +/* soap callback */ +static void soap(int code, upnpc_t * p, upnpc_device_t * d, void * data) +{ + (void)data; (void)p; + + printf("SOAP ! %d\n", code); + if(code == 200) { + switch(state) { + case EGetStatusInfo: + printf("ConnectionStatus=%s\n", GetValueFromNameValueList(&d->soap_response_data, "NewConnectionStatus")); + printf("LastConnectionError=%s\n", GetValueFromNameValueList(&d->soap_response_data, "NewLastConnectionError")); + printf("Uptime=%s\n", GetValueFromNameValueList(&d->soap_response_data, "NewUptime")); + upnpc_get_external_ip_address(d); + state = EGetExtIp; + break; + case EGetExtIp: + printf("ExternalIpAddress=%s\n", GetValueFromNameValueList(&d->soap_response_data, "NewExternalIPAddress")); + upnpc_get_link_layer_max_rate(d); + state = EGetMaxRate; + break; + case EGetMaxRate: + printf("DownStream MaxBitRate = %s\t", GetValueFromNameValueList(&d->soap_response_data, "NewLayer1DownstreamMaxBitRate")); + upnpc_add_port_mapping(d, NULL, 60001, 60002, local_address, "TCP", "test port mapping", 0); + printf("UpStream MaxBitRate = %s\n", GetValueFromNameValueList(&d->soap_response_data, "NewLayer1UpstreamMaxBitRate")); + state = EAddPortMapping; + break; + case EAddPortMapping: + printf("AddPortMapping OK!\n"); + upnpc_delete_port_mapping(d, NULL, 60001, "TCP"); + state = EDeletePortMapping; + break; + case EDeletePortMapping: + printf("DeletePortMapping OK!\n"); + state = EFinished; + break; + default: + printf("EFinished : breaking\n"); + event_base_loopbreak(base); + } + } else { + printf("SOAP error :\n"); + printf(" faultcode='%s'\n", GetValueFromNameValueList(&d->soap_response_data, "faultcode")); + printf(" faultstring='%s'\n", GetValueFromNameValueList(&d->soap_response_data, "faultstring")); + printf(" errorCode=%s\n", GetValueFromNameValueList(&d->soap_response_data, "errorCode")); + printf(" errorDescription='%s'\n", GetValueFromNameValueList(&d->soap_response_data, "errorDescription")); + event_base_loopbreak(base); + } +} + +#ifdef ENABLE_UPNP_EVENTS +/* event callback */ +static void event_callback(upnpc_t * p, upnpc_device_t * d, void * data, + const char * service_id, const char * property_name, const char * property_value) +{ + (void)p; (void)d; (void)data; + printf("PROPERTY VALUE CHANGE (service=%s): %s=%s\n", service_id, property_name, property_value); +} +#endif /* ENABLE_UPNP_EVENTS */ + +/* use a UDP "connection" to 8.8.8.8 + * to retrieve local address */ +int find_local_address(void) +{ + int s; + struct sockaddr_in local, remote; + socklen_t len; + + s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if(s < 0) { + perror("socket"); + return -1; + } + + memset(&local, 0, sizeof(local)); + memset(&remote, 0, sizeof(remote)); + /* bind to local port 4567 */ + local.sin_family = AF_INET; + local.sin_port = htons(4567); + local.sin_addr.s_addr = htonl(INADDR_ANY); + if(bind(s, (struct sockaddr *)&local, sizeof(local)) < 0) { + perror("bind"); + return -1; + } + /* "connect" google's DNS server at 8.8.8.8 port 4567 */ + remote.sin_family = AF_INET; + remote.sin_port = htons(4567); + remote.sin_addr.s_addr = inet_addr("8.8.8.8"); + if(connect(s, (struct sockaddr *)&remote, sizeof(remote)) < 0) { + perror("connect"); + return -1; + } + len = sizeof(local); + if(getsockname(s, (struct sockaddr *)&local, &len) < 0) { + perror("getsockname"); + return -1; + } + if(inet_ntop(AF_INET, &(local.sin_addr), local_address, sizeof(local_address)) == NULL) { + perror("inet_ntop"); + return -1; + } + printf("local address : %s\n", local_address); + close(s); + return 0; +} + +/* program entry point */ + +int main(int argc, char * * argv) +{ + struct sigaction sa; + upnpc_t upnp; + char * multicast_if = NULL; + + if(argc > 1) { + multicast_if = argv[1]; + } + + memset(&sa, 0, sizeof(struct sigaction)); + sa.sa_handler = sighandler; + if(sigaction(SIGINT, &sa, NULL) < 0) { + perror("sigaction"); + } + + if(find_local_address() < 0) { + fprintf(stderr, "failed to get local address\n"); + return 1; + } +#ifdef DEBUG + event_enable_debug_mode(); +#if LIBEVENT_VERSION_NUMBER >= 0x02010100 + event_enable_debug_logging(EVENT_DBG_ALL); /* Libevent 2.1.1 */ +#endif /* LIBEVENT_VERSION_NUMBER >= 0x02010100 */ +#endif /* DEBUG */ + printf("Using libevent %s\n", event_get_version()); + if(LIBEVENT_VERSION_NUMBER != event_get_version_number()) { + fprintf(stderr, "WARNING build using libevent %s", LIBEVENT_VERSION); + } + + base = event_base_new(); + if(base == NULL) { + fprintf(stderr, "event_base_new() failed\n"); + return 1; + } +#ifdef DEBUG + printf("Using Libevent with backend method %s.\n", + event_base_get_method(base)); +#endif /* DEBUG */ + + if(upnpc_init(&upnp, base, multicast_if, ready, soap, &upnp) != UPNPC_OK) { + fprintf(stderr, "upnpc_init() failed\n"); + return 1; + } + upnpc_set_local_address(&upnp, local_address, 50000); +#ifdef ENABLE_UPNP_EVENTS + upnpc_set_event_callback(&upnp, event_callback); +#endif /* ENABLE_UPNP_EVENTS */ + if(upnpc_start(&upnp) != UPNPC_OK) { + fprintf(stderr, "upnp_start() failed\n"); + return 1; + } + + event_base_dispatch(base); /* TODO : check return value */ + printf("finishing...\n"); + + upnpc_finalize(&upnp); + event_base_free(base); + +#if LIBEVENT_VERSION_NUMBER >= 0x02010100 + libevent_global_shutdown(); /* Libevent 2.1.1 */ +#endif + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpc-libevent/upnpreplyparse.c b/src/contrib/miniupnp/miniupnpc-libevent/upnpreplyparse.c new file mode 100644 index 0000000..5de5796 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/upnpreplyparse.c @@ -0,0 +1,197 @@ +/* $Id: upnpreplyparse.c,v 1.19 2015/07/15 10:29:11 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2015 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include + +#include "upnpreplyparse.h" +#include "minixml.h" + +static void +NameValueParserStartElt(void * d, const char * name, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + data->topelt = 1; + if(l>63) + l = 63; + memcpy(data->curelt, name, l); + data->curelt[l] = '\0'; + data->cdata = NULL; + data->cdatalen = 0; +} + +static void +NameValueParserEndElt(void * d, const char * name, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + struct NameValue * nv; + (void)name; + (void)l; + if(!data->topelt) + return; + if(strcmp(data->curelt, "NewPortListing") != 0) + { + int l; + /* standard case. Limited to n chars strings */ + l = data->cdatalen; + nv = malloc(sizeof(struct NameValue)); + if(nv == NULL) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "NameValueParserEndElt"); +#endif /* DEBUG */ + return; + } + if(l>=(int)sizeof(nv->value)) + l = sizeof(nv->value) - 1; + strncpy(nv->name, data->curelt, 64); + nv->name[63] = '\0'; + if(data->cdata != NULL) + { + memcpy(nv->value, data->cdata, l); + nv->value[l] = '\0'; + } + else + { + nv->value[0] = '\0'; + } + nv->l_next = data->l_head; /* insert in list */ + data->l_head = nv; + } + data->cdata = NULL; + data->cdatalen = 0; + data->topelt = 0; +} + +static void +NameValueParserGetData(void * d, const char * datas, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + if(strcmp(data->curelt, "NewPortListing") == 0) + { + /* specific case for NewPortListing which is a XML Document */ + data->portListing = malloc(l + 1); + if(!data->portListing) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "NameValueParserGetData"); +#endif /* DEBUG */ + return; + } + memcpy(data->portListing, datas, l); + data->portListing[l] = '\0'; + data->portListingLength = l; + } + else + { + /* standard case. */ + data->cdata = datas; + data->cdatalen = l; + } +} + +void +ParseNameValue(const char * buffer, int bufsize, + struct NameValueParserData * data) +{ + struct xmlparser parser; + data->l_head = NULL; + data->portListing = NULL; + data->portListingLength = 0; + /* init xmlparser object */ + parser.xmlstart = buffer; + parser.xmlsize = bufsize; + parser.data = data; + parser.starteltfunc = NameValueParserStartElt; + parser.endeltfunc = NameValueParserEndElt; + parser.datafunc = NameValueParserGetData; + parser.attfunc = 0; + parsexml(&parser); +} + +void +ClearNameValueList(struct NameValueParserData * pdata) +{ + struct NameValue * nv; + if(pdata->portListing) + { + free(pdata->portListing); + pdata->portListing = NULL; + pdata->portListingLength = 0; + } + while((nv = pdata->l_head) != NULL) + { + pdata->l_head = nv->l_next; + free(nv); + } +} + +char * +GetValueFromNameValueList(struct NameValueParserData * pdata, + const char * Name) +{ + struct NameValue * nv; + char * p = NULL; + for(nv = pdata->l_head; + (nv != NULL) && (p == NULL); + nv = nv->l_next) + { + if(strcmp(nv->name, Name) == 0) + p = nv->value; + } + return p; +} + +#if 0 +/* useless now that minixml ignores namespaces by itself */ +char * +GetValueFromNameValueListIgnoreNS(struct NameValueParserData * pdata, + const char * Name) +{ + struct NameValue * nv; + char * p = NULL; + char * pname; + for(nv = pdata->head.lh_first; + (nv != NULL) && (p == NULL); + nv = nv->entries.le_next) + { + pname = strrchr(nv->name, ':'); + if(pname) + pname++; + else + pname = nv->name; + if(strcmp(pname, Name)==0) + p = nv->value; + } + return p; +} +#endif + +/* debug all-in-one function + * do parsing then display to stdout */ +#ifdef DEBUG +void +DisplayNameValueList(char * buffer, int bufsize) +{ + struct NameValueParserData pdata; + struct NameValue * nv; + ParseNameValue(buffer, bufsize, &pdata); + for(nv = pdata.l_head; + nv != NULL; + nv = nv->l_next) + { + printf("%s = %s\n", nv->name, nv->value); + } + ClearNameValueList(&pdata); +} +#endif /* DEBUG */ + diff --git a/src/contrib/miniupnp/miniupnpc-libevent/upnpreplyparse.h b/src/contrib/miniupnp/miniupnpc-libevent/upnpreplyparse.h new file mode 100644 index 0000000..6badd15 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc-libevent/upnpreplyparse.h @@ -0,0 +1,63 @@ +/* $Id: upnpreplyparse.h,v 1.19 2014/10/27 16:33:19 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPREPLYPARSE_H_INCLUDED +#define UPNPREPLYPARSE_H_INCLUDED + +#ifdef __cplusplus +extern "C" { +#endif + +struct NameValue { + struct NameValue * l_next; + char name[64]; + char value[128]; +}; + +struct NameValueParserData { + struct NameValue * l_head; + char curelt[64]; + char * portListing; + int portListingLength; + int topelt; + const char * cdata; + int cdatalen; +}; + +/* ParseNameValue() */ +void +ParseNameValue(const char * buffer, int bufsize, + struct NameValueParserData * data); + +/* ClearNameValueList() */ +void +ClearNameValueList(struct NameValueParserData * pdata); + +/* GetValueFromNameValueList() */ +char * +GetValueFromNameValueList(struct NameValueParserData * pdata, + const char * Name); + +#if 0 +/* GetValueFromNameValueListIgnoreNS() */ +char * +GetValueFromNameValueListIgnoreNS(struct NameValueParserData * pdata, + const char * Name); +#endif + +/* DisplayNameValueList() */ +#ifdef DEBUG +void +DisplayNameValueList(char * buffer, int bufsize); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/.gitignore b/src/contrib/miniupnp/miniupnpc/.gitignore new file mode 100644 index 0000000..ba06bcc --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/.gitignore @@ -0,0 +1,37 @@ +deb_dist/ +build/ +*.o +*.a +*.so +*.dll +*.dll.def +*.exe +*.lib +*.dylib +Makefile.bak +miniupnpcstrings.h +pythonmodule +pythonmodule3 +upnpc-shared +upnpc-static +minihttptestserver +minixmlvalid +testminiwget +validateminiwget +validateminixml +java/miniupnpc_*.jar +_jnaerator.* +out.errors.txt +jnaerator-*.jar +miniupnpc.h.bak +testupnpreplyparse +validateupnpreplyparse +testportlistingparse +validateportlistingparse +listdevices +testigddescparse +validateigddescparse +dist/ +miniupnpc.egg-info/ +init +miniupnpc.pc diff --git a/src/contrib/miniupnp/miniupnpc/CMakeLists.txt b/src/contrib/miniupnp/miniupnpc/CMakeLists.txt new file mode 100644 index 0000000..c1061b1 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/CMakeLists.txt @@ -0,0 +1,190 @@ +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +project (miniupnpc C) + +set (MINIUPNPC_VERSION 2.1) +set (MINIUPNPC_API_VERSION 17) + +option (UPNPC_BUILD_STATIC "Build static library" TRUE) +option (UPNPC_BUILD_SHARED "Build shared library" TRUE) +option (UPNPC_BUILD_TESTS "Build test executables" TRUE) +option (UPNPC_BUILD_SAMPLE "Build sample executables" TRUE) +option (NO_GETADDRINFO "Define NO_GETADDRINFO" FALSE) + +if (NOT UPNPC_BUILD_STATIC AND NOT UPNPC_BUILD_SHARED) + message (FATAL "Both shared and static libraries are disabled!") +endif () + +# Interface library for compile definitions, flags and option +add_library(miniupnpc-private INTERFACE) + +if (NO_GETADDRINFO) + target_compile_definitions(miniupnpc-private INTERFACE NO_GETADDRINFO) +endif () + +if (NOT WIN32) + target_compile_definitions(miniupnpc-private INTERFACE + MINIUPNPC_SET_SOCKET_TIMEOUT + _BSD_SOURCE _DEFAULT_SOURCE) + if (NOT CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + # add_definitions (-D_POSIX_C_SOURCE=200112L) + target_compile_definitions(miniupnpc-private INTERFACE _XOPEN_SOURCE=600) + endif () +else () + target_compile_definitions(miniupnpc-private INTERFACE _WIN32_WINNT=0x0501) # XP or higher for getnameinfo and friends +endif () + +if (CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_compile_definitions(miniupnpc-private INTERFACE _DARWIN_C_SOURCE) +endif () + +# Set compiler specific build flags +if (CMAKE_COMPILER_IS_GNUCC AND NOT CMAKE_SYSTEM_NAME STREQUAL "AmigaOS") + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + target_compile_options(miniupnpc-private INTERFACE -Wall) +endif () + +# Suppress noise warnings +if (MSVC) + target_compile_definitions(miniupnpc-private INTERFACE _CRT_SECURE_NO_WARNINGS) +endif() + +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/miniupnpcstrings.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/miniupnpcstrings.h) +target_include_directories(miniupnpc-private INTERFACE ${CMAKE_CURRENT_BINARY_DIR}) + +set (MINIUPNPC_SOURCES + igd_desc_parse.c + miniupnpc.c + minixml.c + minisoap.c + minissdpc.c + miniwget.c + upnpcommands.c + upnpdev.c + upnpreplyparse.c + upnperrors.c + connecthostport.c + portlistingparse.c + receivedata.c + connecthostport.h + igd_desc_parse.h + minisoap.h + minissdpc.h + miniupnpc.h + miniupnpctypes.h + miniwget.h + minixml.h + portlistingparse.h + receivedata.h + upnpcommands.h + upnpdev.h + upnperrors.h + upnpreplyparse.h + ${CMAKE_CURRENT_BINARY_DIR}/miniupnpcstrings.h +) + +if (NOT WIN32 AND NOT CMAKE_SYSTEM_NAME STREQUAL "AmigaOS") + set (MINIUPNPC_SOURCES ${MINIUPNPC_SOURCES} minissdpc.c) +endif () + +if (WIN32) + target_link_libraries(miniupnpc-private INTERFACE ws2_32 iphlpapi) +#elseif (CMAKE_SYSTEM_NAME STREQUAL "Solaris") +# find_library (SOCKET_LIBRARY NAMES socket) +# find_library (NSL_LIBRARY NAMES nsl) +# find_library (RESOLV_LIBRARY NAMES resolv) +# set (LDLIBS ${SOCKET_LIBRARY} ${NSL_LIBRARY} ${RESOLV_LIBRARY} ${LDLIBS}) +endif () + + +if (UPNPC_BUILD_STATIC) + add_library (libminiupnpc-static STATIC ${MINIUPNPC_SOURCES}) + set_target_properties (libminiupnpc-static PROPERTIES OUTPUT_NAME "miniupnpc") + target_link_libraries (libminiupnpc-static PRIVATE miniupnpc-private) + target_include_directories(libminiupnpc-static INTERFACE ../${CMAKE_CURRENT_SOURCE_DIR}) + target_compile_definitions(libminiupnpc-static PUBLIC MINIUPNP_STATICLIB) + + install (TARGETS libminiupnpc-static + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib${LIB_SUFFIX} + ARCHIVE DESTINATION lib${LIB_SUFFIX}) + + if (UPNPC_BUILD_SAMPLE) + add_executable (upnpc-static upnpc.c) + target_link_libraries (upnpc-static PRIVATE libminiupnpc-static) + target_include_directories(upnpc-static PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + endif () +endif () + +if (UPNPC_BUILD_SHARED) + add_library (libminiupnpc-shared SHARED ${MINIUPNPC_SOURCES}) + set_target_properties (libminiupnpc-shared PROPERTIES OUTPUT_NAME "miniupnpc") + set_target_properties (libminiupnpc-shared PROPERTIES VERSION ${MINIUPNPC_VERSION}) + set_target_properties (libminiupnpc-shared PROPERTIES SOVERSION ${MINIUPNPC_API_VERSION}) + target_link_libraries (libminiupnpc-shared PRIVATE miniupnpc-private) + target_compile_definitions(libminiupnpc-shared PRIVATE MINIUPNP_EXPORTS) + + target_include_directories(libminiupnpc-static INTERFACE ../${CMAKE_CURRENT_SOURCE_DIR}) + if (WIN32) + target_link_libraries(libminiupnpc-shared INTERFACE ws2_32 iphlpapi) + endif() + + install (TARGETS libminiupnpc-shared + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib${LIB_SUFFIX} + ARCHIVE DESTINATION lib${LIB_SUFFIX}) + + if (UPNPC_BUILD_SAMPLE) + add_executable (upnpc-shared upnpc.c) + target_link_libraries (upnpc-shared PRIVATE libminiupnpc-shared) + target_include_directories(upnpc-shared PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + endif () +endif () + +if (UPNPC_BUILD_TESTS) + add_library(miniupnpc-tests INTERFACE) + target_link_libraries(miniupnpc-tests INTERFACE miniupnpc-private) + target_compile_definitions(miniupnpc-tests INTERFACE MINIUPNP_STATICLIB) + + add_executable (testminixml testminixml.c minixml.c igd_desc_parse.c) + target_link_libraries (testminixml PRIVATE miniupnpc-tests) + + add_executable (minixmlvalid minixmlvalid.c minixml.c) + target_link_libraries (minixmlvalid PRIVATE miniupnpc-tests) + + add_executable (testupnpreplyparse testupnpreplyparse.c + minixml.c upnpreplyparse.c) + target_link_libraries (testupnpreplyparse PRIVATE miniupnpc-tests) + + add_executable (testigddescparse testigddescparse.c + igd_desc_parse.c minixml.c miniupnpc.c miniwget.c minissdpc.c + upnpcommands.c upnpreplyparse.c minisoap.c connecthostport.c + portlistingparse.c receivedata.c + ) + target_link_libraries (testigddescparse PRIVATE miniupnpc-tests) + + add_executable (testminiwget testminiwget.c + miniwget.c miniupnpc.c minisoap.c upnpcommands.c minissdpc.c + upnpreplyparse.c minixml.c igd_desc_parse.c connecthostport.c + portlistingparse.c receivedata.c + ) + target_link_libraries (testminiwget PRIVATE miniupnpc-tests) + +# set (UPNPC_INSTALL_TARGETS ${UPNPC_INSTALL_TARGETS} testminixml minixmlvalid testupnpreplyparse testigddescparse testminiwget) +endif () + +install (FILES + miniupnpc.h + miniwget.h + upnpcommands.h + igd_desc_parse.h + upnpreplyparse.h + upnperrors.h + upnpdev.h + miniupnpctypes.h + portlistingparse.h + miniupnpc_declspec.h + DESTINATION include/miniupnpc +) + +# vim: ts=2:sw=2 diff --git a/src/contrib/miniupnp/miniupnpc/Changelog.txt b/src/contrib/miniupnp/miniupnpc/Changelog.txt new file mode 100644 index 0000000..bb2c042 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/Changelog.txt @@ -0,0 +1,711 @@ +$Id: Changelog.txt,v 1.235 2018/05/07 11:05:16 nanard Exp $ +miniUPnP client Changelog. + +VERSION 2.1 : released 2018/05/07 + +2018/05/07: + CMake Modernize and cleanup CMakeLists.txt + Update MS Visual Studio projects + +2018/04/30: + listdevices: show devices sorted by XML desc URL + +2018/04/26: + Small fix in miniupnpcmodule.c (python module) + Support cross compiling in Makefile.mingw + +2018/04/06: + Use SOCKET type instead of int (for Win64 compilation) + Increments API_VERSION to 17 + +2018/02/22: + Disable usage of MiniSSDPd when using -m option + +2017/12/11: + Fix buffer over run in minixml.c + Fix uninitialized variable access in upnpreplyparse.c + +2017/05/05: + Fix CVE-2017-8798 Thanks to tin/Team OSTStrom + +2016/11/11: + check strlen before memcmp in XML parsing portlistingparse.c + fix build under SOLARIS and CYGWIN + +2016/10/11: + Add python 3 compatibility to IGD test + +VERSION 2.0 : released 2016/04/19 + +2016/01/24: + change miniwget to return HTTP status code + increments API_VERSION to 16 + +2016/01/22: + Improve UPNPIGD_IsConnected() to check if WAN address is not private. + parse HTTP response status line in miniwget.c + +2015/10/26: + snprintf() overflow check. check overflow in simpleUPnPcommand2() + +2015/10/25: + fix compilation with old macs + fix compilation with mingw32 (for Appveyor) + fix python module for python <= 2.3 + +2015/10/08: + Change sameport to localport + see https://github.com/miniupnp/miniupnp/pull/120 + increments API_VERSION to 15 + +2015/09/15: + Fix buffer overflow in igd_desc_parse.c/IGDstartelt() + Discovered by Aleksandar Nikolic of Cisco Talos + +2015/08/28: + move ssdpDiscoverDevices() to minissdpc.c + +2015/08/27: + avoid unix socket leak in getDevicesFromMiniSSDPD() + +2015/08/16: + Also accept "Up" as ConnectionStatus value + +2015/07/23: + split getDevicesFromMiniSSDPD + add ttl argument to upnpDiscover() functions + increments API_VERSION to 14 + +2015/07/22: + Read USN from SSDP messages. + +2015/07/15: + Check malloc/calloc + +2015/06/16: + update getDevicesFromMiniSSDPD() to process longer minissdpd + responses + +2015/05/22: + add searchalltypes param to upnpDiscoverDevices() + increments API_VERSION to 13 + +2015/04/30: + upnpc: output version on the terminal + +2015/04/27: + _BSD_SOURCE is deprecated in favor of _DEFAULT_SOURCE + fix CMakeLists.txt COMPILE_DEFINITIONS + fix getDevicesFromMiniSSDPD() not setting scope_id + improve -r command of upnpc command line tool + +2014/11/17: + search all : + upnpDiscoverDevices() / upnpDiscoverAll() functions + listdevices executable + increment API_VERSION to 12 + validate igd_desc_parse + +2014/11/13: + increment API_VERSION to 11 + +2014/11/05: + simplified function GetUPNPUrls() + +2014/09/11: + use remoteHost arg of DeletePortMapping + +2014/09/06: + Fix python3 build + +2014/07/01: + Fix parsing of IGD2 root descriptions + +2014/06/10: + rename LIBSPEC to MINIUPNP_LIBSPEC + +2014/05/15: + Add support for IGD2 AddAnyPortMapping and DeletePortMappingRange + +2014/02/05: + handle EINPROGRESS after connect() + +2014/02/03: + minixml now handle XML comments + +VERSION 1.9 : released 2014/01/31 + +2014/01/31: + added argument remoteHost to UPNP_GetSpecificPortMappingEntry() + increment API_VERSION to 10 + +2013/12/09: + --help and -h arguments in upnpc.c + +2013/10/07: + fixed potential buffer overrun in miniwget.c + Modified UPNP_GetValidIGD() to check for ExternalIpAddress + +2013/08/01: + define MAXHOSTNAMELEN if not already done + +2013/06/06: + update upnpreplyparse to allow larger values (128 chars instead of 64) + +2013/05/14: + Update upnpreplyparse to take into account "empty" elements + validate upnpreplyparse.c code with "make check" + +2013/05/03: + Fix Solaris build thanks to Maciej MaÅ‚ecki + +2013/04/27: + Fix testminiwget.sh for BSD + +2013/03/23: + Fixed Makefile for *BSD + +2013/03/11: + Update Makefile to use JNAerator version 0.11 + +2013/02/11: + Fix testminiwget.sh for use with dash + Use $(DESTDIR) in Makefile + +VERSION 1.8 : released 2013/02/06 + +2012/10/16: + fix testminiwget with no IPv6 support + +2012/09/27: + Rename all include guards to not clash with C99 + (7.1.3 Reserved identifiers). + +2012/08/30: + Added -e option to upnpc program (set description for port mappings) + +2012/08/29: + Python 3 support (thanks to Christopher Foo) + +2012/08/11: + Fix a memory link in UPNP_GetValidIGD() + Try to handle scope id in link local IPv6 URL under MS Windows + +2012/07/20: + Disable HAS_IP_MREQN on DragonFly BSD + +2012/06/28: + GetUPNPUrls() now inserts scope into link-local IPv6 addresses + +2012/06/23: + More error return checks in upnpc.c + #define MINIUPNPC_GET_SRC_ADDR enables receivedata() to get scope_id + parseURL() now parses IPv6 addresses scope + new parameter for miniwget() : IPv6 address scope + increment API_VERSION to 9 + +2012/06/20: + fixed CMakeLists.txt + +2012/05/29 + Improvements in testminiwget.sh + +VERSION 1.7 : released 2012/05/24 + +2012/05/01: + Cleanup settings of CFLAGS in Makefile + Fix signed/unsigned integer comparaisons + +2012/04/20: + Allow to specify protocol with TCP or UDP for -A option + +2012/04/09: + Only try to fetch XML description once in UPNP_GetValidIGD() + Added -ansi flag to compilation, and fixed C++ comments to ANSI C comments. + +2012/04/05: + minor improvements to minihttptestserver.c + +2012/03/15: + upnperrors.c returns valid error string for unrecognized error codes + +2012/03/08: + make minihttptestserver listen on loopback interface instead of 0.0.0.0 + +2012/01/25: + Maven installation thanks to Alexey Kuznetsov + +2012/01/21: + Replace WIN32 macro by _WIN32 + +2012/01/19: + Fixes in java wrappers thanks to Alexey Kuznetsov : + https://github.com/axet/miniupnp/tree/fix-javatest/miniupnpc + Make and install .deb packages (python) thanks to Alexey Kuznetsov : + https://github.com/axet/miniupnp/tree/feature-debbuild/miniupnpc + +2012/01/07: + The multicast interface can now be specified by name with IPv4. + +2012/01/02: + Install man page + +2011/11/25: + added header to Port Mappings list in upnpc.c + +2011/10/09: + Makefile : make clean now removes jnaerator generated files. + MINIUPNPC_VERSION in miniupnpc.h (updated by make) + +2011/09/12: + added rootdescURL to UPNPUrls structure. + +VERSION 1.6 : released 2011/07/25 + +2011/07/25: + Update doc for version 1.6 release + +2011/06/18: + Fix for windows in miniwget.c + +2011/06/04: + display remote host in port mapping listing + +2011/06/03: + Fix in make install : there were missing headers + +2011/05/26: + Fix the socket leak in miniwget thanks to Richard Marsh. + Permit to add leaseduration in -a command. Display lease duration. + +2011/05/15: + Try both LinkLocal and SiteLocal multicast address for SSDP in IPv6 + +2011/05/09: + add a test in testminiwget.sh. + more error checking in miniwget.c + +2011/05/06: + Adding some tool to test and validate miniwget.c + simplified and debugged miniwget.c + +2011/04/11: + moving ReceiveData() to a receivedata.c file. + parsing presentation url + adding IGD v2 WANIPv6FirewallControl commands + +2011/04/10: + update of miniupnpcmodule.c + comments in miniwget.c, update in testminiwget + Adding errors codes from IGD v2 + new functions in upnpc.c for IGD v2 + +2011/04/09: + Support for litteral ip v6 address in miniwget + +2011/04/08: + Adding support for urn:schemas-upnp-org:service:WANIPv6FirewallControl:1 + Updating APIVERSION + Supporting IPV6 in upnpDiscover() + Adding a -6 option to upnpc command line tool + +2011/03/18: + miniwget/parseURL() : return an error when url param is null. + fixing GetListOfPortMappings() + +2011/03/14: + upnpDiscover() now reporting an error code. + improvements in comments. + +2011/03/11: + adding miniupnpcstrings.h.cmake and CMakeLists.txt files. + +2011/02/15: + Implementation of GetListOfPortMappings() + +2011/02/07: + updates to minixml to support character data starting with spaces + minixml now support CDATA + upnpreplyparse treats specificaly + change in simpleUPnPcommand to return the buffer (simplification) + +2011/02/06: + Added leaseDuration argument to AddPortMapping() + Starting to implement GetListOfPortMappings() + +2011/01/11: + updating wingenminiupnpcstrings.c + +2011/01/04: + improving updateminiupnpcstrings.sh + +VERSION 1.5 : released 2011/01/01 + +2010/12/21: + use NO_GETADDRINFO macro to disable the use of getaddrinfo/freeaddrinfo + +2010/12/11: + Improvements on getHTTPResponse() code. + +2010/12/09: + new code for miniwget that handle Chunked transfer encoding + using getHTTPResponse() in SOAP call code + Adding MANIFEST.in for 'python setup.py bdist_rpm' + +2010/11/25: + changes to minissdpc.c to compile under Win32. + see http://miniupnp.tuxfamily.org/forum/viewtopic.php?t=729 + +2010/09/17: + Various improvement to Makefile from MichaÅ‚ Górny + +2010/08/05: + Adding the script "external-ip.sh" from Reuben Hawkins + +2010/06/09: + update to python module to match modification made on 2010/04/05 + update to Java test code to match modification made on 2010/04/05 + all UPNP_* function now return an error if the SOAP request failed + at HTTP level. + +2010/04/17: + Using GetBestRoute() under win32 in order to find the + right interface to use. + +2010/04/12: + Retrying with HTTP/1.1 if HTTP/1.0 failed. see + http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=1703 + +2010/04/07: + avoid returning duplicates in upnpDiscover() + +2010/04/05: + Create a connecthostport.h/.c with connecthostport() function + and use it in miniwget and miniupnpc. + Use getnameinfo() instead of inet_ntop or inet_ntoa + Work to make miniupnpc IPV6 compatible... + Add java test code. + Big changes in order to support device having both WANIPConnection + and WANPPPConnection. + +2010/04/04: + Use getaddrinfo() instead of gethostbyname() in miniwget. + +2010/01/06: + #define _DARWIN_C_SOURCE for Mac OS X + +2009/12/19: + Improve MinGW32 build + +2009/12/11: + adding a MSVC9 project to build the static library and executable + +2009/12/10: + Fixing some compilation stuff for Windows/MinGW + +2009/12/07: + adaptations in Makefile and updateminiupnpcstring.sh for AmigaOS + some fixes for Windows when using virtual ethernet adapters (it is the + case with VMWare installed). + +2009/12/04: + some fixes for AmigaOS compilation + Changed HTTP version to HTTP/1.0 for Soap too (to prevent chunked + transfer encoding) + +2009/12/03: + updating printIDG and testigddescparse.c for debug. + modifications to compile under AmigaOS + adding a testminiwget program + Changed miniwget to advertise itself as HTTP/1.0 to prevent chunked + transfer encoding + +2009/11/26: + fixing updateminiupnpcstrings.sh to take into account + which command that does not return an error code. + +VERSION 1.4 : released 2009/10/30 + +2009/10/16: + using Py_BEGIN_ALLOW_THREADS and Py_END_ALLOW_THREADS in python module. + +2009/10/10: + Some fixes for compilation under Solaris + compilation fixes : http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=1464 + +2009/09/21: + fixing the code to ignore EINTR during connect() calls. + +2009/08/07: + Set socket timeout for connect() + Some cleanup in miniwget.c + +2009/08/04: + remove multiple redirections with -d in upnpc.c + Print textual error code in upnpc.c + Ignore EINTR during the connect() and poll() calls. + +2009/07/29: + fix in updateminiupnpcstrings.sh if OS name contains "/" + Sending a correct value for MX: field in SSDP request + +2009/07/20: + Change the Makefile to compile under Mac OS X + Fixed a stackoverflow in getDevicesFromMiniSSDPD() + +2009/07/09: + Compile under Haiku + generate miniupnpcstrings.h.in from miniupnpcstrings.h + +2009/06/04: + patching to compile under CygWin and cross compile for minGW + +VERSION 1.3 : + +2009/04/17: + updating python module + Use strtoull() when using C99 + +2009/02/28: + Fixed miniwget.c for compiling under sun + +2008/12/18: + cleanup in Makefile (thanks to Paul de Weerd) + minissdpc.c : win32 compatibility + miniupnpc.c : changed xmlns prefix from 'm' to 'u' + Removed NDEBUG (using DEBUG) + +2008/10/14: + Added the ExternalHost argument to DeletePortMapping() + +2008/10/11: + Added the ExternalHost argument to AddPortMapping() + Put a correct User-Agent: header in HTTP requests. + +VERSION 1.2 : + +2008/10/07: + Update docs + +2008/09/25: + Integrated sameport patch from Dario Meloni : Added a "sameport" + argument to upnpDiscover(). + +2008/07/18: + small modif to make Clang happy :) + +2008/07/17: + #define SOAPPREFIX "s" in miniupnpc.c in order to remove SOAP-ENV... + +2008/07/14: + include declspec.h in installation (to /usr/include/miniupnpc) + +VERSION 1.1 : + +2008/07/04: + standard options for install/ln instead of gnu-specific stuff. + +2008/07/03: + now builds a .dll and .lib with win32. (mingw32) + +2008/04/28: + make install now install the binary of the upnpc tool + +2008/04/27: + added testupnpigd.py + added error strings for miniupnpc "internal" errors + improved python module error/exception reporting. + +2008/04/23: + Completely rewrite igd_desc_parse.c in order to be compatible with + Linksys WAG200G + Added testigddescparse + updated python module + +VERSION 1.0 : + +2008/02/21: + put some #ifdef DEBUG around DisplayNameValueList() + +2008/02/18: + Improved error reporting in upnpcommands.c + UPNP_GetStatusInfo() returns LastConnectionError + +2008/02/16: + better error handling in minisoap.c + improving display of "valid IGD found" in upnpc.c + +2008/02/03: + Fixing UPNP_GetValidIGD() + improved make install :) + +2007/12/22: + Adding upnperrors.c/h to provide a strupnperror() function + used to translate UPnP error codes to string. + +2007/12/19: + Fixing getDevicesFromMiniSSDPD() + improved error reporting of UPnP functions + +2007/12/18: + It is now possible to specify a different location for MiniSSDPd socket. + working with MiniSSDPd is now more efficient. + python module improved. + +2007/12/16: + improving error reporting + +2007/12/13: + Try to improve compatibility by using HTTP/1.0 instead of 1.1 and + XML a bit different for SOAP. + +2007/11/25: + fixed select() call for linux + +2007/11/15: + Added -fPIC to CFLAG for better shared library code. + +2007/11/02: + Fixed a potential socket leak in miniwget2() + +2007/10/16: + added a parameter to upnpDiscover() in order to allow the use of another + interface than the default multicast interface. + +2007/10/12: + Fixed the creation of symbolic link in Makefile + +2007/10/08: + Added man page + +2007/10/02: + fixed memory bug in GetUPNPUrls() + +2007/10/01: + fixes in the Makefile + Added UPNP_GetIGDFromUrl() and adapted the sample program accordingly. + Added SONAME in the shared library to please debian :) + fixed MS Windows compilation (minissdpd is not available under MS Windows). + +2007/09/25: + small change to Makefile to be able to install in a different location + (default is /usr) + +2007/09/24: + now compiling both shared and static library + +2007/09/19: + Cosmetic changes on upnpc.c + +2007/09/02: + adapting to new miniSSDPd (release version ?) + +2007/08/31: + Usage of miniSSDPd to skip discovery process. + +2007/08/27: + fixed python module to allow compilation with Python older than Python 2.4 + +2007/06/12: + Added a python module. + +2007/05/19: + Fixed compilation under MinGW + +2007/05/15: + fixed a memory leak in AddPortMapping() + Added testupnpreplyparse executable to check the parsing of + upnp soap messages + minixml now ignore namespace prefixes. + +2007/04/26: + upnpc now displays external ip address with -s or -l + +2007/04/11: + changed MINIUPNPC_URL_MAXSIZE to 128 to accommodate the "BT Voyager 210" + +2007/03/19: + cleanup in miniwget.c + +2007/03/01: + Small typo fix... + +2007/01/30: + Now parsing the HTTP header from SOAP responses in order to + get content-length value. + +2007/01/29: + Fixed the Soap Query to speedup the HTTP request. + added some Win32 DLL stuff... + +2007/01/27: + Fixed some WIN32 compatibility issues + +2006/12/14: + Added UPNPIGD_IsConnected() function in miniupnp.c/.h + Added UPNP_GetValidIGD() in miniupnp.c/.h + cleaned upnpc.c main(). now using UPNP_GetValidIGD() + +2006/12/07: + Version 1.0-RC1 released + +2006/12/03: + Minor changes to compile under SunOS/Solaris + +2006/11/30: + made a minixml parser validator program + updated minixml to handle attributes correctly + +2006/11/22: + Added a -r option to the upnpc sample thanks to Alexander Hubmann. + +2006/11/19: + Cleanup code to make it more ANSI C compliant + +2006/11/10: + detect and display local lan address. + +2006/11/04: + Packets and Bytes Sent/Received are now unsigned int. + +2006/11/01: + Bug fix thanks to Giuseppe D'Angelo + +2006/10/31: + C++ compatibility for .h files. + Added a way to get ip Address on the LAN used to reach the IGD. + +2006/10/25: + Added M-SEARCH to the services in the discovery process. + +2006/10/22: + updated the Makefile to use makedepend, added a "make install" + update Makefile + +2006/10/20: + fixing the description url parsing thanks to patch sent by + Wayne Dawe. + Fixed/translated some comments. + Implemented a better discover process, first looking + for IGD then for root devices (as some devices only reply to + M-SEARCH for root devices). + +2006/09/02: + added freeUPNPDevlist() function. + +2006/08/04: + More command line arguments checking + +2006/08/01: + Added the .bat file to compile under Win32 with minGW32 + +2006/07/31: + Fixed the rootdesc parser (igd_desc_parse.c) + +2006/07/20: + parseMSEARCHReply() is now returning the ST: line as well + starting changes to detect several UPnP devices on the network + +2006/07/19: + using GetCommonLinkProperties to get down/upload bitrate + diff --git a/src/contrib/miniupnp/miniupnpc/LICENSE b/src/contrib/miniupnp/miniupnpc/LICENSE new file mode 100644 index 0000000..0816733 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/LICENSE @@ -0,0 +1,27 @@ +MiniUPnPc +Copyright (c) 2005-2016, Thomas BERNARD +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/contrib/miniupnp/miniupnpc/MANIFEST.in b/src/contrib/miniupnp/miniupnpc/MANIFEST.in new file mode 100644 index 0000000..a0ebe2a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/MANIFEST.in @@ -0,0 +1,9 @@ +include README +include VERSION +include LICENSE +include miniupnpcmodule.c +include setup.py +include Makefile +include *.[ch] +include *.h.in +include *.sh diff --git a/src/contrib/miniupnp/miniupnpc/Makefile b/src/contrib/miniupnp/miniupnpc/Makefile new file mode 100644 index 0000000..0102f3b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/Makefile @@ -0,0 +1,404 @@ +# $Id: Makefile,v 1.134 2016/10/07 09:04:36 nanard Exp $ +# MiniUPnP Project +# http://miniupnp.free.fr/ +# http://miniupnp.tuxfamily.org/ +# https://github.com/miniupnp/miniupnp +# (c) 2005-2018 Thomas Bernard +# to install use : +# $ make DESTDIR=/tmp/dummylocation install +# or +# $ INSTALLPREFIX=/usr/local make install +# or +# $ make install (default INSTALLPREFIX is /usr) +OS = $(shell $(CC) -dumpmachine) +VERSION = $(shell cat VERSION) + +ifneq (, $(findstring darwin, $(OS))) +JARSUFFIX=mac +LIBTOOL ?= $(shell which libtool) +endif +ifneq (, $(findstring linux, $(OS))) +JARSUFFIX=linux +endif +ifneq (, $(findstring mingw, $(OS))$(findstring cygwin, $(OS))$(findstring msys, $(OS))) +JARSUFFIX=win32 +endif + +HAVE_IPV6 ?= yes +export HAVE_IPV6 + +CC ?= gcc +#AR = gar +#CFLAGS = -O -g -DDEBUG +CFLAGS ?= -O +CFLAGS += -Wall +CFLAGS += -W -Wstrict-prototypes +CFLAGS += -fno-common +CFLAGS += -DMINIUPNPC_SET_SOCKET_TIMEOUT +CFLAGS += -DMINIUPNPC_GET_SRC_ADDR +CFLAGS += -D_BSD_SOURCE +CFLAGS += -D_DEFAULT_SOURCE +ifneq (, $(findstring netbsd, $(OS))) +CFLAGS += -D_NETBSD_SOURCE +endif +ifeq (, $(findstring freebsd, $(OS))$(findstring darwin, $(OS))) +#CFLAGS += -D_POSIX_C_SOURCE=200112L +CFLAGS += -D_XOPEN_SOURCE=600 +endif +#CFLAGS += -ansi +#CFLAGS += -DNO_GETADDRINFO +INSTALL = install +SH = /bin/sh +JAVA = java +# see http://code.google.com/p/jnaerator/ +#JNAERATOR = jnaerator-0.9.7.jar +#JNAERATOR = jnaerator-0.9.8-shaded.jar +#JNAERATORARGS = -library miniupnpc +#JNAERATOR = jnaerator-0.10-shaded.jar +#JNAERATOR = jnaerator-0.11-shaded.jar +# https://repo1.maven.org/maven2/com/nativelibs4java/jnaerator/0.12/jnaerator-0.12-shaded.jar +JNAERATOR = jnaerator-0.12-shaded.jar +JNAERATORARGS = -mode StandaloneJar -runtime JNAerator -library miniupnpc +#JNAERATORBASEURL = http://jnaerator.googlecode.com/files/ +JNAERATORBASEURL = https://repo1.maven.org/maven2/com/nativelibs4java/jnaerator/0.12 + +ifneq (, $(findstring sun, $(OS))) + LDLIBS=-lsocket -lnsl -lresolv + CFLAGS += -D__EXTENSIONS__ + CFLAGS += -std=c99 +endif + +# APIVERSION is used to build SONAME +APIVERSION = 17 + +SRCS = igd_desc_parse.c miniupnpc.c minixml.c minisoap.c miniwget.c \ + upnpc.c upnpcommands.c upnpreplyparse.c testminixml.c \ + minixmlvalid.c testupnpreplyparse.c minissdpc.c \ + upnperrors.c testigddescparse.c testminiwget.c \ + connecthostport.c portlistingparse.c receivedata.c \ + upnpdev.c testportlistingparse.c miniupnpcmodule.c \ + minihttptestserver.c \ + listdevices.c + +LIBOBJS = miniwget.o minixml.o igd_desc_parse.o minisoap.o \ + miniupnpc.o upnpreplyparse.o upnpcommands.o upnperrors.o \ + connecthostport.o portlistingparse.o receivedata.o upnpdev.o + +ifeq (, $(findstring amiga, $(OS))) +ifeq (, $(findstring mingw, $(OS))$(findstring cygwin, $(OS))$(findstring msys, $(OS))) +CFLAGS := -fPIC $(CFLAGS) +endif +LIBOBJS := $(LIBOBJS) minissdpc.o +endif + +OBJS = $(patsubst %.c,%.o,$(SRCS)) + +# HEADERS to install +HEADERS = miniupnpc.h miniwget.h upnpcommands.h igd_desc_parse.h \ + upnpreplyparse.h upnperrors.h miniupnpctypes.h \ + portlistingparse.h \ + upnpdev.h \ + miniupnpc_declspec.h + +# library names +LIBRARY = libminiupnpc.a +ifneq (, $(findstring darwin, $(OS))) + SHAREDLIBRARY = libminiupnpc.dylib + SONAME = $(basename $(SHAREDLIBRARY)).$(APIVERSION).dylib + CFLAGS := -D_DARWIN_C_SOURCE $(CFLAGS) +else +ifeq ($(JARSUFFIX), win32) + SHAREDLIBRARY = miniupnpc.dll +else + # Linux/BSD/etc. + SHAREDLIBRARY = libminiupnpc.so + SONAME = $(SHAREDLIBRARY).$(APIVERSION) +endif +endif + +EXECUTABLES = upnpc-static listdevices +EXECUTABLES_ADDTESTS = testminixml minixmlvalid testupnpreplyparse \ + testigddescparse testminiwget testportlistingparse + +TESTMINIXMLOBJS = minixml.o igd_desc_parse.o testminixml.o + +TESTMINIWGETOBJS = miniwget.o testminiwget.o connecthostport.o receivedata.o + +TESTUPNPREPLYPARSE = testupnpreplyparse.o minixml.o upnpreplyparse.o + +TESTPORTLISTINGPARSE = testportlistingparse.o minixml.o portlistingparse.o + +TESTIGDDESCPARSE = testigddescparse.o igd_desc_parse.o minixml.o \ + miniupnpc.o miniwget.o upnpcommands.o upnpreplyparse.o \ + minisoap.o connecthostport.o receivedata.o \ + portlistingparse.o + +ifeq (, $(findstring amiga, $(OS))) +EXECUTABLES := $(EXECUTABLES) upnpc-shared +TESTMINIWGETOBJS := $(TESTMINIWGETOBJS) minissdpc.o +TESTIGDDESCPARSE := $(TESTIGDDESCPARSE) minissdpc.o +endif + +LIBDIR ?= lib +# install directories +ifeq ($(strip $(PREFIX)),) +INSTALLPREFIX ?= /usr +else +INSTALLPREFIX ?= $(PREFIX) +endif +INSTALLDIRINC = $(INSTALLPREFIX)/include/miniupnpc +INSTALLDIRLIB = $(INSTALLPREFIX)/$(LIBDIR) +INSTALLDIRBIN = $(INSTALLPREFIX)/bin +INSTALLDIRMAN = $(INSTALLPREFIX)/share/man +PKGCONFIGDIR = $(INSTALLDIRLIB)/pkgconfig + +FILESTOINSTALL = $(LIBRARY) $(EXECUTABLES) +ifeq (, $(findstring amiga, $(OS))) +FILESTOINSTALL := $(FILESTOINSTALL) $(SHAREDLIBRARY) miniupnpc.pc +endif + + +.PHONY: install clean depend all check test everything \ + installpythonmodule updateversion +# validateminixml validateminiwget + +all: $(LIBRARY) $(EXECUTABLES) + +test: check + +check: validateminixml validateminiwget validateupnpreplyparse \ + validateportlistingparse validateigddescparse + +everything: all $(EXECUTABLES_ADDTESTS) + +pythonmodule: $(LIBRARY) miniupnpcmodule.c setup.py + MAKE=$(MAKE) python setup.py build + touch $@ + +installpythonmodule: pythonmodule + MAKE=$(MAKE) python setup.py install + +pythonmodule3: $(LIBRARY) miniupnpcmodule.c setup.py + MAKE=$(MAKE) python3 setup.py build + touch $@ + +installpythonmodule3: pythonmodule3 + MAKE=$(MAKE) python3 setup.py install + +validateminixml: minixmlvalid + @echo "minixml validation test" + ./minixmlvalid + touch $@ + +validateminiwget: testminiwget minihttptestserver testminiwget.sh + @echo "miniwget validation test" + ./testminiwget.sh + touch $@ + +validateupnpreplyparse: testupnpreplyparse testupnpreplyparse.sh + @echo "upnpreplyparse validation test" + ./testupnpreplyparse.sh + touch $@ + +validateportlistingparse: testportlistingparse + @echo "portlistingparse validation test" + ./testportlistingparse + touch $@ + +validateigddescparse: testigddescparse + @echo "igd desc parse validation test" + ./testigddescparse testdesc/new_LiveBox_desc.xml testdesc/new_LiveBox_desc.values + ./testigddescparse testdesc/linksys_WAG200G_desc.xml testdesc/linksys_WAG200G_desc.values + touch $@ + +clean: + $(RM) $(LIBRARY) $(SHAREDLIBRARY) $(EXECUTABLES) $(OBJS) miniupnpcstrings.h + $(RM) $(EXECUTABLES_ADDTESTS) + # clean python stuff + $(RM) pythonmodule pythonmodule3 + $(RM) validateminixml validateminiwget validateupnpreplyparse + $(RM) validateigddescparse + $(RM) minihttptestserver + $(RM) -r build/ dist/ + #python setup.py clean + # clean jnaerator stuff + $(RM) _jnaerator.* java/miniupnpc_$(OS).jar + +distclean: clean + $(RM) $(JNAERATOR) java/*.jar java/*.class out.errors.txt + +updateversion: miniupnpc.h + cp miniupnpc.h miniupnpc.h.bak + sed 's/\(.*MINIUPNPC_API_VERSION\s\+\)[0-9]\+/\1$(APIVERSION)/' < miniupnpc.h.bak > miniupnpc.h + +install: updateversion $(FILESTOINSTALL) + $(INSTALL) -d $(DESTDIR)$(INSTALLDIRINC) + $(INSTALL) -m 644 $(HEADERS) $(DESTDIR)$(INSTALLDIRINC) + $(INSTALL) -d $(DESTDIR)$(INSTALLDIRLIB) + $(INSTALL) -m 644 $(LIBRARY) $(DESTDIR)$(INSTALLDIRLIB) +ifeq (, $(findstring amiga, $(OS))) + $(INSTALL) -m 644 $(SHAREDLIBRARY) $(DESTDIR)$(INSTALLDIRLIB)/$(SONAME) + ln -fs $(SONAME) $(DESTDIR)$(INSTALLDIRLIB)/$(SHAREDLIBRARY) + $(INSTALL) -d $(DESTDIR)$(PKGCONFIGDIR) + $(INSTALL) -m 644 miniupnpc.pc $(DESTDIR)$(PKGCONFIGDIR) +endif + $(INSTALL) -d $(DESTDIR)$(INSTALLDIRBIN) +ifneq (, $(findstring amiga, $(OS))) + $(INSTALL) -m 755 upnpc-static $(DESTDIR)$(INSTALLDIRBIN)/upnpc +else + $(INSTALL) -m 755 upnpc-shared $(DESTDIR)$(INSTALLDIRBIN)/upnpc +endif + $(INSTALL) -m 755 external-ip.sh $(DESTDIR)$(INSTALLDIRBIN)/external-ip +ifeq (, $(findstring amiga, $(OS))) + $(INSTALL) -d $(DESTDIR)$(INSTALLDIRMAN)/man3 + $(INSTALL) -m 644 man3/miniupnpc.3 $(DESTDIR)$(INSTALLDIRMAN)/man3/miniupnpc.3 +ifneq (, $(findstring linux, $(OS))) + gzip -f $(DESTDIR)$(INSTALLDIRMAN)/man3/miniupnpc.3 +endif +endif + +install-static: updateversion $(FILESTOINSTALL) + $(INSTALL) -d $(DESTDIR)$(INSTALLDIRINC) + $(INSTALL) -m 644 $(HEADERS) $(DESTDIR)$(INSTALLDIRINC) + $(INSTALL) -d $(DESTDIR)$(INSTALLDIRLIB) + $(INSTALL) -m 644 $(LIBRARY) $(DESTDIR)$(INSTALLDIRLIB) + $(INSTALL) -d $(DESTDIR)$(INSTALLDIRBIN) + $(INSTALL) -m 755 external-ip.sh $(DESTDIR)$(INSTALLDIRBIN)/external-ip + +cleaninstall: + $(RM) -r $(DESTDIR)$(INSTALLDIRINC) + $(RM) $(DESTDIR)$(INSTALLDIRLIB)/$(LIBRARY) + $(RM) $(DESTDIR)$(INSTALLDIRLIB)/$(SHAREDLIBRARY) + +miniupnpc.pc: VERSION + $(RM) $@ + echo "prefix=$(INSTALLPREFIX)" >> $@ + echo "exec_prefix=\$${prefix}" >> $@ + echo "libdir=\$${exec_prefix}/$(LIBDIR)" >> $@ + echo "includedir=\$${prefix}/include/miniupnpc" >> $@ + echo "" >> $@ + echo "Name: miniUPnPc" >> $@ + echo "Description: UPnP IGD client lightweight library" >> $@ + echo "Version: $(VERSION)" >> $@ + echo "Libs: -L\$${libdir} -lminiupnpc" >> $@ + echo "Cflags: -I\$${includedir}" >> $@ + +depend: + makedepend -Y -- $(CFLAGS) -- $(SRCS) 2>/dev/null + +$(LIBRARY): $(LIBOBJS) +ifneq (, $(findstring darwin, $(OS))) + $(LIBTOOL) -static -o $@ $? +else + $(AR) crs $@ $? +endif + +$(SHAREDLIBRARY): $(LIBOBJS) +ifneq (, $(findstring darwin, $(OS))) +# $(CC) -dynamiclib $(LDFLAGS) -Wl,-install_name,$(SONAME) -o $@ $^ + $(CC) -dynamiclib $(LDFLAGS) -Wl,-install_name,$(INSTALLDIRLIB)/$(SONAME) -o $@ $^ +else + $(CC) -shared $(LDFLAGS) -Wl,-soname,$(SONAME) -o $@ $^ +endif + +upnpc-static: upnpc.o $(LIBRARY) + $(CC) $(LDFLAGS) -o $@ $^ $(LOADLIBES) $(LDLIBS) + +upnpc-shared: upnpc.o $(SHAREDLIBRARY) + $(CC) $(LDFLAGS) -o $@ $^ $(LOADLIBES) $(LDLIBS) + +listdevices: listdevices.o $(LIBRARY) + +testminixml: $(TESTMINIXMLOBJS) + +testminiwget: $(TESTMINIWGETOBJS) + +minixmlvalid: minixml.o minixmlvalid.o + +testupnpreplyparse: $(TESTUPNPREPLYPARSE) + +testigddescparse: $(TESTIGDDESCPARSE) + +testportlistingparse: $(TESTPORTLISTINGPARSE) + +miniupnpcstrings.h: miniupnpcstrings.h.in updateminiupnpcstrings.sh VERSION + $(SH) updateminiupnpcstrings.sh + +# ftp tool supplied with OpenBSD can download files from http. +jnaerator-%.jar: + wget $(JNAERATORBASEURL)/$@ || \ + curl -o $@ $(JNAERATORBASEURL)/$@ || \ + ftp $(JNAERATORBASEURL)/$@ + +jar: $(SHAREDLIBRARY) $(JNAERATOR) + $(JAVA) -jar $(JNAERATOR) $(JNAERATORARGS) \ + miniupnpc.h miniupnpc_declspec.h upnpcommands.h upnpreplyparse.h \ + igd_desc_parse.h miniwget.h upnperrors.h $(SHAREDLIBRARY) \ + -package fr.free.miniupnp -o . -jar java/miniupnpc_$(JARSUFFIX).jar -v + +mvn_install: + mvn install:install-file -Dfile=java/miniupnpc_$(JARSUFFIX).jar \ + -DgroupId=com.github \ + -DartifactId=miniupnp \ + -Dversion=$(VERSION) \ + -Dpackaging=jar \ + -Dclassifier=$(JARSUFFIX) \ + -DgeneratePom=true \ + -DcreateChecksum=true + +# make .deb packages +deb: /usr/share/pyshared/stdeb all + (python setup.py --command-packages=stdeb.command bdist_deb) + +# install .deb packages +ideb: + (sudo dpkg -i deb_dist/*.deb) + +/usr/share/pyshared/stdeb: /usr/share/doc/python-all-dev + (sudo apt-get install python-stdeb) + +/usr/share/doc/python-all-dev: + (sudo apt-get install python-all-dev) + +minihttptestserver: minihttptestserver.o + +# DO NOT DELETE THIS LINE -- make depend depends on it. + +igd_desc_parse.o: igd_desc_parse.h +miniupnpc.o: miniupnpc.h miniupnpc_declspec.h igd_desc_parse.h upnpdev.h +miniupnpc.o: minissdpc.h miniwget.h minisoap.h minixml.h upnpcommands.h +miniupnpc.o: upnpreplyparse.h portlistingparse.h miniupnpctypes.h +miniupnpc.o: connecthostport.h +minixml.o: minixml.h +minisoap.o: minisoap.h miniupnpcstrings.h +miniwget.o: miniupnpcstrings.h miniwget.h miniupnpc_declspec.h +miniwget.o: connecthostport.h receivedata.h +upnpc.o: miniwget.h miniupnpc_declspec.h miniupnpc.h igd_desc_parse.h +upnpc.o: upnpdev.h upnpcommands.h upnpreplyparse.h portlistingparse.h +upnpc.o: miniupnpctypes.h upnperrors.h miniupnpcstrings.h +upnpcommands.o: upnpcommands.h upnpreplyparse.h portlistingparse.h +upnpcommands.o: miniupnpc_declspec.h miniupnpctypes.h miniupnpc.h +upnpcommands.o: igd_desc_parse.h upnpdev.h +upnpreplyparse.o: upnpreplyparse.h minixml.h +testminixml.o: minixml.h igd_desc_parse.h +minixmlvalid.o: minixml.h +testupnpreplyparse.o: upnpreplyparse.h +minissdpc.o: minissdpc.h miniupnpc_declspec.h upnpdev.h miniupnpc.h +minissdpc.o: igd_desc_parse.h receivedata.h codelength.h +upnperrors.o: upnperrors.h miniupnpc_declspec.h upnpcommands.h +upnperrors.o: upnpreplyparse.h portlistingparse.h miniupnpctypes.h +upnperrors.o: miniupnpc.h igd_desc_parse.h upnpdev.h +testigddescparse.o: igd_desc_parse.h minixml.h miniupnpc.h +testigddescparse.o: miniupnpc_declspec.h upnpdev.h +testminiwget.o: miniwget.h miniupnpc_declspec.h +connecthostport.o: connecthostport.h +portlistingparse.o: portlistingparse.h miniupnpc_declspec.h miniupnpctypes.h +portlistingparse.o: minixml.h +receivedata.o: receivedata.h +upnpdev.o: upnpdev.h miniupnpc_declspec.h +testportlistingparse.o: portlistingparse.h miniupnpc_declspec.h +testportlistingparse.o: miniupnpctypes.h +miniupnpcmodule.o: miniupnpc.h miniupnpc_declspec.h igd_desc_parse.h +miniupnpcmodule.o: upnpdev.h upnpcommands.h upnpreplyparse.h +miniupnpcmodule.o: portlistingparse.h miniupnpctypes.h upnperrors.h +listdevices.o: miniupnpc.h miniupnpc_declspec.h igd_desc_parse.h upnpdev.h diff --git a/src/contrib/miniupnp/miniupnpc/Makefile.mingw b/src/contrib/miniupnp/miniupnpc/Makefile.mingw new file mode 100644 index 0000000..a63c6f4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/Makefile.mingw @@ -0,0 +1,121 @@ +# $Id: Makefile.mingw,v 1.21 2015/09/18 12:45:16 nanard Exp $ +# Miniupnp project. +# http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ +# (c) 2005-2018 Thomas Bernard +# This Makefile is made for MinGW +# +# To cross compile on a *nix machine : +# make -f Makefile.mingw DLLWRAP=mingw32-dllwrap CC=mingw32-gcc AR=mingw32-ar +# +CC ?= gcc +DLLWRAP = dllwrap +SH = /bin/sh +ifeq ($(OS),Windows_NT) +RM = del +else +RM = rm -f +endif +#CFLAGS = -Wall -g -DDEBUG -D_WIN32_WINNT=0X501 +CFLAGS = -Wall -Os -DNDEBUG -D_WIN32_WINNT=0X501 +LDLIBS = -lws2_32 -liphlpapi +# -lwsock32 +# -liphlpapi is needed for GetBestRoute() and GetIpAddrTable() +PYTHON=\utils\python25\python +OBJS=miniwget.o minixml.o igd_desc_parse.o minisoap.o \ + minissdpc.o \ + miniupnpc.o upnpreplyparse.o upnpcommands.o upnperrors.o \ + connecthostport.o portlistingparse.o receivedata.o \ + upnpdev.o +OBJSDLL=$(addprefix dll/, $(OBJS)) + +all: init upnpc-static upnpc-shared testminixml libminiupnpc.a \ + miniupnpc.dll listdevices + +init: + mkdir dll + echo init > init + +clean: + $(RM) upnpc testminixml *.o + $(RM) dll\*.o + $(RM) *.exe + $(RM) miniupnpc.dll + $(RM) libminiupnpc.a + +libminiupnpc.a: $(OBJS) + $(AR) cr $@ $? + +pythonmodule: libminiupnpc.a + $(PYTHON) setupmingw32.py build --compiler=mingw32 + $(PYTHON) setupmingw32.py install --skip-build + +miniupnpc.dll: libminiupnpc.a $(OBJSDLL) + $(DLLWRAP) -k --driver-name $(CC) \ + --def miniupnpc.def \ + --output-def miniupnpc.dll.def \ + --implib miniupnpc.lib -o $@ \ + $(OBJSDLL) $(LDLIBS) + +miniupnpc.lib: miniupnpc.dll +# echo $@ generated with $< + +dll/upnpc.o: upnpc.o +# echo $@ generated with $< + +.c.o: + $(CC) $(CFLAGS) -DMINIUPNP_STATICLIB -c -o $@ $< + $(CC) $(CFLAGS) -DMINIUPNP_EXPORTS -c -o dll/$@ $< + +upnpc.o: upnpc.c + $(CC) $(CFLAGS) -DMINIUPNP_STATICLIB -c -o $@ $< + $(CC) $(CFLAGS) -c -o dll/$@ $< + +# --enable-stdcall-fixup +upnpc-static: upnpc.o libminiupnpc.a + $(CC) -o $@ $^ $(LDLIBS) + +upnpc-shared: dll/upnpc.o miniupnpc.lib + $(CC) -o $@ $^ $(LDLIBS) + +listdevices: listdevices.o libminiupnpc.a + $(CC) -o $@ $^ $(LDLIBS) + +wingenminiupnpcstrings: wingenminiupnpcstrings.o + +wingenminiupnpcstrings.o: wingenminiupnpcstrings.c + +# To make miniupnpcstrings.h from miniupnpcstrings.h.in we either +# use a custom executable (if running make under windows) or use +# sed (if cross compiling from another platform). +ifeq ($(OS),Windows_NT) +miniupnpcstrings.h: miniupnpcstrings.h.in wingenminiupnpcstrings + wingenminiupnpcstrings $< $@ +else +miniupnpcstrings.h: miniupnpcstrings.h.in VERSION + sed 's|OS_STRING ".*"|OS_STRING "Windows/Mingw32"|' $< | \ + sed 's|MINIUPNPC_VERSION_STRING ".*"|MINIUPNPC_VERSION_STRING "$(shell cat VERSION)"|' > $@ +endif + +minixml.o: minixml.c minixml.h + +upnpc.o: miniwget.h minisoap.h miniupnpc.h igd_desc_parse.h +upnpc.o: upnpreplyparse.h upnpcommands.h upnperrors.h miniupnpcstrings.h + +miniwget.o: miniwget.c miniwget.h miniupnpcstrings.h connecthostport.h + +minisoap.o: minisoap.c minisoap.h miniupnpcstrings.h + +miniupnpc.o: miniupnpc.c miniupnpc.h minisoap.h miniwget.h minixml.h + +igd_desc_parse.o: igd_desc_parse.c igd_desc_parse.h + +testminixml: minixml.o igd_desc_parse.o testminixml.c + +upnpreplyparse.o: upnpreplyparse.c upnpreplyparse.h minixml.h + +upnpcommands.o: upnpcommands.c upnpcommands.h upnpreplyparse.h miniupnpc.h portlistingparse.h + +minissdpc.o: minissdpc.c minissdpc.h receivedata.h + +upnpdev.o: upnpdev.c upnpdev.h + diff --git a/src/contrib/miniupnp/miniupnpc/README b/src/contrib/miniupnp/miniupnpc/README new file mode 100644 index 0000000..0d3b805 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/README @@ -0,0 +1,63 @@ +Project: miniupnp +Project web page: http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ +github: https://github.com/miniupnp/miniupnp +Author: Thomas Bernard +Copyright (c) 2005-2017 Thomas Bernard +This software is subject to the conditions detailed in the +LICENSE file provided within this distribution. + + +* miniUPnP Client - miniUPnPc * + +To compile, simply run 'gmake' (could be 'make' on your system). +Under win32, to compile with MinGW, type "mingw32make.bat". +MS Visual C solution and project files are supplied in the msvc/ subdirectory. + +The compilation is known to work under linux, FreeBSD, +OpenBSD, MacOS X, AmigaOS and cygwin. +The official AmigaOS4.1 SDK was used for AmigaOS4 and GeekGadgets for AmigaOS3. +upx (http://upx.sourceforge.net) is used to compress the win32 .exe files. + +To install the library and headers on the system use : +> su +> make install +> exit + +alternatively, to install into a specific location, use : +> INSTALLPREFIX=/usr/local make install + +upnpc.c is a sample client using the libminiupnpc. +To use the libminiupnpc in your application, link it with +libminiupnpc.a (or .so) and use the following functions found in miniupnpc.h, +upnpcommands.h and miniwget.h : +- upnpDiscover() +- UPNP_GetValidIGD() +- miniwget() +- parserootdesc() +- GetUPNPUrls() +- UPNP_* (calling UPNP methods) + +Note : use #include etc... for the includes +and -lminiupnpc for the link + +Discovery process is speeded up when MiniSSDPd is running on the machine. + + +* Python module * + +you can build a python module with 'make pythonmodule' +and install it with 'make installpythonmodule'. +setup.py (and setupmingw32.py) are included in the distribution. + + +Feel free to contact me if you have any problem : +e-mail : miniupnp@free.fr + +If you are using libminiupnpc in your application, please +send me an email ! + +For any question, you can use the web forum : +https://miniupnp.tuxfamily.org/forum/ + +Bugs should be reported on github : +https://github.com/miniupnp/miniupnp/issues diff --git a/src/contrib/miniupnp/miniupnpc/VERSION b/src/contrib/miniupnp/miniupnpc/VERSION new file mode 100644 index 0000000..879b416 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/VERSION @@ -0,0 +1 @@ +2.1 diff --git a/src/contrib/miniupnp/miniupnpc/apiversions.txt b/src/contrib/miniupnp/miniupnpc/apiversions.txt new file mode 100644 index 0000000..90d169b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/apiversions.txt @@ -0,0 +1,178 @@ +$Id: apiversions.txt,v 1.9 2016/01/24 17:24:36 nanard Exp $ + +Differences in API between miniUPnPc versions + +API version 17 + change struct UPNPDev + move getHTTPResponse() to miniwget_private.h + updated macro : + #define MINIUPNPC_API_VERSION 17 + +API version 16 + added "status_code" argument to getHTTPResponse(), miniwget() and miniwget_getaddr() + updated macro : + #define MINIUPNPC_API_VERSION 16 + +API version 15 + changed "sameport" argument of upnpDiscover() upnpDiscoverAll() upnpDiscoverDevice() + to "localport". When 0 or 1, behaviour is not changed, but it can take + any other value between 2 and 65535 + Existing programs should be compatible + updated macro : + #define MINIUPNPC_API_VERSION 15 + +API version 14 +miniupnpc.h + add ttl argument to upnpDiscover() upnpDiscoverAll() upnpDiscoverDevice() + upnpDiscoverDevices() + getDevicesFromMiniSSDPD() : + connectToMiniSSDPD() / disconnectFromMiniSSDPD() + requestDevicesFromMiniSSDPD() / receiveDevicesFromMiniSSDPD() + updated macro : + #define MINIUPNPC_API_VERSION 14 + +API version 13 +miniupnpc.h: + add searchalltype param to upnpDiscoverDevices() function + updated macro : + #define MINIUPNPC_API_VERSION 13 + +API version 12 +miniupnpc.h : + add upnpDiscoverAll() / upnpDiscoverDevice() / upnpDiscoverDevices() + functions + updated macros : + #define MINIUPNPC_API_VERSION 12 + +API version 11 + +upnpreplyparse.h / portlistingparse.h : + removed usage of sys/queue.h / bsdqueue.h + +miniupnpc.h: + updated macros : + #define MINIUPNPC_API_VERSION 11 + +====================== miniUPnPc version 1.9 ====================== +API version 10 + +upnpcommands.h: + added argument remoteHost to UPNP_GetSpecificPortMappingEntry() + +miniupnpc.h: + updated macros : + #define MINIUPNPC_VERSION "1.9" + #define MINIUPNPC_API_VERSION 10 + +====================== miniUPnPc version 1.8 ====================== +API version 9 + +miniupnpc.h: + updated macros : + #define MINIUPNPC_VERSION "1.8" + #define MINIUPNPC_API_VERSION 9 + added "unsigned int scope_id;" to struct UPNPDev + added scope_id argument to GetUPNPUrls() + + + +====================== miniUPnPc version 1.7 ====================== +API version 8 + +miniupnpc.h : + add new macros : + #define MINIUPNPC_VERSION "1.7" + #define MINIUPNPC_API_VERSION 8 + add rootdescURL to struct UPNPUrls + + + +====================== miniUPnPc version 1.6 ====================== +API version 8 + +Adding support for IPv6. +igd_desc_parse.h : + struct IGDdatas_service : + add char presentationurl[MINIUPNPC_URL_MAXSIZE]; + struct IGDdatas : + add struct IGDdatas_service IPv6FC; +miniupnpc.h : + new macros : + #define UPNPDISCOVER_SUCCESS (0) + #define UPNPDISCOVER_UNKNOWN_ERROR (-1) + #define UPNPDISCOVER_SOCKET_ERROR (-101) + #define UPNPDISCOVER_MEMORY_ERROR (-102) + simpleUPnPcommand() prototype changed (but is normaly not used by API users) + add arguments ipv6 and error to upnpDiscover() : + struct UPNPDev * + upnpDiscover(int delay, const char * multicastif, + const char * minissdpdsock, int sameport, + int ipv6, + int * error); + add controlURL_6FC member to struct UPNPUrls : + struct UPNPUrls { + char * controlURL; + char * ipcondescURL; + char * controlURL_CIF; + char * controlURL_6FC; + }; + +upnpcommands.h : + add leaseDuration argument to UPNP_AddPortMapping() + add desc, enabled and leaseDuration arguments to UPNP_GetSpecificPortMappingEntry() + add UPNP_GetListOfPortMappings() function (IGDv2) + add IGDv2 IPv6 related functions : + UPNP_GetFirewallStatus() + UPNP_GetOutboundPinholeTimeout() + UPNP_AddPinhole() + UPNP_UpdatePinhole() + UPNP_DeletePinhole() + UPNP_CheckPinholeWorking() + UPNP_GetPinholePackets() + + + +====================== miniUPnPc version 1.5 ====================== +API version 5 + +new function : +int UPNPIGD_IsConnected(struct UPNPUrls *, struct IGDdatas *); +new macro in upnpcommands.h : +#define UPNPCOMMAND_HTTP_ERROR + +====================== miniUPnPc version 1.4 ====================== +Same API as version 1.3 + +====================== miniUPnPc version 1.3 ====================== +API version 4 + +Use UNSIGNED_INTEGER type for +UPNP_GetTotalBytesSent(), UPNP_GetTotalBytesReceived(), +UPNP_GetTotalPacketsSent(), UPNP_GetTotalPacketsReceived() +Add remoteHost argument to UPNP_AddPortMapping() and UPNP_DeletePortMapping() + +====================== miniUPnPc version 1.2 ====================== +API version 3 + +added sameport argument to upnpDiscover() +struct UPNPDev * +upnpDiscover(int delay, const char * multicastif, + const char * minissdpdsock, int sameport); + +====================== miniUPnPc Version 1.1 ====================== +Same API as 1.0 + + +====================== miniUPnPc Version 1.0 ====================== +API version 2 + + +struct UPNPDev { + struct UPNPDev * pNext; + char * descURL; + char * st; + char buffer[2]; +}; +struct UPNPDev * upnpDiscover(int delay, const char * multicastif, + const char * minissdpdsock); + diff --git a/src/contrib/miniupnp/miniupnpc/codelength.h b/src/contrib/miniupnp/miniupnpc/codelength.h new file mode 100644 index 0000000..ea0b005 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/codelength.h @@ -0,0 +1,54 @@ +/* $Id: codelength.h,v 1.3 2011/07/30 13:10:05 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas BERNARD + * copyright (c) 2005-2015 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ +#ifndef CODELENGTH_H_INCLUDED +#define CODELENGTH_H_INCLUDED + +/* Encode length by using 7bit per Byte : + * Most significant bit of each byte specifies that the + * following byte is part of the code */ + +/* n : unsigned + * p : unsigned char * + */ +#define DECODELENGTH(n, p) n = 0; \ + do { n = (n << 7) | (*p & 0x7f); } \ + while((*(p++)&0x80) && (n<(1<<25))); + +/* n : unsigned + * READ : function/macro to read one byte (unsigned char) + */ +#define DECODELENGTH_READ(n, READ) \ + n = 0; \ + do { \ + unsigned char c; \ + READ(c); \ + n = (n << 7) | (c & 0x07f); \ + if(!(c&0x80)) break; \ + } while(n<(1<<25)); + +/* n : unsigned + * p : unsigned char * + * p_limit : unsigned char * + */ +#define DECODELENGTH_CHECKLIMIT(n, p, p_limit) \ + n = 0; \ + do { \ + if((p) >= (p_limit)) break; \ + n = (n << 7) | (*(p) & 0x7f); \ + } while((*((p)++)&0x80) && (n<(1<<25))); + + +/* n : unsigned + * p : unsigned char * + */ +#define CODELENGTH(n, p) if(n>=268435456) *(p++) = (n >> 28) | 0x80; \ + if(n>=2097152) *(p++) = (n >> 21) | 0x80; \ + if(n>=16384) *(p++) = (n >> 14) | 0x80; \ + if(n>=128) *(p++) = (n >> 7) | 0x80; \ + *(p++) = n & 0x7f; + +#endif /* CODELENGTH_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpc/connecthostport.c b/src/contrib/miniupnp/miniupnpc/connecthostport.c new file mode 100644 index 0000000..ea6e4e5 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/connecthostport.c @@ -0,0 +1,264 @@ +/* $Id: connecthostport.c,v 1.15 2015/10/09 16:26:19 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp + * Author : Thomas Bernard + * Copyright (c) 2010-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ + +/* use getaddrinfo() or gethostbyname() + * uncomment the following line in order to use gethostbyname() */ +#ifdef NO_GETADDRINFO +#define USE_GETHOSTBYNAME +#endif + +#include +#include +#ifdef _WIN32 +#include +#include +#include +#define MAXHOSTNAMELEN 64 +#define snprintf _snprintf +#define herror +#define socklen_t int +#else /* #ifdef _WIN32 */ +#include +#include +#ifdef MINIUPNPC_SET_SOCKET_TIMEOUT +#include +#endif /* #ifdef MINIUPNPC_SET_SOCKET_TIMEOUT */ +#include +#include +#include +#define closesocket close +#include +#include +/* defining MINIUPNPC_IGNORE_EINTR enable the ignore of interruptions + * during the connect() call */ +#define MINIUPNPC_IGNORE_EINTR +#include +#include +#endif /* #else _WIN32 */ + +/* definition of PRINT_SOCKET_ERROR */ +#ifdef _WIN32 +#define PRINT_SOCKET_ERROR(x) fprintf(stderr, "Socket error: %s, %d\n", x, WSAGetLastError()); +#else +#define PRINT_SOCKET_ERROR(x) perror(x) +#endif + +#if defined(__amigaos__) || defined(__amigaos4__) +#define herror(A) printf("%s\n", A) +#endif + +#include "connecthostport.h" + +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN 64 +#endif + +/* connecthostport() + * return a socket connected (TCP) to the host and port + * or -1 in case of error */ +SOCKET connecthostport(const char * host, unsigned short port, + unsigned int scope_id) +{ + SOCKET s; + int n; +#ifdef USE_GETHOSTBYNAME + struct sockaddr_in dest; + struct hostent *hp; +#else /* #ifdef USE_GETHOSTBYNAME */ + char tmp_host[MAXHOSTNAMELEN+1]; + char port_str[8]; + struct addrinfo *ai, *p; + struct addrinfo hints; +#endif /* #ifdef USE_GETHOSTBYNAME */ +#ifdef MINIUPNPC_SET_SOCKET_TIMEOUT + struct timeval timeout; +#endif /* #ifdef MINIUPNPC_SET_SOCKET_TIMEOUT */ + +#ifdef USE_GETHOSTBYNAME + hp = gethostbyname(host); + if(hp == NULL) + { + herror(host); + return INVALID_SOCKET; + } + memcpy(&dest.sin_addr, hp->h_addr, sizeof(dest.sin_addr)); + memset(dest.sin_zero, 0, sizeof(dest.sin_zero)); + s = socket(PF_INET, SOCK_STREAM, 0); + if(ISINVALID(s)) + { + PRINT_SOCKET_ERROR("socket"); + return INVALID_SOCKET; + } +#ifdef MINIUPNPC_SET_SOCKET_TIMEOUT + /* setting a 3 seconds timeout for the connect() call */ + timeout.tv_sec = 3; + timeout.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(struct timeval)) < 0) + { + PRINT_SOCKET_ERROR("setsockopt SO_RCVTIMEO"); + } + timeout.tv_sec = 3; + timeout.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)) < 0) + { + PRINT_SOCKET_ERROR("setsockopt SO_SNDTIMEO"); + } +#endif /* #ifdef MINIUPNPC_SET_SOCKET_TIMEOUT */ + dest.sin_family = AF_INET; + dest.sin_port = htons(port); + n = connect(s, (struct sockaddr *)&dest, sizeof(struct sockaddr_in)); +#ifdef MINIUPNPC_IGNORE_EINTR + /* EINTR The system call was interrupted by a signal that was caught + * EINPROGRESS The socket is nonblocking and the connection cannot + * be completed immediately. */ + while(n < 0 && (errno == EINTR || errno == EINPROGRESS)) + { + socklen_t len; + fd_set wset; + int err; + FD_ZERO(&wset); + FD_SET(s, &wset); + if((n = select(s + 1, NULL, &wset, NULL, NULL)) == -1 && errno == EINTR) + continue; + /*len = 0;*/ + /*n = getpeername(s, NULL, &len);*/ + len = sizeof(err); + if(getsockopt(s, SOL_SOCKET, SO_ERROR, &err, &len) < 0) { + PRINT_SOCKET_ERROR("getsockopt"); + closesocket(s); + return INVALID_SOCKET; + } + if(err != 0) { + errno = err; + n = -1; + } + } +#endif /* #ifdef MINIUPNPC_IGNORE_EINTR */ + if(n<0) + { + PRINT_SOCKET_ERROR("connect"); + closesocket(s); + return INVALID_SOCKET; + } +#else /* #ifdef USE_GETHOSTBYNAME */ + /* use getaddrinfo() instead of gethostbyname() */ + memset(&hints, 0, sizeof(hints)); + /* hints.ai_flags = AI_ADDRCONFIG; */ +#ifdef AI_NUMERICSERV + hints.ai_flags = AI_NUMERICSERV; +#endif + hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; /* AF_INET, AF_INET6 or AF_UNSPEC */ + /* hints.ai_protocol = IPPROTO_TCP; */ + snprintf(port_str, sizeof(port_str), "%hu", port); + if(host[0] == '[') + { + /* literal ip v6 address */ + int i, j; + for(i = 0, j = 1; host[j] && (host[j] != ']') && i < MAXHOSTNAMELEN; i++, j++) + { + tmp_host[i] = host[j]; + if(0 == memcmp(host+j, "%25", 3)) /* %25 is just url encoding for '%' */ + j+=2; /* skip "25" */ + } + tmp_host[i] = '\0'; + } + else + { + strncpy(tmp_host, host, MAXHOSTNAMELEN); + } + tmp_host[MAXHOSTNAMELEN] = '\0'; + n = getaddrinfo(tmp_host, port_str, &hints, &ai); + if(n != 0) + { +#ifdef _WIN32 + fprintf(stderr, "getaddrinfo() error : %d\n", n); +#else + fprintf(stderr, "getaddrinfo() error : %s\n", gai_strerror(n)); +#endif + return INVALID_SOCKET; + } + s = -1; + for(p = ai; p; p = p->ai_next) + { + s = socket(p->ai_family, p->ai_socktype, p->ai_protocol); + if(ISINVALID(s)) + continue; + if(p->ai_addr->sa_family == AF_INET6 && scope_id > 0) { + struct sockaddr_in6 * addr6 = (struct sockaddr_in6 *)p->ai_addr; + addr6->sin6_scope_id = scope_id; + } +#ifdef MINIUPNPC_SET_SOCKET_TIMEOUT + /* setting a 3 seconds timeout for the connect() call */ + timeout.tv_sec = 3; + timeout.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(struct timeval)) < 0) + { + PRINT_SOCKET_ERROR("setsockopt"); + } + timeout.tv_sec = 3; + timeout.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)) < 0) + { + PRINT_SOCKET_ERROR("setsockopt"); + } +#endif /* #ifdef MINIUPNPC_SET_SOCKET_TIMEOUT */ + n = connect(s, p->ai_addr, p->ai_addrlen); +#ifdef MINIUPNPC_IGNORE_EINTR + /* EINTR The system call was interrupted by a signal that was caught + * EINPROGRESS The socket is nonblocking and the connection cannot + * be completed immediately. */ + while(n < 0 && (errno == EINTR || errno == EINPROGRESS)) + { + socklen_t len; + fd_set wset; + int err; + FD_ZERO(&wset); + FD_SET(s, &wset); + if((n = select(s + 1, NULL, &wset, NULL, NULL)) == -1 && errno == EINTR) + continue; + /*len = 0;*/ + /*n = getpeername(s, NULL, &len);*/ + len = sizeof(err); + if(getsockopt(s, SOL_SOCKET, SO_ERROR, &err, &len) < 0) { + PRINT_SOCKET_ERROR("getsockopt"); + closesocket(s); + freeaddrinfo(ai); + return INVALID_SOCKET; + } + if(err != 0) { + errno = err; + n = -1; + } + } +#endif /* #ifdef MINIUPNPC_IGNORE_EINTR */ + if(n < 0) + { + closesocket(s); + continue; + } + else + { + break; + } + } + freeaddrinfo(ai); + if(ISINVALID(s)) + { + PRINT_SOCKET_ERROR("socket"); + return INVALID_SOCKET; + } + if(n < 0) + { + PRINT_SOCKET_ERROR("connect"); + return INVALID_SOCKET; + } +#endif /* #ifdef USE_GETHOSTBYNAME */ + return s; +} + diff --git a/src/contrib/miniupnp/miniupnpc/connecthostport.h b/src/contrib/miniupnp/miniupnpc/connecthostport.h new file mode 100644 index 0000000..701816b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/connecthostport.h @@ -0,0 +1,20 @@ +/* $Id: connecthostport.h,v 1.2 2012/06/23 22:32:33 nanard Exp $ */ +/* Project: miniupnp + * http://miniupnp.free.fr/ + * Author: Thomas Bernard + * Copyright (c) 2010-2018 Thomas Bernard + * This software is subjects to the conditions detailed + * in the LICENCE file provided within this distribution */ +#ifndef CONNECTHOSTPORT_H_INCLUDED +#define CONNECTHOSTPORT_H_INCLUDED + +#include "miniupnpc_socketdef.h" + +/* connecthostport() + * return a socket connected (TCP) to the host and port + * or INVALID_SOCKET in case of error */ +SOCKET connecthostport(const char * host, unsigned short port, + unsigned int scope_id); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/external-ip.sh b/src/contrib/miniupnp/miniupnpc/external-ip.sh new file mode 100644 index 0000000..6fd82df --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/external-ip.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# $Id: external-ip.sh,v 1.1 2010/08/05 12:57:41 nanard Exp $ +# (c) 2010 Reuben Hawkins +upnpc -s | sed -n -e 's/^ExternalIPAddress = \([0-9.]*\)$/\1/p' diff --git a/src/contrib/miniupnp/miniupnpc/igd_desc_parse.c b/src/contrib/miniupnp/miniupnpc/igd_desc_parse.c new file mode 100644 index 0000000..d2999ad --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/igd_desc_parse.c @@ -0,0 +1,123 @@ +/* $Id: igd_desc_parse.c,v 1.17 2015/09/15 13:30:04 nanard Exp $ */ +/* Project : miniupnp + * http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005-2015 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ + +#include "igd_desc_parse.h" +#include +#include + +/* Start element handler : + * update nesting level counter and copy element name */ +void IGDstartelt(void * d, const char * name, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + if(l >= MINIUPNPC_URL_MAXSIZE) + l = MINIUPNPC_URL_MAXSIZE-1; + memcpy(datas->cureltname, name, l); + datas->cureltname[l] = '\0'; + datas->level++; + if( (l==7) && !memcmp(name, "service", l) ) { + datas->tmp.controlurl[0] = '\0'; + datas->tmp.eventsuburl[0] = '\0'; + datas->tmp.scpdurl[0] = '\0'; + datas->tmp.servicetype[0] = '\0'; + } +} + +#define COMPARE(str, cstr) (0==memcmp(str, cstr, sizeof(cstr) - 1)) + +/* End element handler : + * update nesting level counter and update parser state if + * service element is parsed */ +void IGDendelt(void * d, const char * name, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + datas->level--; + /*printf("endelt %2d %.*s\n", datas->level, l, name);*/ + if( (l==7) && !memcmp(name, "service", l) ) + { + if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:")) { + memcpy(&datas->CIF, &datas->tmp, sizeof(struct IGDdatas_service)); + } else if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANIPv6FirewallControl:")) { + memcpy(&datas->IPv6FC, &datas->tmp, sizeof(struct IGDdatas_service)); + } else if(COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANIPConnection:") + || COMPARE(datas->tmp.servicetype, + "urn:schemas-upnp-org:service:WANPPPConnection:") ) { + if(datas->first.servicetype[0] == '\0') { + memcpy(&datas->first, &datas->tmp, sizeof(struct IGDdatas_service)); + } else { + memcpy(&datas->second, &datas->tmp, sizeof(struct IGDdatas_service)); + } + } + } +} + +/* Data handler : + * copy data depending on the current element name and state */ +void IGDdata(void * d, const char * data, int l) +{ + struct IGDdatas * datas = (struct IGDdatas *)d; + char * dstmember = 0; + /*printf("%2d %s : %.*s\n", + datas->level, datas->cureltname, l, data); */ + if( !strcmp(datas->cureltname, "URLBase") ) + dstmember = datas->urlbase; + else if( !strcmp(datas->cureltname, "presentationURL") ) + dstmember = datas->presentationurl; + else if( !strcmp(datas->cureltname, "serviceType") ) + dstmember = datas->tmp.servicetype; + else if( !strcmp(datas->cureltname, "controlURL") ) + dstmember = datas->tmp.controlurl; + else if( !strcmp(datas->cureltname, "eventSubURL") ) + dstmember = datas->tmp.eventsuburl; + else if( !strcmp(datas->cureltname, "SCPDURL") ) + dstmember = datas->tmp.scpdurl; +/* else if( !strcmp(datas->cureltname, "deviceType") ) + dstmember = datas->devicetype_tmp;*/ + if(dstmember) + { + if(l>=MINIUPNPC_URL_MAXSIZE) + l = MINIUPNPC_URL_MAXSIZE-1; + memcpy(dstmember, data, l); + dstmember[l] = '\0'; + } +} + +#ifdef DEBUG +void printIGD(struct IGDdatas * d) +{ + printf("urlbase = '%s'\n", d->urlbase); + printf("WAN Device (Common interface config) :\n"); + /*printf(" deviceType = '%s'\n", d->CIF.devicetype);*/ + printf(" serviceType = '%s'\n", d->CIF.servicetype); + printf(" controlURL = '%s'\n", d->CIF.controlurl); + printf(" eventSubURL = '%s'\n", d->CIF.eventsuburl); + printf(" SCPDURL = '%s'\n", d->CIF.scpdurl); + printf("primary WAN Connection Device (IP or PPP Connection):\n"); + /*printf(" deviceType = '%s'\n", d->first.devicetype);*/ + printf(" servicetype = '%s'\n", d->first.servicetype); + printf(" controlURL = '%s'\n", d->first.controlurl); + printf(" eventSubURL = '%s'\n", d->first.eventsuburl); + printf(" SCPDURL = '%s'\n", d->first.scpdurl); + printf("secondary WAN Connection Device (IP or PPP Connection):\n"); + /*printf(" deviceType = '%s'\n", d->second.devicetype);*/ + printf(" servicetype = '%s'\n", d->second.servicetype); + printf(" controlURL = '%s'\n", d->second.controlurl); + printf(" eventSubURL = '%s'\n", d->second.eventsuburl); + printf(" SCPDURL = '%s'\n", d->second.scpdurl); + printf("WAN IPv6 Firewall Control :\n"); + /*printf(" deviceType = '%s'\n", d->IPv6FC.devicetype);*/ + printf(" servicetype = '%s'\n", d->IPv6FC.servicetype); + printf(" controlURL = '%s'\n", d->IPv6FC.controlurl); + printf(" eventSubURL = '%s'\n", d->IPv6FC.eventsuburl); + printf(" SCPDURL = '%s'\n", d->IPv6FC.scpdurl); +} +#endif /* DEBUG */ + diff --git a/src/contrib/miniupnp/miniupnpc/igd_desc_parse.h b/src/contrib/miniupnp/miniupnpc/igd_desc_parse.h new file mode 100644 index 0000000..0de546b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/igd_desc_parse.h @@ -0,0 +1,49 @@ +/* $Id: igd_desc_parse.h,v 1.12 2014/11/17 17:19:13 nanard Exp $ */ +/* Project : miniupnp + * http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005-2014 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef IGD_DESC_PARSE_H_INCLUDED +#define IGD_DESC_PARSE_H_INCLUDED + +/* Structure to store the result of the parsing of UPnP + * descriptions of Internet Gateway Devices */ +#define MINIUPNPC_URL_MAXSIZE (128) +struct IGDdatas_service { + char controlurl[MINIUPNPC_URL_MAXSIZE]; + char eventsuburl[MINIUPNPC_URL_MAXSIZE]; + char scpdurl[MINIUPNPC_URL_MAXSIZE]; + char servicetype[MINIUPNPC_URL_MAXSIZE]; + /*char devicetype[MINIUPNPC_URL_MAXSIZE];*/ +}; + +struct IGDdatas { + char cureltname[MINIUPNPC_URL_MAXSIZE]; + char urlbase[MINIUPNPC_URL_MAXSIZE]; + char presentationurl[MINIUPNPC_URL_MAXSIZE]; + int level; + /*int state;*/ + /* "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1" */ + struct IGDdatas_service CIF; + /* "urn:schemas-upnp-org:service:WANIPConnection:1" + * "urn:schemas-upnp-org:service:WANPPPConnection:1" */ + struct IGDdatas_service first; + /* if both WANIPConnection and WANPPPConnection are present */ + struct IGDdatas_service second; + /* "urn:schemas-upnp-org:service:WANIPv6FirewallControl:1" */ + struct IGDdatas_service IPv6FC; + /* tmp */ + struct IGDdatas_service tmp; +}; + +void IGDstartelt(void *, const char *, int); +void IGDendelt(void *, const char *, int); +void IGDdata(void *, const char *, int); +#ifdef DEBUG +void printIGD(struct IGDdatas *); +#endif /* DEBUG */ + +#endif /* IGD_DESC_PARSE_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpc/java/.gitignore b/src/contrib/miniupnp/miniupnpc/java/.gitignore new file mode 100644 index 0000000..8e9e794 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/java/.gitignore @@ -0,0 +1,2 @@ +*.class +*.jar diff --git a/src/contrib/miniupnp/miniupnpc/java/JavaBridgeTest.java b/src/contrib/miniupnp/miniupnpc/java/JavaBridgeTest.java new file mode 100644 index 0000000..c658c59 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/java/JavaBridgeTest.java @@ -0,0 +1,97 @@ +import java.nio.ByteBuffer; +import java.nio.IntBuffer; + +import fr.free.miniupnp.*; + +/** + * + * @author syuu + */ +public class JavaBridgeTest { + public static void main(String[] args) { + int UPNP_DELAY = 2000; + MiniupnpcLibrary miniupnpc = MiniupnpcLibrary.INSTANCE; + UPNPDev devlist = null; + UPNPUrls urls = new UPNPUrls(); + IGDdatas data = new IGDdatas(); + ByteBuffer lanaddr = ByteBuffer.allocate(16); + ByteBuffer intClient = ByteBuffer.allocate(16); + ByteBuffer intPort = ByteBuffer.allocate(6); + ByteBuffer desc = ByteBuffer.allocate(80); + ByteBuffer enabled = ByteBuffer.allocate(4); + ByteBuffer leaseDuration = ByteBuffer.allocate(16); + int ret; + int i; + + if(args.length < 2) { + System.err.println("Usage : java [...] JavaBridgeTest port protocol"); + System.out.println(" port is numeric, protocol is TCP or UDP"); + return; + } + + devlist = miniupnpc.upnpDiscover(UPNP_DELAY, (String) null, (String) null, 0, 0, (byte)2, IntBuffer.allocate(1)); + if (devlist != null) { + System.out.println("List of UPNP devices found on the network :"); + for (UPNPDev device = devlist; device != null; device = device.pNext) { + System.out.println("desc: " + device.descURL.getString(0) + " st: " + device.st.getString(0)); + } + if ((i = miniupnpc.UPNP_GetValidIGD(devlist, urls, data, lanaddr, 16)) != 0) { + switch (i) { + case 1: + System.out.println("Found valid IGD : " + urls.controlURL.getString(0)); + break; + case 2: + System.out.println("Found a (not connected?) IGD : " + urls.controlURL.getString(0)); + System.out.println("Trying to continue anyway"); + break; + case 3: + System.out.println("UPnP device found. Is it an IGD ? : " + urls.controlURL.getString(0)); + System.out.println("Trying to continue anyway"); + break; + default: + System.out.println("Found device (igd ?) : " + urls.controlURL.getString(0)); + System.out.println("Trying to continue anyway"); + + } + System.out.println("Local LAN ip address : " + new String(lanaddr.array())); + ByteBuffer externalAddress = ByteBuffer.allocate(16); + miniupnpc.UPNP_GetExternalIPAddress(urls.controlURL.getString(0), + new String(data.first.servicetype), externalAddress); + System.out.println("ExternalIPAddress = " + new String(externalAddress.array())); + ret = miniupnpc.UPNP_AddPortMapping( + urls.controlURL.getString(0), // controlURL + new String(data.first.servicetype), // servicetype + args[0], // external Port + args[0], // internal Port + new String(lanaddr.array()), // internal client + "added via miniupnpc/JAVA !", // description + args[1], // protocol UDP or TCP + null, // remote host (useless) + "0"); // leaseDuration + if (ret != MiniupnpcLibrary.UPNPCOMMAND_SUCCESS) + System.out.println("AddPortMapping() failed with code " + ret); + ret = miniupnpc.UPNP_GetSpecificPortMappingEntry( + urls.controlURL.getString(0), new String(data.first.servicetype), + args[0], args[1], null, intClient, intPort, + desc, enabled, leaseDuration); + if (ret != MiniupnpcLibrary.UPNPCOMMAND_SUCCESS) + System.out.println("GetSpecificPortMappingEntry() failed with code " + ret); + System.out.println("InternalIP:Port = " + + new String(intClient.array()) + ":" + new String(intPort.array()) + + " (" + new String(desc.array()) + ")"); + ret = miniupnpc.UPNP_DeletePortMapping( + urls.controlURL.getString(0), + new String(data.first.servicetype), + args[0], args[1], null); + if (ret != MiniupnpcLibrary.UPNPCOMMAND_SUCCESS) + System.out.println("DelPortMapping() failed with code " + ret); + miniupnpc.FreeUPNPUrls(urls); + } else { + System.out.println("No valid UPNP Internet Gateway Device found."); + } + miniupnpc.freeUPNPDevlist(devlist); + } else { + System.out.println("No IGD UPnP Device found on the network !\n"); + } + } +} diff --git a/src/contrib/miniupnp/miniupnpc/java/testjava.bat b/src/contrib/miniupnp/miniupnpc/java/testjava.bat new file mode 100644 index 0000000..b836da1 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/java/testjava.bat @@ -0,0 +1,8 @@ +@echo off +set JAVA=java +set JAVAC=javac +REM notice the semicolon for Windows. Write once, run ... oh nevermind +set CP=miniupnpc_win32.jar;. + +%JAVAC% -cp "%CP%" JavaBridgeTest.java || exit 1 +%JAVA% -cp "%CP%" JavaBridgeTest 12345 UDP || exit 1 diff --git a/src/contrib/miniupnp/miniupnpc/java/testjava.sh b/src/contrib/miniupnp/miniupnpc/java/testjava.sh new file mode 100644 index 0000000..9880523 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/java/testjava.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +JAVA=java +JAVAC=javac +CP=$(for i in *.jar; do echo -n $i:; done). + +$JAVAC -cp $CP JavaBridgeTest.java || exit 1 +$JAVA -cp $CP JavaBridgeTest 12345 UDP || exit 1 diff --git a/src/contrib/miniupnp/miniupnpc/listdevices.c b/src/contrib/miniupnp/miniupnpc/listdevices.c new file mode 100644 index 0000000..bd9ba57 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/listdevices.c @@ -0,0 +1,197 @@ +/* $Id: listdevices.c,v 1.6 2015/07/23 20:40:08 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas Bernard + * Copyright (c) 2013-2015 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ + +#include +#include +#include +#ifdef _WIN32 +#include +#endif /* _WIN32 */ +#include "miniupnpc.h" + +struct upnp_dev_list { + struct upnp_dev_list * next; + char * descURL; + struct UPNPDev * * array; + size_t count; + size_t allocated_count; +}; + +#define ADD_DEVICE_COUNT_STEP 16 + +void add_device(struct upnp_dev_list * * list_head, struct UPNPDev * dev) +{ + struct upnp_dev_list * elt; + size_t i; + + if(dev == NULL) + return; + for(elt = *list_head; elt != NULL; elt = elt->next) { + if(strcmp(elt->descURL, dev->descURL) == 0) { + for(i = 0; i < elt->count; i++) { + if (strcmp(elt->array[i]->st, dev->st) == 0 && strcmp(elt->array[i]->usn, dev->usn) == 0) { + return; /* already found */ + } + } + if(elt->count >= elt->allocated_count) { + struct UPNPDev * * tmp; + elt->allocated_count += ADD_DEVICE_COUNT_STEP; + tmp = realloc(elt->array, elt->allocated_count * sizeof(struct UPNPDev *)); + if(tmp == NULL) { + fprintf(stderr, "Failed to realloc(%p, %lu)\n", elt->array, (unsigned long)(elt->allocated_count * sizeof(struct UPNPDev *))); + return; + } + elt->array = tmp; + } + elt->array[elt->count++] = dev; + return; + } + } + elt = malloc(sizeof(struct upnp_dev_list)); + if(elt == NULL) { + fprintf(stderr, "Failed to malloc(%lu)\n", (unsigned long)sizeof(struct upnp_dev_list)); + return; + } + elt->next = *list_head; + elt->descURL = strdup(dev->descURL); + if(elt->descURL == NULL) { + fprintf(stderr, "Failed to strdup(%s)\n", dev->descURL); + free(elt); + return; + } + elt->allocated_count = ADD_DEVICE_COUNT_STEP; + elt->array = malloc(ADD_DEVICE_COUNT_STEP * sizeof(struct UPNPDev *)); + if(elt->array == NULL) { + fprintf(stderr, "Failed to malloc(%lu)\n", (unsigned long)(ADD_DEVICE_COUNT_STEP * sizeof(struct UPNPDev *))); + free(elt->descURL); + free(elt); + return; + } + elt->array[0] = dev; + elt->count = 1; + *list_head = elt; +} + +void free_device(struct upnp_dev_list * elt) +{ + free(elt->descURL); + free(elt->array); + free(elt); +} + +int main(int argc, char * * argv) +{ + const char * searched_device = NULL; + const char * * searched_devices = NULL; + const char * multicastif = 0; + const char * minissdpdpath = 0; + int ipv6 = 0; + unsigned char ttl = 2; + int error = 0; + struct UPNPDev * devlist = 0; + struct UPNPDev * dev; + struct upnp_dev_list * sorted_list = NULL; + struct upnp_dev_list * dev_array; + int i; + +#ifdef _WIN32 + WSADATA wsaData; + int nResult = WSAStartup(MAKEWORD(2,2), &wsaData); + if(nResult != NO_ERROR) + { + fprintf(stderr, "WSAStartup() failed.\n"); + return -1; + } +#endif + + for(i = 1; i < argc; i++) { + if(strcmp(argv[i], "-6") == 0) + ipv6 = 1; + else if(strcmp(argv[i], "-d") == 0) { + if(++i >= argc) { + fprintf(stderr, "%s option needs one argument\n", "-d"); + return 1; + } + searched_device = argv[i]; + } else if(strcmp(argv[i], "-t") == 0) { + if(++i >= argc) { + fprintf(stderr, "%s option needs one argument\n", "-t"); + return 1; + } + ttl = (unsigned char)atoi(argv[i]); + } else if(strcmp(argv[i], "-l") == 0) { + if(++i >= argc) { + fprintf(stderr, "-l option needs at least one argument\n"); + return 1; + } + searched_devices = (const char * *)(argv + i); + break; + } else if(strcmp(argv[i], "-m") == 0) { + if(++i >= argc) { + fprintf(stderr, "-m option needs one argument\n"); + return 1; + } + multicastif = argv[i]; + } else { + printf("usage : %s [options] [-l ...]\n", argv[0]); + printf("options :\n"); + printf(" -6 : use IPv6\n"); + printf(" -m address/ifname : network interface to use for multicast\n"); + printf(" -d : search only for this type of device\n"); + printf(" -l ... : search only for theses types of device\n"); + printf(" -t ttl : set multicast TTL. Default value is 2.\n"); + printf(" -h : this help\n"); + return 1; + } + } + + if(searched_device) { + printf("searching UPnP device type %s\n", searched_device); + devlist = upnpDiscoverDevice(searched_device, + 2000, multicastif, minissdpdpath, + 0/*localport*/, ipv6, ttl, &error); + } else if(searched_devices) { + printf("searching UPnP device types :\n"); + for(i = 0; searched_devices[i]; i++) + printf("\t%s\n", searched_devices[i]); + devlist = upnpDiscoverDevices(searched_devices, + 2000, multicastif, minissdpdpath, + 0/*localport*/, ipv6, ttl, &error, 1); + } else { + printf("searching all UPnP devices\n"); + devlist = upnpDiscoverAll(2000, multicastif, minissdpdpath, + 0/*localport*/, ipv6, ttl, &error); + } + if(devlist) { + for(dev = devlist, i = 1; dev != NULL; dev = dev->pNext, i++) { + printf("%3d: %-48s\n", i, dev->st); + printf(" %s\n", dev->descURL); + printf(" %s\n", dev->usn); + add_device(&sorted_list, dev); + } + putchar('\n'); + for (dev_array = sorted_list; dev_array != NULL ; dev_array = dev_array->next) { + printf("%s :\n", dev_array->descURL); + for(i = 0; (unsigned)i < dev_array->count; i++) { + printf("%2d: %s\n", i+1, dev_array->array[i]->st); + printf(" %s\n", dev_array->array[i]->usn); + } + putchar('\n'); + } + freeUPNPDevlist(devlist); + while(sorted_list != NULL) { + dev_array = sorted_list; + sorted_list = sorted_list->next; + free_device(dev_array); + } + } else { + printf("no device found.\n"); + } + + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpc/man3/miniupnpc.3 b/src/contrib/miniupnp/miniupnpc/man3/miniupnpc.3 new file mode 100644 index 0000000..e5c3ec8 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/man3/miniupnpc.3 @@ -0,0 +1,55 @@ +.TH MINIUPNPC 3 +.SH NAME +miniupnpc \- UPnP client library +.SH SYNOPSIS +.SH DESCRIPTION +The miniupnpc library implement the UPnP protocol defined +to dialog with Internet Gateway Devices. It also has +the ability to use data gathered by minissdpd(1) about +UPnP devices up on the network in order to skip the +long UPnP device discovery process. +.PP +At first, upnpDiscover(3) has to be used to discover UPnP IGD present +on the network. Then UPNP_GetValidIGD(3) to select the right one. +Alternatively, UPNP_GetIGDFromUrl(3) could be used to bypass discovery +process if the root description url of the device to use is known. +Then all the UPNP_* functions can be used, such as +UPNP_GetConnectionTypeInfo(3), UPNP_AddPortMapping(3), etc... +.SH "HEADER FILES" +.IP miniupnpc.h +That's the main header file for the miniupnpc library API. +It contains all the functions and structures related to device discovery. +.IP upnpcommands.h +This header file contain the UPnP IGD methods that are accessible +through the miniupnpc API. The name of the C functions are matching +the UPnP methods names. ie: GetGenericPortMappingEntry is +UPNP_GetGenericPortMappingEntry. +.SH "API FUNCTIONS" +.IP "struct UPNPDev * upnpDiscover(int delay, const char * multicastif, const char * minissdpdsock, int localport, int ipv6, int * error);" +execute the discovery process. +delay (in millisecond) is the maximum time for waiting any device response. +If available, device list will be obtained from MiniSSDPd. +Default path for minissdpd socket will be used if minissdpdsock argument is NULL. +If multicastif is not NULL, it will be used instead of the default multicast interface for sending SSDP discover packets. +If localport is set to UPNP_LOCAL_PORT_SAME(1) SSDP packets will be sent +from the source port 1900 (same as destination port), if set to +UPNP_LOCAL_PORT_ANY(0) system assign a source port, any other value will +be attempted as the source port. +If ipv6 is not 0, IPv6 is used instead of IPv4 for the discovery process. +.IP "void freeUPNPDevlist(struct UPNPDev * devlist);" +free the list returned by upnpDiscover(). +.IP "int UPNP_GetValidIGD(struct UPNPDev * devlist, struct UPNPUrls * urls, struct IGDdatas * data, char * lanaddr, int lanaddrlen);" +browse the list of device returned by upnpDiscover(), find +a live UPnP internet gateway device and fill structures passed as arguments +with data used for UPNP methods invocation. +.IP "int UPNP_GetIGDFromUrl(const char * rootdescurl, struct UPNPUrls * urls, struct IGDdatas * data, char * lanaddr, int lanaddrlen);" +permit one to bypass the upnpDiscover() call if the xml root description +URL of the UPnP IGD is known. +Fill structures passed as arguments +with data used for UPNP methods invocation. +.IP "void GetUPNPUrls(struct UPNPUrls *, struct IGDdatas *, const char *);" +.IP "void FreeUPNPUrls(struct UPNPUrls *);" + +.SH "SEE ALSO" +minissdpd(1) +.SH BUGS diff --git a/src/contrib/miniupnp/miniupnpc/mingw32make.bat b/src/contrib/miniupnp/miniupnpc/mingw32make.bat new file mode 100644 index 0000000..c5d3cc4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/mingw32make.bat @@ -0,0 +1,8 @@ +@mingw32-make -f Makefile.mingw %1 +@if errorlevel 1 goto end +@if not exist upnpc-static.exe goto end +@strip upnpc-static.exe +@upx --best upnpc-static.exe +@strip upnpc-shared.exe +@upx --best upnpc-shared.exe +:end diff --git a/src/contrib/miniupnp/miniupnpc/minihttptestserver.c b/src/contrib/miniupnp/miniupnpc/minihttptestserver.c new file mode 100644 index 0000000..a252cd4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/minihttptestserver.c @@ -0,0 +1,675 @@ +/* $Id: minihttptestserver.c,v 1.23 2018/01/15 16:20:07 nanard Exp $ */ +/* Project : miniUPnP + * Author : Thomas Bernard + * Copyright (c) 2011-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef INADDR_LOOPBACK +#define INADDR_LOOPBACK 0x7f000001 +#endif + +#define CRAP_LENGTH (2048) + +static int server(unsigned short port, const char * expected_file_name, int ipv6); + +volatile sig_atomic_t quit = 0; +volatile sig_atomic_t child_to_wait_for = 0; + +/** + * signal handler for SIGCHLD (child status has changed) + */ +void handle_signal_chld(int sig) +{ + (void)sig; + /* printf("handle_signal_chld(%d)\n", sig); */ + ++child_to_wait_for; +} + +/** + * signal handler for SIGINT (CRTL C) + */ +void handle_signal_int(int sig) +{ + (void)sig; + /* printf("handle_signal_int(%d)\n", sig); */ + quit = 1; +} + +/** + * build a text/plain content of the specified length + */ +void build_content(char * p, size_t n) +{ + char line_buffer[80]; + int k; + int i = 0; + + while(n > 0) { + k = snprintf(line_buffer, sizeof(line_buffer), + "%04d_ABCDEFGHIJKL_This_line_is_64_bytes_long_ABCDEFGHIJKL_%04d\r\n", + i, i); + if(k != 64) { + fprintf(stderr, "snprintf() returned %d in build_content()\n", k); + } + ++i; + if(n >= 64) { + memcpy(p, line_buffer, 64); + p += 64; + n -= 64; + } else { + memcpy(p, line_buffer, n); + p += n; + n = 0; + } + } +} + +/** + * build crappy content + */ +void build_crap(char * p, size_t n) +{ + static const char crap[] = "_CRAP_\r\n"; + size_t i; + + while(n > 0) { + i = sizeof(crap) - 1; + if(i > n) + i = n; + memcpy(p, crap, i); + p += i; + n -= i; + } +} + +/** + * build chunked response. + * return a malloc'ed buffer + */ +char * build_chunked_response(size_t content_length, size_t * response_len) +{ + char * response_buffer; + char * content_buffer; + size_t buffer_length; + size_t i; + unsigned int n; + + /* allocate to have some margin */ + buffer_length = 256 + content_length + (content_length >> 4); + response_buffer = malloc(buffer_length); + if(response_buffer == NULL) + return NULL; + *response_len = snprintf(response_buffer, buffer_length, + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/plain\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n"); + + /* build the content */ + content_buffer = malloc(content_length); + if(content_buffer == NULL) { + free(response_buffer); + return NULL; + } + build_content(content_buffer, content_length); + + /* chunk it */ + i = 0; + while(i < content_length) { + n = (rand() % 199) + 1; + if(i + n > content_length) { + n = content_length - i; + } + /* TODO : check buffer size ! */ + *response_len += snprintf(response_buffer + *response_len, + buffer_length - *response_len, + "%x\r\n", n); + memcpy(response_buffer + *response_len, content_buffer + i, n); + *response_len += n; + i += n; + response_buffer[(*response_len)++] = '\r'; + response_buffer[(*response_len)++] = '\n'; + } + /* the last chunk : "0\r\n" a empty body and then + * the final "\r\n" */ + memcpy(response_buffer + *response_len, "0\r\n\r\n", 5); + *response_len += 5; + free(content_buffer); + + printf("resp_length=%lu buffer_length=%lu content_length=%lu\n", + *response_len, buffer_length, content_length); + return response_buffer; +} + +/* favicon.ico generator */ +#ifdef OLD_HEADER +#define FAVICON_LENGTH (6 + 16 + 12 + 8 + 32 * 4) +#else +#define FAVICON_LENGTH (6 + 16 + 40 + 8 + 32 * 4) +#endif +void build_favicon_content(unsigned char * p, size_t n) +{ + int i; + if(n < FAVICON_LENGTH) + return; + /* header : 6 bytes */ + *p++ = 0; + *p++ = 0; + *p++ = 1; /* type : ICO */ + *p++ = 0; + *p++ = 1; /* number of images in file */ + *p++ = 0; + /* image directory (1 entry) : 16 bytes */ + *p++ = 16; /* width */ + *p++ = 16; /* height */ + *p++ = 2; /* number of colors in the palette. 0 = no palette */ + *p++ = 0; /* reserved */ + *p++ = 1; /* color planes */ + *p++ = 0; /* " */ + *p++ = 1; /* bpp */ + *p++ = 0; /* " */ +#ifdef OLD_HEADER + *p++ = 12 + 8 + 32 * 4; /* bmp size */ +#else + *p++ = 40 + 8 + 32 * 4; /* bmp size */ +#endif + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 6 + 16; /* bmp offset */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + /* BMP */ +#ifdef OLD_HEADER + /* BITMAPCOREHEADER */ + *p++ = 12; /* size of this header */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 16; /* width */ + *p++ = 0; /* " */ + *p++ = 16 * 2; /* height x 2 ! */ + *p++ = 0; /* " */ + *p++ = 1; /* color planes */ + *p++ = 0; /* " */ + *p++ = 1; /* bpp */ + *p++ = 0; /* " */ +#else + /* BITMAPINFOHEADER */ + *p++ = 40; /* size of this header */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 16; /* width */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 16 * 2; /* height x 2 ! */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 0; /* " */ + *p++ = 1; /* color planes */ + *p++ = 0; /* " */ + *p++ = 1; /* bpp */ + *p++ = 0; /* " */ + /* compression method, image size, ppm x, ppm y */ + /* colors in the palette ? */ + /* important colors */ + for(i = 4 * 6; i > 0; --i) + *p++ = 0; +#endif + /* palette */ + *p++ = 0; /* b */ + *p++ = 0; /* g */ + *p++ = 0; /* r */ + *p++ = 0; /* reserved */ + *p++ = 255; /* b */ + *p++ = 255; /* g */ + *p++ = 255; /* r */ + *p++ = 0; /* reserved */ + /* pixel data */ + for(i = 16; i > 0; --i) { + if(i & 1) { + *p++ = 0125; + *p++ = 0125; + } else { + *p++ = 0252; + *p++ = 0252; + } + *p++ = 0; + *p++ = 0; + } + /* Opacity MASK */ + for(i = 16 * 4; i > 0; --i) { + *p++ = 0; + } +} + +enum modes { + MODE_INVALID, MODE_CHUNKED, MODE_ADDCRAP, MODE_NORMAL, MODE_FAVICON +}; + +const struct { + const enum modes mode; + const char * text; +} modes_array[] = { + {MODE_CHUNKED, "chunked"}, + {MODE_ADDCRAP, "addcrap"}, + {MODE_NORMAL, "normal"}, + {MODE_FAVICON, "favicon.ico"}, + {MODE_INVALID, NULL} +}; + +/** + * write the response with random behaviour ! + */ +void send_response(int c, const char * buffer, size_t len) +{ + ssize_t n; + while(len > 0) { + n = (rand() % 99) + 1; + if((size_t)n > len) + n = len; + n = write(c, buffer, n); + if(n < 0) { + if(errno != EINTR) { + perror("write"); + return; + } + /* if errno == EINTR, try again */ + } else { + len -= n; + buffer += n; + usleep(10000); /* 10ms */ + } + } +} + +/** + * handle the HTTP connection + */ +void handle_http_connection(int c) +{ + char request_buffer[2048]; + size_t request_len = 0; + int headers_found = 0; + ssize_t n, m; + size_t i; + char request_method[16]; + char request_uri[256]; + char http_version[16]; + char * p; + char * response_buffer; + size_t response_len; + enum modes mode; + size_t content_length = 16*1024; + + /* read the request */ + while(request_len < sizeof(request_buffer) && !headers_found) { + n = read(c, + request_buffer + request_len, + sizeof(request_buffer) - request_len); + if(n < 0) { + if(errno == EINTR) + continue; + perror("read"); + return; + } else if(n==0) { + /* remote host closed the connection */ + break; + } else { + request_len += n; + for(i = 0; i < request_len - 3; i++) { + if(0 == memcmp(request_buffer + i, "\r\n\r\n", 4)) { + /* found the end of headers */ + headers_found = 1; + break; + } + } + } + } + if(!headers_found) { + /* error */ + printf("no HTTP header found in the request\n"); + return; + } + printf("headers :\n%.*s", (int)request_len, request_buffer); + /* the request have been received, now parse the request line */ + p = request_buffer; + for(i = 0; i < sizeof(request_method) - 1; i++) { + if(*p == ' ' || *p == '\r') + break; + request_method[i] = *p; + ++p; + } + request_method[i] = '\0'; + while(*p == ' ') + p++; + for(i = 0; i < (int)sizeof(request_uri) - 1; i++) { + if(*p == ' ' || *p == '\r') + break; + request_uri[i] = *p; + ++p; + } + request_uri[i] = '\0'; + while(*p == ' ') + p++; + for(i = 0; i < (int)sizeof(http_version) - 1; i++) { + if(*p == ' ' || *p == '\r') + break; + http_version[i] = *p; + ++p; + } + http_version[i] = '\0'; + printf("Method = %s, URI = %s, %s\n", + request_method, request_uri, http_version); + /* check if the request method is allowed */ + if(0 != strcmp(request_method, "GET")) { + const char response405[] = "HTTP/1.1 405 Method Not Allowed\r\n" + "Allow: GET\r\n\r\n"; + const char * pc; + /* 405 Method Not Allowed */ + /* The response MUST include an Allow header containing a list + * of valid methods for the requested resource. */ + n = sizeof(response405) - 1; + pc = response405; + while(n > 0) { + m = write(c, pc, n); + if(m<0) { + if(errno != EINTR) { + perror("write"); + return; + } + } else { + n -= m; + pc += m; + } + } + return; + } + + mode = MODE_INVALID; + /* use the request URI to know what to do */ + for(i = 0; modes_array[i].mode != MODE_INVALID; i++) { + if(strstr(request_uri, modes_array[i].text)) { + mode = modes_array[i].mode; /* found */ + break; + } + } + + switch(mode) { + case MODE_CHUNKED: + response_buffer = build_chunked_response(content_length, &response_len); + break; + case MODE_ADDCRAP: + response_len = content_length+256; + response_buffer = malloc(response_len); + if(!response_buffer) + break; + n = snprintf(response_buffer, response_len, + "HTTP/1.1 200 OK\r\n" + "Server: minihttptestserver\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: %lu\r\n" + "\r\n", content_length); + response_len = content_length+n+CRAP_LENGTH; + p = realloc(response_buffer, response_len); + if(p == NULL) { + /* error 500 */ + free(response_buffer); + response_buffer = NULL; + break; + } + response_buffer = p; + build_content(response_buffer + n, content_length); + build_crap(response_buffer + n + content_length, CRAP_LENGTH); + break; + case MODE_FAVICON: + content_length = FAVICON_LENGTH; + response_len = content_length + 256; + response_buffer = malloc(response_len); + if(!response_buffer) + break; + n = snprintf(response_buffer, response_len, + "HTTP/1.1 200 OK\r\n" + "Server: minihttptestserver\r\n" + "Content-Type: image/vnd.microsoft.icon\r\n" + "Content-Length: %lu\r\n" + "\r\n", content_length); + /* image/x-icon */ + build_favicon_content((unsigned char *)(response_buffer + n), content_length); + response_len = content_length + n; + break; + default: + response_len = content_length+256; + response_buffer = malloc(response_len); + if(!response_buffer) + break; + n = snprintf(response_buffer, response_len, + "HTTP/1.1 200 OK\r\n" + "Server: minihttptestserver\r\n" + "Content-Type: text/plain\r\n" + "\r\n"); + response_len = content_length+n; + p = realloc(response_buffer, response_len); + if(p == NULL) { + /* Error 500 */ + free(response_buffer); + response_buffer = NULL; + break; + } + response_buffer = p; + build_content(response_buffer + n, response_len - n); + } + + if(response_buffer) { + send_response(c, response_buffer, response_len); + free(response_buffer); + } else { + /* Error 500 */ + } +} + +/** + */ +int main(int argc, char * * argv) { + int ipv6 = 0; + int r, i; + unsigned short port = 0; + const char * expected_file_name = NULL; + + for(i = 1; i < argc; i++) { + if(argv[i][0] == '-') { + switch(argv[i][1]) { + case '6': + ipv6 = 1; + break; + case 'e': + /* write expected file ! */ + expected_file_name = argv[++i]; + break; + case 'p': + /* port */ + if(++i < argc) { + port = (unsigned short)atoi(argv[i]); + } + break; + default: + fprintf(stderr, "unknown command line switch '%s'\n", argv[i]); + } + } else { + fprintf(stderr, "unknown command line argument '%s'\n", argv[i]); + } + } + + srand(time(NULL)); + + r = server(port, expected_file_name, ipv6); + if(r != 0) { + printf("*** ERROR ***\n"); + } + return r; +} + +static int server(unsigned short port, const char * expected_file_name, int ipv6) +{ + int s, c; + int i; + struct sockaddr_storage server_addr; + socklen_t server_addrlen; + struct sockaddr_storage client_addr; + socklen_t client_addrlen; + pid_t pid; + int child = 0; + int status; + struct sigaction sa; + + memset(&sa, 0, sizeof(struct sigaction)); + + /*signal(SIGCHLD, handle_signal_chld);*/ + sa.sa_handler = handle_signal_chld; + if(sigaction(SIGCHLD, &sa, NULL) < 0) { + perror("sigaction"); + return 1; + } + /*signal(SIGINT, handle_signal_int);*/ + sa.sa_handler = handle_signal_int; + if(sigaction(SIGINT, &sa, NULL) < 0) { + perror("sigaction"); + return 1; + } + + s = socket(ipv6 ? AF_INET6 : AF_INET, SOCK_STREAM, 0); + if(s < 0) { + perror("socket"); + return 1; + } + memset(&server_addr, 0, sizeof(struct sockaddr_storage)); + memset(&client_addr, 0, sizeof(struct sockaddr_storage)); + if(ipv6) { + struct sockaddr_in6 * addr = (struct sockaddr_in6 *)&server_addr; + addr->sin6_family = AF_INET6; + addr->sin6_port = htons(port); + addr->sin6_addr = in6addr_loopback; + } else { + struct sockaddr_in * addr = (struct sockaddr_in *)&server_addr; + addr->sin_family = AF_INET; + addr->sin_port = htons(port); + addr->sin_addr.s_addr = htonl(INADDR_LOOPBACK); + } + if(bind(s, (struct sockaddr *)&server_addr, + ipv6 ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)) < 0) { + perror("bind"); + return 1; + } + if(listen(s, 5) < 0) { + perror("listen"); + } + if(port == 0) { + server_addrlen = sizeof(struct sockaddr_storage); + if(getsockname(s, (struct sockaddr *)&server_addr, &server_addrlen) < 0) { + perror("getsockname"); + return 1; + } + if(ipv6) { + struct sockaddr_in6 * addr = (struct sockaddr_in6 *)&server_addr; + port = ntohs(addr->sin6_port); + } else { + struct sockaddr_in * addr = (struct sockaddr_in *)&server_addr; + port = ntohs(addr->sin_port); + } + printf("Listening on port %hu\n", port); + fflush(stdout); + } + + /* write expected file */ + if(expected_file_name) { + FILE * f; + f = fopen(expected_file_name, "wb"); + if(f) { + char * buffer; + buffer = malloc(16*1024); + if(buffer == NULL) { + fprintf(stderr, "memory allocation error\n"); + } else { + build_content(buffer, 16*1024); + i = fwrite(buffer, 1, 16*1024, f); + if(i != 16*1024) { + fprintf(stderr, "error writing to file %s : %dbytes written (out of %d)\n", expected_file_name, i, 16*1024); + } + free(buffer); + } + fclose(f); + } else { + fprintf(stderr, "error opening file %s for writing\n", expected_file_name); + } + } + + /* fork() loop */ + while(!child && !quit) { + while(child_to_wait_for > 0) { + pid = wait(&status); + if(pid < 0) { + perror("wait"); + } else { + printf("child(%d) terminated with status %d\n", (int)pid, status); + } + --child_to_wait_for; + } + client_addrlen = sizeof(struct sockaddr_storage); + c = accept(s, (struct sockaddr *)&client_addr, + &client_addrlen); + if(c < 0) { + if(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) + continue; + perror("accept"); + return 1; + } + printf("accept...\n"); + pid = fork(); + if(pid < 0) { + perror("fork"); + return 1; + } else if(pid == 0) { + /* child */ + child = 1; + close(s); + s = -1; + handle_http_connection(c); + } + close(c); + } + if(s >= 0) { + close(s); + s = -1; + } + if(!child) { + while(child_to_wait_for > 0) { + pid = wait(&status); + if(pid < 0) { + perror("wait"); + } else { + printf("child(%d) terminated with status %d\n", (int)pid, status); + } + --child_to_wait_for; + } + printf("Bye...\n"); + } + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpc/minisoap.c b/src/contrib/miniupnp/miniupnpc/minisoap.c new file mode 100644 index 0000000..520c930 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/minisoap.c @@ -0,0 +1,124 @@ +/* $Id: minisoap.c,v 1.25 2017/04/21 10:03:24 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp + * Author : Thomas Bernard + * Copyright (c) 2005-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * + * Minimal SOAP implementation for UPnP protocol. + */ +#include +#include +#ifdef _WIN32 +#include +#include +#define snprintf _snprintf +#else +#include +#include +#include +#endif +#include "minisoap.h" +#include "miniupnpcstrings.h" + +/* only for malloc */ +#include + +#ifdef _WIN32 +#define PRINT_SOCKET_ERROR(x) fprintf(stderr, "Socket error: %s, %d\n", x, WSAGetLastError()); +#else +#define PRINT_SOCKET_ERROR(x) perror(x) +#endif + +/* httpWrite sends the headers and the body to the socket + * and returns the number of bytes sent */ +static int +httpWrite(SOCKET fd, const char * body, int bodysize, + const char * headers, int headerssize) +{ + int n = 0; + /*n = write(fd, headers, headerssize);*/ + /*if(bodysize>0) + n += write(fd, body, bodysize);*/ + /* Note : my old linksys router only took into account + * soap request that are sent into only one packet */ + char * p; + /* TODO: AVOID MALLOC, we could use writev() for that */ + p = malloc(headerssize+bodysize); + if(!p) + return -1; + memcpy(p, headers, headerssize); + memcpy(p+headerssize, body, bodysize); + /*n = write(fd, p, headerssize+bodysize);*/ + n = send(fd, p, headerssize+bodysize, 0); + if(n<0) { + PRINT_SOCKET_ERROR("send"); + } + /* disable send on the socket */ + /* draytek routers don't seem to like that... */ +#if 0 +#ifdef _WIN32 + if(shutdown(fd, SD_SEND)<0) { +#else + if(shutdown(fd, SHUT_WR)<0) { /*SD_SEND*/ +#endif + PRINT_SOCKET_ERROR("shutdown"); + } +#endif + free(p); + return n; +} + +/* self explanatory */ +int soapPostSubmit(SOCKET fd, + const char * url, + const char * host, + unsigned short port, + const char * action, + const char * body, + const char * httpversion) +{ + int bodysize; + char headerbuf[512]; + int headerssize; + char portstr[8]; + bodysize = (int)strlen(body); + /* We are not using keep-alive HTTP connections. + * HTTP/1.1 needs the header Connection: close to do that. + * This is the default with HTTP/1.0 + * Using HTTP/1.1 means we need to support chunked transfer-encoding : + * When using HTTP/1.1, the router "BiPAC 7404VNOX" always use chunked + * transfer encoding. */ + /* Connection: Close is normally there only in HTTP/1.1 but who knows */ + portstr[0] = '\0'; + if(port != 80) + snprintf(portstr, sizeof(portstr), ":%hu", port); + headerssize = snprintf(headerbuf, sizeof(headerbuf), + "POST %s HTTP/%s\r\n" + "Host: %s%s\r\n" + "User-Agent: " OS_STRING ", " UPNP_VERSION_STRING ", MiniUPnPc/" MINIUPNPC_VERSION_STRING "\r\n" + "Content-Length: %d\r\n" + "Content-Type: text/xml\r\n" + "SOAPAction: \"%s\"\r\n" + "Connection: Close\r\n" + "Cache-Control: no-cache\r\n" /* ??? */ + "Pragma: no-cache\r\n" + "\r\n", + url, httpversion, host, portstr, bodysize, action); + if ((unsigned int)headerssize >= sizeof(headerbuf)) + return -1; +#ifdef DEBUG + /*printf("SOAP request : headersize=%d bodysize=%d\n", + headerssize, bodysize); + */ + printf("SOAP request : POST %s HTTP/%s - Host: %s%s\n", + url, httpversion, host, portstr); + printf("SOAPAction: \"%s\" - Content-Length: %d\n", action, bodysize); + printf("Headers :\n%s", headerbuf); + printf("Body :\n%s\n", body); +#endif + return httpWrite(fd, body, bodysize, headerbuf, headerssize); +} + + diff --git a/src/contrib/miniupnp/miniupnpc/minisoap.h b/src/contrib/miniupnp/miniupnpc/minisoap.h new file mode 100644 index 0000000..d6a45d0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/minisoap.h @@ -0,0 +1,17 @@ +/* $Id: minisoap.h,v 1.4 2010/04/12 20:39:41 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas Bernard + * Copyright (c) 2005-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ +#ifndef MINISOAP_H_INCLUDED +#define MINISOAP_H_INCLUDED + +#include "miniupnpc_socketdef.h" + +/*int httpWrite(int, const char *, int, const char *);*/ +int soapPostSubmit(SOCKET, const char *, const char *, unsigned short, + const char *, const char *, const char *); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/minissdpc.c b/src/contrib/miniupnp/miniupnpc/minissdpc.c new file mode 100644 index 0000000..d76b242 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/minissdpc.c @@ -0,0 +1,888 @@ +/* $Id: minissdpc.c,v 1.32 2016/10/07 09:04:36 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp + * Web : http://miniupnp.free.fr/ + * Author : Thomas BERNARD + * copyright (c) 2005-2018 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ +/*#include */ +#include +#include +#include +#include +#if defined (__NetBSD__) +#include +#endif +#if defined(_WIN32) || defined(__amigaos__) || defined(__amigaos4__) +#ifdef _WIN32 +#include +#include +#include +#include +#define snprintf _snprintf +#if !defined(_MSC_VER) +#include +#else /* !defined(_MSC_VER) */ +typedef unsigned short uint16_t; +#endif /* !defined(_MSC_VER) */ +#ifndef strncasecmp +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#define strncasecmp _memicmp +#else /* defined(_MSC_VER) && (_MSC_VER >= 1400) */ +#define strncasecmp memicmp +#endif /* defined(_MSC_VER) && (_MSC_VER >= 1400) */ +#endif /* #ifndef strncasecmp */ +#endif /* _WIN32 */ +#if defined(__amigaos__) || defined(__amigaos4__) +#include +#endif /* defined(__amigaos__) || defined(__amigaos4__) */ +#if defined(__amigaos__) +#define uint16_t unsigned short +#endif /* defined(__amigaos__) */ +/* Hack */ +#define UNIX_PATH_LEN 108 +struct sockaddr_un { + uint16_t sun_family; + char sun_path[UNIX_PATH_LEN]; +}; +#else /* defined(_WIN32) || defined(__amigaos__) || defined(__amigaos4__) */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define closesocket close +#endif + +#include "miniupnpc_socketdef.h" + +#if !defined(__DragonFly__) && !defined(__OpenBSD__) && !defined(__NetBSD__) && !defined(__APPLE__) && !defined(_WIN32) && !defined(__CYGWIN__) && !defined(__sun) && !defined(__GNU__) && !defined(__FreeBSD_kernel__) +#define HAS_IP_MREQN +#endif + +#if !defined(HAS_IP_MREQN) && !defined(_WIN32) +#include +#if defined(__sun) +#include +#endif +#endif + +#if defined(HAS_IP_MREQN) && defined(NEED_STRUCT_IP_MREQN) +/* Several versions of glibc don't define this structure, + * define it here and compile with CFLAGS NEED_STRUCT_IP_MREQN */ +struct ip_mreqn +{ + struct in_addr imr_multiaddr; /* IP multicast address of group */ + struct in_addr imr_address; /* local IP address of interface */ + int imr_ifindex; /* Interface index */ +}; +#endif + +#if defined(__amigaos__) || defined(__amigaos4__) +/* Amiga OS specific stuff */ +#define TIMEVAL struct timeval +#endif + +#include "minissdpc.h" +#include "miniupnpc.h" +#include "receivedata.h" + +#if !(defined(_WIN32) || defined(__amigaos__) || defined(__amigaos4__)) + +#include "codelength.h" + +struct UPNPDev * +getDevicesFromMiniSSDPD(const char * devtype, const char * socketpath, int * error) +{ + struct UPNPDev * devlist = NULL; + int s; + int res; + + s = connectToMiniSSDPD(socketpath); + if (s < 0) { + if (error) + *error = s; + return NULL; + } + res = requestDevicesFromMiniSSDPD(s, devtype); + if (res < 0) { + if (error) + *error = res; + } else { + devlist = receiveDevicesFromMiniSSDPD(s, error); + } + disconnectFromMiniSSDPD(s); + return devlist; +} + +/* macros used to read from unix socket */ +#define READ_BYTE_BUFFER(c) \ + if((int)bufferindex >= n) { \ + n = read(s, buffer, sizeof(buffer)); \ + if(n<=0) break; \ + bufferindex = 0; \ + } \ + c = buffer[bufferindex++]; + +#ifndef MIN +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif /* MIN */ + +#define READ_COPY_BUFFER(dst, len) \ + for(l = len, p = (unsigned char *)dst; l > 0; ) { \ + unsigned int lcopy; \ + if((int)bufferindex >= n) { \ + n = read(s, buffer, sizeof(buffer)); \ + if(n<=0) break; \ + bufferindex = 0; \ + } \ + lcopy = MIN(l, (n - bufferindex)); \ + memcpy(p, buffer + bufferindex, lcopy); \ + l -= lcopy; \ + p += lcopy; \ + bufferindex += lcopy; \ + } + +#define READ_DISCARD_BUFFER(len) \ + for(l = len; l > 0; ) { \ + unsigned int lcopy; \ + if(bufferindex >= n) { \ + n = read(s, buffer, sizeof(buffer)); \ + if(n<=0) break; \ + bufferindex = 0; \ + } \ + lcopy = MIN(l, (n - bufferindex)); \ + l -= lcopy; \ + bufferindex += lcopy; \ + } + +int +connectToMiniSSDPD(const char * socketpath) +{ + int s; + struct sockaddr_un addr; +#if defined(MINIUPNPC_SET_SOCKET_TIMEOUT) && !defined(__sun) + struct timeval timeout; +#endif /* #ifdef MINIUPNPC_SET_SOCKET_TIMEOUT */ + + s = socket(AF_UNIX, SOCK_STREAM, 0); + if(s < 0) + { + /*syslog(LOG_ERR, "socket(unix): %m");*/ + perror("socket(unix)"); + return MINISSDPC_SOCKET_ERROR; + } +#if defined(MINIUPNPC_SET_SOCKET_TIMEOUT) && !defined(__sun) + /* setting a 3 seconds timeout */ + /* not supported for AF_UNIX sockets under Solaris */ + timeout.tv_sec = 3; + timeout.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(struct timeval)) < 0) + { + perror("setsockopt SO_RCVTIMEO unix"); + } + timeout.tv_sec = 3; + timeout.tv_usec = 0; + if(setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)) < 0) + { + perror("setsockopt SO_SNDTIMEO unix"); + } +#endif /* #ifdef MINIUPNPC_SET_SOCKET_TIMEOUT */ + if(!socketpath) + socketpath = "/var/run/minissdpd.sock"; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, socketpath, sizeof(addr.sun_path)); + /* TODO : check if we need to handle the EINTR */ + if(connect(s, (struct sockaddr *)&addr, sizeof(struct sockaddr_un)) < 0) + { + /*syslog(LOG_WARNING, "connect(\"%s\"): %m", socketpath);*/ + close(s); + return MINISSDPC_SOCKET_ERROR; + } + return s; +} + +int +disconnectFromMiniSSDPD(int s) +{ + if (close(s) < 0) + return MINISSDPC_SOCKET_ERROR; + return MINISSDPC_SUCCESS; +} + +int +requestDevicesFromMiniSSDPD(int s, const char * devtype) +{ + unsigned char buffer[256]; + unsigned char * p; + unsigned int stsize, l; + + stsize = strlen(devtype); + if(stsize == 8 && 0 == memcmp(devtype, "ssdp:all", 8)) + { + buffer[0] = 3; /* request type 3 : everything */ + } + else + { + buffer[0] = 1; /* request type 1 : request devices/services by type */ + } + p = buffer + 1; + l = stsize; CODELENGTH(l, p); + if(p + stsize > buffer + sizeof(buffer)) + { + /* devtype is too long ! */ +#ifdef DEBUG + fprintf(stderr, "devtype is too long ! stsize=%u sizeof(buffer)=%u\n", + stsize, (unsigned)sizeof(buffer)); +#endif /* DEBUG */ + return MINISSDPC_INVALID_INPUT; + } + memcpy(p, devtype, stsize); + p += stsize; + if(write(s, buffer, p - buffer) < 0) + { + /*syslog(LOG_ERR, "write(): %m");*/ + perror("minissdpc.c: write()"); + return MINISSDPC_SOCKET_ERROR; + } + return MINISSDPC_SUCCESS; +} + +struct UPNPDev * +receiveDevicesFromMiniSSDPD(int s, int * error) +{ + struct UPNPDev * tmp; + struct UPNPDev * devlist = NULL; + unsigned char buffer[256]; + ssize_t n; + unsigned char * p; + unsigned char * url; + unsigned char * st; + unsigned int bufferindex; + unsigned int i, ndev; + unsigned int urlsize, stsize, usnsize, l; + + n = read(s, buffer, sizeof(buffer)); + if(n<=0) + { + perror("minissdpc.c: read()"); + if (error) + *error = MINISSDPC_SOCKET_ERROR; + return NULL; + } + ndev = buffer[0]; + bufferindex = 1; + for(i = 0; i < ndev; i++) + { + DECODELENGTH_READ(urlsize, READ_BYTE_BUFFER); + if(n<=0) { + if (error) + *error = MINISSDPC_INVALID_SERVER_REPLY; + return devlist; + } +#ifdef DEBUG + printf(" urlsize=%u", urlsize); +#endif /* DEBUG */ + url = malloc(urlsize); + if(url == NULL) { + if (error) + *error = MINISSDPC_MEMORY_ERROR; + return devlist; + } + READ_COPY_BUFFER(url, urlsize); + if(n<=0) { + if (error) + *error = MINISSDPC_INVALID_SERVER_REPLY; + goto free_url_and_return; + } + DECODELENGTH_READ(stsize, READ_BYTE_BUFFER); + if(n<=0) { + if (error) + *error = MINISSDPC_INVALID_SERVER_REPLY; + goto free_url_and_return; + } +#ifdef DEBUG + printf(" stsize=%u", stsize); +#endif /* DEBUG */ + st = malloc(stsize); + if (st == NULL) { + if (error) + *error = MINISSDPC_MEMORY_ERROR; + goto free_url_and_return; + } + READ_COPY_BUFFER(st, stsize); + if(n<=0) { + if (error) + *error = MINISSDPC_INVALID_SERVER_REPLY; + goto free_url_and_st_and_return; + } + DECODELENGTH_READ(usnsize, READ_BYTE_BUFFER); + if(n<=0) { + if (error) + *error = MINISSDPC_INVALID_SERVER_REPLY; + goto free_url_and_st_and_return; + } +#ifdef DEBUG + printf(" usnsize=%u\n", usnsize); +#endif /* DEBUG */ + tmp = (struct UPNPDev *)malloc(sizeof(struct UPNPDev)+urlsize+stsize+usnsize); + if(tmp == NULL) { + if (error) + *error = MINISSDPC_MEMORY_ERROR; + goto free_url_and_st_and_return; + } + tmp->pNext = devlist; + tmp->descURL = tmp->buffer; + tmp->st = tmp->buffer + 1 + urlsize; + memcpy(tmp->buffer, url, urlsize); + tmp->buffer[urlsize] = '\0'; + memcpy(tmp->st, st, stsize); + tmp->buffer[urlsize+1+stsize] = '\0'; + free(url); + free(st); + url = NULL; + st = NULL; + tmp->usn = tmp->buffer + 1 + urlsize + 1 + stsize; + READ_COPY_BUFFER(tmp->usn, usnsize); + if(n<=0) { + if (error) + *error = MINISSDPC_INVALID_SERVER_REPLY; + goto free_tmp_and_return; + } + tmp->buffer[urlsize+1+stsize+1+usnsize] = '\0'; + tmp->scope_id = 0; /* default value. scope_id is not available with MiniSSDPd */ + devlist = tmp; + } + if (error) + *error = MINISSDPC_SUCCESS; + return devlist; + +free_url_and_st_and_return: + free(st); +free_url_and_return: + free(url); + return devlist; + +free_tmp_and_return: + free(tmp); + return devlist; +} + +#endif /* !(defined(_WIN32) || defined(__amigaos__) || defined(__amigaos4__)) */ + +/* parseMSEARCHReply() + * the last 4 arguments are filled during the parsing : + * - location/locationsize : "location:" field of the SSDP reply packet + * - st/stsize : "st:" field of the SSDP reply packet. + * The strings are NOT null terminated */ +static void +parseMSEARCHReply(const char * reply, int size, + const char * * location, int * locationsize, + const char * * st, int * stsize, + const char * * usn, int * usnsize) +{ + int a, b, i; + i = 0; + a = i; /* start of the line */ + b = 0; /* end of the "header" (position of the colon) */ + while(isin6_family = AF_INET6; + if(localport > 0 && localport < 65536) + p->sin6_port = htons((unsigned short)localport); + p->sin6_addr = in6addr_any; /* in6addr_any is not available with MinGW32 3.4.2 */ + } else { + struct sockaddr_in * p = (struct sockaddr_in *)&sockudp_r; + p->sin_family = AF_INET; + if(localport > 0 && localport < 65536) + p->sin_port = htons((unsigned short)localport); + p->sin_addr.s_addr = INADDR_ANY; + } +#ifdef _WIN32 +/* This code could help us to use the right Network interface for + * SSDP multicast traffic */ +/* Get IP associated with the index given in the ip_forward struct + * in order to give this ip to setsockopt(sudp, IPPROTO_IP, IP_MULTICAST_IF) */ + if(!ipv6 + && (GetBestRoute(inet_addr("223.255.255.255"), 0, &ip_forward) == NO_ERROR)) { + DWORD dwRetVal = 0; + PMIB_IPADDRTABLE pIPAddrTable; + DWORD dwSize = 0; +#ifdef DEBUG + IN_ADDR IPAddr; +#endif + int i; +#ifdef DEBUG + printf("ifIndex=%lu nextHop=%lx \n", ip_forward.dwForwardIfIndex, ip_forward.dwForwardNextHop); +#endif + pIPAddrTable = (MIB_IPADDRTABLE *) malloc(sizeof (MIB_IPADDRTABLE)); + if(pIPAddrTable) { + if (GetIpAddrTable(pIPAddrTable, &dwSize, 0) == ERROR_INSUFFICIENT_BUFFER) { + free(pIPAddrTable); + pIPAddrTable = (MIB_IPADDRTABLE *) malloc(dwSize); + } + } + if(pIPAddrTable) { + dwRetVal = GetIpAddrTable( pIPAddrTable, &dwSize, 0 ); + if (dwRetVal == NO_ERROR) { +#ifdef DEBUG + printf("\tNum Entries: %ld\n", pIPAddrTable->dwNumEntries); +#endif + for (i=0; i < (int) pIPAddrTable->dwNumEntries; i++) { +#ifdef DEBUG + printf("\n\tInterface Index[%d]:\t%ld\n", i, pIPAddrTable->table[i].dwIndex); + IPAddr.S_un.S_addr = (u_long) pIPAddrTable->table[i].dwAddr; + printf("\tIP Address[%d]: \t%s\n", i, inet_ntoa(IPAddr) ); + IPAddr.S_un.S_addr = (u_long) pIPAddrTable->table[i].dwMask; + printf("\tSubnet Mask[%d]: \t%s\n", i, inet_ntoa(IPAddr) ); + IPAddr.S_un.S_addr = (u_long) pIPAddrTable->table[i].dwBCastAddr; + printf("\tBroadCast[%d]: \t%s (%ld)\n", i, inet_ntoa(IPAddr), pIPAddrTable->table[i].dwBCastAddr); + printf("\tReassembly size[%d]:\t%ld\n", i, pIPAddrTable->table[i].dwReasmSize); + printf("\tType and State[%d]:", i); + printf("\n"); +#endif + if (pIPAddrTable->table[i].dwIndex == ip_forward.dwForwardIfIndex) { + /* Set the address of this interface to be used */ + struct in_addr mc_if; + memset(&mc_if, 0, sizeof(mc_if)); + mc_if.s_addr = pIPAddrTable->table[i].dwAddr; + if(setsockopt(sudp, IPPROTO_IP, IP_MULTICAST_IF, (const char *)&mc_if, sizeof(mc_if)) < 0) { + PRINT_SOCKET_ERROR("setsockopt"); + } + ((struct sockaddr_in *)&sockudp_r)->sin_addr.s_addr = pIPAddrTable->table[i].dwAddr; +#ifndef DEBUG + break; +#endif + } + } + } + free(pIPAddrTable); + pIPAddrTable = NULL; + } + } +#endif /* _WIN32 */ + +#ifdef _WIN32 + if (setsockopt(sudp, SOL_SOCKET, SO_REUSEADDR, (const char *)&opt, sizeof (opt)) < 0) +#else + if (setsockopt(sudp, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof (opt)) < 0) +#endif + { + if(error) + *error = MINISSDPC_SOCKET_ERROR; + PRINT_SOCKET_ERROR("setsockopt(SO_REUSEADDR,...)"); + return NULL; + } + + if(ipv6) { +#ifdef _WIN32 + DWORD mcastHops = ttl; + if(setsockopt(sudp, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, (const char *)&mcastHops, sizeof(mcastHops)) < 0) +#else /* _WIN32 */ + int mcastHops = ttl; + if(setsockopt(sudp, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, &mcastHops, sizeof(mcastHops)) < 0) +#endif /* _WIN32 */ + { + PRINT_SOCKET_ERROR("setsockopt(IPV6_MULTICAST_HOPS,...)"); + } + } else { +#ifdef _WIN32 + if(setsockopt(sudp, IPPROTO_IP, IP_MULTICAST_TTL, (const char *)&_ttl, sizeof(_ttl)) < 0) +#else /* _WIN32 */ + if(setsockopt(sudp, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) < 0) +#endif /* _WIN32 */ + { + /* not a fatal error */ + PRINT_SOCKET_ERROR("setsockopt(IP_MULTICAST_TTL,...)"); + } + } + + if(multicastif) + { + if(ipv6) { +#if !defined(_WIN32) + /* according to MSDN, if_nametoindex() is supported since + * MS Windows Vista and MS Windows Server 2008. + * http://msdn.microsoft.com/en-us/library/bb408409%28v=vs.85%29.aspx */ + unsigned int ifindex = if_nametoindex(multicastif); /* eth0, etc. */ + if(setsockopt(sudp, IPPROTO_IPV6, IPV6_MULTICAST_IF, &ifindex, sizeof(ifindex)) < 0) + { + PRINT_SOCKET_ERROR("setsockopt IPV6_MULTICAST_IF"); + } +#else +#ifdef DEBUG + printf("Setting of multicast interface not supported in IPv6 under Windows.\n"); +#endif +#endif + } else { + struct in_addr mc_if; + mc_if.s_addr = inet_addr(multicastif); /* ex: 192.168.x.x */ + if(mc_if.s_addr != INADDR_NONE) + { + ((struct sockaddr_in *)&sockudp_r)->sin_addr.s_addr = mc_if.s_addr; + if(setsockopt(sudp, IPPROTO_IP, IP_MULTICAST_IF, (const char *)&mc_if, sizeof(mc_if)) < 0) + { + PRINT_SOCKET_ERROR("setsockopt IP_MULTICAST_IF"); + } + } else { +#ifdef HAS_IP_MREQN + /* was not an ip address, try with an interface name */ + struct ip_mreqn reqn; /* only defined with -D_BSD_SOURCE or -D_GNU_SOURCE */ + memset(&reqn, 0, sizeof(struct ip_mreqn)); + reqn.imr_ifindex = if_nametoindex(multicastif); + if(setsockopt(sudp, IPPROTO_IP, IP_MULTICAST_IF, (const char *)&reqn, sizeof(reqn)) < 0) + { + PRINT_SOCKET_ERROR("setsockopt IP_MULTICAST_IF"); + } +#elif !defined(_WIN32) + struct ifreq ifr; + int ifrlen = sizeof(ifr); + strncpy(ifr.ifr_name, multicastif, IFNAMSIZ); + ifr.ifr_name[IFNAMSIZ-1] = '\0'; + if(ioctl(sudp, SIOCGIFADDR, &ifr, &ifrlen) < 0) + { + PRINT_SOCKET_ERROR("ioctl(...SIOCGIFADDR...)"); + } + mc_if.s_addr = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr.s_addr; + if(setsockopt(sudp, IPPROTO_IP, IP_MULTICAST_IF, (const char *)&mc_if, sizeof(mc_if)) < 0) + { + PRINT_SOCKET_ERROR("setsockopt IP_MULTICAST_IF"); + } +#else /* _WIN32 */ +#ifdef DEBUG + printf("Setting of multicast interface not supported with interface name.\n"); +#endif +#endif /* #ifdef HAS_IP_MREQN / !defined(_WIN32) */ + } + } + } + + /* Before sending the packed, we first "bind" in order to be able + * to receive the response */ + if (bind(sudp, (const struct sockaddr *)&sockudp_r, + ipv6 ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)) != 0) + { + if(error) + *error = MINISSDPC_SOCKET_ERROR; + PRINT_SOCKET_ERROR("bind"); + closesocket(sudp); + return NULL; + } + + if(error) + *error = MINISSDPC_SUCCESS; + /* Calculating maximum response time in seconds */ + mx = ((unsigned int)delay) / 1000u; + if(mx == 0) { + mx = 1; + delay = 1000; + } + /* receiving SSDP response packet */ + for(deviceIndex = 0; deviceTypes[deviceIndex]; deviceIndex++) { + sentok = 0; + /* sending the SSDP M-SEARCH packet */ + n = snprintf(bufr, sizeof(bufr), + MSearchMsgFmt, + ipv6 ? + (linklocal ? "[" UPNP_MCAST_LL_ADDR "]" : "[" UPNP_MCAST_SL_ADDR "]") + : UPNP_MCAST_ADDR, + deviceTypes[deviceIndex], mx); + if ((unsigned int)n >= sizeof(bufr)) { + if(error) + *error = MINISSDPC_MEMORY_ERROR; + goto error; + } +#ifdef DEBUG + /*printf("Sending %s", bufr);*/ + printf("Sending M-SEARCH request to %s with ST: %s\n", + ipv6 ? + (linklocal ? "[" UPNP_MCAST_LL_ADDR "]" : "[" UPNP_MCAST_SL_ADDR "]") + : UPNP_MCAST_ADDR, + deviceTypes[deviceIndex]); +#endif +#ifdef NO_GETADDRINFO + /* the following code is not using getaddrinfo */ + /* emission */ + memset(&sockudp_w, 0, sizeof(struct sockaddr_storage)); + if(ipv6) { + struct sockaddr_in6 * p = (struct sockaddr_in6 *)&sockudp_w; + p->sin6_family = AF_INET6; + p->sin6_port = htons(SSDP_PORT); + inet_pton(AF_INET6, + linklocal ? UPNP_MCAST_LL_ADDR : UPNP_MCAST_SL_ADDR, + &(p->sin6_addr)); + } else { + struct sockaddr_in * p = (struct sockaddr_in *)&sockudp_w; + p->sin_family = AF_INET; + p->sin_port = htons(SSDP_PORT); + p->sin_addr.s_addr = inet_addr(UPNP_MCAST_ADDR); + } + n = sendto(sudp, bufr, n, 0, &sockudp_w, + ipv6 ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in)); + if (n < 0) { + if(error) + *error = MINISSDPC_SOCKET_ERROR; + PRINT_SOCKET_ERROR("sendto"); + } else { + sentok = 1; + } +#else /* #ifdef NO_GETADDRINFO */ + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; /* AF_INET6 or AF_INET */ + hints.ai_socktype = SOCK_DGRAM; + /*hints.ai_flags = */ + if ((rv = getaddrinfo(ipv6 + ? (linklocal ? UPNP_MCAST_LL_ADDR : UPNP_MCAST_SL_ADDR) + : UPNP_MCAST_ADDR, + XSTR(SSDP_PORT), &hints, &servinfo)) != 0) { + if(error) + *error = MINISSDPC_SOCKET_ERROR; +#ifdef _WIN32 + fprintf(stderr, "getaddrinfo() failed: %d\n", rv); +#else + fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv)); +#endif + break; + } + for(p = servinfo; p; p = p->ai_next) { + n = sendto(sudp, bufr, n, 0, p->ai_addr, p->ai_addrlen); + if (n < 0) { +#ifdef DEBUG + char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; + if (getnameinfo(p->ai_addr, p->ai_addrlen, hbuf, sizeof(hbuf), sbuf, + sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0) { + fprintf(stderr, "host:%s port:%s\n", hbuf, sbuf); + } +#endif + PRINT_SOCKET_ERROR("sendto"); + continue; + } else { + sentok = 1; + } + } + freeaddrinfo(servinfo); + if(!sentok) { + if(error) + *error = MINISSDPC_SOCKET_ERROR; + } +#endif /* #ifdef NO_GETADDRINFO */ + /* Waiting for SSDP REPLY packet to M-SEARCH + * if searchalltypes is set, enter the loop only + * when the last deviceType is reached */ + if((sentok && !searchalltypes) || !deviceTypes[deviceIndex + 1]) do { + n = receivedata(sudp, bufr, sizeof(bufr), delay, &scope_id); + if (n < 0) { + /* error */ + if(error) + *error = MINISSDPC_SOCKET_ERROR; + goto error; + } else if (n == 0) { + /* no data or Time Out */ +#ifdef DEBUG + printf("NODATA or TIMEOUT\n"); +#endif /* DEBUG */ + if (devlist && !searchalltypes) { + /* found some devices, stop now*/ + if(error) + *error = MINISSDPC_SUCCESS; + goto error; + } + } else { + const char * descURL=NULL; + int urlsize=0; + const char * st=NULL; + int stsize=0; + const char * usn=NULL; + int usnsize=0; + parseMSEARCHReply(bufr, n, &descURL, &urlsize, &st, &stsize, &usn, &usnsize); + if(st&&descURL) { +#ifdef DEBUG + printf("M-SEARCH Reply:\n ST: %.*s\n USN: %.*s\n Location: %.*s\n", + stsize, st, usnsize, (usn?usn:""), urlsize, descURL); +#endif /* DEBUG */ + for(tmp=devlist; tmp; tmp = tmp->pNext) { + if(memcmp(tmp->descURL, descURL, urlsize) == 0 && + tmp->descURL[urlsize] == '\0' && + memcmp(tmp->st, st, stsize) == 0 && + tmp->st[stsize] == '\0' && + (usnsize == 0 || memcmp(tmp->usn, usn, usnsize) == 0) && + tmp->usn[usnsize] == '\0') + break; + } + /* at the exit of the loop above, tmp is null if + * no duplicate device was found */ + if(tmp) + continue; + tmp = (struct UPNPDev *)malloc(sizeof(struct UPNPDev)+urlsize+stsize+usnsize); + if(!tmp) { + /* memory allocation error */ + if(error) + *error = MINISSDPC_MEMORY_ERROR; + goto error; + } + tmp->pNext = devlist; + tmp->descURL = tmp->buffer; + tmp->st = tmp->buffer + 1 + urlsize; + tmp->usn = tmp->st + 1 + stsize; + memcpy(tmp->buffer, descURL, urlsize); + tmp->buffer[urlsize] = '\0'; + memcpy(tmp->st, st, stsize); + tmp->buffer[urlsize+1+stsize] = '\0'; + if(usn != NULL) + memcpy(tmp->usn, usn, usnsize); + tmp->buffer[urlsize+1+stsize+1+usnsize] = '\0'; + tmp->scope_id = scope_id; + devlist = tmp; + } + } + } while(n > 0); + if(ipv6) { + /* switch linklocal flag */ + if(linklocal) { + linklocal = 0; + --deviceIndex; + } else { + linklocal = 1; + } + } + } +error: + closesocket(sudp); + return devlist; +} + diff --git a/src/contrib/miniupnp/miniupnpc/minissdpc.h b/src/contrib/miniupnp/miniupnpc/minissdpc.h new file mode 100644 index 0000000..167d897 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/minissdpc.h @@ -0,0 +1,58 @@ +/* $Id: minissdpc.h,v 1.6 2015/09/18 12:45:16 nanard Exp $ */ +/* Project: miniupnp + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author: Thomas Bernard + * Copyright (c) 2005-2015 Thomas Bernard + * This software is subjects to the conditions detailed + * in the LICENCE file provided within this distribution */ +#ifndef MINISSDPC_H_INCLUDED +#define MINISSDPC_H_INCLUDED + +#include "miniupnpc_declspec.h" +#include "upnpdev.h" + +/* error codes : */ +#define MINISSDPC_SUCCESS (0) +#define MINISSDPC_UNKNOWN_ERROR (-1) +#define MINISSDPC_SOCKET_ERROR (-101) +#define MINISSDPC_MEMORY_ERROR (-102) +#define MINISSDPC_INVALID_INPUT (-103) +#define MINISSDPC_INVALID_SERVER_REPLY (-104) + +#ifdef __cplusplus +extern "C" { +#endif + +#if !(defined(_WIN32) || defined(__amigaos__) || defined(__amigaos4__)) + +MINIUPNP_LIBSPEC struct UPNPDev * +getDevicesFromMiniSSDPD(const char * devtype, const char * socketpath, int * error); + +MINIUPNP_LIBSPEC int +connectToMiniSSDPD(const char * socketpath); + +MINIUPNP_LIBSPEC int +disconnectFromMiniSSDPD(int fd); + +MINIUPNP_LIBSPEC int +requestDevicesFromMiniSSDPD(int fd, const char * devtype); + +MINIUPNP_LIBSPEC struct UPNPDev * +receiveDevicesFromMiniSSDPD(int fd, int * error); + +#endif /* !(defined(_WIN32) || defined(__amigaos__) || defined(__amigaos4__)) */ + +MINIUPNP_LIBSPEC struct UPNPDev * +ssdpDiscoverDevices(const char * const deviceTypes[], + int delay, const char * multicastif, + int localport, + int ipv6, unsigned char ttl, + int * error, + int searchalltypes); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpc.c b/src/contrib/miniupnp/miniupnpc/miniupnpc.c new file mode 100644 index 0000000..5d93ef9 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpc.c @@ -0,0 +1,727 @@ +/* $Id: miniupnpc.c,v 1.149 2016/02/09 09:50:46 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp + * Web : http://miniupnp.free.fr/ + * Author : Thomas BERNARD + * copyright (c) 2005-2018 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENSE file. */ +#include +#include +#include +#ifdef _WIN32 +/* Win32 Specific includes and defines */ +#include +#include +#include +#include +#define snprintf _snprintf +#define strdup _strdup +#ifndef strncasecmp +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#define strncasecmp _memicmp +#else /* defined(_MSC_VER) && (_MSC_VER >= 1400) */ +#define strncasecmp memicmp +#endif /* defined(_MSC_VER) && (_MSC_VER >= 1400) */ +#endif /* #ifndef strncasecmp */ +#define MAXHOSTNAMELEN 64 +#else /* #ifdef _WIN32 */ +/* Standard POSIX includes */ +#include +#if defined(__amigaos__) && !defined(__amigaos4__) +/* Amiga OS 3 specific stuff */ +#define socklen_t int +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#if !defined(__amigaos__) && !defined(__amigaos4__) +#include +#endif +#include +#include +#define closesocket close +#endif /* #else _WIN32 */ +#ifdef __GNU__ +#define MAXHOSTNAMELEN 64 +#endif + + +#include "miniupnpc.h" +#include "minissdpc.h" +#include "miniwget.h" +#include "miniwget_private.h" +#include "minisoap.h" +#include "minixml.h" +#include "upnpcommands.h" +#include "connecthostport.h" + +/* compare the beginning of a string with a constant string */ +#define COMPARE(str, cstr) (0==memcmp(str, cstr, sizeof(cstr) - 1)) + +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN 64 +#endif + +#define SOAPPREFIX "s" +#define SERVICEPREFIX "u" +#define SERVICEPREFIX2 'u' + +/* check if an ip address is a private (LAN) address + * see https://tools.ietf.org/html/rfc1918 */ +static int is_rfc1918addr(const char * addr) +{ + /* 192.168.0.0 - 192.168.255.255 (192.168/16 prefix) */ + if(COMPARE(addr, "192.168.")) + return 1; + /* 10.0.0.0 - 10.255.255.255 (10/8 prefix) */ + if(COMPARE(addr, "10.")) + return 1; + /* 172.16.0.0 - 172.31.255.255 (172.16/12 prefix) */ + if(COMPARE(addr, "172.")) { + int i = atoi(addr + 4); + if((16 <= i) && (i <= 31)) + return 1; + } + return 0; +} + +/* root description parsing */ +MINIUPNP_LIBSPEC void parserootdesc(const char * buffer, int bufsize, struct IGDdatas * data) +{ + struct xmlparser parser; + /* xmlparser object */ + parser.xmlstart = buffer; + parser.xmlsize = bufsize; + parser.data = data; + parser.starteltfunc = IGDstartelt; + parser.endeltfunc = IGDendelt; + parser.datafunc = IGDdata; + parser.attfunc = 0; + parsexml(&parser); +#ifdef DEBUG + printIGD(data); +#endif +} + +/* simpleUPnPcommand2 : + * not so simple ! + * return values : + * pointer - OK + * NULL - error */ +static char * +simpleUPnPcommand2(SOCKET s, const char * url, const char * service, + const char * action, struct UPNParg * args, + int * bufsize, const char * httpversion) +{ + char hostname[MAXHOSTNAMELEN+1]; + unsigned short port = 0; + char * path; + char soapact[128]; + char soapbody[2048]; + int soapbodylen; + char * buf; + int n; + int status_code; + + *bufsize = 0; + snprintf(soapact, sizeof(soapact), "%s#%s", service, action); + if(args==NULL) + { + soapbodylen = snprintf(soapbody, sizeof(soapbody), + "\r\n" + "<" SOAPPREFIX ":Envelope " + "xmlns:" SOAPPREFIX "=\"http://schemas.xmlsoap.org/soap/envelope/\" " + SOAPPREFIX ":encodingStyle=\"http://schemas.xmlsoap.org/soap/encoding/\">" + "<" SOAPPREFIX ":Body>" + "<" SERVICEPREFIX ":%s xmlns:" SERVICEPREFIX "=\"%s\">" + "" + "" + "\r\n", action, service, action); + if ((unsigned int)soapbodylen >= sizeof(soapbody)) + return NULL; + } + else + { + char * p; + const char * pe, * pv; + const char * const pend = soapbody + sizeof(soapbody); + soapbodylen = snprintf(soapbody, sizeof(soapbody), + "\r\n" + "<" SOAPPREFIX ":Envelope " + "xmlns:" SOAPPREFIX "=\"http://schemas.xmlsoap.org/soap/envelope/\" " + SOAPPREFIX ":encodingStyle=\"http://schemas.xmlsoap.org/soap/encoding/\">" + "<" SOAPPREFIX ":Body>" + "<" SERVICEPREFIX ":%s xmlns:" SERVICEPREFIX "=\"%s\">", + action, service); + if ((unsigned int)soapbodylen >= sizeof(soapbody)) + return NULL; + p = soapbody + soapbodylen; + while(args->elt) + { + if(p >= pend) /* check for space to write next byte */ + return NULL; + *(p++) = '<'; + + pe = args->elt; + while(p < pend && *pe) + *(p++) = *(pe++); + + if(p >= pend) /* check for space to write next byte */ + return NULL; + *(p++) = '>'; + + if((pv = args->val)) + { + while(p < pend && *pv) + *(p++) = *(pv++); + } + + if((p+2) > pend) /* check for space to write next 2 bytes */ + return NULL; + *(p++) = '<'; + *(p++) = '/'; + + pe = args->elt; + while(p < pend && *pe) + *(p++) = *(pe++); + + if(p >= pend) /* check for space to write next byte */ + return NULL; + *(p++) = '>'; + + args++; + } + if((p+4) > pend) /* check for space to write next 4 bytes */ + return NULL; + *(p++) = '<'; + *(p++) = '/'; + *(p++) = SERVICEPREFIX2; + *(p++) = ':'; + + pe = action; + while(p < pend && *pe) + *(p++) = *(pe++); + + strncpy(p, ">\r\n", + pend - p); + if(soapbody[sizeof(soapbody)-1]) /* strncpy pads buffer with 0s, so if it doesn't end in 0, could not fit full string */ + return NULL; + } + if(!parseURL(url, hostname, &port, &path, NULL)) return NULL; + if(ISINVALID(s)) { + s = connecthostport(hostname, port, 0); + if(ISINVALID(s)) { + /* failed to connect */ + return NULL; + } + } + + n = soapPostSubmit(s, path, hostname, port, soapact, soapbody, httpversion); + if(n<=0) { +#ifdef DEBUG + printf("Error sending SOAP request\n"); +#endif + closesocket(s); + return NULL; + } + + buf = getHTTPResponse(s, bufsize, &status_code); +#ifdef DEBUG + if(*bufsize > 0 && buf) + { + printf("HTTP %d SOAP Response :\n%.*s\n", status_code, *bufsize, buf); + } + else + { + printf("HTTP %d, empty SOAP response. size=%d\n", status_code, *bufsize); + } +#endif + closesocket(s); + return buf; +} + +/* simpleUPnPcommand : + * not so simple ! + * return values : + * pointer - OK + * NULL - error */ +char * +simpleUPnPcommand(int s, const char * url, const char * service, + const char * action, struct UPNParg * args, + int * bufsize) +{ + char * buf; + +#if 1 + buf = simpleUPnPcommand2((SOCKET)s, url, service, action, args, bufsize, "1.1"); +#else + buf = simpleUPnPcommand2((SOCKET)s, url, service, action, args, bufsize, "1.0"); + if (!buf || *bufsize == 0) + { +#if DEBUG + printf("Error or no result from SOAP request; retrying with HTTP/1.1\n"); +#endif + buf = simpleUPnPcommand2((SOCKET)s, url, service, action, args, bufsize, "1.1"); + } +#endif + return buf; +} + +/* upnpDiscoverDevices() : + * return a chained list of all devices found or NULL if + * no devices was found. + * It is up to the caller to free the chained list + * delay is in millisecond (poll). + * UDA v1.1 says : + * The TTL for the IP packet SHOULD default to 2 and + * SHOULD be configurable. */ +MINIUPNP_LIBSPEC struct UPNPDev * +upnpDiscoverDevices(const char * const deviceTypes[], + int delay, const char * multicastif, + const char * minissdpdsock, int localport, + int ipv6, unsigned char ttl, + int * error, + int searchalltypes) +{ + struct UPNPDev * tmp; + struct UPNPDev * devlist = 0; +#if !defined(_WIN32) && !defined(__amigaos__) && !defined(__amigaos4__) + int deviceIndex; +#endif /* !defined(_WIN32) && !defined(__amigaos__) && !defined(__amigaos4__) */ + + if(error) + *error = UPNPDISCOVER_UNKNOWN_ERROR; +#if !defined(_WIN32) && !defined(__amigaos__) && !defined(__amigaos4__) + /* first try to get infos from minissdpd ! */ + if(!minissdpdsock) + minissdpdsock = "/var/run/minissdpd.sock"; + if(minissdpdsock[0] != '\0') { + for(deviceIndex = 0; deviceTypes[deviceIndex]; deviceIndex++) { + struct UPNPDev * minissdpd_devlist; + int only_rootdevice = 1; + minissdpd_devlist = getDevicesFromMiniSSDPD(deviceTypes[deviceIndex], + minissdpdsock, 0); + if(minissdpd_devlist) { +#ifdef DEBUG + printf("returned by MiniSSDPD: %s\t%s\n", + minissdpd_devlist->st, minissdpd_devlist->descURL); +#endif /* DEBUG */ + if(!strstr(minissdpd_devlist->st, "rootdevice")) + only_rootdevice = 0; + for(tmp = minissdpd_devlist; tmp->pNext != NULL; tmp = tmp->pNext) { +#ifdef DEBUG + printf("returned by MiniSSDPD: %s\t%s\n", + tmp->pNext->st, tmp->pNext->descURL); +#endif /* DEBUG */ + if(!strstr(tmp->st, "rootdevice")) + only_rootdevice = 0; + } + tmp->pNext = devlist; + devlist = minissdpd_devlist; + if(!searchalltypes && !only_rootdevice) + break; + } + } + } + for(tmp = devlist; tmp != NULL; tmp = tmp->pNext) { + /* We return what we have found if it was not only a rootdevice */ + if(!strstr(tmp->st, "rootdevice")) { + if(error) + *error = UPNPDISCOVER_SUCCESS; + return devlist; + } + } +#endif /* !defined(_WIN32) && !defined(__amigaos__) && !defined(__amigaos4__) */ + + /* direct discovery if minissdpd responses are not sufficient */ + { + struct UPNPDev * discovered_devlist; + discovered_devlist = ssdpDiscoverDevices(deviceTypes, delay, multicastif, localport, + ipv6, ttl, error, searchalltypes); + if(devlist == NULL) + devlist = discovered_devlist; + else { + for(tmp = devlist; tmp->pNext != NULL; tmp = tmp->pNext); + tmp->pNext = discovered_devlist; + } + } + return devlist; +} + +/* upnpDiscover() Discover IGD device */ +MINIUPNP_LIBSPEC struct UPNPDev * +upnpDiscover(int delay, const char * multicastif, + const char * minissdpdsock, int localport, + int ipv6, unsigned char ttl, + int * error) +{ + static const char * const deviceList[] = { +#if 0 + "urn:schemas-upnp-org:device:InternetGatewayDevice:2", + "urn:schemas-upnp-org:service:WANIPConnection:2", +#endif + "urn:schemas-upnp-org:device:InternetGatewayDevice:1", + "urn:schemas-upnp-org:service:WANIPConnection:1", + "urn:schemas-upnp-org:service:WANPPPConnection:1", + "upnp:rootdevice", + /*"ssdp:all",*/ + 0 + }; + return upnpDiscoverDevices(deviceList, + delay, multicastif, minissdpdsock, localport, + ipv6, ttl, error, 0); +} + +/* upnpDiscoverAll() Discover all UPnP devices */ +MINIUPNP_LIBSPEC struct UPNPDev * +upnpDiscoverAll(int delay, const char * multicastif, + const char * minissdpdsock, int localport, + int ipv6, unsigned char ttl, + int * error) +{ + static const char * const deviceList[] = { + /*"upnp:rootdevice",*/ + "ssdp:all", + 0 + }; + return upnpDiscoverDevices(deviceList, + delay, multicastif, minissdpdsock, localport, + ipv6, ttl, error, 0); +} + +/* upnpDiscoverDevice() Discover a specific device */ +MINIUPNP_LIBSPEC struct UPNPDev * +upnpDiscoverDevice(const char * device, int delay, const char * multicastif, + const char * minissdpdsock, int localport, + int ipv6, unsigned char ttl, + int * error) +{ + const char * const deviceList[] = { + device, + 0 + }; + return upnpDiscoverDevices(deviceList, + delay, multicastif, minissdpdsock, localport, + ipv6, ttl, error, 0); +} + +static char * +build_absolute_url(const char * baseurl, const char * descURL, + const char * url, unsigned int scope_id) +{ + int l, n; + char * s; + const char * base; + char * p; +#if defined(IF_NAMESIZE) && !defined(_WIN32) + char ifname[IF_NAMESIZE]; +#else /* defined(IF_NAMESIZE) && !defined(_WIN32) */ + char scope_str[8]; +#endif /* defined(IF_NAMESIZE) && !defined(_WIN32) */ + + if( (url[0] == 'h') + &&(url[1] == 't') + &&(url[2] == 't') + &&(url[3] == 'p') + &&(url[4] == ':') + &&(url[5] == '/') + &&(url[6] == '/')) + return strdup(url); + base = (baseurl[0] == '\0') ? descURL : baseurl; + n = strlen(base); + if(n > 7) { + p = strchr(base + 7, '/'); + if(p) + n = p - base; + } + l = n + strlen(url) + 1; + if(url[0] != '/') + l++; + if(scope_id != 0) { +#if defined(IF_NAMESIZE) && !defined(_WIN32) + if(if_indextoname(scope_id, ifname)) { + l += 3 + strlen(ifname); /* 3 == strlen(%25) */ + } +#else /* defined(IF_NAMESIZE) && !defined(_WIN32) */ + /* under windows, scope is numerical */ + l += 3 + snprintf(scope_str, sizeof(scope_str), "%u", scope_id); +#endif /* defined(IF_NAMESIZE) && !defined(_WIN32) */ + } + s = malloc(l); + if(s == NULL) return NULL; + memcpy(s, base, n); + if(scope_id != 0) { + s[n] = '\0'; + if(0 == memcmp(s, "http://[fe80:", 13)) { + /* this is a linklocal IPv6 address */ + p = strchr(s, ']'); + if(p) { + /* insert %25 into URL */ +#if defined(IF_NAMESIZE) && !defined(_WIN32) + memmove(p + 3 + strlen(ifname), p, strlen(p) + 1); + memcpy(p, "%25", 3); + memcpy(p + 3, ifname, strlen(ifname)); + n += 3 + strlen(ifname); +#else /* defined(IF_NAMESIZE) && !defined(_WIN32) */ + memmove(p + 3 + strlen(scope_str), p, strlen(p) + 1); + memcpy(p, "%25", 3); + memcpy(p + 3, scope_str, strlen(scope_str)); + n += 3 + strlen(scope_str); +#endif /* defined(IF_NAMESIZE) && !defined(_WIN32) */ + } + } + } + if(url[0] != '/') + s[n++] = '/'; + memcpy(s + n, url, l - n); + return s; +} + +/* Prepare the Urls for usage... + */ +MINIUPNP_LIBSPEC void +GetUPNPUrls(struct UPNPUrls * urls, struct IGDdatas * data, + const char * descURL, unsigned int scope_id) +{ + /* strdup descURL */ + urls->rootdescURL = strdup(descURL); + + /* get description of WANIPConnection */ + urls->ipcondescURL = build_absolute_url(data->urlbase, descURL, + data->first.scpdurl, scope_id); + urls->controlURL = build_absolute_url(data->urlbase, descURL, + data->first.controlurl, scope_id); + urls->controlURL_CIF = build_absolute_url(data->urlbase, descURL, + data->CIF.controlurl, scope_id); + urls->controlURL_6FC = build_absolute_url(data->urlbase, descURL, + data->IPv6FC.controlurl, scope_id); + +#ifdef DEBUG + printf("urls->ipcondescURL='%s'\n", urls->ipcondescURL); + printf("urls->controlURL='%s'\n", urls->controlURL); + printf("urls->controlURL_CIF='%s'\n", urls->controlURL_CIF); + printf("urls->controlURL_6FC='%s'\n", urls->controlURL_6FC); +#endif +} + +MINIUPNP_LIBSPEC void +FreeUPNPUrls(struct UPNPUrls * urls) +{ + if(!urls) + return; + free(urls->controlURL); + urls->controlURL = 0; + free(urls->ipcondescURL); + urls->ipcondescURL = 0; + free(urls->controlURL_CIF); + urls->controlURL_CIF = 0; + free(urls->controlURL_6FC); + urls->controlURL_6FC = 0; + free(urls->rootdescURL); + urls->rootdescURL = 0; +} + +int +UPNPIGD_IsConnected(struct UPNPUrls * urls, struct IGDdatas * data) +{ + char status[64]; + unsigned int uptime; + status[0] = '\0'; + UPNP_GetStatusInfo(urls->controlURL, data->first.servicetype, + status, &uptime, NULL); + if(0 == strcmp("Connected", status)) + return 1; + else if(0 == strcmp("Up", status)) /* Also accept "Up" */ + return 1; + else + return 0; +} + + +/* UPNP_GetValidIGD() : + * return values : + * -1 = Internal error + * 0 = NO IGD found + * 1 = A valid connected IGD has been found + * 2 = A valid IGD has been found but it reported as + * not connected + * 3 = an UPnP device has been found but was not recognized as an IGD + * + * In any positive non zero return case, the urls and data structures + * passed as parameters are set. Don't forget to call FreeUPNPUrls(urls) to + * free allocated memory. + */ +MINIUPNP_LIBSPEC int +UPNP_GetValidIGD(struct UPNPDev * devlist, + struct UPNPUrls * urls, + struct IGDdatas * data, + char * lanaddr, int lanaddrlen) +{ + struct xml_desc { + char * xml; + int size; + int is_igd; + } * desc = NULL; + struct UPNPDev * dev; + int ndev = 0; + int i; + int state = -1; /* state 1 : IGD connected. State 2 : IGD. State 3 : anything */ + int n_igd = 0; + char extIpAddr[16]; + char myLanAddr[40]; + int status_code = -1; + + if(!devlist) + { +#ifdef DEBUG + printf("Empty devlist\n"); +#endif + return 0; + } + /* counting total number of devices in the list */ + for(dev = devlist; dev; dev = dev->pNext) + ndev++; + if(ndev > 0) + { + desc = calloc(ndev, sizeof(struct xml_desc)); + if(!desc) + return -1; /* memory allocation error */ + } + /* Step 1 : downloading descriptions and testing type */ + for(dev = devlist, i = 0; dev; dev = dev->pNext, i++) + { + /* we should choose an internet gateway device. + * with st == urn:schemas-upnp-org:device:InternetGatewayDevice:1 */ + desc[i].xml = miniwget_getaddr(dev->descURL, &(desc[i].size), + myLanAddr, sizeof(myLanAddr), + dev->scope_id, &status_code); +#ifdef DEBUG + if(!desc[i].xml) + { + printf("error getting XML description %s\n", dev->descURL); + } +#endif + if(desc[i].xml) + { + memset(data, 0, sizeof(struct IGDdatas)); + memset(urls, 0, sizeof(struct UPNPUrls)); + parserootdesc(desc[i].xml, desc[i].size, data); + if(COMPARE(data->CIF.servicetype, + "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:")) + { + desc[i].is_igd = 1; + n_igd++; + if(lanaddr) + strncpy(lanaddr, myLanAddr, lanaddrlen); + } + } + } + /* iterate the list to find a device depending on state */ + for(state = 1; state <= 3; state++) + { + for(dev = devlist, i = 0; dev; dev = dev->pNext, i++) + { + if(desc[i].xml) + { + memset(data, 0, sizeof(struct IGDdatas)); + memset(urls, 0, sizeof(struct UPNPUrls)); + parserootdesc(desc[i].xml, desc[i].size, data); + if(desc[i].is_igd || state >= 3 ) + { + int is_connected; + + GetUPNPUrls(urls, data, dev->descURL, dev->scope_id); + + /* in state 2 and 3 we don't test if device is connected ! */ + if(state >= 2) + goto free_and_return; + is_connected = UPNPIGD_IsConnected(urls, data); +#ifdef DEBUG + printf("UPNPIGD_IsConnected(%s) = %d\n", + urls->controlURL, is_connected); +#endif + /* checks that status is connected AND there is a external IP address assigned */ + if(is_connected && + (UPNP_GetExternalIPAddress(urls->controlURL, data->first.servicetype, extIpAddr) == 0)) { + if(!is_rfc1918addr(extIpAddr) && (extIpAddr[0] != '\0') + && (0 != strcmp(extIpAddr, "0.0.0.0"))) + goto free_and_return; + } + FreeUPNPUrls(urls); + if(data->second.servicetype[0] != '\0') { +#ifdef DEBUG + printf("We tried %s, now we try %s !\n", + data->first.servicetype, data->second.servicetype); +#endif + /* swaping WANPPPConnection and WANIPConnection ! */ + memcpy(&data->tmp, &data->first, sizeof(struct IGDdatas_service)); + memcpy(&data->first, &data->second, sizeof(struct IGDdatas_service)); + memcpy(&data->second, &data->tmp, sizeof(struct IGDdatas_service)); + GetUPNPUrls(urls, data, dev->descURL, dev->scope_id); + is_connected = UPNPIGD_IsConnected(urls, data); +#ifdef DEBUG + printf("UPNPIGD_IsConnected(%s) = %d\n", + urls->controlURL, is_connected); +#endif + if(is_connected && + (UPNP_GetExternalIPAddress(urls->controlURL, data->first.servicetype, extIpAddr) == 0)) { + if(!is_rfc1918addr(extIpAddr) && (extIpAddr[0] != '\0') + && (0 != strcmp(extIpAddr, "0.0.0.0"))) + goto free_and_return; + } + FreeUPNPUrls(urls); + } + } + memset(data, 0, sizeof(struct IGDdatas)); + } + } + } + state = 0; +free_and_return: + if(desc) { + for(i = 0; i < ndev; i++) { + if(desc[i].xml) { + free(desc[i].xml); + } + } + free(desc); + } + return state; +} + +/* UPNP_GetIGDFromUrl() + * Used when skipping the discovery process. + * return value : + * 0 - Not ok + * 1 - OK */ +int +UPNP_GetIGDFromUrl(const char * rootdescurl, + struct UPNPUrls * urls, + struct IGDdatas * data, + char * lanaddr, int lanaddrlen) +{ + char * descXML; + int descXMLsize = 0; + + descXML = miniwget_getaddr(rootdescurl, &descXMLsize, + lanaddr, lanaddrlen, 0, NULL); + if(descXML) { + memset(data, 0, sizeof(struct IGDdatas)); + memset(urls, 0, sizeof(struct UPNPUrls)); + parserootdesc(descXML, descXMLsize, data); + free(descXML); + descXML = NULL; + GetUPNPUrls(urls, data, rootdescurl, 0); + return 1; + } else { + return 0; + } +} + diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpc.def b/src/contrib/miniupnp/miniupnpc/miniupnpc.def new file mode 100644 index 0000000..60e0bbe --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpc.def @@ -0,0 +1,45 @@ +LIBRARY +; miniupnpc library + miniupnpc + +EXPORTS +; miniupnpc + upnpDiscover + freeUPNPDevlist + parserootdesc + UPNP_GetValidIGD + UPNP_GetIGDFromUrl + GetUPNPUrls + FreeUPNPUrls +; miniwget + miniwget + miniwget_getaddr +; upnpcommands + UPNP_GetTotalBytesSent + UPNP_GetTotalBytesReceived + UPNP_GetTotalPacketsSent + UPNP_GetTotalPacketsReceived + UPNP_GetStatusInfo + UPNP_GetConnectionTypeInfo + UPNP_GetExternalIPAddress + UPNP_GetLinkLayerMaxBitRates + UPNP_AddPortMapping + UPNP_AddAnyPortMapping + UPNP_DeletePortMapping + UPNP_DeletePortMappingRange + UPNP_GetPortMappingNumberOfEntries + UPNP_GetSpecificPortMappingEntry + UPNP_GetGenericPortMappingEntry + UPNP_GetListOfPortMappings + UPNP_AddPinhole + UPNP_CheckPinholeWorking + UPNP_UpdatePinhole + UPNP_GetPinholePackets + UPNP_DeletePinhole + UPNP_GetFirewallStatus + UPNP_GetOutboundPinholeTimeout +; upnperrors + strupnperror +; portlistingparse + ParsePortListing + FreePortListing diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpc.h b/src/contrib/miniupnp/miniupnpc/miniupnpc.h new file mode 100644 index 0000000..8ddc282 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpc.h @@ -0,0 +1,153 @@ +/* $Id: miniupnpc.h,v 1.53 2018/05/07 11:05:16 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project: miniupnp + * http://miniupnp.free.fr/ + * Author: Thomas Bernard + * Copyright (c) 2005-2018 Thomas Bernard + * This software is subjects to the conditions detailed + * in the LICENCE file provided within this distribution */ +#ifndef MINIUPNPC_H_INCLUDED +#define MINIUPNPC_H_INCLUDED + +#include "miniupnpc_declspec.h" +#include "igd_desc_parse.h" +#include "upnpdev.h" + +/* error codes : */ +#define UPNPDISCOVER_SUCCESS (0) +#define UPNPDISCOVER_UNKNOWN_ERROR (-1) +#define UPNPDISCOVER_SOCKET_ERROR (-101) +#define UPNPDISCOVER_MEMORY_ERROR (-102) + +/* versions : */ +#define MINIUPNPC_VERSION "2.1" +#define MINIUPNPC_API_VERSION 17 + +/* Source port: + Using "1" as an alias for 1900 for backwards compatibility + (presuming one would have used that for the "sameport" parameter) */ +#define UPNP_LOCAL_PORT_ANY 0 +#define UPNP_LOCAL_PORT_SAME 1 + +#ifdef __cplusplus +extern "C" { +#endif + +/* Structures definitions : */ +struct UPNParg { const char * elt; const char * val; }; + +char * +simpleUPnPcommand(int, const char *, const char *, + const char *, struct UPNParg *, + int *); + +/* upnpDiscover() + * discover UPnP devices on the network. + * The discovered devices are returned as a chained list. + * It is up to the caller to free the list with freeUPNPDevlist(). + * delay (in millisecond) is the maximum time for waiting any device + * response. + * If available, device list will be obtained from MiniSSDPd. + * Default path for minissdpd socket will be used if minissdpdsock argument + * is NULL. + * If multicastif is not NULL, it will be used instead of the default + * multicast interface for sending SSDP discover packets. + * If localport is set to UPNP_LOCAL_PORT_SAME(1) SSDP packets will be sent + * from the source port 1900 (same as destination port), if set to + * UPNP_LOCAL_PORT_ANY(0) system assign a source port, any other value will + * be attempted as the source port. + * "searchalltypes" parameter is useful when searching several types, + * if 0, the discovery will stop with the first type returning results. + * TTL should default to 2. */ +MINIUPNP_LIBSPEC struct UPNPDev * +upnpDiscover(int delay, const char * multicastif, + const char * minissdpdsock, int localport, + int ipv6, unsigned char ttl, + int * error); + +MINIUPNP_LIBSPEC struct UPNPDev * +upnpDiscoverAll(int delay, const char * multicastif, + const char * minissdpdsock, int localport, + int ipv6, unsigned char ttl, + int * error); + +MINIUPNP_LIBSPEC struct UPNPDev * +upnpDiscoverDevice(const char * device, int delay, const char * multicastif, + const char * minissdpdsock, int localport, + int ipv6, unsigned char ttl, + int * error); + +MINIUPNP_LIBSPEC struct UPNPDev * +upnpDiscoverDevices(const char * const deviceTypes[], + int delay, const char * multicastif, + const char * minissdpdsock, int localport, + int ipv6, unsigned char ttl, + int * error, + int searchalltypes); + +/* parserootdesc() : + * parse root XML description of a UPnP device and fill the IGDdatas + * structure. */ +MINIUPNP_LIBSPEC void parserootdesc(const char *, int, struct IGDdatas *); + +/* structure used to get fast access to urls + * controlURL: controlURL of the WANIPConnection + * ipcondescURL: url of the description of the WANIPConnection + * controlURL_CIF: controlURL of the WANCommonInterfaceConfig + * controlURL_6FC: controlURL of the WANIPv6FirewallControl + */ +struct UPNPUrls { + char * controlURL; + char * ipcondescURL; + char * controlURL_CIF; + char * controlURL_6FC; + char * rootdescURL; +}; + +/* UPNP_GetValidIGD() : + * return values : + * 0 = NO IGD found + * 1 = A valid connected IGD has been found + * 2 = A valid IGD has been found but it reported as + * not connected + * 3 = an UPnP device has been found but was not recognized as an IGD + * + * In any non zero return case, the urls and data structures + * passed as parameters are set. Donc forget to call FreeUPNPUrls(urls) to + * free allocated memory. + */ +MINIUPNP_LIBSPEC int +UPNP_GetValidIGD(struct UPNPDev * devlist, + struct UPNPUrls * urls, + struct IGDdatas * data, + char * lanaddr, int lanaddrlen); + +/* UPNP_GetIGDFromUrl() + * Used when skipping the discovery process. + * When succeding, urls, data, and lanaddr arguments are set. + * return value : + * 0 - Not ok + * 1 - OK */ +MINIUPNP_LIBSPEC int +UPNP_GetIGDFromUrl(const char * rootdescurl, + struct UPNPUrls * urls, + struct IGDdatas * data, + char * lanaddr, int lanaddrlen); + +MINIUPNP_LIBSPEC void +GetUPNPUrls(struct UPNPUrls *, struct IGDdatas *, + const char *, unsigned int); + +MINIUPNP_LIBSPEC void +FreeUPNPUrls(struct UPNPUrls *); + +/* return 0 or 1 */ +MINIUPNP_LIBSPEC int UPNPIGD_IsConnected(struct UPNPUrls *, struct IGDdatas *); + + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpc_declspec.h b/src/contrib/miniupnp/miniupnpc/miniupnpc_declspec.h new file mode 100644 index 0000000..40adb92 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpc_declspec.h @@ -0,0 +1,21 @@ +#ifndef MINIUPNPC_DECLSPEC_H_INCLUDED +#define MINIUPNPC_DECLSPEC_H_INCLUDED + +#if defined(_WIN32) && !defined(MINIUPNP_STATICLIB) + /* for windows dll */ + #ifdef MINIUPNP_EXPORTS + #define MINIUPNP_LIBSPEC __declspec(dllexport) + #else + #define MINIUPNP_LIBSPEC __declspec(dllimport) + #endif +#else + #if defined(__GNUC__) && __GNUC__ >= 4 + /* fix dynlib for OS X 10.9.2 and Apple LLVM version 5.0 */ + #define MINIUPNP_LIBSPEC __attribute__ ((visibility ("default"))) + #else + #define MINIUPNP_LIBSPEC + #endif +#endif + +#endif /* MINIUPNPC_DECLSPEC_H_INCLUDED */ + diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpc_socketdef.h b/src/contrib/miniupnp/miniupnpc/miniupnpc_socketdef.h new file mode 100644 index 0000000..965d915 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpc_socketdef.h @@ -0,0 +1,37 @@ +/* $Id: miniupnpc_socketdef.h,v 1.1 2018/03/13 23:44:10 nanard Exp $ */ +/* Miniupnp project : http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * Author : Thomas Bernard + * Copyright (c) 2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided within this distribution */ +#ifndef MINIUPNPC_SOCKETDEF_H_INCLUDED +#define MINIUPNPC_SOCKETDEF_H_INCLUDED + +#ifdef _MSC_VER + +#define ISINVALID(s) (INVALID_SOCKET==(s)) + +#else + +#ifndef SOCKET +#define SOCKET int +#endif +#ifndef SSIZE_T +#define SSIZE_T ssize_t +#endif +#ifndef INVALID_SOCKET +#define INVALID_SOCKET (-1) +#endif +#ifndef ISINVALID +#define ISINVALID(s) ((s)<0) +#endif + +#endif + +#ifdef _WIN32 +#define PRINT_SOCKET_ERROR(x) fprintf(stderr, "Socket error: %s, %d\n", x, WSAGetLastError()); +#else +#define PRINT_SOCKET_ERROR(x) perror(x) +#endif + +#endif /* MINIUPNPC_SOCKETDEF_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpcmodule.c b/src/contrib/miniupnp/miniupnpc/miniupnpcmodule.c new file mode 100644 index 0000000..8657a0e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpcmodule.c @@ -0,0 +1,703 @@ +/* $Id: miniupnpcmodule.c,v 1.24 2014/06/10 09:48:11 nanard Exp $*/ +/* Project : miniupnp + * Author : Thomas BERNARD + * website : https://miniupnp.tuxfamily.org/ + * copyright (c) 2007-2018 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ +#include +#define MINIUPNP_STATICLIB +#include "structmember.h" +#include "miniupnpc.h" +#include "upnpcommands.h" +#include "upnperrors.h" + +#ifdef _WIN32 +#include +#endif + +/* for compatibility with Python < 2.4 */ +#ifndef Py_RETURN_NONE +#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None +#endif + +#ifndef Py_RETURN_TRUE +#define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True +#endif + +#ifndef Py_RETURN_FALSE +#define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False +#endif + +/* for compatibility with Python < 3.0 */ +#ifndef PyVarObject_HEAD_INIT +#define PyVarObject_HEAD_INIT(type, size) \ + PyObject_HEAD_INIT(type) size, +#endif + +#ifndef Py_TYPE +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#endif + +typedef struct { + PyObject_HEAD + /* Type-specific fields go here. */ + struct UPNPDev * devlist; + struct UPNPUrls urls; + struct IGDdatas data; + unsigned int discoverdelay; /* value passed to upnpDiscover() */ + unsigned int localport; /* value passed to upnpDiscover() */ + char lanaddr[40]; /* our ip address on the LAN */ + char * multicastif; + char * minissdpdsocket; +} UPnPObject; + +static PyMemberDef UPnP_members[] = { + {"lanaddr", T_STRING_INPLACE, offsetof(UPnPObject, lanaddr), + READONLY, "ip address on the LAN" + }, + {"discoverdelay", T_UINT, offsetof(UPnPObject, discoverdelay), + 0/*READWRITE*/, "value in ms used to wait for SSDP responses" + }, + {"localport", T_UINT, offsetof(UPnPObject, localport), + 0/*READWRITE*/, + "If localport is set to UPNP_LOCAL_PORT_SAME(1) " + "SSDP packets will be sent from the source port " + "1900 (same as destination port), if set to " + "UPNP_LOCAL_PORT_ANY(0) system assign a source " + "port, any other value will be attempted as the " + "source port" + }, + /* T_STRING is allways readonly :( */ + {"multicastif", T_STRING, offsetof(UPnPObject, multicastif), + 0, "IP of the network interface to be used for multicast operations" + }, + {"minissdpdsocket", T_STRING, offsetof(UPnPObject, minissdpdsocket), + 0, "path of the MiniSSDPd unix socket" + }, + {NULL} +}; + + +static int UPnP_init(UPnPObject *self, PyObject *args, PyObject *kwds) +{ + char* multicastif = NULL; + char* minissdpdsocket = NULL; + static char *kwlist[] = { + "multicastif", "minissdpdsocket", "discoverdelay", + "localport", NULL + }; + + if(!PyArg_ParseTupleAndKeywords(args, kwds, "|zzII", kwlist, + &multicastif, + &minissdpdsocket, + &self->discoverdelay, + &self->localport)) + return -1; + + if(self->localport>1 && + (self->localport>65534||self->localport<1024)) { + PyErr_SetString(PyExc_Exception, "Invalid localport value"); + return -1; + } + if(multicastif) + self->multicastif = strdup(multicastif); + if(minissdpdsocket) + self->minissdpdsocket = strdup(minissdpdsocket); + + return 0; +} + +static void +UPnPObject_dealloc(UPnPObject *self) +{ + freeUPNPDevlist(self->devlist); + FreeUPNPUrls(&self->urls); + free(self->multicastif); + free(self->minissdpdsocket); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject * +UPnP_discover(UPnPObject *self) +{ + struct UPNPDev * dev; + int i; + PyObject *res = NULL; + if(self->devlist) + { + freeUPNPDevlist(self->devlist); + self->devlist = 0; + } + Py_BEGIN_ALLOW_THREADS + self->devlist = upnpDiscover((int)self->discoverdelay/*timeout in ms*/, + self->multicastif, + self->minissdpdsocket, + (int)self->localport, + 0/*ip v6*/, + 2/* TTL */, + 0/*error */); + Py_END_ALLOW_THREADS + /* Py_RETURN_NONE ??? */ + for(dev = self->devlist, i = 0; dev; dev = dev->pNext) + i++; + res = Py_BuildValue("i", i); + return res; +} + +static PyObject * +UPnP_selectigd(UPnPObject *self) +{ + int r; +Py_BEGIN_ALLOW_THREADS + r = UPNP_GetValidIGD(self->devlist, &self->urls, &self->data, + self->lanaddr, sizeof(self->lanaddr)); +Py_END_ALLOW_THREADS + if(r) + { + return Py_BuildValue("s", self->urls.controlURL); + } + else + { + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, "No UPnP device discovered"); + return NULL; + } +} + +static PyObject * +UPnP_totalbytesent(UPnPObject *self) +{ + UNSIGNED_INTEGER i; +Py_BEGIN_ALLOW_THREADS + i = UPNP_GetTotalBytesSent(self->urls.controlURL_CIF, + self->data.CIF.servicetype); +Py_END_ALLOW_THREADS +#if (PY_MAJOR_VERSION >= 3) || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION > 3) + return Py_BuildValue("I", i); +#else + return Py_BuildValue("i", (int)i); +#endif +} + +static PyObject * +UPnP_totalbytereceived(UPnPObject *self) +{ + UNSIGNED_INTEGER i; +Py_BEGIN_ALLOW_THREADS + i = UPNP_GetTotalBytesReceived(self->urls.controlURL_CIF, + self->data.CIF.servicetype); +Py_END_ALLOW_THREADS +#if (PY_MAJOR_VERSION >= 3) || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION > 3) + return Py_BuildValue("I", i); +#else + return Py_BuildValue("i", (int)i); +#endif +} + +static PyObject * +UPnP_totalpacketsent(UPnPObject *self) +{ + UNSIGNED_INTEGER i; +Py_BEGIN_ALLOW_THREADS + i = UPNP_GetTotalPacketsSent(self->urls.controlURL_CIF, + self->data.CIF.servicetype); +Py_END_ALLOW_THREADS +#if (PY_MAJOR_VERSION >= 3) || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION > 3) + return Py_BuildValue("I", i); +#else + return Py_BuildValue("i", (int)i); +#endif +} + +static PyObject * +UPnP_totalpacketreceived(UPnPObject *self) +{ + UNSIGNED_INTEGER i; +Py_BEGIN_ALLOW_THREADS + i = UPNP_GetTotalPacketsReceived(self->urls.controlURL_CIF, + self->data.CIF.servicetype); +Py_END_ALLOW_THREADS +#if (PY_MAJOR_VERSION >= 3) || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION > 3) + return Py_BuildValue("I", i); +#else + return Py_BuildValue("i", (int)i); +#endif +} + +static PyObject * +UPnP_statusinfo(UPnPObject *self) +{ + char status[64]; + char lastconnerror[64]; + unsigned int uptime = 0; + int r; + status[0] = '\0'; + lastconnerror[0] = '\0'; +Py_BEGIN_ALLOW_THREADS + r = UPNP_GetStatusInfo(self->urls.controlURL, self->data.first.servicetype, + status, &uptime, lastconnerror); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) { +#if (PY_MAJOR_VERSION >= 3) || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION > 3) + return Py_BuildValue("(s,I,s)", status, uptime, lastconnerror); +#else + return Py_BuildValue("(s,i,s)", status, (int)uptime, lastconnerror); +#endif + } else { + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, strupnperror(r)); + return NULL; + } +} + +static PyObject * +UPnP_connectiontype(UPnPObject *self) +{ + char connectionType[64]; + int r; + connectionType[0] = '\0'; +Py_BEGIN_ALLOW_THREADS + r = UPNP_GetConnectionTypeInfo(self->urls.controlURL, + self->data.first.servicetype, + connectionType); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) { + return Py_BuildValue("s", connectionType); + } else { + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, strupnperror(r)); + return NULL; + } +} + +static PyObject * +UPnP_externalipaddress(UPnPObject *self) +{ + char externalIPAddress[40]; + int r; + externalIPAddress[0] = '\0'; +Py_BEGIN_ALLOW_THREADS + r = UPNP_GetExternalIPAddress(self->urls.controlURL, + self->data.first.servicetype, + externalIPAddress); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) { + return Py_BuildValue("s", externalIPAddress); + } else { + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, strupnperror(r)); + return NULL; + } +} + +/* AddPortMapping(externalPort, protocol, internalHost, internalPort, desc, + * remoteHost) + * protocol is 'UDP' or 'TCP' */ +static PyObject * +UPnP_addportmapping(UPnPObject *self, PyObject *args) +{ + char extPort[6]; + unsigned short ePort; + char inPort[6]; + unsigned short iPort; + const char * proto; + const char * host; + const char * desc; + const char * remoteHost; + const char * leaseDuration = "0"; + int r; + if (!PyArg_ParseTuple(args, "HssHzz", &ePort, &proto, + &host, &iPort, &desc, &remoteHost)) + return NULL; +Py_BEGIN_ALLOW_THREADS + sprintf(extPort, "%hu", ePort); + sprintf(inPort, "%hu", iPort); + r = UPNP_AddPortMapping(self->urls.controlURL, self->data.first.servicetype, + extPort, inPort, host, desc, proto, + remoteHost, leaseDuration); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) + { + Py_RETURN_TRUE; + } + else + { + // TODO: RAISE an Exception. See upnpcommands.h for errors codes. + // upnperrors.c + //Py_RETURN_FALSE; + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, strupnperror(r)); + return NULL; + } +} + +/* AddAnyPortMapping(externalPort, protocol, internalHost, internalPort, desc, + * remoteHost) + * protocol is 'UDP' or 'TCP' */ +static PyObject * +UPnP_addanyportmapping(UPnPObject *self, PyObject *args) +{ + char extPort[6]; + unsigned short ePort; + char inPort[6]; + unsigned short iPort; + char reservedPort[6]; + const char * proto; + const char * host; + const char * desc; + const char * remoteHost; + const char * leaseDuration = "0"; + int r; + if (!PyArg_ParseTuple(args, "HssHzz", &ePort, &proto, &host, &iPort, &desc, &remoteHost)) + return NULL; +Py_BEGIN_ALLOW_THREADS + sprintf(extPort, "%hu", ePort); + sprintf(inPort, "%hu", iPort); + r = UPNP_AddAnyPortMapping(self->urls.controlURL, self->data.first.servicetype, + extPort, inPort, host, desc, proto, + remoteHost, leaseDuration, reservedPort); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) { + return Py_BuildValue("i", atoi(reservedPort)); + } else { + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, strupnperror(r)); + return NULL; + } +} + + +/* DeletePortMapping(extPort, proto, removeHost='') + * proto = 'UDP', 'TCP' */ +static PyObject * +UPnP_deleteportmapping(UPnPObject *self, PyObject *args) +{ + char extPort[6]; + unsigned short ePort; + const char * proto; + const char * remoteHost = ""; + int r; + if(!PyArg_ParseTuple(args, "Hs|z", &ePort, &proto, &remoteHost)) + return NULL; +Py_BEGIN_ALLOW_THREADS + sprintf(extPort, "%hu", ePort); + r = UPNP_DeletePortMapping(self->urls.controlURL, self->data.first.servicetype, + extPort, proto, remoteHost); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) { + Py_RETURN_TRUE; + } else { + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, strupnperror(r)); + return NULL; + } +} + +/* DeletePortMappingRange(extPort, proto, removeHost='') + * proto = 'UDP', 'TCP' */ +static PyObject * +UPnP_deleteportmappingrange(UPnPObject *self, PyObject *args) +{ + char extPortStart[6]; + unsigned short ePortStart; + char extPortEnd[6]; + unsigned short ePortEnd; + const char * proto; + unsigned char manage; + char manageStr[6]; + int r; + if(!PyArg_ParseTuple(args, "HHsb", &ePortStart, &ePortEnd, &proto, &manage)) + return NULL; +Py_BEGIN_ALLOW_THREADS + sprintf(extPortStart, "%hu", ePortStart); + sprintf(extPortEnd, "%hu", ePortEnd); + sprintf(manageStr, "%hu", (unsigned short)manage); + r = UPNP_DeletePortMappingRange(self->urls.controlURL, self->data.first.servicetype, + extPortStart, extPortEnd, proto, manageStr); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) { + Py_RETURN_TRUE; + } else { + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, strupnperror(r)); + return NULL; + } +} + +static PyObject * +UPnP_getportmappingnumberofentries(UPnPObject *self) +{ + unsigned int n = 0; + int r; +Py_BEGIN_ALLOW_THREADS + r = UPNP_GetPortMappingNumberOfEntries(self->urls.controlURL, + self->data.first.servicetype, + &n); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) { +#if (PY_MAJOR_VERSION >= 3) || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION > 3) + return Py_BuildValue("I", n); +#else + return Py_BuildValue("i", (int)n); +#endif + } else { + /* TODO: have our own exception type ! */ + PyErr_SetString(PyExc_Exception, strupnperror(r)); + return NULL; + } +} + +/* GetSpecificPortMapping(ePort, proto, remoteHost='') + * proto = 'UDP' or 'TCP' */ +static PyObject * +UPnP_getspecificportmapping(UPnPObject *self, PyObject *args) +{ + char extPort[6]; + unsigned short ePort; + const char * proto; + const char * remoteHost = ""; + char intClient[40]; + char intPort[6]; + unsigned short iPort; + char desc[80]; + char enabled[4]; + char leaseDuration[16]; + if(!PyArg_ParseTuple(args, "Hs|z", &ePort, &proto, &remoteHost)) + return NULL; + extPort[0] = '\0'; intClient[0] = '\0'; intPort[0] = '\0'; + desc[0] = '\0'; enabled[0] = '\0'; leaseDuration[0] = '\0'; +Py_BEGIN_ALLOW_THREADS + sprintf(extPort, "%hu", ePort); + UPNP_GetSpecificPortMappingEntry(self->urls.controlURL, + self->data.first.servicetype, + extPort, proto, remoteHost, + intClient, intPort, + desc, enabled, leaseDuration); +Py_END_ALLOW_THREADS + if(intClient[0]) + { + iPort = (unsigned short)atoi(intPort); + return Py_BuildValue("(s,H,s,O,i)", + intClient, iPort, desc, + PyBool_FromLong(atoi(enabled)), + atoi(leaseDuration)); + } + else + { + Py_RETURN_NONE; + } +} + +/* GetGenericPortMapping(index) */ +static PyObject * +UPnP_getgenericportmapping(UPnPObject *self, PyObject *args) +{ + int i, r; + char index[8]; + char intClient[40]; + char intPort[6]; + unsigned short iPort; + char extPort[6]; + unsigned short ePort; + char protocol[4]; + char desc[80]; + char enabled[6]; + char rHost[64]; + char duration[16]; /* lease duration */ + unsigned int dur; + if(!PyArg_ParseTuple(args, "i", &i)) + return NULL; +Py_BEGIN_ALLOW_THREADS + snprintf(index, sizeof(index), "%d", i); + rHost[0] = '\0'; enabled[0] = '\0'; + duration[0] = '\0'; desc[0] = '\0'; + extPort[0] = '\0'; intPort[0] = '\0'; intClient[0] = '\0'; + r = UPNP_GetGenericPortMappingEntry(self->urls.controlURL, + self->data.first.servicetype, + index, + extPort, intClient, intPort, + protocol, desc, enabled, rHost, + duration); +Py_END_ALLOW_THREADS + if(r==UPNPCOMMAND_SUCCESS) + { + ePort = (unsigned short)atoi(extPort); + iPort = (unsigned short)atoi(intPort); + dur = (unsigned int)strtoul(duration, 0, 0); +#if (PY_MAJOR_VERSION >= 3) || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION > 3) + return Py_BuildValue("(H,s,(s,H),s,s,s,I)", + ePort, protocol, intClient, iPort, + desc, enabled, rHost, dur); +#else + return Py_BuildValue("(i,s,(s,i),s,s,s,i)", + (int)ePort, protocol, intClient, (int)iPort, + desc, enabled, rHost, (int)dur); +#endif + } + else + { + Py_RETURN_NONE; + } +} + +/* miniupnpc.UPnP object Method Table */ +static PyMethodDef UPnP_methods[] = { + {"discover", (PyCFunction)UPnP_discover, METH_NOARGS, + "discover UPnP IGD devices on the network" + }, + {"selectigd", (PyCFunction)UPnP_selectigd, METH_NOARGS, + "select a valid UPnP IGD among discovered devices" + }, + {"totalbytesent", (PyCFunction)UPnP_totalbytesent, METH_NOARGS, + "return the total number of bytes sent by UPnP IGD" + }, + {"totalbytereceived", (PyCFunction)UPnP_totalbytereceived, METH_NOARGS, + "return the total number of bytes received by UPnP IGD" + }, + {"totalpacketsent", (PyCFunction)UPnP_totalpacketsent, METH_NOARGS, + "return the total number of packets sent by UPnP IGD" + }, + {"totalpacketreceived", (PyCFunction)UPnP_totalpacketreceived, METH_NOARGS, + "return the total number of packets received by UPnP IGD" + }, + {"statusinfo", (PyCFunction)UPnP_statusinfo, METH_NOARGS, + "return status and uptime" + }, + {"connectiontype", (PyCFunction)UPnP_connectiontype, METH_NOARGS, + "return IGD WAN connection type" + }, + {"externalipaddress", (PyCFunction)UPnP_externalipaddress, METH_NOARGS, + "return external IP address" + }, + {"addportmapping", (PyCFunction)UPnP_addportmapping, METH_VARARGS, + "add a port mapping" + }, + {"addanyportmapping", (PyCFunction)UPnP_addanyportmapping, METH_VARARGS, + "add a port mapping, IGD to select alternative if necessary" + }, + {"deleteportmapping", (PyCFunction)UPnP_deleteportmapping, METH_VARARGS, + "delete a port mapping" + }, + {"deleteportmappingrange", (PyCFunction)UPnP_deleteportmappingrange, METH_VARARGS, + "delete a range of port mappings" + }, + {"getportmappingnumberofentries", (PyCFunction)UPnP_getportmappingnumberofentries, METH_NOARGS, + "-- non standard --" + }, + {"getspecificportmapping", (PyCFunction)UPnP_getspecificportmapping, METH_VARARGS, + "get details about a specific port mapping entry" + }, + {"getgenericportmapping", (PyCFunction)UPnP_getgenericportmapping, METH_VARARGS, + "get all details about the port mapping at index" + }, + {NULL} /* Sentinel */ +}; + +static PyTypeObject UPnPType = { + PyVarObject_HEAD_INIT(NULL, + 0) /*ob_size*/ + "miniupnpc.UPnP", /*tp_name*/ + sizeof(UPnPObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)UPnPObject_dealloc,/*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "UPnP objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + UPnP_methods, /* tp_methods */ + UPnP_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)UPnP_init, /* tp_init */ + 0, /* tp_alloc */ +#ifndef _WIN32 + PyType_GenericNew,/*UPnP_new,*/ /* tp_new */ +#else + 0, +#endif +}; + +/* module methods */ +static PyMethodDef miniupnpc_methods[] = { + {NULL} /* Sentinel */ +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "miniupnpc", /* m_name */ + "miniupnpc module.", /* m_doc */ + -1, /* m_size */ + miniupnpc_methods, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ +}; +#endif + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif + +PyMODINIT_FUNC +#if PY_MAJOR_VERSION >= 3 +PyInit_miniupnpc(void) +#else +initminiupnpc(void) +#endif +{ + PyObject* m; + +#ifdef _WIN32 + /* initialize Winsock. */ + WSADATA wsaData; + int nResult = WSAStartup(MAKEWORD(2,2), &wsaData); + + UPnPType.tp_new = PyType_GenericNew; +#endif + if (PyType_Ready(&UPnPType) < 0) +#if PY_MAJOR_VERSION >= 3 + return 0; +#else + return; +#endif + +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&moduledef); +#else + m = Py_InitModule3("miniupnpc", miniupnpc_methods, + "miniupnpc module."); +#endif + + Py_INCREF(&UPnPType); + PyModule_AddObject(m, "UPnP", (PyObject *)&UPnPType); + +#if PY_MAJOR_VERSION >= 3 + return m; +#endif +} + diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpcstrings.h.cmake b/src/contrib/miniupnp/miniupnpc/miniupnpcstrings.h.cmake new file mode 100644 index 0000000..78c8fe9 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpcstrings.h.cmake @@ -0,0 +1,15 @@ +#ifndef MINIUPNPCSTRINGS_H_INCLUDED +#define MINIUPNPCSTRINGS_H_INCLUDED + +#define OS_STRING "${CMAKE_SYSTEM_NAME}" +#define MINIUPNPC_VERSION_STRING "${MINIUPNPC_VERSION}" + +#if 0 +/* according to "UPnP Device Architecture 1.0" */ +#define UPNP_VERSION_STRING "UPnP/1.0" +#else +/* according to "UPnP Device Architecture 1.1" */ +#define UPNP_VERSION_STRING "UPnP/1.1" +#endif + +#endif diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpcstrings.h.in b/src/contrib/miniupnp/miniupnpc/miniupnpcstrings.h.in new file mode 100644 index 0000000..68bf429 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpcstrings.h.in @@ -0,0 +1,23 @@ +/* $Id: miniupnpcstrings.h.in,v 1.6 2014/11/04 22:31:55 nanard Exp $ */ +/* Project: miniupnp + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author: Thomas Bernard + * Copyright (c) 2005-2014 Thomas Bernard + * This software is subjects to the conditions detailed + * in the LICENCE file provided within this distribution */ +#ifndef MINIUPNPCSTRINGS_H_INCLUDED +#define MINIUPNPCSTRINGS_H_INCLUDED + +#define OS_STRING "OS/version" +#define MINIUPNPC_VERSION_STRING "version" + +#if 0 +/* according to "UPnP Device Architecture 1.0" */ +#define UPNP_VERSION_STRING "UPnP/1.0" +#else +/* according to "UPnP Device Architecture 1.1" */ +#define UPNP_VERSION_STRING "UPnP/1.1" +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/miniupnpctypes.h b/src/contrib/miniupnp/miniupnpc/miniupnpctypes.h new file mode 100644 index 0000000..307ce39 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniupnpctypes.h @@ -0,0 +1,19 @@ +/* $Id: miniupnpctypes.h,v 1.1 2011/02/15 11:10:40 nanard Exp $ */ +/* Miniupnp project : http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org + * Author : Thomas Bernard + * Copyright (c) 2011 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided within this distribution */ +#ifndef MINIUPNPCTYPES_H_INCLUDED +#define MINIUPNPCTYPES_H_INCLUDED + +#if (defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L) +#define UNSIGNED_INTEGER unsigned long long +#define STRTOUI strtoull +#else +#define UNSIGNED_INTEGER unsigned int +#define STRTOUI strtoul +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/miniwget.c b/src/contrib/miniupnp/miniupnpc/miniwget.c new file mode 100644 index 0000000..a46ba76 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniwget.c @@ -0,0 +1,662 @@ +/* $Id: miniwget.c,v 1.78 2018/03/13 23:22:18 nanard Exp $ */ +/* Project : miniupnp + * Website : http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ + +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#include +#define MAXHOSTNAMELEN 64 +#define snprintf _snprintf +#define socklen_t int +#ifndef strncasecmp +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#define strncasecmp _memicmp +#else /* defined(_MSC_VER) && (_MSC_VER >= 1400) */ +#define strncasecmp memicmp +#endif /* defined(_MSC_VER) && (_MSC_VER >= 1400) */ +#endif /* #ifndef strncasecmp */ +#else /* #ifdef _WIN32 */ +#include +#include +#if defined(__amigaos__) && !defined(__amigaos4__) +#define socklen_t int +#else /* #if defined(__amigaos__) && !defined(__amigaos4__) */ +#include +#endif /* #else defined(__amigaos__) && !defined(__amigaos4__) */ +#include +#include +#include +#include +#include +#define closesocket close +#include +#endif /* #else _WIN32 */ +#ifdef __GNU__ +#define MAXHOSTNAMELEN 64 +#endif /* __GNU__ */ + +#ifndef MIN +#define MIN(x,y) (((x)<(y))?(x):(y)) +#endif /* MIN */ + + +#include "miniupnpcstrings.h" +#include "miniwget.h" +#include "connecthostport.h" +#include "receivedata.h" + +#ifndef MAXHOSTNAMELEN +#define MAXHOSTNAMELEN 64 +#endif + +/* + * Read a HTTP response from a socket. + * Process Content-Length and Transfer-encoding headers. + * return a pointer to the content buffer, which length is saved + * to the length parameter. + */ +void * +getHTTPResponse(SOCKET s, int * size, int * status_code) +{ + char buf[2048]; + int n; + int endofheaders = 0; + int chunked = 0; + int content_length = -1; + unsigned int chunksize = 0; + unsigned int bytestocopy = 0; + /* buffers : */ + char * header_buf; + unsigned int header_buf_len = 2048; + unsigned int header_buf_used = 0; + char * content_buf; + unsigned int content_buf_len = 2048; + unsigned int content_buf_used = 0; + char chunksize_buf[32]; + unsigned int chunksize_buf_index; +#ifdef DEBUG + char * reason_phrase = NULL; + int reason_phrase_len = 0; +#endif + + if(status_code) *status_code = -1; + header_buf = malloc(header_buf_len); + if(header_buf == NULL) + { +#ifdef DEBUG + fprintf(stderr, "%s: Memory allocation error\n", "getHTTPResponse"); +#endif /* DEBUG */ + *size = -1; + return NULL; + } + content_buf = malloc(content_buf_len); + if(content_buf == NULL) + { + free(header_buf); +#ifdef DEBUG + fprintf(stderr, "%s: Memory allocation error\n", "getHTTPResponse"); +#endif /* DEBUG */ + *size = -1; + return NULL; + } + chunksize_buf[0] = '\0'; + chunksize_buf_index = 0; + + while((n = receivedata(s, buf, sizeof(buf), 5000, NULL)) > 0) + { + if(endofheaders == 0) + { + int i; + int linestart=0; + int colon=0; + int valuestart=0; + if(header_buf_used + n > header_buf_len) { + char * tmp = realloc(header_buf, header_buf_used + n); + if(tmp == NULL) { + /* memory allocation error */ + free(header_buf); + free(content_buf); + *size = -1; + return NULL; + } + header_buf = tmp; + header_buf_len = header_buf_used + n; + } + memcpy(header_buf + header_buf_used, buf, n); + header_buf_used += n; + /* search for CR LF CR LF (end of headers) + * recognize also LF LF */ + i = 0; + while(i < ((int)header_buf_used-1) && (endofheaders == 0)) { + if(header_buf[i] == '\r') { + i++; + if(header_buf[i] == '\n') { + i++; + if(i < (int)header_buf_used && header_buf[i] == '\r') { + i++; + if(i < (int)header_buf_used && header_buf[i] == '\n') { + endofheaders = i+1; + } + } + } + } else if(header_buf[i] == '\n') { + i++; + if(header_buf[i] == '\n') { + endofheaders = i+1; + } + } + i++; + } + if(endofheaders == 0) + continue; + /* parse header lines */ + for(i = 0; i < endofheaders - 1; i++) { + if(linestart > 0 && colon <= linestart && header_buf[i]==':') + { + colon = i; + while(i < (endofheaders-1) + && (header_buf[i+1] == ' ' || header_buf[i+1] == '\t')) + i++; + valuestart = i + 1; + } + /* detecting end of line */ + else if(header_buf[i]=='\r' || header_buf[i]=='\n') + { + if(linestart == 0 && status_code) + { + /* Status line + * HTTP-Version SP Status-Code SP Reason-Phrase CRLF */ + int sp; + for(sp = 0; sp < i; sp++) + if(header_buf[sp] == ' ') + { + if(*status_code < 0) + *status_code = atoi(header_buf + sp + 1); + else + { +#ifdef DEBUG + reason_phrase = header_buf + sp + 1; + reason_phrase_len = i - sp - 1; +#endif + break; + } + } +#ifdef DEBUG + printf("HTTP status code = %d, Reason phrase = %.*s\n", + *status_code, reason_phrase_len, reason_phrase); +#endif + } + else if(colon > linestart && valuestart > colon) + { +#ifdef DEBUG + printf("header='%.*s', value='%.*s'\n", + colon-linestart, header_buf+linestart, + i-valuestart, header_buf+valuestart); +#endif + if(0==strncasecmp(header_buf+linestart, "content-length", colon-linestart)) + { + content_length = atoi(header_buf+valuestart); +#ifdef DEBUG + printf("Content-Length: %d\n", content_length); +#endif + } + else if(0==strncasecmp(header_buf+linestart, "transfer-encoding", colon-linestart) + && 0==strncasecmp(header_buf+valuestart, "chunked", 7)) + { +#ifdef DEBUG + printf("chunked transfer-encoding!\n"); +#endif + chunked = 1; + } + } + while((i < (int)header_buf_used) && (header_buf[i]=='\r' || header_buf[i] == '\n')) + i++; + linestart = i; + colon = linestart; + valuestart = 0; + } + } + /* copy the remaining of the received data back to buf */ + n = header_buf_used - endofheaders; + memcpy(buf, header_buf + endofheaders, n); + /* if(headers) */ + } + /* if we get there, endofheaders != 0. + * In the other case, there was a continue above */ + /* content */ + if(chunked) + { + int i = 0; + while(i < n) + { + if(chunksize == 0) + { + /* reading chunk size */ + if(chunksize_buf_index == 0) { + /* skipping any leading CR LF */ + if(i= '0' + && chunksize_buf[j] <= '9') + chunksize = (chunksize << 4) + (chunksize_buf[j] - '0'); + else + chunksize = (chunksize << 4) + ((chunksize_buf[j] | 32) - 'a' + 10); + } + chunksize_buf[0] = '\0'; + chunksize_buf_index = 0; + i++; + } else { + /* not finished to get chunksize */ + continue; + } +#ifdef DEBUG + printf("chunksize = %u (%x)\n", chunksize, chunksize); +#endif + if(chunksize == 0) + { +#ifdef DEBUG + printf("end of HTTP content - %d %d\n", i, n); + /*printf("'%.*s'\n", n-i, buf+i);*/ +#endif + goto end_of_stream; + } + } + /* it is guaranteed that (n >= i) */ + bytestocopy = (chunksize < (unsigned int)(n - i))?chunksize:(unsigned int)(n - i); + if((content_buf_used + bytestocopy) > content_buf_len) + { + char * tmp; + if((content_length >= 0) && ((unsigned int)content_length >= (content_buf_used + bytestocopy))) { + content_buf_len = content_length; + } else { + content_buf_len = content_buf_used + bytestocopy; + } + tmp = realloc(content_buf, content_buf_len); + if(tmp == NULL) { + /* memory allocation error */ + free(content_buf); + free(header_buf); + *size = -1; + return NULL; + } + content_buf = tmp; + } + memcpy(content_buf + content_buf_used, buf + i, bytestocopy); + content_buf_used += bytestocopy; + i += bytestocopy; + chunksize -= bytestocopy; + } + } + else + { + /* not chunked */ + if(content_length > 0 + && (content_buf_used + n) > (unsigned int)content_length) { + /* skipping additional bytes */ + n = content_length - content_buf_used; + } + if(content_buf_used + n > content_buf_len) + { + char * tmp; + if(content_length >= 0 + && (unsigned int)content_length >= (content_buf_used + n)) { + content_buf_len = content_length; + } else { + content_buf_len = content_buf_used + n; + } + tmp = realloc(content_buf, content_buf_len); + if(tmp == NULL) { + /* memory allocation error */ + free(content_buf); + free(header_buf); + *size = -1; + return NULL; + } + content_buf = tmp; + } + memcpy(content_buf + content_buf_used, buf, n); + content_buf_used += n; + } + /* use the Content-Length header value if available */ + if(content_length > 0 && content_buf_used >= (unsigned int)content_length) + { +#ifdef DEBUG + printf("End of HTTP content\n"); +#endif + break; + } + } +end_of_stream: + free(header_buf); header_buf = NULL; + *size = content_buf_used; + if(content_buf_used == 0) + { + free(content_buf); + content_buf = NULL; + } + return content_buf; +} + +/* miniwget3() : + * do all the work. + * Return NULL if something failed. */ +static void * +miniwget3(const char * host, + unsigned short port, const char * path, + int * size, char * addr_str, int addr_str_len, + const char * httpversion, unsigned int scope_id, + int * status_code) +{ + char buf[2048]; + SOCKET s; + int n; + int len; + int sent; + void * content; + + *size = 0; + s = connecthostport(host, port, scope_id); + if(ISINVALID(s)) + return NULL; + + /* get address for caller ! */ + if(addr_str) + { + struct sockaddr_storage saddr; + socklen_t saddrlen; + + saddrlen = sizeof(saddr); + if(getsockname(s, (struct sockaddr *)&saddr, &saddrlen) < 0) + { + perror("getsockname"); + } + else + { +#if defined(__amigaos__) && !defined(__amigaos4__) + /* using INT WINAPI WSAAddressToStringA(LPSOCKADDR, DWORD, LPWSAPROTOCOL_INFOA, LPSTR, LPDWORD); + * But his function make a string with the port : nn.nn.nn.nn:port */ +/* if(WSAAddressToStringA((SOCKADDR *)&saddr, sizeof(saddr), + NULL, addr_str, (DWORD *)&addr_str_len)) + { + printf("WSAAddressToStringA() failed : %d\n", WSAGetLastError()); + }*/ + /* the following code is only compatible with ip v4 addresses */ + strncpy(addr_str, inet_ntoa(((struct sockaddr_in *)&saddr)->sin_addr), addr_str_len); +#else +#if 0 + if(saddr.sa_family == AF_INET6) { + inet_ntop(AF_INET6, + &(((struct sockaddr_in6 *)&saddr)->sin6_addr), + addr_str, addr_str_len); + } else { + inet_ntop(AF_INET, + &(((struct sockaddr_in *)&saddr)->sin_addr), + addr_str, addr_str_len); + } +#endif + /* getnameinfo return ip v6 address with the scope identifier + * such as : 2a01:e35:8b2b:7330::%4281128194 */ + n = getnameinfo((const struct sockaddr *)&saddr, saddrlen, + addr_str, addr_str_len, + NULL, 0, + NI_NUMERICHOST | NI_NUMERICSERV); + if(n != 0) { +#ifdef _WIN32 + fprintf(stderr, "getnameinfo() failed : %d\n", n); +#else + fprintf(stderr, "getnameinfo() failed : %s\n", gai_strerror(n)); +#endif + } +#endif + } +#ifdef DEBUG + printf("address miniwget : %s\n", addr_str); +#endif + } + + len = snprintf(buf, sizeof(buf), + "GET %s HTTP/%s\r\n" + "Host: %s:%d\r\n" + "Connection: Close\r\n" + "User-Agent: " OS_STRING ", " UPNP_VERSION_STRING ", MiniUPnPc/" MINIUPNPC_VERSION_STRING "\r\n" + + "\r\n", + path, httpversion, host, port); + if ((unsigned int)len >= sizeof(buf)) + { + closesocket(s); + return NULL; + } + sent = 0; + /* sending the HTTP request */ + while(sent < len) + { + n = send(s, buf+sent, len-sent, 0); + if(n < 0) + { + perror("send"); + closesocket(s); + return NULL; + } + else + { + sent += n; + } + } + content = getHTTPResponse(s, size, status_code); + closesocket(s); + return content; +} + +/* miniwget2() : + * Call miniwget3(); retry with HTTP/1.1 if 1.0 fails. */ +static void * +miniwget2(const char * host, + unsigned short port, const char * path, + int * size, char * addr_str, int addr_str_len, + unsigned int scope_id, int * status_code) +{ + char * respbuffer; + +#if 1 + respbuffer = miniwget3(host, port, path, size, + addr_str, addr_str_len, "1.1", + scope_id, status_code); +#else + respbuffer = miniwget3(host, port, path, size, + addr_str, addr_str_len, "1.0", + scope_id, status_code); + if (*size == 0) + { +#ifdef DEBUG + printf("Retrying with HTTP/1.1\n"); +#endif + free(respbuffer); + respbuffer = miniwget3(host, port, path, size, + addr_str, addr_str_len, "1.1", + scope_id, status_code); + } +#endif + return respbuffer; +} + + + + +/* parseURL() + * arguments : + * url : source string not modified + * hostname : hostname destination string (size of MAXHOSTNAMELEN+1) + * port : port (destination) + * path : pointer to the path part of the URL + * + * Return values : + * 0 - Failure + * 1 - Success */ +int +parseURL(const char * url, + char * hostname, unsigned short * port, + char * * path, unsigned int * scope_id) +{ + char * p1, *p2, *p3; + if(!url) + return 0; + p1 = strstr(url, "://"); + if(!p1) + return 0; + p1 += 3; + if( (url[0]!='h') || (url[1]!='t') + ||(url[2]!='t') || (url[3]!='p')) + return 0; + memset(hostname, 0, MAXHOSTNAMELEN + 1); + if(*p1 == '[') + { + /* IP v6 : http://[2a00:1450:8002::6a]/path/abc */ + char * scope; + scope = strchr(p1, '%'); + p2 = strchr(p1, ']'); + if(p2 && scope && scope < p2 && scope_id) { + /* parse scope */ +#ifdef IF_NAMESIZE + char tmp[IF_NAMESIZE]; + int l; + scope++; + /* "%25" is just '%' in URL encoding */ + if(scope[0] == '2' && scope[1] == '5') + scope += 2; /* skip "25" */ + l = p2 - scope; + if(l >= IF_NAMESIZE) + l = IF_NAMESIZE - 1; + memcpy(tmp, scope, l); + tmp[l] = '\0'; + *scope_id = if_nametoindex(tmp); + if(*scope_id == 0) { + *scope_id = (unsigned int)strtoul(tmp, NULL, 10); + } +#else + /* under windows, scope is numerical */ + char tmp[8]; + int l; + scope++; + /* "%25" is just '%' in URL encoding */ + if(scope[0] == '2' && scope[1] == '5') + scope += 2; /* skip "25" */ + l = p2 - scope; + if(l >= sizeof(tmp)) + l = sizeof(tmp) - 1; + memcpy(tmp, scope, l); + tmp[l] = '\0'; + *scope_id = (unsigned int)strtoul(tmp, NULL, 10); +#endif + } + p3 = strchr(p1, '/'); + if(p2 && p3) + { + p2++; + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p2-p1))); + if(*p2 == ':') + { + *port = 0; + p2++; + while( (*p2 >= '0') && (*p2 <= '9')) + { + *port *= 10; + *port += (unsigned short)(*p2 - '0'); + p2++; + } + } + else + { + *port = 80; + } + *path = p3; + return 1; + } + } + p2 = strchr(p1, ':'); + p3 = strchr(p1, '/'); + if(!p3) + return 0; + if(!p2 || (p2>p3)) + { + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p3-p1))); + *port = 80; + } + else + { + strncpy(hostname, p1, MIN(MAXHOSTNAMELEN, (int)(p2-p1))); + *port = 0; + p2++; + while( (*p2 >= '0') && (*p2 <= '9')) + { + *port *= 10; + *port += (unsigned short)(*p2 - '0'); + p2++; + } + } + *path = p3; + return 1; +} + +void * +miniwget(const char * url, int * size, + unsigned int scope_id, int * status_code) +{ + unsigned short port; + char * path; + /* protocol://host:port/chemin */ + char hostname[MAXHOSTNAMELEN+1]; + *size = 0; + if(!parseURL(url, hostname, &port, &path, &scope_id)) + return NULL; +#ifdef DEBUG + printf("parsed url : hostname='%s' port=%hu path='%s' scope_id=%u\n", + hostname, port, path, scope_id); +#endif + return miniwget2(hostname, port, path, size, 0, 0, scope_id, status_code); +} + +void * +miniwget_getaddr(const char * url, int * size, + char * addr, int addrlen, unsigned int scope_id, + int * status_code) +{ + unsigned short port; + char * path; + /* protocol://host:port/path */ + char hostname[MAXHOSTNAMELEN+1]; + *size = 0; + if(addr) + addr[0] = '\0'; + if(!parseURL(url, hostname, &port, &path, &scope_id)) + return NULL; +#ifdef DEBUG + printf("parsed url : hostname='%s' port=%hu path='%s' scope_id=%u\n", + hostname, port, path, scope_id); +#endif + return miniwget2(hostname, port, path, size, addr, addrlen, scope_id, status_code); +} + diff --git a/src/contrib/miniupnp/miniupnpc/miniwget.h b/src/contrib/miniupnp/miniupnpc/miniwget.h new file mode 100644 index 0000000..f5572c2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniwget.h @@ -0,0 +1,27 @@ +/* $Id: miniwget.h,v 1.12 2016/01/24 17:24:36 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas Bernard + * Copyright (c) 2005-2016 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef MINIWGET_H_INCLUDED +#define MINIWGET_H_INCLUDED + +#include "miniupnpc_declspec.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MINIUPNP_LIBSPEC void * miniwget(const char *, int *, unsigned int, int *); + +MINIUPNP_LIBSPEC void * miniwget_getaddr(const char *, int *, char *, int, unsigned int, int *); + +int parseURL(const char *, char *, unsigned short *, char * *, unsigned int *); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/contrib/miniupnp/miniupnpc/miniwget_private.h b/src/contrib/miniupnp/miniupnpc/miniwget_private.h new file mode 100644 index 0000000..e4eaac8 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/miniwget_private.h @@ -0,0 +1,15 @@ +/* $Id: miniwget_private.h,v 1.1 2018/04/06 10:17:58 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas Bernard + * Copyright (c) 2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef MINIWGET_INTERNAL_H_INCLUDED +#define MINIWGET_INTERNAL_H_INCLUDED + +#include "miniupnpc_socketdef.h" + +void * getHTTPResponse(SOCKET s, int * size, int * status_code); + +#endif diff --git a/src/contrib/miniupnp/miniupnpc/minixml.c b/src/contrib/miniupnp/miniupnpc/minixml.c new file mode 100644 index 0000000..ed2d3c7 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/minixml.c @@ -0,0 +1,231 @@ +/* $Id: minixml.c,v 1.10 2012/03/05 19:42:47 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * minixml.c : the minimum size a xml parser can be ! */ +/* Project : miniupnp + * webpage: http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author : Thomas Bernard + +Copyright (c) 2005-2017, Thomas BERNARD +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include "minixml.h" + +/* parseatt : used to parse the argument list + * return 0 (false) in case of success and -1 (true) if the end + * of the xmlbuffer is reached. */ +static int parseatt(struct xmlparser * p) +{ + const char * attname; + int attnamelen; + const char * attvalue; + int attvaluelen; + while(p->xml < p->xmlend) + { + if(*p->xml=='/' || *p->xml=='>') + return 0; + if( !IS_WHITE_SPACE(*p->xml) ) + { + char sep; + attname = p->xml; + attnamelen = 0; + while(*p->xml!='=' && !IS_WHITE_SPACE(*p->xml) ) + { + attnamelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + while(*(p->xml++) != '=') + { + if(p->xml >= p->xmlend) + return -1; + } + while(IS_WHITE_SPACE(*p->xml)) + { + p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + sep = *p->xml; + if(sep=='\'' || sep=='\"') + { + p->xml++; + if(p->xml >= p->xmlend) + return -1; + attvalue = p->xml; + attvaluelen = 0; + while(*p->xml != sep) + { + attvaluelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + } + else + { + attvalue = p->xml; + attvaluelen = 0; + while( !IS_WHITE_SPACE(*p->xml) + && *p->xml != '>' && *p->xml != '/') + { + attvaluelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + } + /*printf("%.*s='%.*s'\n", + attnamelen, attname, attvaluelen, attvalue);*/ + if(p->attfunc) + p->attfunc(p->data, attname, attnamelen, attvalue, attvaluelen); + } + p->xml++; + } + return -1; +} + +/* parseelt parse the xml stream and + * call the callback functions when needed... */ +static void parseelt(struct xmlparser * p) +{ + int i; + const char * elementname; + while(p->xml < (p->xmlend - 1)) + { + if((p->xml + 4) <= p->xmlend && (0 == memcmp(p->xml, "", 3) != 0); + p->xml += 3; + } + else if((p->xml)[0]=='<' && (p->xml)[1]!='?') + { + i = 0; elementname = ++p->xml; + while( !IS_WHITE_SPACE(*p->xml) + && (*p->xml!='>') && (*p->xml!='/') + ) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + /* to ignore namespace : */ + if(*p->xml==':') + { + i = 0; + elementname = ++p->xml; + } + } + if(i>0) + { + if(p->starteltfunc) + p->starteltfunc(p->data, elementname, i); + if(parseatt(p)) + return; + if(*p->xml!='/') + { + const char * data; + i = 0; data = ++p->xml; + if (p->xml >= p->xmlend) + return; + while( IS_WHITE_SPACE(*p->xml) ) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + } + /* CDATA are at least 9 + 3 characters long : */ + if((p->xmlend >= (p->xml + (9 + 3))) && (memcmp(p->xml, "xml += 9; + data = p->xml; + i = 0; + while(memcmp(p->xml, "]]>", 3) != 0) + { + i++; p->xml++; + if ((p->xml + 3) >= p->xmlend) + return; + } + if(i>0 && p->datafunc) + p->datafunc(p->data, data, i); + while(*p->xml!='<') + { + p->xml++; + if (p->xml >= p->xmlend) + return; + } + } + else + { + while(*p->xml!='<') + { + i++; p->xml++; + if ((p->xml + 1) >= p->xmlend) + return; + } + if(i>0 && p->datafunc && *(p->xml + 1) == '/') + p->datafunc(p->data, data, i); + } + } + } + else if(*p->xml == '/') + { + i = 0; elementname = ++p->xml; + if (p->xml >= p->xmlend) + return; + while((*p->xml != '>')) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + } + if(p->endeltfunc) + p->endeltfunc(p->data, elementname, i); + p->xml++; + } + } + else + { + p->xml++; + } + } +} + +/* the parser must be initialized before calling this function */ +void parsexml(struct xmlparser * parser) +{ + parser->xml = parser->xmlstart; + parser->xmlend = parser->xmlstart + parser->xmlsize; + parseelt(parser); +} + + diff --git a/src/contrib/miniupnp/miniupnpc/minixml.h b/src/contrib/miniupnp/miniupnpc/minixml.h new file mode 100644 index 0000000..19e6f51 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/minixml.h @@ -0,0 +1,37 @@ +/* $Id: minixml.h,v 1.6 2006/11/30 11:47:21 nanard Exp $ */ +/* minimal xml parser + * + * Project : miniupnp + * Website : http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef MINIXML_H_INCLUDED +#define MINIXML_H_INCLUDED +#define IS_WHITE_SPACE(c) ((c==' ') || (c=='\t') || (c=='\r') || (c=='\n')) + +/* if a callback function pointer is set to NULL, + * the function is not called */ +struct xmlparser { + const char *xmlstart; + const char *xmlend; + const char *xml; /* pointer to current character */ + int xmlsize; + void * data; + void (*starteltfunc) (void *, const char *, int); + void (*endeltfunc) (void *, const char *, int); + void (*datafunc) (void *, const char *, int); + void (*attfunc) (void *, const char *, int, const char *, int); +}; + +/* parsexml() + * the xmlparser structure must be initialized before the call + * the following structure members have to be initialized : + * xmlstart, xmlsize, data, *func + * xml is for internal usage, xmlend is computed automatically */ +void parsexml(struct xmlparser *); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/minixmlvalid.c b/src/contrib/miniupnp/miniupnpc/minixmlvalid.c new file mode 100644 index 0000000..dad1488 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/minixmlvalid.c @@ -0,0 +1,163 @@ +/* $Id: minixmlvalid.c,v 1.7 2015/07/15 12:41:15 nanard Exp $ */ +/* MiniUPnP Project + * http://miniupnp.tuxfamily.org/ or http://miniupnp.free.fr/ + * minixmlvalid.c : + * validation program for the minixml parser + * + * (c) 2006-2011 Thomas Bernard */ + +#include +#include +#include +#include "minixml.h" + +/* xml event structure */ +struct event { + enum { ELTSTART, ELTEND, ATT, CHARDATA } type; + const char * data; + int len; +}; + +struct eventlist { + int n; + struct event * events; +}; + +/* compare 2 xml event lists + * return 0 if the two lists are equals */ +int evtlistcmp(struct eventlist * a, struct eventlist * b) +{ + int i; + struct event * ae, * be; + if(a->n != b->n) + { + printf("event number not matching : %d != %d\n", a->n, b->n); + /*return 1;*/ + } + for(i=0; in; i++) + { + ae = a->events + i; + be = b->events + i; + if( (ae->type != be->type) + ||(ae->len != be->len) + ||memcmp(ae->data, be->data, ae->len)) + { + printf("Found a difference : %d '%.*s' != %d '%.*s'\n", + ae->type, ae->len, ae->data, + be->type, be->len, be->data); + return 1; + } + } + return 0; +} + +/* Test data */ +static const char xmldata[] = +"\n" +" " +"character data" +" \n \t" +"" +"\nstuff !\n ]]> \n\n" +" \tchardata1 chardata2 " +""; + +static const struct event evtref[] = +{ + {ELTSTART, "xmlroot", 7}, + {ELTSTART, "elt1", 4}, + /* attributes */ + {CHARDATA, "character data", 14}, + {ELTEND, "elt1", 4}, + {ELTSTART, "elt1b", 5}, + {ELTSTART, "elt1", 4}, + {CHARDATA, " stuff !\n ", 16}, + {ELTEND, "elt1", 4}, + {ELTSTART, "elt2a", 5}, + {ELTSTART, "elt2b", 5}, + {CHARDATA, "chardata1", 9}, + {ELTEND, "elt2b", 5}, + {ELTSTART, "elt2b", 5}, + {CHARDATA, " chardata2 ", 11}, + {ELTEND, "elt2b", 5}, + {ELTEND, "elt2a", 5}, + {ELTEND, "xmlroot", 7} +}; + +void startelt(void * data, const char * p, int l) +{ + struct eventlist * evtlist = data; + struct event * evt; + evt = evtlist->events + evtlist->n; + /*printf("startelt : %.*s\n", l, p);*/ + evt->type = ELTSTART; + evt->data = p; + evt->len = l; + evtlist->n++; +} + +void endelt(void * data, const char * p, int l) +{ + struct eventlist * evtlist = data; + struct event * evt; + evt = evtlist->events + evtlist->n; + /*printf("endelt : %.*s\n", l, p);*/ + evt->type = ELTEND; + evt->data = p; + evt->len = l; + evtlist->n++; +} + +void chardata(void * data, const char * p, int l) +{ + struct eventlist * evtlist = data; + struct event * evt; + evt = evtlist->events + evtlist->n; + /*printf("chardata : '%.*s'\n", l, p);*/ + evt->type = CHARDATA; + evt->data = p; + evt->len = l; + evtlist->n++; +} + +int testxmlparser(const char * xml, int size) +{ + int r; + struct eventlist evtlist; + struct eventlist evtlistref; + struct xmlparser parser; + evtlist.n = 0; + evtlist.events = malloc(sizeof(struct event)*100); + if(evtlist.events == NULL) + { + fprintf(stderr, "Memory allocation error.\n"); + return -1; + } + memset(&parser, 0, sizeof(parser)); + parser.xmlstart = xml; + parser.xmlsize = size; + parser.data = &evtlist; + parser.starteltfunc = startelt; + parser.endeltfunc = endelt; + parser.datafunc = chardata; + parsexml(&parser); + printf("%d events\n", evtlist.n); + /* compare */ + evtlistref.n = sizeof(evtref)/sizeof(struct event); + evtlistref.events = (struct event *)evtref; + r = evtlistcmp(&evtlistref, &evtlist); + free(evtlist.events); + return r; +} + +int main(int argc, char * * argv) +{ + int r; + (void)argc; (void)argv; + + r = testxmlparser(xmldata, sizeof(xmldata)-1); + if(r) + printf("minixml validation test failed\n"); + return r; +} + diff --git a/src/contrib/miniupnp/miniupnpc/msvc/.gitignore b/src/contrib/miniupnp/miniupnpc/msvc/.gitignore new file mode 100644 index 0000000..ec67f23 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/.gitignore @@ -0,0 +1,8 @@ +*.db +.vs +Debug/ +Release/ +*.user +*.suo +*.sdf +ipch/ diff --git a/src/contrib/miniupnp/miniupnpc/msvc/genminiupnpcstrings.vbs b/src/contrib/miniupnp/miniupnpc/msvc/genminiupnpcstrings.vbs new file mode 100644 index 0000000..c1f4a58 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/genminiupnpcstrings.vbs @@ -0,0 +1,53 @@ +' VBScript to generate miniupnpcstrings.h +' Copyright 2018 Thomas Bernard +'Set WshShell = CreateObject("WScript.Shell") +Set FSO = CreateObject("Scripting.FileSystemObject") +versionfile = "..\version" +infile = "..\miniupnpcstrings.h.in" +outfile = "..\miniupnpcstrings.h" + +On Error Resume Next + +'Wscript.Echo revision + +Err.Clear +Set f = FSO.OpenTextFile(versionfile, 1, False) ' 1 = Read +If Err.Number = 0 Then + version = f.ReadLine + f.Close +Else + ' Exit error + WScript.Quit 1 +End If + +os_version = "0.0.0" +strComputer = "." +Set objWMIService = GetObject("winmgmts:" & "{impersonationLevel=impersonate}!\\" & strComputer & "\root\cimv2") +Set colOperatingSystems = objWMIService.ExecQuery ("Select * from Win32_OperatingSystem") +For Each objOperatingSystem in colOperatingSystems + 'Wscript.Echo objOperatingSystem.Caption & " -- " + os_version = objOperatingSystem.Version +Next + +Dim array + +Set f_in = FSO.OpenTextFile(infile, 1, False) +If Err.Number = 0 Then + Set f_out = FSO.OpenTextFile(outfile, 2, True) ' 2 = Write + Do Until f_in.AtEndOfStream + line = f_in.ReadLine + If Len(line) > 0 Then + array = Split(line, " ") + If UBound(array) >= 2 And array(0) = "#define" Then + If array(1) = "OS_STRING" Then + line = "#define OS_STRING " & Chr(34) & "MSWindows/" & os_version & Chr(34) + ElseIf array(1) = "MINIUPNPC_VERSION_STRING" Then + line = "#define MINIUPNPC_VERSION_STRING " & Chr(34) & version & Chr(34) + End if + End if + End If + f_out.WriteLine line + Loop + f_in.Close + f_out.Close +End If diff --git a/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.sln b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.sln new file mode 100644 index 0000000..b3da191 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.sln @@ -0,0 +1,29 @@ + +Microsoft Visual Studio Solution File, Format Version 10.00 +# Visual C++ Express 2008 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "miniupnpc", "miniupnpc.vcproj", "{D28CE435-CB33-4BAE-8A52-C6EF915956F5}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "upnpc-static", "upnpc-static.vcproj", "{469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}" + ProjectSection(ProjectDependencies) = postProject + {D28CE435-CB33-4BAE-8A52-C6EF915956F5} = {D28CE435-CB33-4BAE-8A52-C6EF915956F5} + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Debug|Win32.ActiveCfg = Debug|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Debug|Win32.Build.0 = Debug|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Release|Win32.ActiveCfg = Release|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Release|Win32.Build.0 = Release|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Debug|Win32.ActiveCfg = Debug|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Debug|Win32.Build.0 = Debug|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Release|Win32.ActiveCfg = Release|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcproj b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcproj new file mode 100644 index 0000000..fb301e3 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcproj @@ -0,0 +1,283 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcxproj b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcxproj new file mode 100644 index 0000000..a1569bf --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcxproj @@ -0,0 +1,117 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {D28CE435-CB33-4BAE-8A52-C6EF915956F5} + miniupnpc + Win32Proj + + + + StaticLibrary + v140 + Unicode + true + + + StaticLibrary + v140 + Unicode + + + + + + + + + + + + + <_ProjectFileVersion>14.0.25123.0 + + + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + + + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + + + + Disabled + _CRT_SECURE_NO_WARNINGS;MINIUPNP_STATICLIB;DEBUG;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + + Level3 + EditAndContinue + + + genminiupnpcstrings.vbs + + + + + MaxSpeed + true + _CRT_SECURE_NO_WARNINGS;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) + MultiThreadedDLL + true + + Level3 + ProgramDatabase + + + genminiupnpcstrings.vbs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcxproj.filters b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcxproj.filters new file mode 100644 index 0000000..01a4dbe --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc.vcxproj.filters @@ -0,0 +1,108 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav + + + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + Fichiers sources + + + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + Fichiers d%27en-tête + + + \ No newline at end of file diff --git a/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.sln b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.sln new file mode 100644 index 0000000..65a8734 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.sln @@ -0,0 +1,26 @@ + +Microsoft Visual Studio Solution File, Format Version 11.00 +# Visual C++ Express 2010 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "miniupnpc", "miniupnpc_vs2010.vcxproj", "{D28CE435-CB33-4BAE-8A52-C6EF915956F5}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "upnpc-static", "upnpc-static_vs2010.vcxproj", "{469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Debug|Win32.ActiveCfg = Debug|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Debug|Win32.Build.0 = Debug|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Release|Win32.ActiveCfg = Release|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Release|Win32.Build.0 = Release|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Debug|Win32.ActiveCfg = Debug|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Debug|Win32.Build.0 = Debug|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Release|Win32.ActiveCfg = Release|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.vcxproj b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.vcxproj new file mode 100644 index 0000000..7f83cab --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.vcxproj @@ -0,0 +1,118 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {D28CE435-CB33-4BAE-8A52-C6EF915956F5} + miniupnpc + Win32Proj + + + + StaticLibrary + Unicode + true + + + StaticLibrary + Unicode + + + + + + + + + + + + + <_ProjectFileVersion>10.0.40219.1 + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + miniupnpc + miniupnpc + + + + Disabled + _CRT_SECURE_NO_WARNINGS;MINIUPNP_STATICLIB;DEBUG;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + + + Level3 + EditAndContinue + + + genminiupnpcstrings.vbs + + + + + MaxSpeed + true + _CRT_SECURE_NO_WARNINGS;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) + MultiThreadedDLL + true + + + Level3 + ProgramDatabase + + + genminiupnpcstrings.vbs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.vcxproj.filters b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.vcxproj.filters new file mode 100644 index 0000000..9a75e57 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2010.vcxproj.filters @@ -0,0 +1,111 @@ + + + + + sources + + + sources + + + sources + + + sources + + + sources + + + sources + + + sources + + + sources + + + sources + + + sources + + + sources + + + sources + + + sources + + + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + headers + + + + + {f2cbd46b-f63f-412e-80e5-b7c9048a1add} + + + {2b3996de-1bc4-418b-8a83-a5f34fdf0df5} + + + \ No newline at end of file diff --git a/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2015.sln b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2015.sln new file mode 100644 index 0000000..27a43f6 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/miniupnpc_vs2015.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.25123.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "miniupnpc", "miniupnpc.vcxproj", "{D28CE435-CB33-4BAE-8A52-C6EF915956F5}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "upnpc-static", "upnpc-static.vcxproj", "{469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Debug|Win32.ActiveCfg = Debug|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Debug|Win32.Build.0 = Debug|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Release|Win32.ActiveCfg = Release|Win32 + {D28CE435-CB33-4BAE-8A52-C6EF915956F5}.Release|Win32.Build.0 = Release|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Debug|Win32.ActiveCfg = Debug|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Debug|Win32.Build.0 = Debug|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Release|Win32.ActiveCfg = Release|Win32 + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcproj b/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcproj new file mode 100644 index 0000000..c88c9a6 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcproj @@ -0,0 +1,195 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcxproj b/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcxproj new file mode 100644 index 0000000..44dea81 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcxproj @@ -0,0 +1,103 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1} + upnpcstatic + Win32Proj + + + + Application + v140 + Unicode + true + + + Application + v140 + Unicode + + + + + + + + + + + + + <_ProjectFileVersion>14.0.25123.0 + + + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + true + + + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + false + + + + Disabled + _DEBUG;_CONSOLE;MINIUPNP_STATICLIB;DEBUG;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + + Level3 + EditAndContinue + + + ws2_32.lib;IPHlpApi.Lib;Debug\miniupnpc.lib;%(AdditionalDependencies) + true + Console + MachineX86 + + + + + MaxSpeed + true + NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) + MultiThreadedDLL + true + + Level3 + ProgramDatabase + + + ws2_32.lib;IPHlpApi.Lib;Release\miniupnpc.lib;%(AdditionalDependencies) + true + Console + true + true + MachineX86 + + + + + + + + {d28ce435-cb33-4bae-8a52-c6ef915956f5} + false + + + + + + \ No newline at end of file diff --git a/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcxproj.filters b/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcxproj.filters new file mode 100644 index 0000000..2e75de0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static.vcxproj.filters @@ -0,0 +1,22 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav + + + + + Fichiers sources + + + \ No newline at end of file diff --git a/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static_vs2010.vcxproj b/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static_vs2010.vcxproj new file mode 100644 index 0000000..e815024 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/msvc/upnpc-static_vs2010.vcxproj @@ -0,0 +1,101 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {469E1CF6-08A2-4B7B-A2AA-5BDB089857C1} + upnpcstatic + Win32Proj + + + + Application + Unicode + true + + + Application + Unicode + + + + + + + + + + + + + <_ProjectFileVersion>10.0.40219.1 + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + true + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + false + upnpc-static + upnpc-static + + + + Disabled + _DEBUG;_CONSOLE;MINIUPNP_STATICLIB;DEBUG;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebugDLL + + + Level3 + EditAndContinue + + + ws2_32.lib;IPHlpApi.Lib;Debug\miniupnpc.lib;%(AdditionalDependencies) + true + Console + MachineX86 + + + + + MaxSpeed + true + NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;MINIUPNP_STATICLIB;%(PreprocessorDefinitions) + MultiThreadedDLL + true + + + Level3 + ProgramDatabase + + + ws2_32.lib;IPHlpApi.Lib;Release\miniupnpc.lib;%(AdditionalDependencies) + true + Console + true + true + MachineX86 + + + + + + + + {d28ce435-cb33-4bae-8a52-c6ef915956f5} + false + + + + + + \ No newline at end of file diff --git a/src/contrib/miniupnp/miniupnpc/portlistingparse.c b/src/contrib/miniupnp/miniupnpc/portlistingparse.c new file mode 100644 index 0000000..55859f2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/portlistingparse.c @@ -0,0 +1,172 @@ +/* $Id: portlistingparse.c,v 1.9 2015/07/15 12:41:13 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011-2016 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#include +#include +#ifdef DEBUG +#include +#endif /* DEBUG */ +#include "portlistingparse.h" +#include "minixml.h" + +/* list of the elements */ +static const struct { + const portMappingElt code; + const char * const str; +} elements[] = { + { PortMappingEntry, "PortMappingEntry"}, + { NewRemoteHost, "NewRemoteHost"}, + { NewExternalPort, "NewExternalPort"}, + { NewProtocol, "NewProtocol"}, + { NewInternalPort, "NewInternalPort"}, + { NewInternalClient, "NewInternalClient"}, + { NewEnabled, "NewEnabled"}, + { NewDescription, "NewDescription"}, + { NewLeaseTime, "NewLeaseTime"}, + { PortMappingEltNone, NULL} +}; + +/* Helper function */ +static UNSIGNED_INTEGER +atoui(const char * p, int l) +{ + UNSIGNED_INTEGER r = 0; + while(l > 0 && *p) + { + if(*p >= '0' && *p <= '9') + r = r*10 + (*p - '0'); + else + break; + p++; + l--; + } + return r; +} + +/* Start element handler */ +static void +startelt(void * d, const char * name, int l) +{ + int i; + struct PortMappingParserData * pdata = (struct PortMappingParserData *)d; + pdata->curelt = PortMappingEltNone; + for(i = 0; elements[i].str; i++) + { + if(strlen(elements[i].str) == (size_t)l && memcmp(name, elements[i].str, l) == 0) + { + pdata->curelt = elements[i].code; + break; + } + } + if(pdata->curelt == PortMappingEntry) + { + struct PortMapping * pm; + pm = calloc(1, sizeof(struct PortMapping)); + if(pm == NULL) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "startelt"); +#endif /* DEBUG */ + return; + } + pm->l_next = pdata->l_head; /* insert in list */ + pdata->l_head = pm; + } +} + +/* End element handler */ +static void +endelt(void * d, const char * name, int l) +{ + struct PortMappingParserData * pdata = (struct PortMappingParserData *)d; + (void)name; + (void)l; + pdata->curelt = PortMappingEltNone; +} + +/* Data handler */ +static void +data(void * d, const char * data, int l) +{ + struct PortMapping * pm; + struct PortMappingParserData * pdata = (struct PortMappingParserData *)d; + pm = pdata->l_head; + if(!pm) + return; + if(l > 63) + l = 63; + switch(pdata->curelt) + { + case NewRemoteHost: + memcpy(pm->remoteHost, data, l); + pm->remoteHost[l] = '\0'; + break; + case NewExternalPort: + pm->externalPort = (unsigned short)atoui(data, l); + break; + case NewProtocol: + if(l > 3) + l = 3; + memcpy(pm->protocol, data, l); + pm->protocol[l] = '\0'; + break; + case NewInternalPort: + pm->internalPort = (unsigned short)atoui(data, l); + break; + case NewInternalClient: + memcpy(pm->internalClient, data, l); + pm->internalClient[l] = '\0'; + break; + case NewEnabled: + pm->enabled = (unsigned char)atoui(data, l); + break; + case NewDescription: + memcpy(pm->description, data, l); + pm->description[l] = '\0'; + break; + case NewLeaseTime: + pm->leaseTime = atoui(data, l); + break; + default: + break; + } +} + + +/* Parse the PortMappingList XML document for IGD version 2 + */ +void +ParsePortListing(const char * buffer, int bufsize, + struct PortMappingParserData * pdata) +{ + struct xmlparser parser; + + memset(pdata, 0, sizeof(struct PortMappingParserData)); + /* init xmlparser */ + parser.xmlstart = buffer; + parser.xmlsize = bufsize; + parser.data = pdata; + parser.starteltfunc = startelt; + parser.endeltfunc = endelt; + parser.datafunc = data; + parser.attfunc = 0; + parsexml(&parser); +} + +void +FreePortListing(struct PortMappingParserData * pdata) +{ + struct PortMapping * pm; + while((pm = pdata->l_head) != NULL) + { + /* remove from list */ + pdata->l_head = pm->l_next; + free(pm); + } +} + diff --git a/src/contrib/miniupnp/miniupnpc/portlistingparse.h b/src/contrib/miniupnp/miniupnpc/portlistingparse.h new file mode 100644 index 0000000..e3957a3 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/portlistingparse.h @@ -0,0 +1,65 @@ +/* $Id: portlistingparse.h,v 1.10 2014/11/01 10:37:32 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011-2015 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef PORTLISTINGPARSE_H_INCLUDED +#define PORTLISTINGPARSE_H_INCLUDED + +#include "miniupnpc_declspec.h" +/* for the definition of UNSIGNED_INTEGER */ +#include "miniupnpctypes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* sample of PortMappingEntry : + + 202.233.2.1 + 2345 + TCP + 2345 + 192.168.1.137 + 1 + dooom + 345 + + */ +typedef enum { PortMappingEltNone, + PortMappingEntry, NewRemoteHost, + NewExternalPort, NewProtocol, + NewInternalPort, NewInternalClient, + NewEnabled, NewDescription, + NewLeaseTime } portMappingElt; + +struct PortMapping { + struct PortMapping * l_next; /* list next element */ + UNSIGNED_INTEGER leaseTime; + unsigned short externalPort; + unsigned short internalPort; + char remoteHost[64]; + char internalClient[64]; + char description[64]; + char protocol[4]; + unsigned char enabled; +}; + +struct PortMappingParserData { + struct PortMapping * l_head; /* list head */ + portMappingElt curelt; +}; + +MINIUPNP_LIBSPEC void +ParsePortListing(const char * buffer, int bufsize, + struct PortMappingParserData * pdata); + +MINIUPNP_LIBSPEC void +FreePortListing(struct PortMappingParserData * pdata); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/contrib/miniupnp/miniupnpc/pymoduletest.py b/src/contrib/miniupnp/miniupnpc/pymoduletest.py new file mode 100644 index 0000000..9de14a6 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/pymoduletest.py @@ -0,0 +1,88 @@ +#! /usr/bin/python +# vim: tabstop=2 shiftwidth=2 expandtab +# MiniUPnP project +# Author : Thomas Bernard +# This Sample code is public domain. +# website : https://miniupnp.tuxfamily.org/ + +# import the python miniupnpc module +import miniupnpc +import sys + +try: + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--multicastif') + parser.add_argument('-p', '--minissdpdsocket') + parser.add_argument('-d', '--discoverdelay', type=int, default=200) + parser.add_argument('-z', '--localport', type=int, default=0) + # create the object + u = miniupnpc.UPnP(**vars(parser.parse_args())) +except: + print 'argparse not available' + i = 1 + multicastif = None + minissdpdsocket = None + discoverdelay = 200 + localport = 0 + while i < len(sys.argv): + print sys.argv[i] + if sys.argv[i] == '-m' or sys.argv[i] == '--multicastif': + multicastif = sys.argv[i+1] + elif sys.argv[i] == '-p' or sys.argv[i] == '--minissdpdsocket': + minissdpdsocket = sys.argv[i+1] + elif sys.argv[i] == '-d' or sys.argv[i] == '--discoverdelay': + discoverdelay = int(sys.argv[i+1]) + elif sys.argv[i] == '-z' or sys.argv[i] == '--localport': + localport = int(sys.argv[i+1]) + else: + raise Exception('invalid argument %s' % sys.argv[i]) + i += 2 + # create the object + u = miniupnpc.UPnP(multicastif, minissdpdsocket, discoverdelay, localport) + +print 'inital(default) values :' +print ' discoverdelay', u.discoverdelay +print ' lanaddr', u.lanaddr +print ' multicastif', u.multicastif +print ' minissdpdsocket', u.minissdpdsocket +#u.minissdpdsocket = '../minissdpd/minissdpd.sock' +# discovery process, it usually takes several seconds (2 seconds or more) +print 'Discovering... delay=%ums' % u.discoverdelay +print u.discover(), 'device(s) detected' +# select an igd +try: + u.selectigd() +except Exception, e: + print 'Exception :', e + sys.exit(1) +# display information about the IGD and the internet connection +print 'local ip address :', u.lanaddr +print 'external ip address :', u.externalipaddress() +print u.statusinfo(), u.connectiontype() +print 'total bytes : sent', u.totalbytesent(), 'received', u.totalbytereceived() +print 'total packets : sent', u.totalpacketsent(), 'received', u.totalpacketreceived() + +#print u.addportmapping(64000, 'TCP', +# '192.168.1.166', 63000, 'port mapping test', '') +#print u.deleteportmapping(64000, 'TCP') + +port = 0 +proto = 'UDP' +# list the redirections : +i = 0 +while True: + p = u.getgenericportmapping(i) + if p==None: + break + print i, p + (port, proto, (ihost,iport), desc, c, d, e) = p + #print port, desc + i = i + 1 + +print u.getspecificportmapping(port, proto) +try: + print u.getportmappingnumberofentries() +except Exception, e: + print 'GetPortMappingNumberOfEntries() is not supported :', e + diff --git a/src/contrib/miniupnp/miniupnpc/pymoduletest3.py b/src/contrib/miniupnp/miniupnpc/pymoduletest3.py new file mode 100644 index 0000000..1fcab4e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/pymoduletest3.py @@ -0,0 +1,52 @@ +#! /usr/bin/python3 +# MiniUPnP project +# Author : Thomas Bernard +# This Sample code is public domain. +# website : http://miniupnp.tuxfamily.org/ + +# import the python miniupnpc module +import miniupnpc +import sys + +# create the object +u = miniupnpc.UPnP() +print('inital(default) values :') +print(' discoverdelay', u.discoverdelay) +print(' lanaddr', u.lanaddr) +print(' multicastif', u.multicastif) +print(' minissdpdsocket', u.minissdpdsocket) +u.discoverdelay = 200; +#u.minissdpdsocket = '../minissdpd/minissdpd.sock' +# discovery process, it usually takes several seconds (2 seconds or more) +print('Discovering... delay=%ums' % u.discoverdelay) +print(u.discover(), 'device(s) detected') +# select an igd +try: + u.selectigd() +except Exception as e: + print('Exception :', e) + sys.exit(1) +# display information about the IGD and the internet connection +print('local ip address :', u.lanaddr) +print('external ip address :', u.externalipaddress()) +print(u.statusinfo(), u.connectiontype()) + +#print u.addportmapping(64000, 'TCP', +# '192.168.1.166', 63000, 'port mapping test', '') +#print u.deleteportmapping(64000, 'TCP') + +port = 0 +proto = 'UDP' +# list the redirections : +i = 0 +while True: + p = u.getgenericportmapping(i) + if p==None: + break + print(i, p) + (port, proto, (ihost,iport), desc, c, d, e) = p + #print port, desc + i = i + 1 + +print(u.getspecificportmapping(port, proto)) + diff --git a/src/contrib/miniupnp/miniupnpc/receivedata.c b/src/contrib/miniupnp/miniupnpc/receivedata.c new file mode 100644 index 0000000..7b9cc5b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/receivedata.c @@ -0,0 +1,99 @@ +/* $Id: receivedata.c,v 1.7 2015/11/09 21:51:41 nanard Exp $ */ +/* Project : miniupnp + * Website : http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2011-2014 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ + +#include +#include +#ifdef _WIN32 +#include +#include +#else /* _WIN32 */ +#include +#if defined(__amigaos__) && !defined(__amigaos4__) +#define socklen_t int +#else /* #if defined(__amigaos__) && !defined(__amigaos4__) */ +#include +#endif /* #else defined(__amigaos__) && !defined(__amigaos4__) */ +#include +#include +#if !defined(__amigaos__) && !defined(__amigaos4__) +#include +#endif /* !defined(__amigaos__) && !defined(__amigaos4__) */ +#include +#define MINIUPNPC_IGNORE_EINTR +#endif /* _WIN32 */ + +#include "receivedata.h" + +int +receivedata(SOCKET socket, + char * data, int length, + int timeout, unsigned int * scope_id) +{ +#ifdef MINIUPNPC_GET_SRC_ADDR + struct sockaddr_storage src_addr; + socklen_t src_addr_len = sizeof(src_addr); +#endif /* MINIUPNPC_GET_SRC_ADDR */ + int n; +#if !defined(_WIN32) && !defined(__amigaos__) && !defined(__amigaos4__) + /* using poll */ + struct pollfd fds[1]; /* for the poll */ +#ifdef MINIUPNPC_IGNORE_EINTR + do { +#endif /* MINIUPNPC_IGNORE_EINTR */ + fds[0].fd = socket; + fds[0].events = POLLIN; + n = poll(fds, 1, timeout); +#ifdef MINIUPNPC_IGNORE_EINTR + } while(n < 0 && errno == EINTR); +#endif /* MINIUPNPC_IGNORE_EINTR */ + if(n < 0) { + PRINT_SOCKET_ERROR("poll"); + return -1; + } else if(n == 0) { + /* timeout */ + return 0; + } +#else /* !defined(_WIN32) && !defined(__amigaos__) && !defined(__amigaos4__) */ + /* using select under _WIN32 and amigaos */ + fd_set socketSet; + TIMEVAL timeval; + FD_ZERO(&socketSet); + FD_SET(socket, &socketSet); + timeval.tv_sec = timeout / 1000; + timeval.tv_usec = (timeout % 1000) * 1000; + n = select(FD_SETSIZE, &socketSet, NULL, NULL, &timeval); + if(n < 0) { + PRINT_SOCKET_ERROR("select"); + return -1; + } else if(n == 0) { + return 0; + } +#endif /* !defined(_WIN32) && !defined(__amigaos__) && !defined(__amigaos4__) */ +#ifdef MINIUPNPC_GET_SRC_ADDR + memset(&src_addr, 0, sizeof(src_addr)); + n = recvfrom(socket, data, length, 0, + (struct sockaddr *)&src_addr, &src_addr_len); +#else /* MINIUPNPC_GET_SRC_ADDR */ + n = recv(socket, data, length, 0); +#endif /* MINIUPNPC_GET_SRC_ADDR */ + if(n<0) { + PRINT_SOCKET_ERROR("recv"); + } +#ifdef MINIUPNPC_GET_SRC_ADDR + if (src_addr.ss_family == AF_INET6) { + const struct sockaddr_in6 * src_addr6 = (struct sockaddr_in6 *)&src_addr; +#ifdef DEBUG + printf("scope_id=%u\n", src_addr6->sin6_scope_id); +#endif /* DEBUG */ + if(scope_id) + *scope_id = src_addr6->sin6_scope_id; + } +#endif /* MINIUPNPC_GET_SRC_ADDR */ + return n; +} + diff --git a/src/contrib/miniupnp/miniupnpc/receivedata.h b/src/contrib/miniupnp/miniupnpc/receivedata.h new file mode 100644 index 0000000..c9fdc56 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/receivedata.h @@ -0,0 +1,21 @@ +/* $Id: receivedata.h,v 1.3 2012/06/23 22:34:47 nanard Exp $ */ +/* Project: miniupnp + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author: Thomas Bernard + * Copyright (c) 2011-2018 Thomas Bernard + * This software is subjects to the conditions detailed + * in the LICENCE file provided within this distribution */ +#ifndef RECEIVEDATA_H_INCLUDED +#define RECEIVEDATA_H_INCLUDED + +#include "miniupnpc_socketdef.h" + +/* Reads data from the specified socket. + * Returns the number of bytes read if successful, zero if no bytes were + * read or if we timed out. Returns negative if there was an error. */ +int receivedata(SOCKET socket, + char * data, int length, + int timeout, unsigned int * scope_id); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/setup.py b/src/contrib/miniupnp/miniupnpc/setup.py new file mode 100644 index 0000000..24a676d --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/setup.py @@ -0,0 +1,35 @@ +#! /usr/bin/python +# vim: tabstop=8 shiftwidth=8 expandtab +# $Id: setup.py,v 1.9 2012/05/23 08:50:10 nanard Exp $ +# the MiniUPnP Project (c) 2007-2017 Thomas Bernard +# http://miniupnp.tuxfamily.org/ or http://miniupnp.free.fr/ +# +# python script to build the miniupnpc module under unix +# +# Uses MAKE environment variable (defaulting to 'make') + +from setuptools import setup, Extension +from setuptools.command import build_ext +import subprocess +import os + +EXT = ['libminiupnpc.a'] + +class make_then_build_ext(build_ext.build_ext): + def run(self): + subprocess.check_call([os.environ.get('MAKE', 'make')] + EXT) + build_ext.build_ext.run(self) + +setup(name="miniupnpc", + version=open('VERSION').read().strip(), + author='Thomas BERNARD', + author_email='miniupnp@free.fr', + license=open('LICENSE').read(), + url='http://miniupnp.free.fr/', + description='miniUPnP client', + cmdclass={'build_ext': make_then_build_ext}, + ext_modules=[ + Extension(name="miniupnpc", sources=["miniupnpcmodule.c"], + extra_objects=EXT) + ]) + diff --git a/src/contrib/miniupnp/miniupnpc/setupmingw32.py b/src/contrib/miniupnp/miniupnpc/setupmingw32.py new file mode 100644 index 0000000..7e3c382 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/setupmingw32.py @@ -0,0 +1,28 @@ +#! /usr/bin/python +# vim: tabstop=8 shiftwidth=8 expandtab +# $Id: setupmingw32.py,v 1.8 2012/05/23 08:50:10 nanard Exp $ +# the MiniUPnP Project (c) 2007-2014 Thomas Bernard +# http://miniupnp.tuxfamily.org/ or http://miniupnp.free.fr/ +# +# python script to build the miniupnpc module under windows (using mingw32) +# +try: + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension +from distutils import sysconfig +sysconfig.get_config_vars()["OPT"] = '' +sysconfig.get_config_vars()["CFLAGS"] = '' +setup(name="miniupnpc", + version=open('VERSION').read().strip(), + author='Thomas BERNARD', + author_email='miniupnp@free.fr', + license=open('LICENSE').read(), + url='http://miniupnp.free.fr/', + description='miniUPnP client', + ext_modules=[ + Extension(name="miniupnpc", sources=["miniupnpcmodule.c"], + libraries=["ws2_32", "iphlpapi"], + extra_objects=["libminiupnpc.a"]) + ]) + diff --git a/src/contrib/miniupnp/miniupnpc/testdesc/linksys_WAG200G_desc.values b/src/contrib/miniupnp/miniupnpc/testdesc/linksys_WAG200G_desc.values new file mode 100644 index 0000000..cf42221 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testdesc/linksys_WAG200G_desc.values @@ -0,0 +1,14 @@ +# values for linksys_WAG200G_desc.xml + +CIF: + servicetype = urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1 + controlurl = /upnp/control/WANCommonIFC1 + eventsuburl = /upnp/event/WANCommonIFC1 + scpdurl = /cmnicfg.xml + +first: + servicetype = urn:schemas-upnp-org:service:WANPPPConnection:1 + controlurl = /upnp/control/WANPPPConn1 + eventsuburl = /upnp/event/WANPPPConn1 + scpdurl = /pppcfg.xml + diff --git a/src/contrib/miniupnp/miniupnpc/testdesc/linksys_WAG200G_desc.xml b/src/contrib/miniupnp/miniupnpc/testdesc/linksys_WAG200G_desc.xml new file mode 100644 index 0000000..d428d73 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testdesc/linksys_WAG200G_desc.xml @@ -0,0 +1,110 @@ + + + +1 +0 + +http://192.168.1.1:49152 + +urn:schemas-upnp-org:device:InternetGatewayDevice:1 +LINKSYS WAG200G Gateway +LINKSYS +http://www.linksys.com +LINKSYS WAG200G Gateway +Wireless-G ADSL Home Gateway +WAG200G +http://www.linksys.com +123456789 +uuid:8ca2eb37-1dd2-11b2-86f1-001a709b5aa8 +WAG200G + + +urn:schemas-upnp-org:service:Layer3Forwarding:1 +urn:upnp-org:serviceId:L3Forwarding1 +/upnp/control/L3Forwarding1 +/upnp/event/L3Forwarding1 +/l3frwd.xml + + + + +urn:schemas-upnp-org:device:WANDevice:1 +WANDevice +LINKSYS +http://www.linksys.com/ +Residential Gateway +Internet Connection Sharing +1 +http://www.linksys.com/ +0000001 +uuid:8ca2eb36-1dd2-11b2-86f1-001a709b5aa8 +WAG200G + + +urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1 +urn:upnp-org:serviceId:WANCommonIFC1 +/upnp/control/WANCommonIFC1 +/upnp/event/WANCommonIFC1 +/cmnicfg.xml + + + + +urn:schemas-upnp-org:device:WANConnectionDevice:1 +WANConnectionDevice +LINKSYS +http://www.linksys.com/ +Residential Gateway +Internet Connection Sharing +1 +http://www.linksys.com/ +0000001 +uuid:8ca2eb37-1dd2-11b2-86f0-001a709b5aa8 +WAG200G + + +urn:schemas-upnp-org:service:WANEthernetLinkConfig:1 +urn:upnp-org:serviceId:WANEthLinkC1 +/upnp/control/WANEthLinkC1 +/upnp/event/WANEthLinkC1 +/wanelcfg.xml + + +urn:schemas-upnp-org:service:WANPPPConnection:1 +urn:upnp-org:serviceId:WANPPPConn1 +/upnp/control/WANPPPConn1 +/upnp/event/WANPPPConn1 +/pppcfg.xml + + + + + + +urn:schemas-upnp-org:device:LANDevice:1 +LANDevice +LINKSYS +http://www.linksys.com/ +Residential Gateway +Residential Gateway +1 +http://www.linksys.com/ +0000001 +uuid:8ca2eb36-1dd2-11b2-86f0-001a709b5aa +8 +WAG200G + + +urn:schemas-upnp-org:service:LANHostConfigManagement:1 +urn:upnp-org:serviceId:LANHostCfg1 +/upnp/control/LANHostCfg1 +/upnp/event/LANHostCfg1 +/lanhostc.xml + + + + +http://192.168.1.1/index.htm + + + diff --git a/src/contrib/miniupnp/miniupnpc/testdesc/new_LiveBox_desc.values b/src/contrib/miniupnp/miniupnpc/testdesc/new_LiveBox_desc.values new file mode 100644 index 0000000..c55552e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testdesc/new_LiveBox_desc.values @@ -0,0 +1,20 @@ +# values for new_LiveBox_desc.xml + +CIF: + servicetype = urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1 + controlurl = /87895a19/upnp/control/WANCommonIFC1 + eventsuburl = /87895a19/upnp/control/WANCommonIFC1 + scpdurl = /87895a19/gateicfgSCPD.xml + +first: + servicetype = urn:schemas-upnp-org:service:WANPPPConnection:2 + controlurl = /87895a19/upnp/control/WANIPConn1 + eventsuburl = /87895a19/upnp/control/WANIPConn1 + scpdurl = /87895a19/gateconnSCPD_PPP.xml + +IPv6FC: + servicetype = urn:schemas-upnp-org:service:WANIPv6FirewallControl:1 + controlurl = /87895a19/upnp/control/WANIPv6FwCtrl1 + eventsuburl = /87895a19/upnp/control/WANIPv6FwCtrl1 + scpdurl = /87895a19/wanipv6fwctrlSCPD.xml + diff --git a/src/contrib/miniupnp/miniupnpc/testdesc/new_LiveBox_desc.xml b/src/contrib/miniupnp/miniupnpc/testdesc/new_LiveBox_desc.xml new file mode 100644 index 0000000..3e522df --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testdesc/new_LiveBox_desc.xml @@ -0,0 +1,90 @@ + + + + 1 + 0 + + + VEN_0129&DEV_0000&SUBSYS_03&REV_250417 + GenericUmPass + NetworkInfrastructure.Gateway + Network.Gateway + urn:schemas-upnp-org:device:InternetGatewayDevice:2 + Orange Livebox + Sagemcom + http://www.sagemcom.com/ + Residential Livebox,(DSL,WAN Ethernet) + uuid:87895a19-50f9-3736-a87f-115c230155f8 + Sagemcom,fr,SG30_sip-fr-4.28.35.1 + 3 + LK14129DP441489 + http://192.168.1.1 + + + + image/png + 16 + 16 + 8 + /87895a19/ligd.png + + + + + urn:schemas-upnp-org:device:WANDevice:2 + WANDevice + Sagemcom + http://www.sagemcom.com/ + WAN Device on Sagemcom,fr,SG30_sip-fr-4.28.35.1 + Residential Livebox,(DSL,WAN Ethernet) + 3 + http://www.sagemcom.com/ + LK14129DP441489 + http://192.168.1.1 + uuid:e2397374-53d8-3fc6-8306-593ba1a34625 + + + + urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1 + urn:upnp-org:serviceId:WANCommonIFC1 + /87895a19/upnp/control/WANCommonIFC1 + /87895a19/upnp/control/WANCommonIFC1 + /87895a19/gateicfgSCPD.xml + + + + + urn:schemas-upnp-org:device:WANConnectionDevice:2 + WANConnectionDevice + Sagemcom + http://www.sagemcom.com/ + WanConnectionDevice on Sagemcom,fr,SG30_sip-fr-4.28.35.1 + Residential Livebox,(DSL,WAN Ethernet) + 3 + http://www.sagemcom.com/ + LK14129DP441489 + http://192.168.1.1 + uuid:44598a08-288e-32c9-8a4d-d3c008ede331 + + + + urn:schemas-upnp-org:service:WANPPPConnection:2 + urn:upnp-org:serviceId:WANIPConn1 + /87895a19/upnp/control/WANIPConn1 + /87895a19/upnp/control/WANIPConn1 + /87895a19/gateconnSCPD_PPP.xml + + + urn:schemas-upnp-org:service:WANIPv6FirewallControl:1 + urn:upnp-org:serviceId:WANIPv6FwCtrl1 + /87895a19/upnp/control/WANIPv6FwCtrl1 + /87895a19/upnp/control/WANIPv6FwCtrl1 + /87895a19/wanipv6fwctrlSCPD.xml + + + + + + + + \ No newline at end of file diff --git a/src/contrib/miniupnp/miniupnpc/testigddescparse.c b/src/contrib/miniupnp/miniupnpc/testigddescparse.c new file mode 100644 index 0000000..cb7e94a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testigddescparse.c @@ -0,0 +1,187 @@ +/* $Id: testigddescparse.c,v 1.8 2015/02/08 08:46:06 nanard Exp $ */ +/* Project : miniupnp + * http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2008-2015 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#include +#include +#include +#include "igd_desc_parse.h" +#include "minixml.h" +#include "miniupnpc.h" + +/* count number of differences */ +int compare_service(struct IGDdatas_service * s, FILE * f) +{ + int n = 0; + char line[1024]; + + while(fgets(line, sizeof(line), f)) { + char * value; + char * equal; + char * name; + char * parsedvalue; + int l; + l = strlen(line); + while((l > 0) && ((line[l-1] == '\r') || (line[l-1] == '\n') || (line[l-1] == ' '))) + line[--l] = '\0'; + if(l == 0) + break; /* end on blank line */ + if(line[0] == '#') + continue; /* skip comments */ + equal = strchr(line, '='); + if(equal == NULL) { + fprintf(stderr, "Warning, line does not contain '=' : %s\n", line); + continue; + } + *equal = '\0'; + name = line; + while(*name == ' ' || *name == '\t') + name++; + l = strlen(name); + while((l > 0) && (name[l-1] == ' ' || name[l-1] == '\t')) + name[--l] = '\0'; + value = equal + 1; + while(*value == ' ' || *value == '\t') + value++; + if(strcmp(name, "controlurl") == 0) + parsedvalue = s->controlurl; + else if(strcmp(name, "eventsuburl") == 0) + parsedvalue = s->eventsuburl; + else if(strcmp(name, "scpdurl") == 0) + parsedvalue = s->scpdurl; + else if(strcmp(name, "servicetype") == 0) + parsedvalue = s->servicetype; + else { + fprintf(stderr, "unknown field '%s'\n", name); + continue; + } + if(0 != strcmp(parsedvalue, value)) { + fprintf(stderr, "difference : '%s' != '%s'\n", parsedvalue, value); + n++; + } + } + return n; +} + +int compare_igd(struct IGDdatas * p, FILE * f) +{ + int n = 0; + char line[1024]; + struct IGDdatas_service * s; + + while(fgets(line, sizeof(line), f)) { + char * colon; + int l = (int)strlen(line); + while((l > 0) && (line[l-1] == '\r' || (line[l-1] == '\n'))) + line[--l] = '\0'; + if(l == 0 || line[0] == '#') + continue; /* skip blank lines and comments */ + colon = strchr(line, ':'); + if(colon == NULL) { + fprintf(stderr, "Warning, no ':' : %s\n", line); + continue; + } + s = NULL; + *colon = '\0'; + if(strcmp(line, "CIF") == 0) + s = &p->CIF; + else if(strcmp(line, "first") == 0) + s = &p->first; + else if(strcmp(line, "second") == 0) + s = &p->second; + else if(strcmp(line, "IPv6FC") == 0) + s = &p->IPv6FC; + else { + s = NULL; + fprintf(stderr, "*** unknown service '%s' ***\n", line); + n++; + continue; + } + n += compare_service(s, f); + } + if(n > 0) + fprintf(stderr, "*** %d difference%s ***\n", n, (n > 1) ? "s" : ""); + return n; +} + +int test_igd_desc_parse(char * buffer, int len, FILE * f) +{ + int n; + struct IGDdatas igd; + struct xmlparser parser; + struct UPNPUrls urls; + + memset(&igd, 0, sizeof(struct IGDdatas)); + memset(&parser, 0, sizeof(struct xmlparser)); + parser.xmlstart = buffer; + parser.xmlsize = len; + parser.data = &igd; + parser.starteltfunc = IGDstartelt; + parser.endeltfunc = IGDendelt; + parser.datafunc = IGDdata; + parsexml(&parser); +#ifdef DEBUG + printIGD(&igd); +#endif /* DEBUG */ + GetUPNPUrls(&urls, &igd, "http://fake/desc/url/file.xml", 0); + printf("ipcondescURL='%s'\n", urls.ipcondescURL); + printf("controlURL='%s'\n", urls.controlURL); + printf("controlURL_CIF='%s'\n", urls.controlURL_CIF); + n = f ? compare_igd(&igd, f) : 0; + FreeUPNPUrls(&urls); + return n; +} + +int main(int argc, char * * argv) +{ + FILE * f; + char * buffer; + int len; + int r; + if(argc<2) { + fprintf(stderr, "Usage: %s file.xml [file.values]\n", argv[0]); + return 1; + } + f = fopen(argv[1], "r"); + if(!f) { + fprintf(stderr, "Cannot open %s for reading.\n", argv[1]); + return 1; + } + fseek(f, 0, SEEK_END); + len = ftell(f); + fseek(f, 0, SEEK_SET); + buffer = malloc(len); + if(!buffer) { + fprintf(stderr, "Memory allocation error.\n"); + fclose(f); + return 1; + } + r = (int)fread(buffer, 1, len, f); + if(r != len) { + fprintf(stderr, "Failed to read file %s. %d out of %d bytes.\n", + argv[1], r, len); + fclose(f); + free(buffer); + return 1; + } + fclose(f); + f = NULL; + if(argc > 2) { + f = fopen(argv[2], "r"); + if(!f) { + fprintf(stderr, "Cannot open %s for reading.\n", argv[2]); + free(buffer); + return 1; + } + } + r = test_igd_desc_parse(buffer, len, f); + free(buffer); + if(f) + fclose(f); + return r; +} + diff --git a/src/contrib/miniupnp/miniupnpc/testminiwget.c b/src/contrib/miniupnp/miniupnpc/testminiwget.c new file mode 100644 index 0000000..fba39d5 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testminiwget.c @@ -0,0 +1,56 @@ +/* $Id: testminiwget.c,v 1.6 2017/11/02 16:52:37 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp + * Author : Thomas Bernard + * Copyright (c) 2005-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#include +#include +#include "miniwget.h" + +/** + * This program uses the miniwget / miniwget_getaddr function + * from miniwget.c in order to retrieve a web ressource using + * a GET HTTP method, and store it in a file. + */ +int main(int argc, char * * argv) +{ + void * data; + int size, writtensize; + FILE *f; + char addr[64]; + int status_code = -1; + + if(argc < 3) { + fprintf(stderr, "Usage:\t%s url file\n", argv[0]); + fprintf(stderr, "Example:\t%s http://www.google.com/ out.html\n", argv[0]); + return 1; + } + data = miniwget_getaddr(argv[1], &size, addr, sizeof(addr), 0, &status_code); + if(!data || (status_code != 200)) { + if(data) free(data); + fprintf(stderr, "Error %d fetching %s\n", status_code, argv[1]); + return 1; + } + printf("local address : %s\n", addr); + printf("got %d bytes\n", size); + f = fopen(argv[2], "wb"); + if(!f) { + fprintf(stderr, "Cannot open file %s for writing\n", argv[2]); + free(data); + return 1; + } + writtensize = fwrite(data, 1, size, f); + if(writtensize != size) { + fprintf(stderr, "Could only write %d bytes out of %d to %s\n", + writtensize, size, argv[2]); + } else { + printf("%d bytes written to %s\n", writtensize, argv[2]); + } + fclose(f); + free(data); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpc/testminiwget.sh b/src/contrib/miniupnp/miniupnpc/testminiwget.sh new file mode 100644 index 0000000..32154e0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testminiwget.sh @@ -0,0 +1,111 @@ +#!/bin/sh +# $Id: testminiwget.sh,v 1.15 2017/11/02 17:36:26 nanard Exp $ +# vim: tabstop=4 shiftwidth=4 noexpandtab +# project miniupnp : http://miniupnp.free.fr/ +# (c) 2011-2018 Thomas Bernard +# +# test program for miniwget.c +# is usually invoked by "make check" +# +# This test program : +# 1 - launches a local HTTP server (minihttptestserver) +# 2 - uses testminiwget to retrieve data from this server +# 3 - compares served and received data +# 4 - kills the local HTTP server and exits +# +# The script was tested and works with ksh, bash +# it should now also run with dash + +TMPD=`mktemp -d -t miniwgetXXXXXXXXXX` +HTTPSERVEROUT="${TMPD}/httpserverout" +EXPECTEDFILE="${TMPD}/expectedfile" +DOWNLOADEDFILE="${TMPD}/downloadedfile" +PORT= +RET=0 +IPCONFIG=$(which ifconfig) +if [ -z "$IPCONFIG" ] ; then + IPCONFIG="/sbin/ifconfig" +fi + +if ! $IPCONFIG -a | grep inet6 ; then + HAVE_IPV6=no +fi + +case "$HAVE_IPV6" in + n|no|0) + ADDR=localhost + SERVERARGS="" + ;; + *) + ADDR="[::1]" + SERVERARGS="-6" + ;; + +esac + +#make minihttptestserver +#make testminiwget + +# launching the test HTTP server +./minihttptestserver $SERVERARGS -e $EXPECTEDFILE > $HTTPSERVEROUT & +SERVERPID=$! +while [ -z "$PORT" ]; do + sleep 1 + PORT=`cat $HTTPSERVEROUT | sed 's/Listening on port \([0-9]*\)/\1/' ` +done +if [ "$PORT" = "*** ERROR ***" ]; then + echo "HTTP test server error" + echo "Network config :" + $IPCONFIG -a + exit 2 +fi +echo "Test HTTP server is listening on $PORT" + +URL1="http://$ADDR:$PORT/index.html" +URL2="http://$ADDR:$PORT/chunked" +URL3="http://$ADDR:$PORT/addcrap" + +echo "standard test ..." +./testminiwget $URL1 "${DOWNLOADEDFILE}.1" +if cmp $EXPECTEDFILE "${DOWNLOADEDFILE}.1" ; then + echo "ok" +else + echo "standard test FAILED" + RET=1 +fi + +echo "chunked transfert encoding test ..." +./testminiwget $URL2 "${DOWNLOADEDFILE}.2" +if cmp $EXPECTEDFILE "${DOWNLOADEDFILE}.2" ; then + echo "ok" +else + echo "chunked transfert encoding test FAILED" + RET=1 +fi + +echo "response too long test ..." +./testminiwget $URL3 "${DOWNLOADEDFILE}.3" +if cmp $EXPECTEDFILE "${DOWNLOADEDFILE}.3" ; then + echo "ok" +else + echo "response too long test FAILED" + RET=1 +fi + +# kill the test HTTP server +kill $SERVERPID +wait $SERVERPID + +# remove temporary files (for success cases) +if [ $RET -eq 0 ]; then + rm -f "${DOWNLOADEDFILE}.1" + rm -f "${DOWNLOADEDFILE}.2" + rm -f "${DOWNLOADEDFILE}.3" + rm -f $EXPECTEDFILE $HTTPSERVEROUT + rmdir ${TMPD} +else + echo "at least one of the test FAILED" + echo "directory ${TMPD} is left intact" +fi +exit $RET + diff --git a/src/contrib/miniupnp/miniupnpc/testminixml.c b/src/contrib/miniupnp/miniupnpc/testminixml.c new file mode 100644 index 0000000..57c4a85 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testminixml.c @@ -0,0 +1,89 @@ +/* $Id: testminixml.c,v 1.10 2014/11/17 17:19:13 nanard Exp $ + * MiniUPnP project + * Website : http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author : Thomas Bernard. + * Copyright (c) 2005-2014 Thomas Bernard + * + * testminixml.c + * test program for the "minixml" functions. + */ +#include +#include +#include +#include "minixml.h" +#include "igd_desc_parse.h" + +/* ---------------------------------------------------------------------- */ +void printeltname1(void * d, const char * name, int l) +{ + int i; + (void)d; + printf("element "); + for(i=0;i +#include +#include "portlistingparse.h" + +struct port_mapping { + unsigned int leasetime; + unsigned short externalport; + unsigned short internalport; + const char * remotehost; + const char * client; + const char * proto; + const char * desc; + unsigned char enabled; +}; + +/* return the number of differences */ +int test(const char * portListingXml, int portListingXmlLen, + const struct port_mapping * ref, int count) +{ + int i; + int r = 0; + struct PortMappingParserData data; + struct PortMapping * pm; + + memset(&data, 0, sizeof(data)); + ParsePortListing(portListingXml, portListingXmlLen, &data); + for(i = 0, pm = data.l_head; + (pm != NULL) && (i < count); + i++, pm = pm->l_next) { + printf("%2d %s %5hu->%s:%-5hu '%s' '%s' %u\n", + i, pm->protocol, pm->externalPort, pm->internalClient, + pm->internalPort, + pm->description, pm->remoteHost, + (unsigned)pm->leaseTime); + if(0 != strcmp(pm->protocol, ref[i].proto)) { + printf("protocol : '%s' != '%s'\n", pm->protocol, ref[i].proto); + r++; + } + if(pm->externalPort != ref[i].externalport) { + printf("externalPort : %hu != %hu\n", + pm->externalPort, ref[i].externalport); + r++; + } + if(0 != strcmp(pm->internalClient, ref[i].client)) { + printf("client : '%s' != '%s'\n", + pm->internalClient, ref[i].client); + r++; + } + if(pm->internalPort != ref[i].internalport) { + printf("internalPort : %hu != %hu\n", + pm->internalPort, ref[i].internalport); + r++; + } + if(0 != strcmp(pm->description, ref[i].desc)) { + printf("description : '%s' != '%s'\n", + pm->description, ref[i].desc); + r++; + } + if(0 != strcmp(pm->remoteHost, ref[i].remotehost)) { + printf("remoteHost : '%s' != '%s'\n", + pm->remoteHost, ref[i].remotehost); + r++; + } + if((unsigned)pm->leaseTime != ref[i].leasetime) { + printf("leaseTime : %u != %u\n", + (unsigned)pm->leaseTime, ref[i].leasetime); + r++; + } + if(pm->enabled != ref[i].enabled) { + printf("enabled : %d != %d\n", + (int)pm->enabled, (int)ref[i].enabled); + r++; + } + } + if((i != count) || (pm != NULL)) { + printf("count mismatch : i=%d count=%d pm=%p\n", i, count, pm); + r++; + } + FreePortListing(&data); + return r; +} + +const char test_document[] = +"\n" +"\n" +" \n" +" \n" +" 5002\n" +" UDP\n" +" 4001\n" +" 192.168.1.123\n" +" 1\n" +" xxx\n" +" 0\n" +" \n" +" \n" +" 202.233.2.1\n" +" 2345\n" +" TCP\n" +" 2349\n" +" 192.168.1.137\n" +" 1\n" +" dooom\n" +" 346\n" +" \n" +" \n" +" 134.231.2.11\n" +" 12345\n" +" TCP\n" +" 12345\n" +" 192.168.1.137\n" +" 1\n" +" dooom A\n" +" 347\n" +" \n" +""; + +#define PORT_MAPPINGS_COUNT 3 +const struct port_mapping port_mappings[PORT_MAPPINGS_COUNT] = { +{347, 12345, 12345, "134.231.2.11", "192.168.1.137", "TCP", "dooom A", 1}, +{346, 2345, 2349, "202.233.2.1", "192.168.1.137", "TCP", "dooom", 1}, +{0, 5002, 4001, "", "192.168.1.123", "UDP", "xxx", 1} +}; + +/* --- main --- */ +int main(void) +{ + int r; + r = test(test_document, sizeof(test_document) - 1, + port_mappings, PORT_MAPPINGS_COUNT); + if(r == 0) { + printf("test of portlistingparse OK\n"); + return 0; + } else { + printf("test FAILED (%d differences counted)\n", r); + return 1; + } +} + diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/DeletePortMapping.namevalue b/src/contrib/miniupnp/miniupnpc/testreplyparse/DeletePortMapping.namevalue new file mode 100644 index 0000000..48ca0cc --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/DeletePortMapping.namevalue @@ -0,0 +1,3 @@ +NewRemoteHost= +NewExternalPort=123 +NewProtocol=TCP diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/DeletePortMapping.xml b/src/contrib/miniupnp/miniupnpc/testreplyparse/DeletePortMapping.xml new file mode 100644 index 0000000..a955c53 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/DeletePortMapping.xml @@ -0,0 +1,6 @@ + +123 +TCP + + + diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/GetExternalIPAddress.namevalue b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetExternalIPAddress.namevalue new file mode 100644 index 0000000..5aa75f8 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetExternalIPAddress.namevalue @@ -0,0 +1,2 @@ +NewExternalIPAddress=1.2.3.4 + diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/GetExternalIPAddress.xml b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetExternalIPAddress.xml new file mode 100644 index 0000000..db7ec1f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetExternalIPAddress.xml @@ -0,0 +1,2 @@ +1.2.3.4 + diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryReq.namevalue b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryReq.namevalue new file mode 100644 index 0000000..26b169c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryReq.namevalue @@ -0,0 +1,3 @@ +NewProtocol=UDP +NewExternalPort=12345 +NewRemoteHost= diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryReq.xml b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryReq.xml new file mode 100644 index 0000000..bbb540e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryReq.xml @@ -0,0 +1,3 @@ + +12345UDP + diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryResp.namevalue b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryResp.namevalue new file mode 100644 index 0000000..2189789 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryResp.namevalue @@ -0,0 +1,5 @@ +NewInternalPort=12345 +NewInternalClient=192.168.10.110 +NewEnabled=1 +NewPortMappingDescription=libminiupnpc +NewLeaseDuration=0 diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryResp.xml b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryResp.xml new file mode 100644 index 0000000..77e8d9c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/GetSpecificPortMappingEntryResp.xml @@ -0,0 +1,2 @@ +12345192.168.10.1101libminiupnpc0 + diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/SetDefaultConnectionService.namevalue b/src/contrib/miniupnp/miniupnpc/testreplyparse/SetDefaultConnectionService.namevalue new file mode 100644 index 0000000..f78c7e2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/SetDefaultConnectionService.namevalue @@ -0,0 +1 @@ +NewDefaultConnectionService=uuid:c6c05a33-f704-48df-9910-e099b3471d81:WANConnectionDevice:1,INVALID_SERVICE_ID diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/SetDefaultConnectionService.xml b/src/contrib/miniupnp/miniupnpc/testreplyparse/SetDefaultConnectionService.xml new file mode 100644 index 0000000..ac04c07 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/SetDefaultConnectionService.xml @@ -0,0 +1 @@ +uuid:c6c05a33-f704-48df-9910-e099b3471d81:WANConnectionDevice:1,INVALID_SERVICE_ID diff --git a/src/contrib/miniupnp/miniupnpc/testreplyparse/readme.txt b/src/contrib/miniupnp/miniupnpc/testreplyparse/readme.txt new file mode 100644 index 0000000..3eb1f01 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testreplyparse/readme.txt @@ -0,0 +1,7 @@ +This directory contains files used for validation of upnpreplyparse.c code. + +Each .xml file to parse should give the results which are in the .namevalue +file. + +A .namevalue file contain name=value lines. + diff --git a/src/contrib/miniupnp/miniupnpc/testupnpigd.py b/src/contrib/miniupnp/miniupnpc/testupnpigd.py new file mode 100644 index 0000000..33019bd --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testupnpigd.py @@ -0,0 +1,88 @@ +#! /usr/bin/python +# $Id: testupnpigd.py,v 1.4 2008/10/11 10:27:20 nanard Exp $ +# MiniUPnP project +# Author : Thomas Bernard +# This Sample code is public domain. +# website : http://miniupnp.tuxfamily.org/ + +# import the python miniupnpc module +import miniupnpc +import socket + +try: + from http.server import BaseHTTPRequestHandler, HTTPServer +except ImportError: + from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + +# function definition +def list_redirections(): + i = 0 + while True: + p = u.getgenericportmapping(i) + if p==None: + break + print(i, p) + i = i + 1 + +#define the handler class for HTTP connections +class handler_class(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write("OK MON GARS") + +# create the object +u = miniupnpc.UPnP() +#print 'inital(default) values :' +#print ' discoverdelay', u.discoverdelay +#print ' lanaddr', u.lanaddr +#print ' multicastif', u.multicastif +#print ' minissdpdsocket', u.minissdpdsocket +u.discoverdelay = 200; + +try: + print('Discovering... delay=%ums' % u.discoverdelay) + ndevices = u.discover() + print(ndevices, 'device(s) detected') + + # select an igd + u.selectigd() + # display information about the IGD and the internet connection + print('local ip address :', u.lanaddr) + externalipaddress = u.externalipaddress() + print('external ip address :', externalipaddress) + print(u.statusinfo(), u.connectiontype()) + + #instanciate a HTTPd object. The port is assigned by the system. + httpd = HTTPServer((u.lanaddr, 0), handler_class) + eport = httpd.server_port + + # find a free port for the redirection + r = u.getspecificportmapping(eport, 'TCP') + while r != None and eport < 65536: + eport = eport + 1 + r = u.getspecificportmapping(eport, 'TCP') + + print('trying to redirect %s port %u TCP => %s port %u TCP' % (externalipaddress, eport, u.lanaddr, httpd.server_port)) + + b = u.addportmapping(eport, 'TCP', u.lanaddr, httpd.server_port, + 'UPnP IGD Tester port %u' % eport, '') + if b: + print('Success. Now waiting for some HTTP request on http://%s:%u' % (externalipaddress ,eport)) + try: + httpd.handle_request() + httpd.server_close() + except KeyboardInterrupt as details: + print("CTRL-C exception!", details) + b = u.deleteportmapping(eport, 'TCP') + if b: + print('Successfully deleted port mapping') + else: + print('Failed to remove port mapping') + else: + print('Failed') + + httpd.server_close() + +except Exception as e: + print('Exception :', e) diff --git a/src/contrib/miniupnp/miniupnpc/testupnpreplyparse.c b/src/contrib/miniupnp/miniupnpc/testupnpreplyparse.c new file mode 100644 index 0000000..bfe5f0d --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testupnpreplyparse.c @@ -0,0 +1,115 @@ +/* $Id: testupnpreplyparse.c,v 1.4 2014/01/27 11:45:19 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#include +#include +#include +#include "upnpreplyparse.h" + +int +test_parsing(const char * buf, int len, FILE * f) +{ + char line[1024]; + struct NameValueParserData pdata; + int ok = 1; + ParseNameValue(buf, len, &pdata); + /* check result */ + if(f != NULL) + { + while(fgets(line, sizeof(line), f)) + { + char * value; + char * equal; + char * parsedvalue; + int l; + l = strlen(line); + while((l > 0) && ((line[l-1] == '\r') || (line[l-1] == '\n'))) + line[--l] = '\0'; + /* skip empty lines */ + if(l == 0) + continue; + equal = strchr(line, '='); + if(equal == NULL) + { + fprintf(stderr, "Warning, line does not contain '=' : %s\n", line); + continue; + } + *equal = '\0'; + value = equal + 1; + parsedvalue = GetValueFromNameValueList(&pdata, line); + if((parsedvalue == NULL) || (strcmp(parsedvalue, value) != 0)) + { + fprintf(stderr, "Element <%s> : expecting value '%s', got '%s'\n", + line, value, parsedvalue ? parsedvalue : ""); + ok = 0; + } + } + } + ClearNameValueList(&pdata); + return ok; +} + +int main(int argc, char * * argv) +{ + FILE * f; + char * buffer; + long l; + int ok; + + if(argc<2) + { + fprintf(stderr, "Usage: %s file.xml [file.namevalues]\n", argv[0]); + return 1; + } + f = fopen(argv[1], "r"); + if(!f) + { + fprintf(stderr, "Error : can not open file %s\n", argv[1]); + return 2; + } + if(fseek(f, 0, SEEK_END) < 0) { + perror("fseek"); + return 1; + } + l = (int)ftell(f); + if(l < 0) { + perror("ftell"); + return 1; + } + if(fseek(f, 0, SEEK_SET) < 0) { + perror("fseek"); + return 1; + } + buffer = malloc(l + 1); + if(buffer == NULL) { + fprintf(stderr, "Error: failed to allocate %ld bytes\n", l+1); + return 1; + } + l = fread(buffer, 1, l, f); + fclose(f); + f = NULL; + buffer[l] = '\0'; + if(argc > 2) + { + f = fopen(argv[2], "r"); + if(!f) + { + fprintf(stderr, "Error : can not open file %s\n", argv[2]); + return 2; + } + } +#ifdef DEBUG + DisplayNameValueList(buffer, l); +#endif + ok = test_parsing(buffer, l, f); + if(f) + { + fclose(f); + } + free(buffer); + return ok ? 0 : 3; +} + diff --git a/src/contrib/miniupnp/miniupnpc/testupnpreplyparse.sh b/src/contrib/miniupnp/miniupnpc/testupnpreplyparse.sh new file mode 100644 index 0000000..992930b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/testupnpreplyparse.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +for f in testreplyparse/*.xml ; do + bf="`dirname $f`/`basename $f .xml`" + if ./testupnpreplyparse $f $bf.namevalue ; then + echo "$f : passed" + else + echo "$f : FAILED" + exit 1 + fi +done + +exit 0 + diff --git a/src/contrib/miniupnp/miniupnpc/updateminiupnpcstrings.sh b/src/contrib/miniupnp/miniupnpc/updateminiupnpcstrings.sh new file mode 100644 index 0000000..dde4354 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/updateminiupnpcstrings.sh @@ -0,0 +1,53 @@ +#! /bin/sh +# $Id: updateminiupnpcstrings.sh,v 1.7 2011/01/04 11:41:53 nanard Exp $ +# project miniupnp : http://miniupnp.free.fr/ +# (c) 2009 Thomas Bernard + +FILE=miniupnpcstrings.h +TMPFILE=miniupnpcstrings.h.tmp +TEMPLATE_FILE=${FILE}.in + +# detecting the OS name and version +OS_NAME=`uname -s` +OS_VERSION=`uname -r` +if [ -f /etc/debian_version ]; then + OS_NAME=Debian + OS_VERSION=`cat /etc/debian_version` +fi +# use lsb_release (Linux Standard Base) when available +LSB_RELEASE=`which lsb_release` +if [ 0 -eq $? -a -x "${LSB_RELEASE}" ]; then + OS_NAME=`${LSB_RELEASE} -i -s` + OS_VERSION=`${LSB_RELEASE} -r -s` + case $OS_NAME in + Debian) + #OS_VERSION=`${LSB_RELEASE} -c -s` + ;; + Ubuntu) + #OS_VERSION=`${LSB_RELEASE} -c -s` + ;; + esac +fi + +# on AmigaOS 3, uname -r returns "unknown", so we use uname -v +if [ "$OS_NAME" = "AmigaOS" ]; then + if [ "$OS_VERSION" = "unknown" ]; then + OS_VERSION=`uname -v` + fi +fi + +echo "Detected OS [$OS_NAME] version [$OS_VERSION]" +MINIUPNPC_VERSION=`cat VERSION` +echo "MiniUPnPc version [${MINIUPNPC_VERSION}]" + +EXPR="s|OS_STRING \".*\"|OS_STRING \"${OS_NAME}/${OS_VERSION}\"|" +#echo $EXPR +test -f ${FILE}.in +echo "setting OS_STRING macro value to ${OS_NAME}/${OS_VERSION} in $FILE." +sed -e "$EXPR" < $TEMPLATE_FILE > $TMPFILE + +EXPR="s|MINIUPNPC_VERSION_STRING \".*\"|MINIUPNPC_VERSION_STRING \"${MINIUPNPC_VERSION}\"|" +echo "setting MINIUPNPC_VERSION_STRING macro value to ${MINIUPNPC_VERSION} in $FILE." +sed -e "$EXPR" < $TMPFILE > $FILE +rm $TMPFILE + diff --git a/src/contrib/miniupnp/miniupnpc/upnpc.c b/src/contrib/miniupnp/miniupnpc/upnpc.c new file mode 100644 index 0000000..0c65cbe --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/upnpc.c @@ -0,0 +1,861 @@ +/* $Id: upnpc.c,v 1.119 2018/03/13 23:34:46 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas Bernard + * Copyright (c) 2005-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. */ + +#include +#include +#include +#include +#ifdef _WIN32 +#include +#define snprintf _snprintf +#else +/* for IPPROTO_TCP / IPPROTO_UDP */ +#include +#endif +#include +#include "miniwget.h" +#include "miniupnpc.h" +#include "upnpcommands.h" +#include "portlistingparse.h" +#include "upnperrors.h" +#include "miniupnpcstrings.h" + +/* protofix() checks if protocol is "UDP" or "TCP" + * returns NULL if not */ +const char * protofix(const char * proto) +{ + static const char proto_tcp[4] = { 'T', 'C', 'P', 0}; + static const char proto_udp[4] = { 'U', 'D', 'P', 0}; + int i, b; + for(i=0, b=1; i<4; i++) + b = b && ( (proto[i] == proto_tcp[i]) + || (proto[i] == (proto_tcp[i] | 32)) ); + if(b) + return proto_tcp; + for(i=0, b=1; i<4; i++) + b = b && ( (proto[i] == proto_udp[i]) + || (proto[i] == (proto_udp[i] | 32)) ); + if(b) + return proto_udp; + return 0; +} + +/* is_int() checks if parameter is an integer or not + * 1 for integer + * 0 for not an integer */ +int is_int(char const* s) +{ + if(s == NULL) + return 0; + while(*s) { + /* #define isdigit(c) ((c) >= '0' && (c) <= '9') */ + if(!isdigit(*s)) + return 0; + s++; + } + return 1; +} + +static void DisplayInfos(struct UPNPUrls * urls, + struct IGDdatas * data) +{ + char externalIPAddress[40]; + char connectionType[64]; + char status[64]; + char lastconnerr[64]; + unsigned int uptime = 0; + unsigned int brUp, brDown; + time_t timenow, timestarted; + int r; + if(UPNP_GetConnectionTypeInfo(urls->controlURL, + data->first.servicetype, + connectionType) != UPNPCOMMAND_SUCCESS) + printf("GetConnectionTypeInfo failed.\n"); + else + printf("Connection Type : %s\n", connectionType); + if(UPNP_GetStatusInfo(urls->controlURL, data->first.servicetype, + status, &uptime, lastconnerr) != UPNPCOMMAND_SUCCESS) + printf("GetStatusInfo failed.\n"); + else + printf("Status : %s, uptime=%us, LastConnectionError : %s\n", + status, uptime, lastconnerr); + if(uptime > 0) { + timenow = time(NULL); + timestarted = timenow - uptime; + printf(" Time started : %s", ctime(×tarted)); + } + if(UPNP_GetLinkLayerMaxBitRates(urls->controlURL_CIF, data->CIF.servicetype, + &brDown, &brUp) != UPNPCOMMAND_SUCCESS) { + printf("GetLinkLayerMaxBitRates failed.\n"); + } else { + printf("MaxBitRateDown : %u bps", brDown); + if(brDown >= 1000000) { + printf(" (%u.%u Mbps)", brDown / 1000000, (brDown / 100000) % 10); + } else if(brDown >= 1000) { + printf(" (%u Kbps)", brDown / 1000); + } + printf(" MaxBitRateUp %u bps", brUp); + if(brUp >= 1000000) { + printf(" (%u.%u Mbps)", brUp / 1000000, (brUp / 100000) % 10); + } else if(brUp >= 1000) { + printf(" (%u Kbps)", brUp / 1000); + } + printf("\n"); + } + r = UPNP_GetExternalIPAddress(urls->controlURL, + data->first.servicetype, + externalIPAddress); + if(r != UPNPCOMMAND_SUCCESS) { + printf("GetExternalIPAddress failed. (errorcode=%d)\n", r); + } else { + printf("ExternalIPAddress = %s\n", externalIPAddress); + } +} + +static void GetConnectionStatus(struct UPNPUrls * urls, + struct IGDdatas * data) +{ + unsigned int bytessent, bytesreceived, packetsreceived, packetssent; + DisplayInfos(urls, data); + bytessent = UPNP_GetTotalBytesSent(urls->controlURL_CIF, data->CIF.servicetype); + bytesreceived = UPNP_GetTotalBytesReceived(urls->controlURL_CIF, data->CIF.servicetype); + packetssent = UPNP_GetTotalPacketsSent(urls->controlURL_CIF, data->CIF.servicetype); + packetsreceived = UPNP_GetTotalPacketsReceived(urls->controlURL_CIF, data->CIF.servicetype); + printf("Bytes: Sent: %8u\tRecv: %8u\n", bytessent, bytesreceived); + printf("Packets: Sent: %8u\tRecv: %8u\n", packetssent, packetsreceived); +} + +static void ListRedirections(struct UPNPUrls * urls, + struct IGDdatas * data) +{ + int r; + int i = 0; + char index[6]; + char intClient[40]; + char intPort[6]; + char extPort[6]; + char protocol[4]; + char desc[80]; + char enabled[6]; + char rHost[64]; + char duration[16]; + /*unsigned int num=0; + UPNP_GetPortMappingNumberOfEntries(urls->controlURL, data->servicetype, &num); + printf("PortMappingNumberOfEntries : %u\n", num);*/ + printf(" i protocol exPort->inAddr:inPort description remoteHost leaseTime\n"); + do { + snprintf(index, 6, "%d", i); + rHost[0] = '\0'; enabled[0] = '\0'; + duration[0] = '\0'; desc[0] = '\0'; + extPort[0] = '\0'; intPort[0] = '\0'; intClient[0] = '\0'; + r = UPNP_GetGenericPortMappingEntry(urls->controlURL, + data->first.servicetype, + index, + extPort, intClient, intPort, + protocol, desc, enabled, + rHost, duration); + if(r==0) + /* + printf("%02d - %s %s->%s:%s\tenabled=%s leaseDuration=%s\n" + " desc='%s' rHost='%s'\n", + i, protocol, extPort, intClient, intPort, + enabled, duration, + desc, rHost); + */ + printf("%2d %s %5s->%s:%-5s '%s' '%s' %s\n", + i, protocol, extPort, intClient, intPort, + desc, rHost, duration); + else + printf("GetGenericPortMappingEntry() returned %d (%s)\n", + r, strupnperror(r)); + i++; + } while(r==0); +} + +static void NewListRedirections(struct UPNPUrls * urls, + struct IGDdatas * data) +{ + int r; + int i = 0; + struct PortMappingParserData pdata; + struct PortMapping * pm; + + memset(&pdata, 0, sizeof(struct PortMappingParserData)); + r = UPNP_GetListOfPortMappings(urls->controlURL, + data->first.servicetype, + "0", + "65535", + "TCP", + "1000", + &pdata); + if(r == UPNPCOMMAND_SUCCESS) + { + printf(" i protocol exPort->inAddr:inPort description remoteHost leaseTime\n"); + for(pm = pdata.l_head; pm != NULL; pm = pm->l_next) + { + printf("%2d %s %5hu->%s:%-5hu '%s' '%s' %u\n", + i, pm->protocol, pm->externalPort, pm->internalClient, + pm->internalPort, + pm->description, pm->remoteHost, + (unsigned)pm->leaseTime); + i++; + } + FreePortListing(&pdata); + } + else + { + printf("GetListOfPortMappings() returned %d (%s)\n", + r, strupnperror(r)); + } + r = UPNP_GetListOfPortMappings(urls->controlURL, + data->first.servicetype, + "0", + "65535", + "UDP", + "1000", + &pdata); + if(r == UPNPCOMMAND_SUCCESS) + { + for(pm = pdata.l_head; pm != NULL; pm = pm->l_next) + { + printf("%2d %s %5hu->%s:%-5hu '%s' '%s' %u\n", + i, pm->protocol, pm->externalPort, pm->internalClient, + pm->internalPort, + pm->description, pm->remoteHost, + (unsigned)pm->leaseTime); + i++; + } + FreePortListing(&pdata); + } + else + { + printf("GetListOfPortMappings() returned %d (%s)\n", + r, strupnperror(r)); + } +} + +/* Test function + * 1 - get connection type + * 2 - get extenal ip address + * 3 - Add port mapping + * 4 - get this port mapping from the IGD */ +static int SetRedirectAndTest(struct UPNPUrls * urls, + struct IGDdatas * data, + const char * iaddr, + const char * iport, + const char * eport, + const char * proto, + const char * leaseDuration, + const char * description, + int addAny) +{ + char externalIPAddress[40]; + char intClient[40]; + char intPort[6]; + char reservedPort[6]; + char duration[16]; + int r; + + if(!iaddr || !iport || !eport || !proto) + { + fprintf(stderr, "Wrong arguments\n"); + return -1; + } + proto = protofix(proto); + if(!proto) + { + fprintf(stderr, "invalid protocol\n"); + return -1; + } + + r = UPNP_GetExternalIPAddress(urls->controlURL, + data->first.servicetype, + externalIPAddress); + if(r!=UPNPCOMMAND_SUCCESS) + printf("GetExternalIPAddress failed.\n"); + else + printf("ExternalIPAddress = %s\n", externalIPAddress); + + if (addAny) { + r = UPNP_AddAnyPortMapping(urls->controlURL, data->first.servicetype, + eport, iport, iaddr, description, + proto, 0, leaseDuration, reservedPort); + if(r==UPNPCOMMAND_SUCCESS) + eport = reservedPort; + else + printf("AddAnyPortMapping(%s, %s, %s) failed with code %d (%s)\n", + eport, iport, iaddr, r, strupnperror(r)); + } else { + r = UPNP_AddPortMapping(urls->controlURL, data->first.servicetype, + eport, iport, iaddr, description, + proto, 0, leaseDuration); + if(r!=UPNPCOMMAND_SUCCESS) { + printf("AddPortMapping(%s, %s, %s) failed with code %d (%s)\n", + eport, iport, iaddr, r, strupnperror(r)); + return -2; + } + } + + r = UPNP_GetSpecificPortMappingEntry(urls->controlURL, + data->first.servicetype, + eport, proto, NULL/*remoteHost*/, + intClient, intPort, NULL/*desc*/, + NULL/*enabled*/, duration); + if(r!=UPNPCOMMAND_SUCCESS) { + printf("GetSpecificPortMappingEntry() failed with code %d (%s)\n", + r, strupnperror(r)); + return -2; + } else { + printf("InternalIP:Port = %s:%s\n", intClient, intPort); + printf("external %s:%s %s is redirected to internal %s:%s (duration=%s)\n", + externalIPAddress, eport, proto, intClient, intPort, duration); + } + return 0; +} + +static int +RemoveRedirect(struct UPNPUrls * urls, + struct IGDdatas * data, + const char * eport, + const char * proto, + const char * remoteHost) +{ + int r; + if(!proto || !eport) + { + fprintf(stderr, "invalid arguments\n"); + return -1; + } + proto = protofix(proto); + if(!proto) + { + fprintf(stderr, "protocol invalid\n"); + return -1; + } + r = UPNP_DeletePortMapping(urls->controlURL, data->first.servicetype, eport, proto, remoteHost); + if(r!=UPNPCOMMAND_SUCCESS) { + printf("UPNP_DeletePortMapping() failed with code : %d\n", r); + return -2; + }else { + printf("UPNP_DeletePortMapping() returned : %d\n", r); + } + return 0; +} + +static int +RemoveRedirectRange(struct UPNPUrls * urls, + struct IGDdatas * data, + const char * ePortStart, char const * ePortEnd, + const char * proto, const char * manage) +{ + int r; + + if (!manage) + manage = "0"; + + if(!proto || !ePortStart || !ePortEnd) + { + fprintf(stderr, "invalid arguments\n"); + return -1; + } + proto = protofix(proto); + if(!proto) + { + fprintf(stderr, "protocol invalid\n"); + return -1; + } + r = UPNP_DeletePortMappingRange(urls->controlURL, data->first.servicetype, ePortStart, ePortEnd, proto, manage); + if(r!=UPNPCOMMAND_SUCCESS) { + printf("UPNP_DeletePortMappingRange() failed with code : %d\n", r); + return -2; + }else { + printf("UPNP_DeletePortMappingRange() returned : %d\n", r); + } + return 0; +} + +/* IGD:2, functions for service WANIPv6FirewallControl:1 */ +static void GetFirewallStatus(struct UPNPUrls * urls, struct IGDdatas * data) +{ + unsigned int bytessent, bytesreceived, packetsreceived, packetssent; + int firewallEnabled = 0, inboundPinholeAllowed = 0; + + UPNP_GetFirewallStatus(urls->controlURL_6FC, data->IPv6FC.servicetype, &firewallEnabled, &inboundPinholeAllowed); + printf("FirewallEnabled: %d & Inbound Pinhole Allowed: %d\n", firewallEnabled, inboundPinholeAllowed); + printf("GetFirewallStatus:\n Firewall Enabled: %s\n Inbound Pinhole Allowed: %s\n", (firewallEnabled)? "Yes":"No", (inboundPinholeAllowed)? "Yes":"No"); + + bytessent = UPNP_GetTotalBytesSent(urls->controlURL_CIF, data->CIF.servicetype); + bytesreceived = UPNP_GetTotalBytesReceived(urls->controlURL_CIF, data->CIF.servicetype); + packetssent = UPNP_GetTotalPacketsSent(urls->controlURL_CIF, data->CIF.servicetype); + packetsreceived = UPNP_GetTotalPacketsReceived(urls->controlURL_CIF, data->CIF.servicetype); + printf("Bytes: Sent: %8u\tRecv: %8u\n", bytessent, bytesreceived); + printf("Packets: Sent: %8u\tRecv: %8u\n", packetssent, packetsreceived); +} + +/* Test function + * 1 - Add pinhole + * 2 - Check if pinhole is working from the IGD side */ +static void SetPinholeAndTest(struct UPNPUrls * urls, struct IGDdatas * data, + const char * remoteaddr, const char * eport, + const char * intaddr, const char * iport, + const char * proto, const char * lease_time) +{ + char uniqueID[8]; + /*int isWorking = 0;*/ + int r; + char proto_tmp[8]; + + if(!intaddr || !remoteaddr || !iport || !eport || !proto || !lease_time) + { + fprintf(stderr, "Wrong arguments\n"); + return; + } + if(atoi(proto) == 0) + { + const char * protocol; + protocol = protofix(proto); + if(protocol && (strcmp("TCP", protocol) == 0)) + { + snprintf(proto_tmp, sizeof(proto_tmp), "%d", IPPROTO_TCP); + proto = proto_tmp; + } + else if(protocol && (strcmp("UDP", protocol) == 0)) + { + snprintf(proto_tmp, sizeof(proto_tmp), "%d", IPPROTO_UDP); + proto = proto_tmp; + } + else + { + fprintf(stderr, "invalid protocol\n"); + return; + } + } + r = UPNP_AddPinhole(urls->controlURL_6FC, data->IPv6FC.servicetype, remoteaddr, eport, intaddr, iport, proto, lease_time, uniqueID); + if(r!=UPNPCOMMAND_SUCCESS) + printf("AddPinhole([%s]:%s -> [%s]:%s) failed with code %d (%s)\n", + remoteaddr, eport, intaddr, iport, r, strupnperror(r)); + else + { + printf("AddPinhole: ([%s]:%s -> [%s]:%s) / Pinhole ID = %s\n", + remoteaddr, eport, intaddr, iport, uniqueID); + /*r = UPNP_CheckPinholeWorking(urls->controlURL_6FC, data->servicetype_6FC, uniqueID, &isWorking); + if(r!=UPNPCOMMAND_SUCCESS) + printf("CheckPinholeWorking() failed with code %d (%s)\n", r, strupnperror(r)); + printf("CheckPinholeWorking: Pinhole ID = %s / IsWorking = %s\n", uniqueID, (isWorking)? "Yes":"No");*/ + } +} + +/* Test function + * 1 - Check if pinhole is working from the IGD side + * 2 - Update pinhole */ +static void GetPinholeAndUpdate(struct UPNPUrls * urls, struct IGDdatas * data, + const char * uniqueID, const char * lease_time) +{ + int isWorking = 0; + int r; + + if(!uniqueID || !lease_time) + { + fprintf(stderr, "Wrong arguments\n"); + return; + } + r = UPNP_CheckPinholeWorking(urls->controlURL_6FC, data->IPv6FC.servicetype, uniqueID, &isWorking); + printf("CheckPinholeWorking: Pinhole ID = %s / IsWorking = %s\n", uniqueID, (isWorking)? "Yes":"No"); + if(r!=UPNPCOMMAND_SUCCESS) + printf("CheckPinholeWorking() failed with code %d (%s)\n", r, strupnperror(r)); + if(isWorking || r==709) + { + r = UPNP_UpdatePinhole(urls->controlURL_6FC, data->IPv6FC.servicetype, uniqueID, lease_time); + printf("UpdatePinhole: Pinhole ID = %s with Lease Time: %s\n", uniqueID, lease_time); + if(r!=UPNPCOMMAND_SUCCESS) + printf("UpdatePinhole: ID (%s) failed with code %d (%s)\n", uniqueID, r, strupnperror(r)); + } +} + +/* Test function + * Get pinhole timeout + */ +static void GetPinholeOutboundTimeout(struct UPNPUrls * urls, struct IGDdatas * data, + const char * remoteaddr, const char * eport, + const char * intaddr, const char * iport, + const char * proto) +{ + int timeout = 0; + int r; + + if(!intaddr || !remoteaddr || !iport || !eport || !proto) + { + fprintf(stderr, "Wrong arguments\n"); + return; + } + + r = UPNP_GetOutboundPinholeTimeout(urls->controlURL_6FC, data->IPv6FC.servicetype, remoteaddr, eport, intaddr, iport, proto, &timeout); + if(r!=UPNPCOMMAND_SUCCESS) + printf("GetOutboundPinholeTimeout([%s]:%s -> [%s]:%s) failed with code %d (%s)\n", + intaddr, iport, remoteaddr, eport, r, strupnperror(r)); + else + printf("GetOutboundPinholeTimeout: ([%s]:%s -> [%s]:%s) / Timeout = %d\n", intaddr, iport, remoteaddr, eport, timeout); +} + +static void +GetPinholePackets(struct UPNPUrls * urls, + struct IGDdatas * data, const char * uniqueID) +{ + int r, pinholePackets = 0; + if(!uniqueID) + { + fprintf(stderr, "invalid arguments\n"); + return; + } + r = UPNP_GetPinholePackets(urls->controlURL_6FC, data->IPv6FC.servicetype, uniqueID, &pinholePackets); + if(r!=UPNPCOMMAND_SUCCESS) + printf("GetPinholePackets() failed with code %d (%s)\n", r, strupnperror(r)); + else + printf("GetPinholePackets: Pinhole ID = %s / PinholePackets = %d\n", uniqueID, pinholePackets); +} + +static void +CheckPinhole(struct UPNPUrls * urls, + struct IGDdatas * data, const char * uniqueID) +{ + int r, isWorking = 0; + if(!uniqueID) + { + fprintf(stderr, "invalid arguments\n"); + return; + } + r = UPNP_CheckPinholeWorking(urls->controlURL_6FC, data->IPv6FC.servicetype, uniqueID, &isWorking); + if(r!=UPNPCOMMAND_SUCCESS) + printf("CheckPinholeWorking() failed with code %d (%s)\n", r, strupnperror(r)); + else + printf("CheckPinholeWorking: Pinhole ID = %s / IsWorking = %s\n", uniqueID, (isWorking)? "Yes":"No"); +} + +static void +RemovePinhole(struct UPNPUrls * urls, + struct IGDdatas * data, const char * uniqueID) +{ + int r; + if(!uniqueID) + { + fprintf(stderr, "invalid arguments\n"); + return; + } + r = UPNP_DeletePinhole(urls->controlURL_6FC, data->IPv6FC.servicetype, uniqueID); + printf("UPNP_DeletePinhole() returned : %d\n", r); +} + + +/* sample upnp client program */ +int main(int argc, char ** argv) +{ + char command = 0; + char ** commandargv = 0; + int commandargc = 0; + struct UPNPDev * devlist = 0; + char lanaddr[64] = "unset"; /* my ip address on the LAN */ + int i; + const char * rootdescurl = 0; + const char * multicastif = 0; + const char * minissdpdpath = 0; + int localport = UPNP_LOCAL_PORT_ANY; + int retcode = 0; + int error = 0; + int ipv6 = 0; + unsigned char ttl = 2; /* defaulting to 2 */ + const char * description = 0; + +#ifdef _WIN32 + WSADATA wsaData; + int nResult = WSAStartup(MAKEWORD(2,2), &wsaData); + if(nResult != NO_ERROR) + { + fprintf(stderr, "WSAStartup() failed.\n"); + return -1; + } +#endif + printf("upnpc : miniupnpc library test client, version %s.\n", MINIUPNPC_VERSION_STRING); + printf(" (c) 2005-2018 Thomas Bernard.\n"); + printf("Go to http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/\n" + "for more information.\n"); + /* command line processing */ + for(i=1; i65535 || + (localport >1 && localport < 1024)) + { + fprintf(stderr, "Invalid localport '%s'\n", argv[i]); + localport = UPNP_LOCAL_PORT_ANY; + break; + } + } + else if(argv[i][1] == 'p') + minissdpdpath = argv[++i]; + else if(argv[i][1] == '6') + ipv6 = 1; + else if(argv[i][1] == 'e') + description = argv[++i]; + else if(argv[i][1] == 't') + ttl = (unsigned char)atoi(argv[++i]); + else + { + command = argv[i][1]; + i++; + commandargv = argv + i; + commandargc = argc - i; + break; + } + } + else + { + fprintf(stderr, "option '%s' invalid\n", argv[i]); + } + } + + if(!command + || (command == 'a' && commandargc<4) + || (command == 'd' && argc<2) + || (command == 'r' && argc<2) + || (command == 'A' && commandargc<6) + || (command == 'U' && commandargc<2) + || (command == 'D' && commandargc<1)) + { + fprintf(stderr, "Usage :\t%s [options] -a ip port external_port protocol [duration]\n\t\tAdd port redirection\n", argv[0]); + fprintf(stderr, " \t%s [options] -d external_port protocol \n\t\tDelete port redirection\n", argv[0]); + fprintf(stderr, " \t%s [options] -s\n\t\tGet Connection status\n", argv[0]); + fprintf(stderr, " \t%s [options] -l\n\t\tList redirections\n", argv[0]); + fprintf(stderr, " \t%s [options] -L\n\t\tList redirections (using GetListOfPortMappings (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -n ip port external_port protocol [duration]\n\t\tAdd (any) port redirection allowing IGD to use alternative external_port (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -N external_port_start external_port_end protocol [manage]\n\t\tDelete range of port redirections (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -r port1 [external_port1] protocol1 [port2 [external_port2] protocol2] [...]\n\t\tAdd all redirections to the current host\n", argv[0]); + fprintf(stderr, " \t%s [options] -A remote_ip remote_port internal_ip internal_port protocol lease_time\n\t\tAdd Pinhole (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -U uniqueID new_lease_time\n\t\tUpdate Pinhole (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -C uniqueID\n\t\tCheck if Pinhole is Working (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -K uniqueID\n\t\tGet Number of packets going through the rule (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -D uniqueID\n\t\tDelete Pinhole (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -S\n\t\tGet Firewall status (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -G remote_ip remote_port internal_ip internal_port protocol\n\t\tGet Outbound Pinhole Timeout (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -P\n\t\tGet Presentation url\n", argv[0]); + fprintf(stderr, "\nprotocol is UDP or TCP\n"); + fprintf(stderr, "Options:\n"); + fprintf(stderr, " -e description : set description for port mapping.\n"); + fprintf(stderr, " -6 : use ip v6 instead of ip v4.\n"); + fprintf(stderr, " -u url : bypass discovery process by providing the XML root description url.\n"); + fprintf(stderr, " -m address/interface : provide ip address (ip v4) or interface name (ip v4 or v6) to use for sending SSDP multicast packets.\n"); + fprintf(stderr, " -z localport : SSDP packets local (source) port (1024-65535).\n"); + fprintf(stderr, " -p path : use this path for MiniSSDPd socket.\n"); + fprintf(stderr, " -t ttl : set multicast TTL. Default value is 2.\n"); + return 1; + } + + if( rootdescurl + || (devlist = upnpDiscover(2000, multicastif, minissdpdpath, + localport, ipv6, ttl, &error))) + { + struct UPNPDev * device; + struct UPNPUrls urls; + struct IGDdatas data; + if(devlist) + { + printf("List of UPNP devices found on the network :\n"); + for(device = devlist; device; device = device->pNext) + { + printf(" desc: %s\n st: %s\n\n", + device->descURL, device->st); + } + } + else if(!rootdescurl) + { + printf("upnpDiscover() error code=%d\n", error); + } + i = 1; + if( (rootdescurl && UPNP_GetIGDFromUrl(rootdescurl, &urls, &data, lanaddr, sizeof(lanaddr))) + || (i = UPNP_GetValidIGD(devlist, &urls, &data, lanaddr, sizeof(lanaddr)))) + { + switch(i) { + case 1: + printf("Found valid IGD : %s\n", urls.controlURL); + break; + case 2: + printf("Found a (not connected?) IGD : %s\n", urls.controlURL); + printf("Trying to continue anyway\n"); + break; + case 3: + printf("UPnP device found. Is it an IGD ? : %s\n", urls.controlURL); + printf("Trying to continue anyway\n"); + break; + default: + printf("Found device (igd ?) : %s\n", urls.controlURL); + printf("Trying to continue anyway\n"); + } + printf("Local LAN ip address : %s\n", lanaddr); + #if 0 + printf("getting \"%s\"\n", urls.ipcondescURL); + descXML = miniwget(urls.ipcondescURL, &descXMLsize); + if(descXML) + { + /*fwrite(descXML, 1, descXMLsize, stdout);*/ + free(descXML); descXML = NULL; + } + #endif + + switch(command) + { + case 'l': + DisplayInfos(&urls, &data); + ListRedirections(&urls, &data); + break; + case 'L': + NewListRedirections(&urls, &data); + break; + case 'a': + if (SetRedirectAndTest(&urls, &data, + commandargv[0], commandargv[1], + commandargv[2], commandargv[3], + (commandargc > 4)?commandargv[4]:"0", + description, 0) < 0) + retcode = 2; + break; + case 'd': + if (RemoveRedirect(&urls, &data, commandargv[0], commandargv[1], + commandargc > 2 ? commandargv[2] : NULL) < 0) + retcode = 2; + break; + case 'n': /* aNy */ + if (SetRedirectAndTest(&urls, &data, + commandargv[0], commandargv[1], + commandargv[2], commandargv[3], + (commandargc > 4)?commandargv[4]:"0", + description, 1) < 0) + retcode = 2; + break; + case 'N': + if (commandargc < 3) + fprintf(stderr, "too few arguments\n"); + + if (RemoveRedirectRange(&urls, &data, commandargv[0], commandargv[1], commandargv[2], + commandargc > 3 ? commandargv[3] : NULL) < 0) + retcode = 2; + break; + case 's': + GetConnectionStatus(&urls, &data); + break; + case 'r': + i = 0; + while(i */ + if (SetRedirectAndTest(&urls, &data, + lanaddr, commandargv[i], + commandargv[i+1], commandargv[i+2], "0", + description, 0) < 0) + retcode = 2; + i+=3; /* 3 parameters parsed */ + } else { + /* 2nd parameter not an integer : */ + if (SetRedirectAndTest(&urls, &data, + lanaddr, commandargv[i], + commandargv[i], commandargv[i+1], "0", + description, 0) < 0) + retcode = 2; + i+=2; /* 2 parameters parsed */ + } + } + break; + case 'A': + SetPinholeAndTest(&urls, &data, + commandargv[0], commandargv[1], + commandargv[2], commandargv[3], + commandargv[4], commandargv[5]); + break; + case 'U': + GetPinholeAndUpdate(&urls, &data, + commandargv[0], commandargv[1]); + break; + case 'C': + for(i=0; i +#include +#include +#include "upnpcommands.h" +#include "miniupnpc.h" +#include "portlistingparse.h" +#include "upnpreplyparse.h" + +static UNSIGNED_INTEGER +my_atoui(const char * s) +{ + return s ? ((UNSIGNED_INTEGER)STRTOUI(s, NULL, 0)) : 0; +} + +/* + * */ +MINIUPNP_LIBSPEC UNSIGNED_INTEGER +UPNP_GetTotalBytesSent(const char * controlURL, + const char * servicetype) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + unsigned int r = 0; + char * p; + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetTotalBytesSent", 0, &bufsize))) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + /*DisplayNameValueList(buffer, bufsize);*/ + free(buffer); buffer = NULL; + p = GetValueFromNameValueList(&pdata, "NewTotalBytesSent"); + r = my_atoui(p); + ClearNameValueList(&pdata); + return r; +} + +/* + * */ +MINIUPNP_LIBSPEC UNSIGNED_INTEGER +UPNP_GetTotalBytesReceived(const char * controlURL, + const char * servicetype) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + unsigned int r = 0; + char * p; + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetTotalBytesReceived", 0, &bufsize))) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + /*DisplayNameValueList(buffer, bufsize);*/ + free(buffer); buffer = NULL; + p = GetValueFromNameValueList(&pdata, "NewTotalBytesReceived"); + r = my_atoui(p); + ClearNameValueList(&pdata); + return r; +} + +/* + * */ +MINIUPNP_LIBSPEC UNSIGNED_INTEGER +UPNP_GetTotalPacketsSent(const char * controlURL, + const char * servicetype) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + unsigned int r = 0; + char * p; + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetTotalPacketsSent", 0, &bufsize))) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + /*DisplayNameValueList(buffer, bufsize);*/ + free(buffer); buffer = NULL; + p = GetValueFromNameValueList(&pdata, "NewTotalPacketsSent"); + r = my_atoui(p); + ClearNameValueList(&pdata); + return r; +} + +/* + * */ +MINIUPNP_LIBSPEC UNSIGNED_INTEGER +UPNP_GetTotalPacketsReceived(const char * controlURL, + const char * servicetype) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + unsigned int r = 0; + char * p; + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetTotalPacketsReceived", 0, &bufsize))) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + /*DisplayNameValueList(buffer, bufsize);*/ + free(buffer); buffer = NULL; + p = GetValueFromNameValueList(&pdata, "NewTotalPacketsReceived"); + r = my_atoui(p); + ClearNameValueList(&pdata); + return r; +} + +/* UPNP_GetStatusInfo() call the corresponding UPNP method + * returns the current status and uptime */ +MINIUPNP_LIBSPEC int +UPNP_GetStatusInfo(const char * controlURL, + const char * servicetype, + char * status, + unsigned int * uptime, + char * lastconnerror) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + char * p; + char * up; + char * err; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + + if(!status && !uptime) + return UPNPCOMMAND_INVALID_ARGS; + + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetStatusInfo", 0, &bufsize))) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + /*DisplayNameValueList(buffer, bufsize);*/ + free(buffer); buffer = NULL; + up = GetValueFromNameValueList(&pdata, "NewUptime"); + p = GetValueFromNameValueList(&pdata, "NewConnectionStatus"); + err = GetValueFromNameValueList(&pdata, "NewLastConnectionError"); + if(p && up) + ret = UPNPCOMMAND_SUCCESS; + + if(status) { + if(p){ + strncpy(status, p, 64 ); + status[63] = '\0'; + }else + status[0]= '\0'; + } + + if(uptime) { + if(up) + sscanf(up,"%u",uptime); + else + *uptime = 0; + } + + if(lastconnerror) { + if(err) { + strncpy(lastconnerror, err, 64 ); + lastconnerror[63] = '\0'; + } else + lastconnerror[0] = '\0'; + } + + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + ClearNameValueList(&pdata); + return ret; +} + +/* UPNP_GetConnectionTypeInfo() call the corresponding UPNP method + * returns the connection type */ +MINIUPNP_LIBSPEC int +UPNP_GetConnectionTypeInfo(const char * controlURL, + const char * servicetype, + char * connectionType) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + char * p; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + + if(!connectionType) + return UPNPCOMMAND_INVALID_ARGS; + + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetConnectionTypeInfo", 0, &bufsize))) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + p = GetValueFromNameValueList(&pdata, "NewConnectionType"); + /*p = GetValueFromNameValueList(&pdata, "NewPossibleConnectionTypes");*/ + /* PossibleConnectionTypes will have several values.... */ + if(p) { + strncpy(connectionType, p, 64 ); + connectionType[63] = '\0'; + ret = UPNPCOMMAND_SUCCESS; + } else + connectionType[0] = '\0'; + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + ClearNameValueList(&pdata); + return ret; +} + +/* UPNP_GetLinkLayerMaxBitRate() call the corresponding UPNP method. + * Returns 2 values: Downloadlink bandwidth and Uplink bandwidth. + * One of the values can be null + * Note : GetLinkLayerMaxBitRates belongs to WANPPPConnection:1 only + * We can use the GetCommonLinkProperties from WANCommonInterfaceConfig:1 */ +MINIUPNP_LIBSPEC int +UPNP_GetLinkLayerMaxBitRates(const char * controlURL, + const char * servicetype, + unsigned int * bitrateDown, + unsigned int * bitrateUp) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + char * down; + char * up; + char * p; + + if(!bitrateDown && !bitrateUp) + return UPNPCOMMAND_INVALID_ARGS; + + /* shouldn't we use GetCommonLinkProperties ? */ + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetCommonLinkProperties", 0, &bufsize))) { + /*"GetLinkLayerMaxBitRates", 0, &bufsize);*/ + return UPNPCOMMAND_HTTP_ERROR; + } + /*DisplayNameValueList(buffer, bufsize);*/ + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + /*down = GetValueFromNameValueList(&pdata, "NewDownstreamMaxBitRate");*/ + /*up = GetValueFromNameValueList(&pdata, "NewUpstreamMaxBitRate");*/ + down = GetValueFromNameValueList(&pdata, "NewLayer1DownstreamMaxBitRate"); + up = GetValueFromNameValueList(&pdata, "NewLayer1UpstreamMaxBitRate"); + /*GetValueFromNameValueList(&pdata, "NewWANAccessType");*/ + /*GetValueFromNameValueList(&pdata, "NewPhysicalLinkStatus");*/ + if(down && up) + ret = UPNPCOMMAND_SUCCESS; + + if(bitrateDown) { + if(down) + sscanf(down,"%u",bitrateDown); + else + *bitrateDown = 0; + } + + if(bitrateUp) { + if(up) + sscanf(up,"%u",bitrateUp); + else + *bitrateUp = 0; + } + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + ClearNameValueList(&pdata); + return ret; +} + + +/* UPNP_GetExternalIPAddress() call the corresponding UPNP method. + * if the third arg is not null the value is copied to it. + * at least 16 bytes must be available + * + * Return values : + * 0 : SUCCESS + * NON ZERO : ERROR Either an UPnP error code or an unknown error. + * + * 402 Invalid Args - See UPnP Device Architecture section on Control. + * 501 Action Failed - See UPnP Device Architecture section on Control. + */ +MINIUPNP_LIBSPEC int +UPNP_GetExternalIPAddress(const char * controlURL, + const char * servicetype, + char * extIpAdd) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + char * p; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + + if(!extIpAdd || !controlURL || !servicetype) + return UPNPCOMMAND_INVALID_ARGS; + + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetExternalIPAddress", 0, &bufsize))) { + return UPNPCOMMAND_HTTP_ERROR; + } + /*DisplayNameValueList(buffer, bufsize);*/ + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + /*printf("external ip = %s\n", GetValueFromNameValueList(&pdata, "NewExternalIPAddress") );*/ + p = GetValueFromNameValueList(&pdata, "NewExternalIPAddress"); + if(p) { + strncpy(extIpAdd, p, 16 ); + extIpAdd[15] = '\0'; + ret = UPNPCOMMAND_SUCCESS; + } else + extIpAdd[0] = '\0'; + + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_AddPortMapping(const char * controlURL, const char * servicetype, + const char * extPort, + const char * inPort, + const char * inClient, + const char * desc, + const char * proto, + const char * remoteHost, + const char * leaseDuration) +{ + struct UPNParg * AddPortMappingArgs; + char * buffer; + int bufsize; + struct NameValueParserData pdata; + const char * resVal; + int ret; + + if(!inPort || !inClient || !proto || !extPort) + return UPNPCOMMAND_INVALID_ARGS; + + AddPortMappingArgs = calloc(9, sizeof(struct UPNParg)); + if(AddPortMappingArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + AddPortMappingArgs[0].elt = "NewRemoteHost"; + AddPortMappingArgs[0].val = remoteHost; + AddPortMappingArgs[1].elt = "NewExternalPort"; + AddPortMappingArgs[1].val = extPort; + AddPortMappingArgs[2].elt = "NewProtocol"; + AddPortMappingArgs[2].val = proto; + AddPortMappingArgs[3].elt = "NewInternalPort"; + AddPortMappingArgs[3].val = inPort; + AddPortMappingArgs[4].elt = "NewInternalClient"; + AddPortMappingArgs[4].val = inClient; + AddPortMappingArgs[5].elt = "NewEnabled"; + AddPortMappingArgs[5].val = "1"; + AddPortMappingArgs[6].elt = "NewPortMappingDescription"; + AddPortMappingArgs[6].val = desc?desc:"libminiupnpc"; + AddPortMappingArgs[7].elt = "NewLeaseDuration"; + AddPortMappingArgs[7].val = leaseDuration?leaseDuration:"0"; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "AddPortMapping", AddPortMappingArgs, + &bufsize); + free(AddPortMappingArgs); + if(!buffer) { + return UPNPCOMMAND_HTTP_ERROR; + } + /*DisplayNameValueList(buffer, bufsize);*/ + /*buffer[bufsize] = '\0';*/ + /*puts(buffer);*/ + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + resVal = GetValueFromNameValueList(&pdata, "errorCode"); + if(resVal) { + /*printf("AddPortMapping errorCode = '%s'\n", resVal); */ + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(resVal, "%d", &ret); + } else { + ret = UPNPCOMMAND_SUCCESS; + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_AddAnyPortMapping(const char * controlURL, const char * servicetype, + const char * extPort, + const char * inPort, + const char * inClient, + const char * desc, + const char * proto, + const char * remoteHost, + const char * leaseDuration, + char * reservedPort) +{ + struct UPNParg * AddPortMappingArgs; + char * buffer; + int bufsize; + struct NameValueParserData pdata; + const char * resVal; + int ret; + + if(!inPort || !inClient || !proto || !extPort) + return UPNPCOMMAND_INVALID_ARGS; + + AddPortMappingArgs = calloc(9, sizeof(struct UPNParg)); + if(AddPortMappingArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + AddPortMappingArgs[0].elt = "NewRemoteHost"; + AddPortMappingArgs[0].val = remoteHost; + AddPortMappingArgs[1].elt = "NewExternalPort"; + AddPortMappingArgs[1].val = extPort; + AddPortMappingArgs[2].elt = "NewProtocol"; + AddPortMappingArgs[2].val = proto; + AddPortMappingArgs[3].elt = "NewInternalPort"; + AddPortMappingArgs[3].val = inPort; + AddPortMappingArgs[4].elt = "NewInternalClient"; + AddPortMappingArgs[4].val = inClient; + AddPortMappingArgs[5].elt = "NewEnabled"; + AddPortMappingArgs[5].val = "1"; + AddPortMappingArgs[6].elt = "NewPortMappingDescription"; + AddPortMappingArgs[6].val = desc?desc:"libminiupnpc"; + AddPortMappingArgs[7].elt = "NewLeaseDuration"; + AddPortMappingArgs[7].val = leaseDuration?leaseDuration:"0"; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "AddAnyPortMapping", AddPortMappingArgs, + &bufsize); + free(AddPortMappingArgs); + if(!buffer) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + resVal = GetValueFromNameValueList(&pdata, "errorCode"); + if(resVal) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(resVal, "%d", &ret); + } else { + char *p; + + p = GetValueFromNameValueList(&pdata, "NewReservedPort"); + if(p) { + strncpy(reservedPort, p, 6); + reservedPort[5] = '\0'; + ret = UPNPCOMMAND_SUCCESS; + } else { + ret = UPNPCOMMAND_INVALID_RESPONSE; + } + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_DeletePortMapping(const char * controlURL, const char * servicetype, + const char * extPort, const char * proto, + const char * remoteHost) +{ + /*struct NameValueParserData pdata;*/ + struct UPNParg * DeletePortMappingArgs; + char * buffer; + int bufsize; + struct NameValueParserData pdata; + const char * resVal; + int ret; + + if(!extPort || !proto) + return UPNPCOMMAND_INVALID_ARGS; + + DeletePortMappingArgs = calloc(4, sizeof(struct UPNParg)); + if(DeletePortMappingArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + DeletePortMappingArgs[0].elt = "NewRemoteHost"; + DeletePortMappingArgs[0].val = remoteHost; + DeletePortMappingArgs[1].elt = "NewExternalPort"; + DeletePortMappingArgs[1].val = extPort; + DeletePortMappingArgs[2].elt = "NewProtocol"; + DeletePortMappingArgs[2].val = proto; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "DeletePortMapping", + DeletePortMappingArgs, &bufsize); + free(DeletePortMappingArgs); + if(!buffer) { + return UPNPCOMMAND_HTTP_ERROR; + } + /*DisplayNameValueList(buffer, bufsize);*/ + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + resVal = GetValueFromNameValueList(&pdata, "errorCode"); + if(resVal) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(resVal, "%d", &ret); + } else { + ret = UPNPCOMMAND_SUCCESS; + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_DeletePortMappingRange(const char * controlURL, const char * servicetype, + const char * extPortStart, const char * extPortEnd, + const char * proto, + const char * manage) +{ + struct UPNParg * DeletePortMappingArgs; + char * buffer; + int bufsize; + struct NameValueParserData pdata; + const char * resVal; + int ret; + + if(!extPortStart || !extPortEnd || !proto || !manage) + return UPNPCOMMAND_INVALID_ARGS; + + DeletePortMappingArgs = calloc(5, sizeof(struct UPNParg)); + if(DeletePortMappingArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + DeletePortMappingArgs[0].elt = "NewStartPort"; + DeletePortMappingArgs[0].val = extPortStart; + DeletePortMappingArgs[1].elt = "NewEndPort"; + DeletePortMappingArgs[1].val = extPortEnd; + DeletePortMappingArgs[2].elt = "NewProtocol"; + DeletePortMappingArgs[2].val = proto; + DeletePortMappingArgs[3].elt = "NewManage"; + DeletePortMappingArgs[3].val = manage; + + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "DeletePortMappingRange", + DeletePortMappingArgs, &bufsize); + free(DeletePortMappingArgs); + if(!buffer) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + resVal = GetValueFromNameValueList(&pdata, "errorCode"); + if(resVal) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(resVal, "%d", &ret); + } else { + ret = UPNPCOMMAND_SUCCESS; + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_GetGenericPortMappingEntry(const char * controlURL, + const char * servicetype, + const char * index, + char * extPort, + char * intClient, + char * intPort, + char * protocol, + char * desc, + char * enabled, + char * rHost, + char * duration) +{ + struct NameValueParserData pdata; + struct UPNParg * GetPortMappingArgs; + char * buffer; + int bufsize; + char * p; + int r = UPNPCOMMAND_UNKNOWN_ERROR; + if(!index) + return UPNPCOMMAND_INVALID_ARGS; + intClient[0] = '\0'; + intPort[0] = '\0'; + GetPortMappingArgs = calloc(2, sizeof(struct UPNParg)); + if(GetPortMappingArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + GetPortMappingArgs[0].elt = "NewPortMappingIndex"; + GetPortMappingArgs[0].val = index; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetGenericPortMappingEntry", + GetPortMappingArgs, &bufsize); + free(GetPortMappingArgs); + if(!buffer) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + + p = GetValueFromNameValueList(&pdata, "NewRemoteHost"); + if(p && rHost) + { + strncpy(rHost, p, 64); + rHost[63] = '\0'; + } + p = GetValueFromNameValueList(&pdata, "NewExternalPort"); + if(p && extPort) + { + strncpy(extPort, p, 6); + extPort[5] = '\0'; + r = UPNPCOMMAND_SUCCESS; + } + p = GetValueFromNameValueList(&pdata, "NewProtocol"); + if(p && protocol) + { + strncpy(protocol, p, 4); + protocol[3] = '\0'; + } + p = GetValueFromNameValueList(&pdata, "NewInternalClient"); + if(p) + { + strncpy(intClient, p, 16); + intClient[15] = '\0'; + r = 0; + } + p = GetValueFromNameValueList(&pdata, "NewInternalPort"); + if(p) + { + strncpy(intPort, p, 6); + intPort[5] = '\0'; + } + p = GetValueFromNameValueList(&pdata, "NewEnabled"); + if(p && enabled) + { + strncpy(enabled, p, 4); + enabled[3] = '\0'; + } + p = GetValueFromNameValueList(&pdata, "NewPortMappingDescription"); + if(p && desc) + { + strncpy(desc, p, 80); + desc[79] = '\0'; + } + p = GetValueFromNameValueList(&pdata, "NewLeaseDuration"); + if(p && duration) + { + strncpy(duration, p, 16); + duration[15] = '\0'; + } + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) { + r = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &r); + } + ClearNameValueList(&pdata); + return r; +} + +MINIUPNP_LIBSPEC int +UPNP_GetPortMappingNumberOfEntries(const char * controlURL, + const char * servicetype, + unsigned int * numEntries) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + char* p; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + if(!(buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetPortMappingNumberOfEntries", 0, + &bufsize))) { + return UPNPCOMMAND_HTTP_ERROR; + } +#ifdef DEBUG + DisplayNameValueList(buffer, bufsize); +#endif + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + + p = GetValueFromNameValueList(&pdata, "NewPortMappingNumberOfEntries"); + if(numEntries && p) { + *numEntries = 0; + sscanf(p, "%u", numEntries); + ret = UPNPCOMMAND_SUCCESS; + } + + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + + ClearNameValueList(&pdata); + return ret; +} + +/* UPNP_GetSpecificPortMappingEntry retrieves an existing port mapping + * the result is returned in the intClient and intPort strings + * please provide 16 and 6 bytes of data */ +MINIUPNP_LIBSPEC int +UPNP_GetSpecificPortMappingEntry(const char * controlURL, + const char * servicetype, + const char * extPort, + const char * proto, + const char * remoteHost, + char * intClient, + char * intPort, + char * desc, + char * enabled, + char * leaseDuration) +{ + struct NameValueParserData pdata; + struct UPNParg * GetPortMappingArgs; + char * buffer; + int bufsize; + char * p; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + + if(!intPort || !intClient || !extPort || !proto) + return UPNPCOMMAND_INVALID_ARGS; + + GetPortMappingArgs = calloc(4, sizeof(struct UPNParg)); + if(GetPortMappingArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + GetPortMappingArgs[0].elt = "NewRemoteHost"; + GetPortMappingArgs[0].val = remoteHost; + GetPortMappingArgs[1].elt = "NewExternalPort"; + GetPortMappingArgs[1].val = extPort; + GetPortMappingArgs[2].elt = "NewProtocol"; + GetPortMappingArgs[2].val = proto; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetSpecificPortMappingEntry", + GetPortMappingArgs, &bufsize); + free(GetPortMappingArgs); + if(!buffer) { + return UPNPCOMMAND_HTTP_ERROR; + } + /*DisplayNameValueList(buffer, bufsize);*/ + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + + p = GetValueFromNameValueList(&pdata, "NewInternalClient"); + if(p) { + strncpy(intClient, p, 16); + intClient[15] = '\0'; + ret = UPNPCOMMAND_SUCCESS; + } else + intClient[0] = '\0'; + + p = GetValueFromNameValueList(&pdata, "NewInternalPort"); + if(p) { + strncpy(intPort, p, 6); + intPort[5] = '\0'; + } else + intPort[0] = '\0'; + + p = GetValueFromNameValueList(&pdata, "NewEnabled"); + if(p && enabled) { + strncpy(enabled, p, 4); + enabled[3] = '\0'; + } + + p = GetValueFromNameValueList(&pdata, "NewPortMappingDescription"); + if(p && desc) { + strncpy(desc, p, 80); + desc[79] = '\0'; + } + + p = GetValueFromNameValueList(&pdata, "NewLeaseDuration"); + if(p && leaseDuration) + { + strncpy(leaseDuration, p, 16); + leaseDuration[15] = '\0'; + } + + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + + ClearNameValueList(&pdata); + return ret; +} + +/* UPNP_GetListOfPortMappings() + * + * Possible UPNP Error codes : + * 606 Action not Authorized + * 730 PortMappingNotFound - no port mapping is found in the specified range. + * 733 InconsistantParameters - NewStartPort and NewEndPort values are not + * consistent. + */ +MINIUPNP_LIBSPEC int +UPNP_GetListOfPortMappings(const char * controlURL, + const char * servicetype, + const char * startPort, + const char * endPort, + const char * protocol, + const char * numberOfPorts, + struct PortMappingParserData * data) +{ + struct NameValueParserData pdata; + struct UPNParg * GetListOfPortMappingsArgs; + const char * p; + char * buffer; + int bufsize; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + + if(!startPort || !endPort || !protocol) + return UPNPCOMMAND_INVALID_ARGS; + + GetListOfPortMappingsArgs = calloc(6, sizeof(struct UPNParg)); + if(GetListOfPortMappingsArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + GetListOfPortMappingsArgs[0].elt = "NewStartPort"; + GetListOfPortMappingsArgs[0].val = startPort; + GetListOfPortMappingsArgs[1].elt = "NewEndPort"; + GetListOfPortMappingsArgs[1].val = endPort; + GetListOfPortMappingsArgs[2].elt = "NewProtocol"; + GetListOfPortMappingsArgs[2].val = protocol; + GetListOfPortMappingsArgs[3].elt = "NewManage"; + GetListOfPortMappingsArgs[3].val = "1"; + GetListOfPortMappingsArgs[4].elt = "NewNumberOfPorts"; + GetListOfPortMappingsArgs[4].val = numberOfPorts?numberOfPorts:"1000"; + + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetListOfPortMappings", + GetListOfPortMappingsArgs, &bufsize); + free(GetListOfPortMappingsArgs); + if(!buffer) { + return UPNPCOMMAND_HTTP_ERROR; + } + + /*DisplayNameValueList(buffer, bufsize);*/ + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + + /*p = GetValueFromNameValueList(&pdata, "NewPortListing");*/ + /*if(p) { + printf("NewPortListing : %s\n", p); + }*/ + /*printf("NewPortListing(%d chars) : %s\n", + pdata.portListingLength, pdata.portListing);*/ + if(pdata.portListing) + { + /*struct PortMapping * pm; + int i = 0;*/ + ParsePortListing(pdata.portListing, pdata.portListingLength, + data); + ret = UPNPCOMMAND_SUCCESS; + /* + for(pm = data->head.lh_first; pm != NULL; pm = pm->entries.le_next) + { + printf("%2d %s %5hu->%s:%-5hu '%s' '%s'\n", + i, pm->protocol, pm->externalPort, pm->internalClient, + pm->internalPort, + pm->description, pm->remoteHost); + i++; + } + */ + /*FreePortListing(&data);*/ + } + + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + ClearNameValueList(&pdata); + + /*printf("%.*s", bufsize, buffer);*/ + + return ret; +} + +/* IGD:2, functions for service WANIPv6FirewallControl:1 */ +MINIUPNP_LIBSPEC int +UPNP_GetFirewallStatus(const char * controlURL, + const char * servicetype, + int * firewallEnabled, + int * inboundPinholeAllowed) +{ + struct NameValueParserData pdata; + char * buffer; + int bufsize; + char * fe, *ipa, *p; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + + if(!firewallEnabled || !inboundPinholeAllowed) + return UPNPCOMMAND_INVALID_ARGS; + + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetFirewallStatus", 0, &bufsize); + if(!buffer) { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + fe = GetValueFromNameValueList(&pdata, "FirewallEnabled"); + ipa = GetValueFromNameValueList(&pdata, "InboundPinholeAllowed"); + if(ipa && fe) + ret = UPNPCOMMAND_SUCCESS; + if(fe) + *firewallEnabled = my_atoui(fe); + /*else + *firewallEnabled = 0;*/ + if(ipa) + *inboundPinholeAllowed = my_atoui(ipa); + /*else + *inboundPinholeAllowed = 0;*/ + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) + { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_GetOutboundPinholeTimeout(const char * controlURL, const char * servicetype, + const char * remoteHost, + const char * remotePort, + const char * intClient, + const char * intPort, + const char * proto, + int * opTimeout) +{ + struct UPNParg * GetOutboundPinholeTimeoutArgs; + char * buffer; + int bufsize; + struct NameValueParserData pdata; + const char * resVal; + char * p; + int ret; + + if(!intPort || !intClient || !proto || !remotePort || !remoteHost) + return UPNPCOMMAND_INVALID_ARGS; + + GetOutboundPinholeTimeoutArgs = calloc(6, sizeof(struct UPNParg)); + if(GetOutboundPinholeTimeoutArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + GetOutboundPinholeTimeoutArgs[0].elt = "RemoteHost"; + GetOutboundPinholeTimeoutArgs[0].val = remoteHost; + GetOutboundPinholeTimeoutArgs[1].elt = "RemotePort"; + GetOutboundPinholeTimeoutArgs[1].val = remotePort; + GetOutboundPinholeTimeoutArgs[2].elt = "Protocol"; + GetOutboundPinholeTimeoutArgs[2].val = proto; + GetOutboundPinholeTimeoutArgs[3].elt = "InternalPort"; + GetOutboundPinholeTimeoutArgs[3].val = intPort; + GetOutboundPinholeTimeoutArgs[4].elt = "InternalClient"; + GetOutboundPinholeTimeoutArgs[4].val = intClient; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetOutboundPinholeTimeout", GetOutboundPinholeTimeoutArgs, &bufsize); + free(GetOutboundPinholeTimeoutArgs); + if(!buffer) + return UPNPCOMMAND_HTTP_ERROR; + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + resVal = GetValueFromNameValueList(&pdata, "errorCode"); + if(resVal) + { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(resVal, "%d", &ret); + } + else + { + ret = UPNPCOMMAND_SUCCESS; + p = GetValueFromNameValueList(&pdata, "OutboundPinholeTimeout"); + if(p) + *opTimeout = my_atoui(p); + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_AddPinhole(const char * controlURL, const char * servicetype, + const char * remoteHost, + const char * remotePort, + const char * intClient, + const char * intPort, + const char * proto, + const char * leaseTime, + char * uniqueID) +{ + struct UPNParg * AddPinholeArgs; + char * buffer; + int bufsize; + struct NameValueParserData pdata; + const char * resVal; + char * p; + int ret; + + if(!intPort || !intClient || !proto || !remoteHost || !remotePort || !leaseTime) + return UPNPCOMMAND_INVALID_ARGS; + + AddPinholeArgs = calloc(7, sizeof(struct UPNParg)); + if(AddPinholeArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + /* RemoteHost can be wilcarded */ + if(strncmp(remoteHost, "empty", 5)==0) + { + AddPinholeArgs[0].elt = "RemoteHost"; + AddPinholeArgs[0].val = ""; + } + else + { + AddPinholeArgs[0].elt = "RemoteHost"; + AddPinholeArgs[0].val = remoteHost; + } + AddPinholeArgs[1].elt = "RemotePort"; + AddPinholeArgs[1].val = remotePort; + AddPinholeArgs[2].elt = "Protocol"; + AddPinholeArgs[2].val = proto; + AddPinholeArgs[3].elt = "InternalPort"; + AddPinholeArgs[3].val = intPort; + if(strncmp(intClient, "empty", 5)==0) + { + AddPinholeArgs[4].elt = "InternalClient"; + AddPinholeArgs[4].val = ""; + } + else + { + AddPinholeArgs[4].elt = "InternalClient"; + AddPinholeArgs[4].val = intClient; + } + AddPinholeArgs[5].elt = "LeaseTime"; + AddPinholeArgs[5].val = leaseTime; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "AddPinhole", AddPinholeArgs, &bufsize); + free(AddPinholeArgs); + if(!buffer) + return UPNPCOMMAND_HTTP_ERROR; + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + p = GetValueFromNameValueList(&pdata, "UniqueID"); + if(p) + { + strncpy(uniqueID, p, 8); + uniqueID[7] = '\0'; + } + resVal = GetValueFromNameValueList(&pdata, "errorCode"); + if(resVal) + { + /*printf("AddPortMapping errorCode = '%s'\n", resVal);*/ + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(resVal, "%d", &ret); + } + else + { + ret = UPNPCOMMAND_SUCCESS; + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_UpdatePinhole(const char * controlURL, const char * servicetype, + const char * uniqueID, + const char * leaseTime) +{ + struct UPNParg * UpdatePinholeArgs; + char * buffer; + int bufsize; + struct NameValueParserData pdata; + const char * resVal; + int ret; + + if(!uniqueID || !leaseTime) + return UPNPCOMMAND_INVALID_ARGS; + + UpdatePinholeArgs = calloc(3, sizeof(struct UPNParg)); + if(UpdatePinholeArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + UpdatePinholeArgs[0].elt = "UniqueID"; + UpdatePinholeArgs[0].val = uniqueID; + UpdatePinholeArgs[1].elt = "NewLeaseTime"; + UpdatePinholeArgs[1].val = leaseTime; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "UpdatePinhole", UpdatePinholeArgs, &bufsize); + free(UpdatePinholeArgs); + if(!buffer) + return UPNPCOMMAND_HTTP_ERROR; + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + resVal = GetValueFromNameValueList(&pdata, "errorCode"); + if(resVal) + { + /*printf("AddPortMapping errorCode = '%s'\n", resVal); */ + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(resVal, "%d", &ret); + } + else + { + ret = UPNPCOMMAND_SUCCESS; + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_DeletePinhole(const char * controlURL, const char * servicetype, const char * uniqueID) +{ + /*struct NameValueParserData pdata;*/ + struct UPNParg * DeletePinholeArgs; + char * buffer; + int bufsize; + struct NameValueParserData pdata; + const char * resVal; + int ret; + + if(!uniqueID) + return UPNPCOMMAND_INVALID_ARGS; + + DeletePinholeArgs = calloc(2, sizeof(struct UPNParg)); + if(DeletePinholeArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + DeletePinholeArgs[0].elt = "UniqueID"; + DeletePinholeArgs[0].val = uniqueID; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "DeletePinhole", DeletePinholeArgs, &bufsize); + free(DeletePinholeArgs); + if(!buffer) + return UPNPCOMMAND_HTTP_ERROR; + /*DisplayNameValueList(buffer, bufsize);*/ + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + resVal = GetValueFromNameValueList(&pdata, "errorCode"); + if(resVal) + { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(resVal, "%d", &ret); + } + else + { + ret = UPNPCOMMAND_SUCCESS; + } + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_CheckPinholeWorking(const char * controlURL, const char * servicetype, + const char * uniqueID, int * isWorking) +{ + struct NameValueParserData pdata; + struct UPNParg * CheckPinholeWorkingArgs; + char * buffer; + int bufsize; + char * p; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + + if(!uniqueID) + return UPNPCOMMAND_INVALID_ARGS; + + CheckPinholeWorkingArgs = calloc(4, sizeof(struct UPNParg)); + if(CheckPinholeWorkingArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + CheckPinholeWorkingArgs[0].elt = "UniqueID"; + CheckPinholeWorkingArgs[0].val = uniqueID; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "CheckPinholeWorking", CheckPinholeWorkingArgs, &bufsize); + free(CheckPinholeWorkingArgs); + if(!buffer) + { + return UPNPCOMMAND_HTTP_ERROR; + } + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + + p = GetValueFromNameValueList(&pdata, "IsWorking"); + if(p) + { + *isWorking=my_atoui(p); + ret = UPNPCOMMAND_SUCCESS; + } + else + *isWorking = 0; + + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) + { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + + ClearNameValueList(&pdata); + return ret; +} + +MINIUPNP_LIBSPEC int +UPNP_GetPinholePackets(const char * controlURL, const char * servicetype, + const char * uniqueID, int * packets) +{ + struct NameValueParserData pdata; + struct UPNParg * GetPinholePacketsArgs; + char * buffer; + int bufsize; + char * p; + int ret = UPNPCOMMAND_UNKNOWN_ERROR; + + if(!uniqueID) + return UPNPCOMMAND_INVALID_ARGS; + + GetPinholePacketsArgs = calloc(4, sizeof(struct UPNParg)); + if(GetPinholePacketsArgs == NULL) + return UPNPCOMMAND_MEM_ALLOC_ERROR; + GetPinholePacketsArgs[0].elt = "UniqueID"; + GetPinholePacketsArgs[0].val = uniqueID; + buffer = simpleUPnPcommand(-1, controlURL, servicetype, + "GetPinholePackets", GetPinholePacketsArgs, &bufsize); + free(GetPinholePacketsArgs); + if(!buffer) + return UPNPCOMMAND_HTTP_ERROR; + ParseNameValue(buffer, bufsize, &pdata); + free(buffer); buffer = NULL; + + p = GetValueFromNameValueList(&pdata, "PinholePackets"); + if(p) + { + *packets=my_atoui(p); + ret = UPNPCOMMAND_SUCCESS; + } + + p = GetValueFromNameValueList(&pdata, "errorCode"); + if(p) + { + ret = UPNPCOMMAND_UNKNOWN_ERROR; + sscanf(p, "%d", &ret); + } + + ClearNameValueList(&pdata); + return ret; +} + + diff --git a/src/contrib/miniupnp/miniupnpc/upnpcommands.h b/src/contrib/miniupnp/miniupnpc/upnpcommands.h new file mode 100644 index 0000000..0c6d501 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/upnpcommands.h @@ -0,0 +1,348 @@ +/* $Id: upnpcommands.h,v 1.32 2018/03/13 23:34:47 nanard Exp $ */ +/* Miniupnp project : http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided within this distribution */ +#ifndef UPNPCOMMANDS_H_INCLUDED +#define UPNPCOMMANDS_H_INCLUDED + +#include "miniupnpc_declspec.h" +#include "miniupnpctypes.h" + +/* MiniUPnPc return codes : */ +#define UPNPCOMMAND_SUCCESS (0) +#define UPNPCOMMAND_UNKNOWN_ERROR (-1) +#define UPNPCOMMAND_INVALID_ARGS (-2) +#define UPNPCOMMAND_HTTP_ERROR (-3) +#define UPNPCOMMAND_INVALID_RESPONSE (-4) +#define UPNPCOMMAND_MEM_ALLOC_ERROR (-5) + +#ifdef __cplusplus +extern "C" { +#endif + +struct PortMappingParserData; + +MINIUPNP_LIBSPEC UNSIGNED_INTEGER +UPNP_GetTotalBytesSent(const char * controlURL, + const char * servicetype); + +MINIUPNP_LIBSPEC UNSIGNED_INTEGER +UPNP_GetTotalBytesReceived(const char * controlURL, + const char * servicetype); + +MINIUPNP_LIBSPEC UNSIGNED_INTEGER +UPNP_GetTotalPacketsSent(const char * controlURL, + const char * servicetype); + +MINIUPNP_LIBSPEC UNSIGNED_INTEGER +UPNP_GetTotalPacketsReceived(const char * controlURL, + const char * servicetype); + +/* UPNP_GetStatusInfo() + * status and lastconnerror are 64 byte buffers + * Return values : + * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR + * or a UPnP Error code */ +MINIUPNP_LIBSPEC int +UPNP_GetStatusInfo(const char * controlURL, + const char * servicetype, + char * status, + unsigned int * uptime, + char * lastconnerror); + +/* UPNP_GetConnectionTypeInfo() + * argument connectionType is a 64 character buffer + * Return Values : + * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR + * or a UPnP Error code */ +MINIUPNP_LIBSPEC int +UPNP_GetConnectionTypeInfo(const char * controlURL, + const char * servicetype, + char * connectionType); + +/* UPNP_GetExternalIPAddress() call the corresponding UPNP method. + * if the third arg is not null the value is copied to it. + * at least 16 bytes must be available + * + * Return values : + * 0 : SUCCESS + * NON ZERO : ERROR Either an UPnP error code or an unknown error. + * + * possible UPnP Errors : + * 402 Invalid Args - See UPnP Device Architecture section on Control. + * 501 Action Failed - See UPnP Device Architecture section on Control. */ +MINIUPNP_LIBSPEC int +UPNP_GetExternalIPAddress(const char * controlURL, + const char * servicetype, + char * extIpAdd); + +/* UPNP_GetLinkLayerMaxBitRates() + * call WANCommonInterfaceConfig:1#GetCommonLinkProperties + * + * return values : + * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR + * or a UPnP Error Code. */ +MINIUPNP_LIBSPEC int +UPNP_GetLinkLayerMaxBitRates(const char* controlURL, + const char* servicetype, + unsigned int * bitrateDown, + unsigned int * bitrateUp); + +/* UPNP_AddPortMapping() + * if desc is NULL, it will be defaulted to "libminiupnpc" + * remoteHost is usually NULL because IGD don't support it. + * + * Return values : + * 0 : SUCCESS + * NON ZERO : ERROR. Either an UPnP error code or an unknown error. + * + * List of possible UPnP errors for AddPortMapping : + * errorCode errorDescription (short) - Description (long) + * 402 Invalid Args - See UPnP Device Architecture section on Control. + * 501 Action Failed - See UPnP Device Architecture section on Control. + * 606 Action not authorized - The action requested REQUIRES authorization and + * the sender was not authorized. + * 715 WildCardNotPermittedInSrcIP - The source IP address cannot be + * wild-carded + * 716 WildCardNotPermittedInExtPort - The external port cannot be wild-carded + * 718 ConflictInMappingEntry - The port mapping entry specified conflicts + * with a mapping assigned previously to another client + * 724 SamePortValuesRequired - Internal and External port values + * must be the same + * 725 OnlyPermanentLeasesSupported - The NAT implementation only supports + * permanent lease times on port mappings + * 726 RemoteHostOnlySupportsWildcard - RemoteHost must be a wildcard + * and cannot be a specific IP address or DNS name + * 727 ExternalPortOnlySupportsWildcard - ExternalPort must be a wildcard and + * cannot be a specific port value + * 728 NoPortMapsAvailable - There are not enough free ports available to + * complete port mapping. + * 729 ConflictWithOtherMechanisms - Attempted port mapping is not allowed + * due to conflict with other mechanisms. + * 732 WildCardNotPermittedInIntPort - The internal port cannot be wild-carded + */ +MINIUPNP_LIBSPEC int +UPNP_AddPortMapping(const char * controlURL, const char * servicetype, + const char * extPort, + const char * inPort, + const char * inClient, + const char * desc, + const char * proto, + const char * remoteHost, + const char * leaseDuration); + +/* UPNP_AddAnyPortMapping() + * if desc is NULL, it will be defaulted to "libminiupnpc" + * remoteHost is usually NULL because IGD don't support it. + * + * Return values : + * 0 : SUCCESS + * NON ZERO : ERROR. Either an UPnP error code or an unknown error. + * + * List of possible UPnP errors for AddPortMapping : + * errorCode errorDescription (short) - Description (long) + * 402 Invalid Args - See UPnP Device Architecture section on Control. + * 501 Action Failed - See UPnP Device Architecture section on Control. + * 606 Action not authorized - The action requested REQUIRES authorization and + * the sender was not authorized. + * 715 WildCardNotPermittedInSrcIP - The source IP address cannot be + * wild-carded + * 716 WildCardNotPermittedInExtPort - The external port cannot be wild-carded + * 728 NoPortMapsAvailable - There are not enough free ports available to + * complete port mapping. + * 729 ConflictWithOtherMechanisms - Attempted port mapping is not allowed + * due to conflict with other mechanisms. + * 732 WildCardNotPermittedInIntPort - The internal port cannot be wild-carded + */ +MINIUPNP_LIBSPEC int +UPNP_AddAnyPortMapping(const char * controlURL, const char * servicetype, + const char * extPort, + const char * inPort, + const char * inClient, + const char * desc, + const char * proto, + const char * remoteHost, + const char * leaseDuration, + char * reservedPort); + +/* UPNP_DeletePortMapping() + * Use same argument values as what was used for AddPortMapping(). + * remoteHost is usually NULL because IGD don't support it. + * Return Values : + * 0 : SUCCESS + * NON ZERO : error. Either an UPnP error code or an undefined error. + * + * List of possible UPnP errors for DeletePortMapping : + * 402 Invalid Args - See UPnP Device Architecture section on Control. + * 606 Action not authorized - The action requested REQUIRES authorization + * and the sender was not authorized. + * 714 NoSuchEntryInArray - The specified value does not exist in the array */ +MINIUPNP_LIBSPEC int +UPNP_DeletePortMapping(const char * controlURL, const char * servicetype, + const char * extPort, const char * proto, + const char * remoteHost); + +/* UPNP_DeletePortRangeMapping() + * Use same argument values as what was used for AddPortMapping(). + * remoteHost is usually NULL because IGD don't support it. + * Return Values : + * 0 : SUCCESS + * NON ZERO : error. Either an UPnP error code or an undefined error. + * + * List of possible UPnP errors for DeletePortMapping : + * 606 Action not authorized - The action requested REQUIRES authorization + * and the sender was not authorized. + * 730 PortMappingNotFound - This error message is returned if no port + * mapping is found in the specified range. + * 733 InconsistentParameters - NewStartPort and NewEndPort values are not consistent. */ +MINIUPNP_LIBSPEC int +UPNP_DeletePortMappingRange(const char * controlURL, const char * servicetype, + const char * extPortStart, const char * extPortEnd, + const char * proto, + const char * manage); + +/* UPNP_GetPortMappingNumberOfEntries() + * not supported by all routers */ +MINIUPNP_LIBSPEC int +UPNP_GetPortMappingNumberOfEntries(const char* controlURL, + const char* servicetype, + unsigned int * num); + +/* UPNP_GetSpecificPortMappingEntry() + * retrieves an existing port mapping + * params : + * in extPort + * in proto + * in remoteHost + * out intClient (16 bytes) + * out intPort (6 bytes) + * out desc (80 bytes) + * out enabled (4 bytes) + * out leaseDuration (16 bytes) + * + * return value : + * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR + * or a UPnP Error Code. + * + * List of possible UPnP errors for _GetSpecificPortMappingEntry : + * 402 Invalid Args - See UPnP Device Architecture section on Control. + * 501 Action Failed - See UPnP Device Architecture section on Control. + * 606 Action not authorized - The action requested REQUIRES authorization + * and the sender was not authorized. + * 714 NoSuchEntryInArray - The specified value does not exist in the array. + */ +MINIUPNP_LIBSPEC int +UPNP_GetSpecificPortMappingEntry(const char * controlURL, + const char * servicetype, + const char * extPort, + const char * proto, + const char * remoteHost, + char * intClient, + char * intPort, + char * desc, + char * enabled, + char * leaseDuration); + +/* UPNP_GetGenericPortMappingEntry() + * params : + * in index + * out extPort (6 bytes) + * out intClient (16 bytes) + * out intPort (6 bytes) + * out protocol (4 bytes) + * out desc (80 bytes) + * out enabled (4 bytes) + * out rHost (64 bytes) + * out duration (16 bytes) + * + * return value : + * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR + * or a UPnP Error Code. + * + * Possible UPNP Error codes : + * 402 Invalid Args - See UPnP Device Architecture section on Control. + * 606 Action not authorized - The action requested REQUIRES authorization + * and the sender was not authorized. + * 713 SpecifiedArrayIndexInvalid - The specified array index is out of bounds + */ +MINIUPNP_LIBSPEC int +UPNP_GetGenericPortMappingEntry(const char * controlURL, + const char * servicetype, + const char * index, + char * extPort, + char * intClient, + char * intPort, + char * protocol, + char * desc, + char * enabled, + char * rHost, + char * duration); + +/* UPNP_GetListOfPortMappings() Available in IGD v2 + * + * + * Possible UPNP Error codes : + * 606 Action not Authorized + * 730 PortMappingNotFound - no port mapping is found in the specified range. + * 733 InconsistantParameters - NewStartPort and NewEndPort values are not + * consistent. + */ +MINIUPNP_LIBSPEC int +UPNP_GetListOfPortMappings(const char * controlURL, + const char * servicetype, + const char * startPort, + const char * endPort, + const char * protocol, + const char * numberOfPorts, + struct PortMappingParserData * data); + +/* IGD:2, functions for service WANIPv6FirewallControl:1 */ +MINIUPNP_LIBSPEC int +UPNP_GetFirewallStatus(const char * controlURL, + const char * servicetype, + int * firewallEnabled, + int * inboundPinholeAllowed); + +MINIUPNP_LIBSPEC int +UPNP_GetOutboundPinholeTimeout(const char * controlURL, const char * servicetype, + const char * remoteHost, + const char * remotePort, + const char * intClient, + const char * intPort, + const char * proto, + int * opTimeout); + +MINIUPNP_LIBSPEC int +UPNP_AddPinhole(const char * controlURL, const char * servicetype, + const char * remoteHost, + const char * remotePort, + const char * intClient, + const char * intPort, + const char * proto, + const char * leaseTime, + char * uniqueID); + +MINIUPNP_LIBSPEC int +UPNP_UpdatePinhole(const char * controlURL, const char * servicetype, + const char * uniqueID, + const char * leaseTime); + +MINIUPNP_LIBSPEC int +UPNP_DeletePinhole(const char * controlURL, const char * servicetype, const char * uniqueID); + +MINIUPNP_LIBSPEC int +UPNP_CheckPinholeWorking(const char * controlURL, const char * servicetype, + const char * uniqueID, int * isWorking); + +MINIUPNP_LIBSPEC int +UPNP_GetPinholePackets(const char * controlURL, const char * servicetype, + const char * uniqueID, int * packets); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/upnpdev.c b/src/contrib/miniupnp/miniupnpc/upnpdev.c new file mode 100644 index 0000000..d89a993 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/upnpdev.c @@ -0,0 +1,23 @@ +/* $Id: upnpdev.c,v 1.1 2015/08/28 12:14:19 nanard Exp $ */ +/* Project : miniupnp + * Web : http://miniupnp.free.fr/ + * Author : Thomas BERNARD + * copyright (c) 2005-2015 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENSE file. */ +#include +#include "upnpdev.h" + +/* freeUPNPDevlist() should be used to + * free the chained list returned by upnpDiscover() */ +void freeUPNPDevlist(struct UPNPDev * devlist) +{ + struct UPNPDev * next; + while(devlist) + { + next = devlist->pNext; + free(devlist); + devlist = next; + } +} + diff --git a/src/contrib/miniupnp/miniupnpc/upnpdev.h b/src/contrib/miniupnp/miniupnpc/upnpdev.h new file mode 100644 index 0000000..f4ae174 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/upnpdev.h @@ -0,0 +1,36 @@ +/* $Id: upnpdev.h,v 1.1 2015/08/28 12:14:19 nanard Exp $ */ +/* Project : miniupnp + * Web : http://miniupnp.free.fr/ + * Author : Thomas BERNARD + * copyright (c) 2005-2018 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENSE file. */ +#ifndef UPNPDEV_H_INCLUDED +#define UPNPDEV_H_INCLUDED + +#include "miniupnpc_declspec.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct UPNPDev { + struct UPNPDev * pNext; + char * descURL; + char * st; + char * usn; + unsigned int scope_id; + char buffer[3]; +}; + +/* freeUPNPDevlist() + * free list returned by upnpDiscover() */ +MINIUPNP_LIBSPEC void freeUPNPDevlist(struct UPNPDev * devlist); + + +#ifdef __cplusplus +} +#endif + + +#endif /* UPNPDEV_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpc/upnperrors.c b/src/contrib/miniupnp/miniupnpc/upnperrors.c new file mode 100644 index 0000000..40a2e78 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/upnperrors.c @@ -0,0 +1,107 @@ +/* $Id: upnperrors.c,v 1.5 2011/04/10 11:19:36 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas BERNARD + * copyright (c) 2007 Thomas Bernard + * All Right reserved. + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ +#include +#include "upnperrors.h" +#include "upnpcommands.h" +#include "miniupnpc.h" + +const char * strupnperror(int err) +{ + const char * s = NULL; + switch(err) { + case UPNPCOMMAND_SUCCESS: + s = "Success"; + break; + case UPNPCOMMAND_UNKNOWN_ERROR: + s = "Miniupnpc Unknown Error"; + break; + case UPNPCOMMAND_INVALID_ARGS: + s = "Miniupnpc Invalid Arguments"; + break; + case UPNPCOMMAND_INVALID_RESPONSE: + s = "Miniupnpc Invalid response"; + break; + case UPNPDISCOVER_SOCKET_ERROR: + s = "Miniupnpc Socket error"; + break; + case UPNPDISCOVER_MEMORY_ERROR: + s = "Miniupnpc Memory allocation error"; + break; + case 401: + s = "Invalid Action"; + break; + case 402: + s = "Invalid Args"; + break; + case 501: + s = "Action Failed"; + break; + case 606: + s = "Action not authorized"; + break; + case 701: + s = "PinholeSpaceExhausted"; + break; + case 702: + s = "FirewallDisabled"; + break; + case 703: + s = "InboundPinholeNotAllowed"; + break; + case 704: + s = "NoSuchEntry"; + break; + case 705: + s = "ProtocolNotSupported"; + break; + case 706: + s = "InternalPortWildcardingNotAllowed"; + break; + case 707: + s = "ProtocolWildcardingNotAllowed"; + break; + case 708: + s = "WildcardNotPermittedInSrcIP"; + break; + case 709: + s = "NoPacketSent"; + break; + case 713: + s = "SpecifiedArrayIndexInvalid"; + break; + case 714: + s = "NoSuchEntryInArray"; + break; + case 715: + s = "WildCardNotPermittedInSrcIP"; + break; + case 716: + s = "WildCardNotPermittedInExtPort"; + break; + case 718: + s = "ConflictInMappingEntry"; + break; + case 724: + s = "SamePortValuesRequired"; + break; + case 725: + s = "OnlyPermanentLeasesSupported"; + break; + case 726: + s = "RemoteHostOnlySupportsWildcard"; + break; + case 727: + s = "ExternalPortOnlySupportsWildcard"; + break; + default: + s = "UnknownError"; + break; + } + return s; +} diff --git a/src/contrib/miniupnp/miniupnpc/upnperrors.h b/src/contrib/miniupnp/miniupnpc/upnperrors.h new file mode 100644 index 0000000..8499d9a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/upnperrors.h @@ -0,0 +1,26 @@ +/* $Id: upnperrors.h,v 1.2 2008/07/02 23:31:15 nanard Exp $ */ +/* (c) 2007-2015 Thomas Bernard + * All rights reserved. + * MiniUPnP Project. + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ +#ifndef UPNPERRORS_H_INCLUDED +#define UPNPERRORS_H_INCLUDED + +#include "miniupnpc_declspec.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* strupnperror() + * Return a string description of the UPnP error code + * or NULL for undefinded errors */ +MINIUPNP_LIBSPEC const char * strupnperror(int err); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/contrib/miniupnp/miniupnpc/upnpreplyparse.c b/src/contrib/miniupnp/miniupnpc/upnpreplyparse.c new file mode 100644 index 0000000..68a47c0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/upnpreplyparse.c @@ -0,0 +1,196 @@ +/* $Id: upnpreplyparse.c,v 1.19 2015/07/15 10:29:11 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include + +#include "upnpreplyparse.h" +#include "minixml.h" + +static void +NameValueParserStartElt(void * d, const char * name, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + data->topelt = 1; + if(l>63) + l = 63; + memcpy(data->curelt, name, l); + data->curelt[l] = '\0'; + data->cdata = NULL; + data->cdatalen = 0; +} + +static void +NameValueParserEndElt(void * d, const char * name, int namelen) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + struct NameValue * nv; + (void)name; + (void)namelen; + if(!data->topelt) + return; + if(strcmp(data->curelt, "NewPortListing") != 0) + { + int l; + /* standard case. Limited to n chars strings */ + l = data->cdatalen; + nv = malloc(sizeof(struct NameValue)); + if(nv == NULL) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "NameValueParserEndElt"); +#endif /* DEBUG */ + return; + } + if(l>=(int)sizeof(nv->value)) + l = sizeof(nv->value) - 1; + strncpy(nv->name, data->curelt, 64); + nv->name[63] = '\0'; + if(data->cdata != NULL) + { + memcpy(nv->value, data->cdata, l); + nv->value[l] = '\0'; + } + else + { + nv->value[0] = '\0'; + } + nv->l_next = data->l_head; /* insert in list */ + data->l_head = nv; + } + data->cdata = NULL; + data->cdatalen = 0; + data->topelt = 0; +} + +static void +NameValueParserGetData(void * d, const char * datas, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + if(strcmp(data->curelt, "NewPortListing") == 0) + { + /* specific case for NewPortListing which is a XML Document */ + data->portListing = malloc(l + 1); + if(!data->portListing) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "NameValueParserGetData"); +#endif /* DEBUG */ + return; + } + memcpy(data->portListing, datas, l); + data->portListing[l] = '\0'; + data->portListingLength = l; + } + else + { + /* standard case. */ + data->cdata = datas; + data->cdatalen = l; + } +} + +void +ParseNameValue(const char * buffer, int bufsize, + struct NameValueParserData * data) +{ + struct xmlparser parser; + memset(data, 0, sizeof(struct NameValueParserData)); + /* init xmlparser object */ + parser.xmlstart = buffer; + parser.xmlsize = bufsize; + parser.data = data; + parser.starteltfunc = NameValueParserStartElt; + parser.endeltfunc = NameValueParserEndElt; + parser.datafunc = NameValueParserGetData; + parser.attfunc = 0; + parsexml(&parser); +} + +void +ClearNameValueList(struct NameValueParserData * pdata) +{ + struct NameValue * nv; + if(pdata->portListing) + { + free(pdata->portListing); + pdata->portListing = NULL; + pdata->portListingLength = 0; + } + while((nv = pdata->l_head) != NULL) + { + pdata->l_head = nv->l_next; + free(nv); + } +} + +char * +GetValueFromNameValueList(struct NameValueParserData * pdata, + const char * Name) +{ + struct NameValue * nv; + char * p = NULL; + for(nv = pdata->l_head; + (nv != NULL) && (p == NULL); + nv = nv->l_next) + { + if(strcmp(nv->name, Name) == 0) + p = nv->value; + } + return p; +} + +#if 0 +/* useless now that minixml ignores namespaces by itself */ +char * +GetValueFromNameValueListIgnoreNS(struct NameValueParserData * pdata, + const char * Name) +{ + struct NameValue * nv; + char * p = NULL; + char * pname; + for(nv = pdata->head.lh_first; + (nv != NULL) && (p == NULL); + nv = nv->entries.le_next) + { + pname = strrchr(nv->name, ':'); + if(pname) + pname++; + else + pname = nv->name; + if(strcmp(pname, Name)==0) + p = nv->value; + } + return p; +} +#endif + +/* debug all-in-one function + * do parsing then display to stdout */ +#ifdef DEBUG +void +DisplayNameValueList(char * buffer, int bufsize) +{ + struct NameValueParserData pdata; + struct NameValue * nv; + ParseNameValue(buffer, bufsize, &pdata); + for(nv = pdata.l_head; + nv != NULL; + nv = nv->l_next) + { + printf("%s = %s\n", nv->name, nv->value); + } + ClearNameValueList(&pdata); +} +#endif /* DEBUG */ + diff --git a/src/contrib/miniupnp/miniupnpc/upnpreplyparse.h b/src/contrib/miniupnp/miniupnpc/upnpreplyparse.h new file mode 100644 index 0000000..6badd15 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/upnpreplyparse.h @@ -0,0 +1,63 @@ +/* $Id: upnpreplyparse.h,v 1.19 2014/10/27 16:33:19 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPREPLYPARSE_H_INCLUDED +#define UPNPREPLYPARSE_H_INCLUDED + +#ifdef __cplusplus +extern "C" { +#endif + +struct NameValue { + struct NameValue * l_next; + char name[64]; + char value[128]; +}; + +struct NameValueParserData { + struct NameValue * l_head; + char curelt[64]; + char * portListing; + int portListingLength; + int topelt; + const char * cdata; + int cdatalen; +}; + +/* ParseNameValue() */ +void +ParseNameValue(const char * buffer, int bufsize, + struct NameValueParserData * data); + +/* ClearNameValueList() */ +void +ClearNameValueList(struct NameValueParserData * pdata); + +/* GetValueFromNameValueList() */ +char * +GetValueFromNameValueList(struct NameValueParserData * pdata, + const char * Name); + +#if 0 +/* GetValueFromNameValueListIgnoreNS() */ +char * +GetValueFromNameValueListIgnoreNS(struct NameValueParserData * pdata, + const char * Name); +#endif + +/* DisplayNameValueList() */ +#ifdef DEBUG +void +DisplayNameValueList(char * buffer, int bufsize); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpc/wingenminiupnpcstrings.c b/src/contrib/miniupnp/miniupnpc/wingenminiupnpcstrings.c new file mode 100644 index 0000000..50df06a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpc/wingenminiupnpcstrings.c @@ -0,0 +1,83 @@ +/* $Id: wingenminiupnpcstrings.c,v 1.4 2015/02/08 08:46:06 nanard Exp $ */ +/* Project: miniupnp + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author: Thomas Bernard + * Copyright (c) 2005-2015 Thomas Bernard + * This software is subjects to the conditions detailed + * in the LICENSE file provided within this distribution */ +#include +#include + +/* This program display the Windows version and is used to + * generate the miniupnpcstrings.h + * wingenminiupnpcstrings miniupnpcstrings.h.in miniupnpcstrings.h + */ +int main(int argc, char * * argv) { + char buffer[256]; + OSVERSIONINFO osvi; + FILE * fin; + FILE * fout; + int n; + char miniupnpcVersion[32]; + /* dwMajorVersion : + The major version number of the operating system. For more information, see Remarks. + dwMinorVersion : + The minor version number of the operating system. For more information, see Remarks. + dwBuildNumber : + The build number of the operating system. + dwPlatformId + The operating system platform. This member can be the following value. + szCSDVersion + A null-terminated string, such as "Service Pack 3", that indicates the + latest Service Pack installed on the system. If no Service Pack has + been installed, the string is empty. + */ + ZeroMemory(&osvi, sizeof(OSVERSIONINFO)); + osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); + + GetVersionEx(&osvi); + + printf("Windows %lu.%lu Build %lu %s\n", + osvi.dwMajorVersion, osvi.dwMinorVersion, + osvi.dwBuildNumber, (const char *)&(osvi.szCSDVersion)); + + fin = fopen("VERSION", "r"); + fgets(miniupnpcVersion, sizeof(miniupnpcVersion), fin); + fclose(fin); + for(n = 0; n < sizeof(miniupnpcVersion); n++) { + if(miniupnpcVersion[n] < ' ') + miniupnpcVersion[n] = '\0'; + } + printf("MiniUPnPc version %s\n", miniupnpcVersion); + + if(argc >= 3) { + fin = fopen(argv[1], "r"); + if(!fin) { + fprintf(stderr, "Cannot open %s for reading.\n", argv[1]); + return 1; + } + fout = fopen(argv[2], "w"); + if(!fout) { + fprintf(stderr, "Cannot open %s for writing.\n", argv[2]); + fclose(fin); + return 1; + } + n = 0; + while(fgets(buffer, sizeof(buffer), fin)) { + if(0 == memcmp(buffer, "#define OS_STRING \"OS/version\"", 30)) { + sprintf(buffer, "#define OS_STRING \"MSWindows/%ld.%ld.%ld\"\n", + osvi.dwMajorVersion, osvi.dwMinorVersion, osvi.dwBuildNumber); + } else if(0 == memcmp(buffer, "#define MINIUPNPC_VERSION_STRING \"version\"", 42)) { + sprintf(buffer, "#define MINIUPNPC_VERSION_STRING \"%s\"\n", + miniupnpcVersion); + } + /*fputs(buffer, stdout);*/ + fputs(buffer, fout); + n++; + } + fclose(fin); + fclose(fout); + printf("%d lines written to %s.\n", n, argv[2]); + } + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/.gitignore b/src/contrib/miniupnp/miniupnpd/.gitignore new file mode 100644 index 0000000..fe899b2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/.gitignore @@ -0,0 +1,26 @@ +*.o +*.bak +config.h +ipfw/testipfwrdr +miniupnpd +miniupnpdctl +testgetifaddr +testgetifstats +testupnpdescgen +testupnppermissions +testgetroute +testasyncsendto +testportinuse +netfilter/testiptcrdr +netfilter/testiptcrdr_dscp +netfilter/testiptcrdr_peer +testdescs +validateupnppermissions +validategetifaddr +testssdppktgen +validatessdppktgen +.depend +pf/testobsdrdr +pf/testpfpinhole +netfilter/test_nfct_get +testminissdp diff --git a/src/contrib/miniupnp/miniupnpd/Changelog.txt b/src/contrib/miniupnp/miniupnpd/Changelog.txt new file mode 100644 index 0000000..7dd3b02 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/Changelog.txt @@ -0,0 +1,1270 @@ +$Id: Changelog.txt,v 1.439 2018/04/12 09:32:22 nanard Exp $ + +2018/05/02: + option to store remaining time in leasefile + +2018/04/12: + pf: set dst address in rule if use_ext_ip_addr is set + +2018/04/06: + Add options for netfilter scripts + +2018/03/13: + Use monotonic clock for timeouts, etc. + +2018/02/22: + Add option force_igd_desc_v1 to force devices and services versions + to 1 in IGD v2 mode + +2017/12/12: + Fix a few buffer overrun in SSDP and SOAP parsing + +2017/11/02: + PCP : reset epoch after address change + +2017/05/26: + merge https://github.com/miniupnp/miniupnp/tree/randomize_url branch + +2017/05/24: + get SSDP packet receiving interface index and use it to check if the + packet is from a LAN + +2017/03/13: + default to client address for AddPortMapping when + is empty + pass ext_if_name to add_pinhole() + +2016/12/23: + Fix UDA-1.2.10 Man header empty or invalid + +2016/12/16: + Do not try to open IPv6 sockets once it is disabled + +2016/12/01: + Fix "AddPinhole Twice" test + +2016/11/11: + fixes build for Solaris/SunOS + +2016/07/23: + fixes build error on DragonFly BSD + +VERSION 2.0 : released on 2016/04/19 + +2016/04/18: + linux/netfilter: fix compile time detection of iptables version >= 1.4.3 + +2016/03/08: + linux/netfilter: do not add MASQUERADE rule if ports are equals + +2016/02/19: + set IPv6 Hop limit to 10 + fix HOST: header of event notifications in IPv6 + be more compliant on 64bit machines : ui4 in [0;2^32-1] + +2016/02/16: + minor changes to follow UDA 1.1 more closely. + more argument checking in Soap methods. + +2016/02/12: + return error 729 - ConflictWithOtherMechanisms if IGD v2 is enabled. + add iptc_init() check in iptcrdr.c/init_redirect() + add update_portmapping() / update_portmapping_desc_timestamp() functions + +2016/02/11: + use Linux libuuid uuid_generate() / BSD uuid_create() API + +2016/01/28: + renamed iptables chain MINIUPNPD-PCP-PEER to MINIUPNPD-POSTROUTING + implemented "IGD2 Port Triggering" with netfilter/iptables + +2016/01/18: + fix pcpserver.c CreatePCPMap_FW() : check pinhole before adding + +2015/12/16: + improve syslog message for incoming HTTP requests + +2015/12/13: + --disable-pppconn to disable WanPPPConnection + more fixes in DeviceProtection service + +2015/12/12: + add commandline option to genconfig.sh to set UPnP (UDA) version + advertise correct service and device versions when IGDv2 is enabled + fix action arguments for DeviceProtection service + fix event subscription renewal (include SID in response) + +2015/11/16: + Fix bsd/getroute.c get_src_for_route_to() when args are NULL + +2015/11/02: + use LOG_INFO instead of LOG_ERR for PCP PEER and MAP success + +2015/10/30: + fix : properly call find_ipv6_addr() with the 1st LAN interface + use name server from query in SOAP responses (continued) + +2015/10/24: + move SSDP_PACKET_MAX_LEN definition to config.h. also set default to 1024. + +2015/09/22: + cleanup UPNP_VERSION macro / add UPNP_VERSION_MAJOR, UPNP_VERSION_MINOR + Don't use packed structs anymore to read/write PCP messages + +2015/09/15: + use name server from query in SOAP responses + +2015/09/14: + Randomize URLs to avoid http://www.filet-o-firewall.com/ + https://github.com/filetofirewall/fof + (specific branch, merged later, see above) + +2015/08/25: + better bind socket to right interface(s), + using struct ip_mreqn, SO_BINDTODEVICE + +2015/04/30: + Adding linux/nftables support + +2015/04/26: + Remove dependency to libnfnetlink + fix typos in miniupnpd.conf + +2015/03/09: + fix get_portmappings_in_range() for linux/netfilter + +2015/03/07: + don't die when IPv6 is enabled and interface has no IPv4 address + +2015/02/10: + IP wildcard for AddPinhole() is empty string + +2014/12/10: + Checking Host: HTTP request header to prevent DNS rebinding attack + configurable BOOTID.UPNP.ORG SSDP header + use time for BOOTID.UPNP.ORG value + +2014/12/09: + fix upnp_add_inboundpinhole() : check inet_pton() return + fix upnp_redirect() : check inet_aton() return + fix potential memory corruption in upnpsoap.c/GetListOfPortMappings() + fix buffer overrun in ParseHttpHeaders() if Content-Length doesn't contain any digit ! + check if BuildHeader_upnphttp() failed to allocate memory + Credits goes to Stephen Röttger of the Google Security Team for identifying + the vulnerabilities + +2014/12/04: + check "sysctl -n net.ipv6.bindv6only" for linux + +2014/11/28: + fixes ExecuteSoapAction if SoapAction value is not enclosed into + double quotes + +2014/11/07: + sockaddr_to_string() includes scope in IPv6 addresses + +VERSION 1.9 : released on 2014/10/27 + +2014/10/23: + Properly implements NAT-PMP mapping removal according to RCF6886 + +2014/10/22: + Discard NAT-PMP packets coming from the WAN + Send SSDP announces to IPv6 link-local, site-local + and global multicast addresses + +2014/10/21: + small modifications to compile with exotic C libraries + +2014/10/14: + add comments in miniupnpd.conf regarding security + +2014/09/25: + DeletePortMapping now checks for client IP in Securemode + +2014/06/xx: + Various fixes : + e->ipv6.flags |= IP6T_F_PROTO; (netfilter) + fix natpmp.c byte order conversion + add small delay before SSDP response to prevent flooding + +2014/05/22: + Add ipv6_bind_address (option "ipv6_listening_ip") + disable IPv6 when socket(PF_INTET6, ...) errors with EAFNOSUPPORT + Add IPV6 multicast membership only on selected "LAN" interfaces + +2014/05/20: + be more strict when parsing LAN addresses / interface names + +2014/05/19: + set source address for IPV6 packets sendto_schedule2() etc. + +2014/05/15: + Fix deletePortMappingRange() + +2014/04/21: + Fix PCP when request contain 0 IPv4 external address + Remove pointer casting in natpmp.c + +2014/04/15: + rewrite iptables_*.sh scripts + +2014/04/12: + Add FreeBSD support for CHECK_PORTINUSE + Add PCP support for CHECK_PORTINUSE + +2014/04/09: + Add HTTPS support and skeleton of DeviceProtection implementation + +2014/03/24: + start work to enable IPv6 PCP operations + +2014/03/14: + reject renewal of subscribtion that already timeouted + Support for multiple URL in Callback: header (SUBSCRIBE) + +2014/03/13: + fix getifaddr_in6() (used for PCP) + implement permissions with PCP Map + fix upnp_event_notify_connect() when ENABLE_IPV6 is set + +2014/03/10: + Enable PCP by default. + Work in IPv6 on system where PF_INET6 are restricted to IPv6 only + change ipv6_enabled/ipv6fc_inbound_pinhole_allowed/ipv6fc_firewall_enabled + global vars to flags in runtime_flags + +2014/03/09: + IPv6 support in testgetifaddr + +2014/03/07: + NAT-PMP search an allowed eport instead of returning an error + if the original eport is not allowed. + +2014/03/06: + Fix add_filter_rule2() for pf. + +2014/02/28: + log message when shutting down + natpmp : avoid hang when all external ports in use + +2014/02/25: + add implementation of scheduled sendto (asyncsendto) in order + to retry failed sendto() calls or schedule sending of packets + +2014/02/24: + Defaulting to SSDP_RESPOND_SAME_VERSION + +2014/02/11: + Fix PCP Map renewal + +2014/02/06: + possibility to disable ipv6 at runtime + +2014/02/03: + PCP : Add support for ANNOUNCE requests + minixml now handle XML comments + +2013/12/16: + Attempt to compile with OS X/pf + +2013/12/13: + Make all manufacturer info configurable thanks to Leo Moll + Merge PCP support (see https://github.com/miniupnp/miniupnp) + +2013/06/13: + Have 3 UUID for the 3 devices (IGD, WAN Device, WAN Connection Device) + +2013/06/06: + update upnpreplyparse to allow larger values (128 chars instead of 64) + +2013/06/05: + check Service ID in SetDefaultConnectionService method + Don't advertise WANPPPConnection in UPNP_STRICT mode + +2013/05/29: + Remove namespace from variable name elements in Events "propertyset" + to comply with UPNP DeviceArchitecture v1.1. + +2013/05/20: + Adding support for IP Filter version 5.x + +2013/05/16: + refuses non integer values + +2013/05/14: + Update upnpreplyparse to take into account "empty" elements + +2013/05/03: + Use pkg-config under linux to find libiptc. Thanks to Olivier Langlois + +2013/04/29: + Add warning message when using IPv4 address for listening_ip with IPv6 enabled + +2013/04/27: + Uses ifr_addr if ifr_netmask is not defined in struct ifreq + +2013/04/26: + Correctly handle truncated snprintf() in SSDP code + +2013/04/24: + to avoid build race conditions, genconfig.sh now uses a temporary file + +2013/04/20: + use scope in get_lan_for_peer() for IPv6 addresses + +2013/03/23: + autodetect LAN interface netmask instead of defaulting to /24 + +2013/02/11: + Use $(DESTDIR) in Makefile.linux. + see https://github.com/miniupnp/miniupnp/issues/26 + +2013/02/07: + Add DATE: header in SSDP packets + Fix SSDP packets sent with uuid as ST: header to conform to UDA + ignore SSDP packets missing the MX: header in UPNP_STRICT mode + Added Ext: header to HTTP responses to conform to UDA + Refactored SendSSDPNotifies() and SendSSDPGoodbye() and add + missing ssdp:alive and ssdp:byebye with NT uuid value. + +VERSION 1.8 : released on 2013/02/06 + +2013/02/06: + Check source address of incomining HTTP connections and SSDP + packets in order to filter out WAN SSDP and HTTP traffic. + Implement get_src_for_route_to() for *BSD + fix 2 potential memory leaks in GetListOfPortMappings() + +2013/01/29: + upnphttp.c: Fix and comment the findendheaders() function + upnphttp.c: remove strchr() call in ParseHttpHeaders() + add comments to explain how buffer is checked before calls + to ParseHttpHeaders() + +2013/01/27: + upnphttp.c: ParseHttpHeaders() now checks atoi() return + +2012/12/11: + More return value check for malloc() and realloc() + +2012/10/23: + minor modifications to linux/getroute.c and testgetroute.c + +2012/10/04: + updated DEFAULTCONNECTIONSERVICE_MAGICALVALUE for IGDv2 + increased default buffer size for HTTP response + More argument check for SOAP actions in UPNP_STRICT mode + Better error checking after connect() in upnpevent + +2012/10/03: + Fix atoi() on null pointer in upnpsoap.c + properly set service/device version in SSDP messages + fix newSubscriber() for IP6FirewallControl and DeviceProtection services + Enforce compliance for SUBSCRIBE messages (UPNP_STRICT mode) + Enforce compliance for UNSUBSCRIBE messages (UPNP_STRICT mode) + Ignore "-Wmissing-field-initializers" in upnpdescgen.c + check size of h->res_buf before building HTTP response + ENABLE_HTTP_DATE : add a Date: header to all HTTP responses + +2012/09/27: + Fixes with DISABLE_CONFIG_FILE + and UPNP_STRICT + UPC must be a 12 decimal digit code + SetDefaultConnectionService() checks its argumnents in UPNP_STRICT mode + Support for Accept-Language/Content-Language HTTP headers + Content-Type is now text/xml; charset="utf-8" to conform with UDA v1.1 + Support Expect: 100-continue for POST HTTP requests + Manage services/devices versions in minissdp.c + Rename all include guards to not clash with C99. + (7.1.3 Reserved identifiers) + +2012/09/20: + Cleaning code in ipfw (Jardel Weyrich) + +2012/09/18: + Fixing a bug in clean_pinhole_list() under linux/netfilter + +2012/09/15: + Adding an informational message at startup + +2012/08/24: + Moved man page to section 8. miniupnpd.1 => miniupnpd.8 + Added install of miniupnpd.8 man page in Makefile.linux + +2012/08/10: + improved SubmitServicesToMiniSSDPD() function fiability + +2012/07/17: + Add -A command line option to add permission rules + +2012/07/14: + Add -z command line option to change friendly name (thanks to Shawn Fisher) + +2012/07/10: + Detect port in use - patch by David Kerr + +2012/06/29: + added DISABLE_CONFIG_FILE in options.h to disable miniupnpd.conf parsing + Add command line parsing for clean_ruleset_interval option + +2012/06/28: + Only activate -L option for PF and IPF + -a option takes two arguments with MULTIPLE_EXTERNAL_IP defined + +2012/06/23: + in UPNP_STRICT mode, the literal IPv6 address in "location:" of SSDP + messages is the source address used to send the message + +2012/06/08: + Disable -ansi CFLAGS in Makefile.linux because recent iptables headers + make use of typeof keyword which is a GCC extension. + +2012/05/31: + Improvements in autodetecting firewall under (Free)BSD + +2012/05/28: + Cleanup HTTP request handling. Answer 405 when relevant + +VERSION 1.7 : released the 2012/05/28 + +2012/05/28: + clean linux/ifacewatcher.c + set natpmp socket non blocking + +2012/05/24: + More solaris fixes + +2012/05/21: + Clean signal handling + +2012/05/08: + Clean expired IPv6 pinholes correctly. and also with linux/netfilter. + +2012/05/07: + Finalizing netfilter version of get_pinhole_info() + +2012/05/01: + Move IPv6FirewallControl related code from upnpredirect.c to upnppinhole.c + Add netfilter implementation for + delete_pinhole()/update_pinhole()/get_pinhole_info() + +2012/04/30: + Clean up settings of CFLAGS in Makefile's + Remove Warnings caused by signed/unsigned integer comparaisons + Also fix a couple of integer/pointer comparaisons. + Add UNUSED(arg) macro to remove unused argument warning. + Fix error handling in upnpevents.c (was causing segfault on Solaris !) + +2012/04/26: + Started to implement add_pinhole() for netfilter (linux) + +2012/04/25: + Fixed a bug in upnphttp that happened when POST is received in several + recv() calls and realloc() is called so the buffer used is moved. + +2012/04/23: + Implement CheckPinholeWorking GetPinholePackets. WANIPv6FirewallControl + UpdatePinhole still to be done. And also netfilter/ipf/ipfw versions + +2012/04/20: + Enough WANIPv6FirewallControl is implemented on pf so that AddPinhole() and + DeletePinhole() works ! + +2012/04/19: + First working experiment of IPv6 "pinhole" with pf + +2012/04/15: + More C++ => ANSI C comments to compile with -ansi option + Add command line arguments to genconfig.sh config script. + +2012/04/12: + Set TTL on SSDP Notify sockets (IPv4). TTL is set to 2 (recommendation from + UPnP Device Architecture v1.1) + +2012/04/06: + Implementing IPv6 support : + Send SSDP NOTIFY ssdp:alive and ssdp:goodbye messages in IPv6. + Use UPnP/1.1 in SERVER: string as required in UPnP Device architecture 1.1. + Allow LAN interface to be given as interface names, instead of interface + IP addresses. It will allow IPv6 operations. + fix linux/getifstats.c when bitrate is unknown + +2012/03/31: + Only remove pidfile if one was written in the first place. + +2012/03/19: + Fix ipfilter support (thanks dhowland https://github.com/dhowland) + +2012/03/14: + Changes to miniupnpd.init.d.script by Shawn Landden + +2012/03/05: + fixed reload_from_lease_file(). + +2012/02/15: + Change parselanaddr() function to allow 192.168.1.1/255.255.255.0 in + configuration file. + Change read_permission_line() to allow 192.168.1.1/255.255.255.0 in + permission line (in configuration file). + +2012/02/12: + More syntax checks in upnppermissions.c + +2012/02/11: + Fix ipfw/Mac OS X specific source files to compile ok with -ansi flag + +2012/02/09: + Make HTTP listen socket non blocking (so accept() can't block) + Make SSDP receive sockets non blocking + use sockaddr_to_string() in SendSSDPAnnonce2 to handle IPv6 addresses + +2012/02/06: + Make HTTP (SOAP) sockets non blocking. + +2012/02/05: + Compile ok with -ansi flag. + Save a few bytes in options.c using a string repository, instead of a fixed size + buffer for each option value. + +2012/02/04: + Added friendly_name= option to config file + +2012/02/03: + Anchor name (PF) is now configurable through the config file with anchor= + Added test of presence of /lib/libip4tc.so and /lib/libip6tc.so files in + Makefile.linux in order to add -lip4tc and -lip6tc to LIBS accordingly. + +2012/02/01: + always handle EAGAIN, EWOULDBLOCK and EINTR after recv()/recvfrom() calls + +2012/01/20: + Always #include before #include (for OpenBSD) + .onrdomain field was added in pf with OpenBSD 5.0. Add PFRULE_HAS_ONRDOMAIN + +2012/01/02: + Fixing netfilter/iptables_*.sh scripts for new ifconfig output format. + getifaddr.c: added additional checks on structure returned by getifaddrs() + Fixing Mac OS X makefile for installation + +2011/11/18: + avoid infinite loop in SendResp_upnphttp() in case of error + Replaced SendResp_upnphttp() + CloseSocket_upnphttp() by + SendRespAndClose_upnphttp() + Tomato specifics in genconfig.sh + +2011/07/30: + netfilter : Added a tiny_nf_nat.h file to compile with iptables + installed headers. + include xtables.h instead of iptables.h + +VERSION 1.6 : released the 2011/07/25 + +2011/07/25: + Update doc for version 1.6 + +2011/07/15: + Fixing code with MULTIPLE_EXTERNAL_IP defined. + +2011/06/27: + IPv6 support for UPnP events. + Security checks in UPnP events. + +2011/06/22: + Remote host for GetListOfPortMappings + Remote host support for ipfw (tested on Mac OS X) + +2011/06/20: + support for iptables-1.4.11.1 + +2011/06/18: + Remote host support for pf version + +2011/06/04: + Supporting RemoteHost (mandatory in IGD v2) + +2011/06/03: + Enabling events by default + +2011/06/01: + Fixing Timeout missing in SUBSCRIBE renewal responses + (thanks to Pranesh Kulkarni) + Added comments about changes between IGD v1 and IGD v2 + +2011/05/28: + Description and leaseduration kept in ipfw version of the code. + Fixing ipfw code after testing under Mac OS X 10.6.7 (darwin 10.7.0) + +2011/05/27: + Finishing and testing LeaseDuration support under OpenBSD. + Changing NAT-PMP port mapping lifetime support to match + lease duration support. + NAT-PMP address change announce broadcasted to both port + 5350 and 5351 to be compatible with client following the + version of NAT PMP specification from 2008 or earlier. + writepidfile() Overwrite file if already existing + +2011/05/26: + fix in linux/getifstats.c. + See http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=2212 + Implementation of LeaseDuration support. + +2011/05/23: + added get_wan_connection_status_str() + +2011/05/20: + adding ifacewatcher thanks to Alexey Osipov + GET /DP.xml is now available. The description has to be completed. + +2011/05/19: + Add getconnstatus.c/.h. Don't always have ConnectionStatus to "Connected" + Events for WANIPv6FirewallControll + +2011/05/16: + patches for gentoo linux. + generation of the DeviceProtection service description. + +2011/05/15: + Making the SSDP receiving socket work in IPv6 ! + +2011/05/14: + Support for HTTP in both IPv6 and IPv4. + IPv6 for SSDP receiving socket. + +2011/05/13: + add new options in genconfig.sh (IGD_V2, ENABLE_DP_SERVICE) + add global vars ipv6fc_firewall_enabled and ipv6fc_inbound_pinhole_allowed + have MACROS for magical values in upnpdescgen.c, add eventing vars for WanIPv6FirewallControl. + applied 0001-Cosmetic-changes.patch(see http://miniupnp.tuxfamily.org/forum/viewtopic.php?t=764) + applied 0002-Remove-lan-addresses-limit-by-changing-storage-type-.patch + replaced some of the urn:schemas-upnp-org:device:* literal strings by macros. + adding some support for IP v6. #define ENABLE_IPV6 + added -fno-strict-aliasing to compile options. + +2011/05/09: + updating upnp descriptions for IGDv2 + +2011/05/07: + Adding WANIPv6FirewallContro to upnp description + +2011/04/30: + adding a UPNP_STRICT config macro. Use it now for checking RemoteHost. + ENABLE_6FC_SERVICE : add the implementations of WANIPv6FirewallControl actions + +2011/04/11: + preparing getifaddr() for IP v6 + preparing SSDP stuff for IP v6. Trying to conform to UDA v1.1 + +2011/03/09: + Some modifications thanks to Daniel Dickinson to improve OpenWRT + build. + Fixed some warnings. + +2011/03/03: + Added code to generate devices/services descriptions for IGD v2 + (to be continued) + +2011/03/02: + improved netfilter/delete_redirect_and_filter_rules() in order + to remove the right filter rule, even if it has another index than + the nat rule. + +2011/03/01: + clean up an fixes to make netfilter/testiptcrdr compile + +2011/02/21: + Make "Makefile" work under Mac OS X with bsdmake. + added get_portmappings_in_range() in ipfwrdr.c + +2011/02/07: + added get_portmappings_in_range() / upnp_get_portmappings_in_range() + +2011/02/06: + Implementation of GetListOfPortMappings + +2011/01/27: + Reverting "fixes" done in linux/iptables code the 2010/09/27. + see http://miniupnp.tuxfamily.org/forum/viewtopic.php?t=741 + +2011/01/04: + added MINIUPNPD_VERSION in config.h. Taken from VERSION file. + +VERSION 1.5 : released the 2011/01/01 + +2011/01/01: + Started to implement some of the new methods from WANIPConnection v2 + +2010/09/27: + Some fixes in the linux/iptables code when + miniupnpd_nat_chain <> miniupnpd_forward_chain + +2010/09/21: + Patch to support nfqueue thanks to Colin McFarlane + +2010/08/07: + Update Mac OS X / ipfw stuff from Jardel Weyrich + Fix in Makefile.linux for x86_64 + +2010/05/06: + Bugfix un CleanNATPMPRules() : see http://miniupnp.tuxfamily.org/forum/viewtopic.php?t=640 + +2010/03/14: + Fixing natpmp sockets. + +2010/03/08: + Fix Makefile.linux to compile properly under Mandriva/rh/Fedora with + Iptables >= 1.4.3 + Workaround for bad uptime when started with a bad time set. + +2010/03/07: + Tried to make a OpenBSD version 4.7 compatible code... still some + issues. + +2010/03/06: + updates to testobsdrdr + +2010/03/03: + -lip4tc in Makefile.linux. + +2010/02/15: + some more error handling in set_startup_time() + silencing some warnings + +2010/01/14: + Open Several sockets for NAT-PMP to make sure the source address + of NAT-PMP replies is right. + see http://miniupnp.tuxfamily.org/forum/viewtopic.php?t=609 + +2009/12/31: + miniupnpdctl now output command line arguments. + added a -h option to get help. improved help. + +2009/12/22: + using PRIu64 format to printf u_int64_t + Fixing calls to get_redirect_rule_by_index() : ifname should be initialized. + Add header lines to miniupnpdctl output + +2009/11/06: + implementing sending of ip address change notification when receiving + the signal SIGUSR1 + +VERSION 1.4 : released the 2009/10/30 + +2009/10/10: + Integrate IPfilter patch from Roy Marples. + Fix Netfilter code for old netfilter : + see http://miniupnp.tuxfamily.org/forum/viewtopic.php?t=584 + trim the description string in reload_from_lease_file() + +2009/09/21: + Fixing unclosed raw sockets bug with netfilter code. + +2009/09/04: + Fixes in ipf code thanks to Roy Marples + Enable DragonFly BSD Support thanks to Roy Marples. + Allow packager to define default location of config file via CFLAGS + Respect $DESTDIR when installing + +2009/08/20: + Adding some support for MacOS X and IPFW + SO_REUSEADDR in minissdp.c for SSDP listening socket + +2009/06/05: + unlink lease file in reload_from_lease_file() + +2009/05/16: + Fixed a buffer overflow in ProcessSSDPRequest() + +2009/05/11: + improving genconfig.sh for NetBSD : detecting use of pf or ipf + +VERSION 1.3 : +2009/04/17: + working support for iptables >= 1.4.3 + +2009/04/13: + work to support iptables-1.4.3 and up + +2009/04/10: + fix in upnpevents_removeSubscriber() + +2009/02/14: + added reload_from_lease_file() + +2009/02/13: + Changes in upnpdescgen.c to allow to remove empty elements + strcasecmp instead of strcmp on path comparaisons to allow + bugged clients to work + +2009/01/29: + Some minor changes to Makefile + improving Makefile.linux in order to build with iptables not properly + installed on the system. + +2009/01/23: + Fixing upnpevents thanks to Justin Maggard + +2008/10/15: + getifstats() return -1 when supplied with bad arguments + +2008/10/11: + Fixed NAT-PMP response when IP not allocated to external interface + +2008/10/09: + adding testgetifaddr + Reporting Unconnected status when the "external interface" has + no IP address assigned. Also added some comments + +VERSION 1.2 : + +2008/10/07: + updating docs + +2008/10/06: + MiniUPnPd is now able to use MiniSSDPd to manage SSDP M-SEARCH answering + +2008/10/03: + You can now let miniupnpd choose itself the HTTP port used. + +2008/10/01: + Improvements in genconfig.sh for detecting ipf or pf (under FreeBSD) + and improve debian/ubuntu stuff. + custom chain name patch from : + http://miniupnp.tuxfamily.org/forum/viewtopic.php?t=493 + +2008/08/24: + added USE_IFNAME_IN_RULES macro that can be disabled in order to + remove interface name from rules. + +2008/07/10: + Fixed compilation without ENABLE_L3F_SERVICE + +2008/04/27: + correct UNSUBSCRIBE processing + +2008/04/25(bis): + changed iptables_removeall.sh and iptables_init.sh in order + to remove IP from the rules + +VERSION 1.1 : + +2008/04/25: + Eventing is allmost completly implemented + +2008/04/24: + Correct event handling ? + +2008/04/08: + enabling tag in PF rules. quick can be set off. + +2008/03/13: + implementing event notify + +2008/03/11: + fixing a command line parsing error + +2008/03/09: + optimisations in upnpsoap.c + +2008/03/08: + optimizing upnpsoap.c for size + +2008/03/06: + Worked on the Eventing : generating XML event notifications + Send initial notification after subscribe + Improved pretty print of testupnpdescgen + Reduced Memory usage of upnpdescgen + fixed a small bug in the description + +2008/03/03: + Fixed miniupnpd.c for compiling without natpmp support + fixed presentationURL not there with L3F + fixing lease file creation/modification + +2008/02/25: + Rewrite of Send501() and Send404() + More work on events + genconfig.sh autodetects pf/ipf + +2008/02/24: + Started to implement UPnP Events. do NOT use it at the moment ! + +2008/02/21: + Added support for the Layer3Forwarding Service + added init_redirect() and shutdown_redirect() functions + +2008/02/20: + Removed Ext: HTTP header when useless + enabled the dummy service by default to please windows XP ! + +2008/02/07: + upnp_enable patch by Nikos Mavrogiannopoulos. + lease_file patch by Nikos Mavrogiannopoulos. + +2008/01/29: + some changes to Makefile.openwrt + use daemon() - daemonize() is still available for systems lacking daemon() + +VERSION 1.0 : +2008/01/27: + moved lan_addr to upnpglobalvars.h/.c + Adding experimental multiple external IP support. + +2008/01/22: + removed dummy service from description to improve compatibility + with emule client + Add "secure mode". put runtime flags in the same variable + +2008/01/14: + Fixed a bug in options.c for the parsing of empty lines. + +2008/01/03: + Fixed CleanExpiredNATPMP() + +2008/01/02: + Adding a queue parameter for setting ALTQ in pf + +2007/12/27: + improving some stuff with the PF_ENABLE_FILTER_RULE. + +2007/12/22: + Adding a runtime option to enable/disable NAT-PMP + +2007/12/20: + Added a cache in linux getifstats(). Please enable by editing config.h + +2007/12/14: + Updating an existing NAT-PMP mapping now works + +2007/12/13: + NAT-PMP code now remove expired mappings + TCP/UDP where swapped in NAT-PMP code + +2007/12/04: + Adding details to the error message for sendto(udp_notify) + +2007/11/27: + pf code doesn't generate filter rules by default anymore. The + #ifdef PF_ENABLE_FILTER_RULES must be uncommented in config.h. + +2007/11/02: + moved some of the prototypes common to all firewalls to commonrdr.h + Added functionalities to NAT-PMP + +2007/11/01: + Debugged NAT-PMP code + +2007/10/28: + Cleaning and improving NAT-PMP code + +2007/10/25: + improved the NAT-PMP experimental support + updated README and INSTALL files + +2007/10/24: + Adding support for NAT-PMP (from apple !) + +2007/10/11: + Checking the commandline for errors. + +2007/10/08: + Improved the BSD/Solaris Makefile + Merging last code from Darren Reed. Solaris/IPF should work now ! + added a man page. + +2007/10/07: + Adding Darren Reed code for ipf. + +2007/10/06: + Adding SunOS support thanks to Darren Reed. + Reorganizing os/firewall dependent code thanks to Darren Reed. + +2007/09/27: + linux make install support PREFIX variable + +2007/09/25: + reorganizing LAN sockets/address to improve multi LAN support. + SSDP announces are sent to all configured networks. + SSDP responses are "customized" by subnetwork. + +2007/09/24: + prototype code to remove unused rules + miniupnpdctl now display current rules + synchronised add_filter_rule2() prototype between pf and netfilter code. + +2007/09/19: + Correctly filling the Cache-control header in SSDP packets + +2007/08/28: + update PFRULE_INOUT_COUNTS detection for FreeBSD + +2007/08/27: + update version in genconfig.sh + do not error when a duplicate redirection is requested. + +2007/07/16: + really fixed the compilation bug with linux>=2.6.22 + +2007/07/04: + fixed an error in options.c that prevented to use packet_log option + +2007/07/03: + improved genconfig.sh + fixed a compilation bug with linux>=2.6.22 + +2007/06/22: + added PFRULE_INOUT_COUNTS macro to enable separate in/out packet and + bytes counts in pf for OpenBSD >= 3.8 + +2007/06/15: + removed a possible racecondition in writepidfile() + +2007/06/12: + improved genconfig.sh : no more "echo -e", use lsb_release when available + +2007/06/11: + get_redirect_rule*() functions now return some statistics about + rule usage (bytes and packets) + +2007/06/07: + Fixed the get_redirect_desc() in the linux/netfilter code + +2007/06/05: + Clean up init code in miniupnpd.c + Added a syslog message in SoapError() + +2007/06/04: + Now store redirection descriptions in the linux/netfilter code + +2007/05/21: + Answers to SSDP M-SEARCH requests with ST: ssdp:all + added make install to Makefile.linux + +2007/05/10: + Fixed a bug int the DeletePortMapping linux/netfilter implementation + It was allways the 1st rule that was deleted. + +2007/04/26: + Fixed config.h.openwrt + +2007/04/16: + added something in the INSTALL file about the FreeBSD send(udp_notify) + problem fix (allowing 239.0.0.0/8 explicitely in pf.conf) + +2007/03/30: + added setsockopt(s, SOL_SOCKET, SO_BROADCAST ...) for broadcasting + socket + +2007/03/17: + Fixed filter rule under linux : it was using wrong port ! + thanks to Wesley W. Terpstra + +2007/03/01: + Moved some of the SSDP code from miniupnpd.c to minissdp.c + +2007/02/28: + creating miniupnpdctl + +2007/02/26: + use LOG_MINIUPNPD macro for openlog() + simplify miniupndShutdown() + +2007/02/09: + improved genconfig.h + Added stuff to change the pf rule "rdr" to "rdr pass" + +2007/02/07: + Corrected Bytes per seconds to bits per second. + Ryan cleaned up comments and typos. + Ryan cleaned up daemonize stuff. + Ryan added possibility to configure model number and serial number + +2007/01/30: + ryan improved the robustness of most UPnP Soap methods + I added a target in the Makefiles to properly generate an uuid using + command line tools. + Improved configuration file parsing. + +2007/01/29: + Adding uuid option in miniupnpd.conf + +2007/01/27: + Added upnppermissions stuff : adding some security to UPnP ! + fixed XML description thanks to Ryan Wagoner + improved QueryStateVariable thanks to Ryan Wagoner + +2007/01/22: + use getifaddr() for each GetExtenalIPAddress() Call. + We can change the ip during execution without pb + +2007/01/17: + Lots of code cleanup + +2007/01/12: + Fixed a nasty bug in the linux/netfilter version of get_filter_rule() + +2007/01/11: + Improved the handling of the miniupnpd.conf file. + added -f option to choose which config file to read. + +2007/01/10: + Fixed potential bugs with ClearNameValueList() + +2007/01/08: + All by Ryan Wagoner : + - coding style and comments cleanup + - using now option file miniupnpd.conf + +2007/01/03: + changed "xx active incoming HTTP connections" msg + +2007/01/02: + Patch from Ryan Wagoner : + - no need to open sockets if we can't set the error handlers + - format the usage so it fits nicely on a standard size terminal + - fix up log_err message so they have the same format and you know what + they are related to + - use same "white space" style throughout + - on shutdown no need to continue if opening socket or setsockopt fails + +2006/12/14: + reduce amount of log lines (keeping the same information) + +2006/12/07: + Fixed Makefiles + fixed typos in logs + version 1.0-RC1 released + +2006/12/02: + moved strings from upnpdescgen.c to upnpdescstrings.h for + easier modification + Server: HTTP header now comes from a #define + added a compilation-time generated config.h + +2006/11/30: + minixml updated. should have no impact + Added support for presentationURL with -w switch + implemented getifstats() for linux. Added testgetifstats program + improved error handling in getifstats() BSD + +2006/11/26: + no need to have miniupnpc sources to compile miniupnpd. + Makefile.openwrt updated + Closing sockets on exit thanks to Ryan Wagoner + +2006/11/23: + now handling signal SIGINT + setting HTTP socket with REUSEADDR thanks to Ryan Wagoner + daemon now tested on a Linksys WRT54G device running OpenWRT ! + +2006/11/21: + disabling rtableid in pf code. + +2006/11/22: + Also responds on M-SEARCH with the uuid + +2006/11/20: + gaining some space in upnpsoap.c + +2006/11/19: + Cleaning up code to comply with ANSI C89 + +2006/11/17: + Linux version now deleting both nat and accept rules + implemented -U option under Linux + +2006/11/16: + implemented delete_redirect_rule() for linux + returning error 714 in DeletePortMapping() when needed + +2006/11/12: + The linux/netfilter version should now WORK ! + fix in the writepidfile() function. open with a mode ! + +2006/11/10: + fixing the XML description generation for big endian machines + working on the linux/netfilter port + +2006/11/09: + improved a lot the handling of HTTP error cases + +2006/11/08: + Tried to make the Makefile compatible with both BSDmake + and GNUmake. It was hard because of $^ and $< + +2006/11/07: + Makefile compatible with BSD make + make install target. + getifstats.c compatible with both OpenBSD and FreeBSD. + +2006/11/06: + added getifstats.c for openBSD. May not work under FreeBSD ? + now reports bytes/packets sent/received + reporting bitrates + possibility to report system uptime + +2006/10/29: + added a -L option to enable loggin (is off by default now). + +2006/10/28: + Patch by Ryan Wagoner to correct the XML description (was NewUpTime + instead of NewUptime) and implement uptime. + Trying to fix the memory leak. Added some comments + added a -d option for debugging purpose + Tnaks to valgrind (under linux!) I removed a small memory access error. + +2006/10/27: + Thanks to a patch sent by Michael van Tellingen, miniupnpd is + now ignoring NOTIFY packets sent by other devices and is + writing is own pid to /var/run/miniupnpd.pid + +2006/10/23: + Allways set sendEvents="no" in XML description (was causing + pb with winXP as SUBSCRIBE is not implemented) + +2006/10/22: + added translation from hostname to IP in the AddPortMapping() method + Thanks to Ryan Wagoner. + +2006/10/18: + Added an INSTALL file + +2006/10/13: + Added the possibility to change the notify interval + +2006/09/29: + Improved compliance of the XML Descriptions + pretty print for testupnpdescgen + +2006/09/25: + improved the Error 404 response. + Better serviceType and serviceId for dummy service... + +2006/09/24: + updating the XML description generator + +2006/09/18: + Thanks to Rick Richard, support for SSDP "alive" and "byebye" notifications + was added. The -u options was also added. The SSDP response are now + improved. + The -o option is now working (to force a specific external IP address). + The Soap Methods errors are correctly responded (401 Invalid Action) + +2006/09/09: + Added code to handle filter rules. Thanks to Seth Mos (pfsense.com) + storing the descriptions in the label of the rule + +2006/09/02: + improved the generation of the XML descriptions. + I still need to add allowed values to variables. + +2006/07/29: + filtering SSDP requests and responding with same ST: field + +2006/07/25: + Added a dummy description for the WANDevice + +2006/07/20: + Command line arguments processing + Added possibility to listen internally on several interfaces + diff --git a/src/contrib/miniupnp/miniupnpd/INSTALL b/src/contrib/miniupnp/miniupnpd/INSTALL new file mode 100644 index 0000000..6358183 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/INSTALL @@ -0,0 +1,195 @@ +MiniUPnP project. +(c) 2006-2017 Thomas Bernard +Homepage : http://miniupnp.free.fr/ +Mirror: https://miniupnp.tuxfamily.org/ +github: https://github.com/miniupnp/miniupnp + +miniupnpd is still under developpement. This documentation is +likely to be a little outdated when you read it. So please go on the +web forum https://miniupnp.tuxfamily.org/ if you need more information. + +================================ *BSD/pf ================================= +To Build and Install : + +- use BSD make to compile. +- you can first 'make config.h' then edit config.h to your preferences and + finally 'make' + Alternatively to editing config.h, options can be passed to genconfig.sh + For more details : + > ./genconfig.sh -h +- add "rdr-anchor miniupnpd" or/and "anchor miniupnpd" lines to /etc/pf.conf + (Since OpenBSD 4.7, rdr-anchor lines are no longer used and should be + removed, leaving only the anchor lines). +- some FreeBSD users reported that it is also necessary for them + to explicitly allow udp traffic on 239.0.0.0/8 by adding the two following + lines to /etc/pf.conf : + pass out on $int_if from any to 239.0.0.0/8 keep state + pass in on $int_if from any to 239.0.0.0/8 keep state +- don't forget to " pfctl -f /etc/pf.conf " +- you can check your modifications are taken into account with + "pfctl -s nat" and "pfctl -s rule". Look for the "rdr-anchor miniupnpd" + (if applicable) and/or "anchor miniupnpd" lines. +- OpenBSD users may need to add a multicast_host= line to /etc/rc.conf.local + see $man 8 netstart +- install as root using : + # make install + or + # PREFIX=/usr/local make install +- run as root : The daemon needs rights to modify pf rules. + +=========================== *BSD,*Solaris/ipf ============================= + +genconfig.sh and the Makefile try to detect wether ipf or pf should be +used. If it fails, edit config.h and Makefile by hand. +In Makefile, the FWNAME variable value should be pf or ipf. +Installation steps are allmost the same as with pf. + +*Solaris users would be interested in reading informations from : +http://blogs.sun.com/avalon/category/IPFilter + +============================= Mac OS X/ipfw =============================== + +- To enable non standard compilation options, + > ./genconfig.sh -h + Or edit config.h after it has been generated by genconfig.sh +- use 'bsdmake' (if available) or 'make -f Makefile.macosx' to build + +============================== Mac OS X/pf ================================ + +Starting with Mac OS X 10.7 Lion, pf replaced ipfw as the OS X firewall. +also bsdmake is not available anymore. +Make sure you have installed the Xcode commande line tools (from the +Xcode Preferences menu or using 'xcode-select --install' command) + +You'll need to download xnu sources : https://github.com/opensource-apple/xnu +> INCLUDES="-I.../xnu/bsd -I.../xnu/libkern" make -f Makefile.macosx + +============================ Linux/netfilter ============================== +To Build and install : + +- make sure you have libiptc available on your system : + if you are using debian, "apt-get install iptables-dev" + Some versions of the iptables-dev package don't include the + necessary files : read "how to get libiptc with its headers on debian" below. + In anycase, libiptc is available in iptables sources packages + from http://netfilter.org +- edit and run netfilter/iptables_init.sh shell script. + This script must allways be run before the daemon + to set up initial rules and chains. +- Build and edit the config.h file + > make -f Makefile.linux config.h + > vi config.h +- Build the daemon + > make -f Makefile.linux + If not using iptables from your system, + > IPTABLESPATH=/path/to/iptables-1.4.1 make -f Makefile.linux +- install as root using : + > make -f Makefile.linux install +- A miniupnpd script should be installed to /etc/init.d + and the configuration files to /etc/miniupnpd +- anytime, you can use the netfilter/iptables_flush.sh + script to flush all rules added by the daemon. +- after killing the daemon, you can get back to + iptables initial state by running the netfilter/iptables_removeall.sh + script. Don't forget to edit the script to your convenience. + +NOTE: a /etc/init.d/miniupnpd script will be installed. + If it suits you, you can use is with start, stop or restart argument. + # /etc/init.d/miniupnpd restart + + +How to get libiptc with its headers on debian : +(Note: that should be useless now that netfilter/tiny_nf_nat.h is included) +- Use apt-get to get sources : + > apt-get source iptables + you should then have an iptables-x.x.x/ directory. +- configure and compile : + > cd iptables-x.x.x/ + > ./configure --enable-static + > make +- it is now possible to compile miniupnpd using the following command : + > IPTABLESPATH=/path/to/iptables-x.x.x make -f Makefile.linux + +======================== Linux/netfilter nftables ========================= + +work is in progress. To build : + > make -f Makefile.linux_nft + +see : +https://miniupnp.tuxfamily.org/forum/viewtopic.php?p=4370 +https://github.com/miniupnp/miniupnp/pull/114 + +=========================== Configuration ============================= +Edit the /etc/miniupnpd.conf file to set options. Almost all options are +also available through command line switches. + +A basic configuration would set : +ext_ifname : WAN network interface (interface connected to the internet) +listening_ip : LAN network interface (network where to supply NAT traversal) +enable_natpmp=yes +enable_upnp=yes +and the permission rules (see below). + +Historically, LAN had to be specified by IP/mask, such as +listening_ip=192.168.0.1/24 +but if you compiled with IPv6 support, you need to specify an interface name : +listening_ip=eth0 +The current code assumes there is only one IPv4 address assigned to LAN +interfaces. That is not the case with some CARP setup, there is then a risk +the wrong mask would be picked. You can force the mask when using interface +names : +listening_ip=eth0/24 + +Miniupnpd supports some kind of security check for allowing or disallowing +redirection to be made. The UPnP permission rules are read from the +miniupnpd.conf configuration file. +When a new redirection is requested, permission rules are evaluated in +top-down order and the first permission rule matched gives the response : +redirection allowed or denied. If no rule is matching, the redirection is +allowed, so it is a good practice to have a "catch all" deny permission +rule at the end of your permission ruleset. +Sample permission ruleset : +allow 4662-4672 192.168.1.34/32 4662-4672 +deny 0-65535 192.168.1.34/32 0-65535 +allow 1024-65535 192.168.1.0/24 1024-65535 +deny 0-65535 0.0.0.0/0 0-65535 +With this ruleset, redirections are allowed only for host on the subnet +192.168.1.0/255.255.255.0 for the ports 1024 or above. There is an exception +for the host 192.168.1.34 for which only redirections from/to port 4662 to +4672 are allowed. + +You can generate the uuid for your UPnP device with the uuidgen available +under linux. The following following OpenBSD package is also providing +a "uuid" tool : +http://www.openbsd.org/4.0_packages/i386/uuid-1.5.0.tgz-long.html +An web based uuid generator is also available : +http://kruithof.xs4all.nl/uuid/uuidgen + +On linux systems, one could also use the command +'cat /proc/sys/kernel/random/uuid' to generate an uuid. + +More simple, use the genuuid makefile target : +> make genuuid +or +> make -f Makefile.linux genuuid +This target is needed by the "install" target, so it is done automatically +during install. + +To stop the daemon use : + # kill `cat /var/run/miniupnpd.pid` +or if your linux system use /etc/init.d/ + # /etc/init.d/miniupnpd stop + + +* Signals : +miniupnpd handles the following signals : +SIGUSR1: Send public IP address change notification +SIGUSR2: Handle special actions in Tomato Firmware version + Or rewrite the lease_file +SIGINT: Close gracefully +SIGTERM: Close gracefully +SIGPIPE: Ignore + +There is code to detect change in network interfaces bsd/ifacewatcher.c and +linux/ifacewatcher.c, but if that code doesn't work for you, you may want to +send SIGUSR1 to miniupnpd if you public IP address changed. diff --git a/src/contrib/miniupnp/miniupnpd/LICENSE b/src/contrib/miniupnp/miniupnpd/LICENSE new file mode 100644 index 0000000..c818145 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/LICENSE @@ -0,0 +1,27 @@ +MiniUPnPd +Copyright (c) 2006-2017, Thomas BERNARD +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/contrib/miniupnp/miniupnpd/Makefile b/src/contrib/miniupnp/miniupnpd/Makefile new file mode 100644 index 0000000..a9cd6b4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/Makefile @@ -0,0 +1,274 @@ +# $Id: Makefile,v 1.88 2016/02/10 20:32:43 nanard Exp $ +# MiniUPnP project +# http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ +# Author: Thomas Bernard +# +# Makefile for miniupnpd (MiniUPnP daemon) +# +# This Makefile should work for *BSD and SunOS/Solaris. +# On Mac OS X, use "bsdmake" to build. +# This Makefile is NOT compatible with GNU Make. +# Linux users, please use Makefile.linux : +# make -f Makefile.linux +# +# options can be passed to genconfig.sh through CONFIG_OPTIONS : +# $ CONFIG_OPTIONS="--ipv6 --igd2" make +# + +CFLAGS ?= -pipe -Os +#CFLAGS = -pipe -O -g -DDEBUG +#CFLAGS += -ansi +CFLAGS += -Wall +CFLAGS += -W +CFLAGS += -Wstrict-prototypes +#CFLAGS += -Wdeclaration-after-statement +#CFLAGS += -Wno-missing-field-initializers +CFLAGS += -fno-common +CC ?= gcc +RM = rm -f +MV = mv +INSTALL = install +STRIP = strip + +# OSNAME and FWNAME are used for building OS or FW dependent code. +OSNAME != uname -s +ARCH != uname -m +.ifndef FWNAME +#.if exists(/usr/include/net/pfvar.h) +#FWNAME = pf +#.else +#FWNAME = ipf +#.endif + +.if $(OSNAME) == "OpenBSD" +FWNAME = pf +.endif + +# better way to find if we are using ipf or pf +.if $(OSNAME) == "FreeBSD" +.if exists(/etc/rc.subr) && exists(/etc/default/rc.conf) +FWNAME != . /etc/rc.subr; . /etc/default/rc.conf; \ + if checkyesno ipfilter_enable; then \ + echo "ipf"; elif checkyesno pf_enable; then \ + echo "pf"; elif checkyesno firewall_enable; then \ + echo "ipfw"; else echo "pf"; fi +.else +FWNAME = pf +.endif +.endif + +.if $(OSNAME) == "NetBSD" +.if exists(/etc/rc.subr) && exists(/etc/rc.conf) +FWNAME != . /etc/rc.subr; . /etc/rc.conf; \ + if checkyesno pf; then \ + echo "pf"; elif checkyesno ipfilter; then \ + echo "ipf"; else echo "pf"; fi +.else +FWNAME = pf +.endif +.endif + +.if $(OSNAME) == "DragonFly" +.if exists(/etc/rc.subr) && exists(/etc/rc.conf) +FWNAME != . /etc/rc.subr; . /etc/rc.conf; \ + if checkyesno pf; then \ + echo "pf"; elif checkyesno ipfilter; then \ + echo "ipf"; else echo "pf"; fi +.else +FWNAME = pf +.endif +.endif + +.if $(OSNAME) == "Darwin" +# Firewall is ipfw up to OS X 10.6 Snow Leopard +# and pf since OS X 10.7 Lion (Darwin 11.0) +FWNAME != [ `uname -r | cut -d. -f1` -ge 11 ] && echo "pf" || echo "ipfw" +.endif + +.endif + +# Solaris specific CFLAGS +.if $(OSNAME) == "SunOS" +CFLAGS += -DSOLARIS2=`uname -r | cut -d. -f2` +.if $(ARCH) == "amd64" +CFLAGS += -m64 -mcmodel=kernel -mno-red-zone -ffreestanding +.elif $(ARCH) == "sparc64" +CFLAGS += -m64 -mcmodel=medlow +.endif +.endif + +STDOBJS = miniupnpd.o upnphttp.o upnpdescgen.o upnpsoap.o \ + upnpredirect.o getifaddr.o daemonize.o upnpglobalvars.o \ + options.o upnppermissions.o minissdp.o natpmp.o pcpserver.o \ + upnpevents.o upnputils.o getconnstatus.o \ + upnppinhole.o asyncsendto.o portinuse.o +BSDOBJS = bsd/getifstats.o bsd/ifacewatcher.o bsd/getroute.o +SUNOSOBJS = solaris/getifstats.o bsd/ifacewatcher.o bsd/getroute.o +MACOBJS = mac/getifstats.o bsd/ifacewatcher.o bsd/getroute.o +PFOBJS = pf/obsdrdr.o pf/pfpinhole.o +IPFOBJS = ipf/ipfrdr.o +IPFWOBJS = ipfw/ipfwrdr.o ipfw/ipfwaux.o +MISCOBJS = upnpreplyparse.o minixml.o + +ALLOBJS = $(STDOBJS) $(MISCOBJS) +.if $(OSNAME) == "SunOS" +ALLOBJS += $(SUNOSOBJS) +TESTGETIFSTATSOBJS = testgetifstats.o solaris/getifstats.o +TESTGETROUTEOBJS = testgetroute.o upnputils.o bsd/getroute.o +.elif $(OSNAME) == "Darwin" +ALLOBJS += $(MACOBJS) +TESTGETIFSTATSOBJS = testgetifstats.o mac/getifstats.o +TESTGETROUTEOBJS = testgetroute.o upnputils.o bsd/getroute.o +.else +ALLOBJS += $(BSDOBJS) +TESTGETIFSTATSOBJS = testgetifstats.o bsd/getifstats.o +TESTGETROUTEOBJS = testgetroute.o upnputils.o bsd/getroute.o +.endif + +.if $(FWNAME) == "pf" +ALLOBJS += $(PFOBJS) +.elif $(FWNAME) == "ipfw" +ALLOBJS += $(IPFWOBJS) +.else +ALLOBJS += $(IPFOBJS) +.endif + +TESTUPNPDESCGENOBJS = testupnpdescgen.o upnpdescgen.o +TESTUPNPPERMISSIONSOBJS = testupnppermissions.o upnppermissions.o +TESTGETIFADDROBJS = testgetifaddr.o getifaddr.o +MINIUPNPDCTLOBJS = miniupnpdctl.o +TESTASYNCSENDTOOBJS = testasyncsendto.o asyncsendto.o upnputils.o bsd/getroute.o +TESTPORTINUSEOBJS = testportinuse.o portinuse.o getifaddr.o +TESTMINISSDPOBJS = testminissdp.o minissdp.o upnputils.o upnpglobalvars.o \ + asyncsendto.o bsd/getroute.o + +EXECUTABLES = miniupnpd testupnpdescgen testgetifstats \ + testupnppermissions miniupnpdctl \ + testgetifaddr testgetroute testasyncsendto \ + testportinuse testssdppktgen testminissdp + +.if $(OSNAME) == "Darwin" +LIBS = +.else +LIBS = -lkvm +.endif +.if $(OSNAME) == "SunOS" +LIBS += -lsocket -lnsl -lkstat -lresolv +.endif + +LIBS += -lssl -lcrypto + +# set PREFIX variable to install in the wanted place + +INSTALLBINDIR = $(PREFIX)/sbin +INSTALLETCDIR = $(PREFIX)/etc +# INSTALLMANDIR = $(PREFIX)/man +INSTALLMANDIR = /usr/share/man + +all: $(EXECUTABLES) + +clean: + $(RM) $(STDOBJS) $(BSDOBJS) $(SUNOSOBJS) $(MACOBJS) $(EXECUTABLES) \ + testupnpdescgen.o \ + $(MISCOBJS) config.h testgetifstats.o testupnppermissions.o \ + miniupnpdctl.o testgetifaddr.o testgetroute.o testasyncsendto.o \ + testportinuse.o \ + $(PFOBJS) $(IPFOBJS) $(IPFWOBJS) + $(RM) validateupnppermissions validategetifaddr validatessdppktgen + +install: miniupnpd genuuid + $(STRIP) miniupnpd + $(INSTALL) -d $(DESTDIR)$(INSTALLBINDIR) + $(INSTALL) -m 755 miniupnpd $(DESTDIR)$(INSTALLBINDIR) + $(INSTALL) -d $(DESTDIR)$(INSTALLETCDIR) + $(INSTALL) -b miniupnpd.conf $(DESTDIR)$(INSTALLETCDIR) + # TODO : install man page correctly +# $(INSTALL) -d $(INSTALLMANDIR) +# $(INSTALL) miniupnpd.8 $(INSTALLMANDIR)/cat8/miniupnpd.0 + +# genuuid is using the uuid cli tool available under OpenBSD 4.0 in +# the uuid-1.5.0 package +# any other cli tool returning a uuid on stdout should work. +UUID != if which uuidgen 2>&1 > /dev/null; then \ + echo `uuidgen` ; \ + elif which uuid 2>&1 > /dev/null; then \ + echo `uuid` ; \ + else echo "00000000-0000-0000-0000-000000000000"; \ + fi + +# bash or ksh +SH != which bash || which ksh + +genuuid: + $(MV) miniupnpd.conf miniupnpd.conf.before + sed -e "s/^uuid=[-0-9a-fA-F]*/uuid=$(UUID)/" miniupnpd.conf.before > miniupnpd.conf + $(RM) miniupnpd.conf.before + +check: validateupnppermissions validategetifaddr validatessdppktgen + +validateupnppermissions: testupnppermissions testupnppermissions.sh + $(SH) ./testupnppermissions.sh + touch $@ + +validategetifaddr: testgetifaddr testgetifaddr.sh + ./testgetifaddr.sh + touch $@ + +validatessdppktgen: testssdppktgen + ./testssdppktgen + touch $@ + +depend: config.h + mkdep $(ALLOBJS:.o=.c) testupnpdescgen.c testgetifstats.c \ + testupnppermissions.c miniupnpdctl.c testgetifaddr.c \ + testgetroute.c testportinuse.c testasyncsendto.c \ + testssdppktgen.c + +miniupnpd: config.h $(ALLOBJS) + $(CC) $(LDFLAGS) -o $@ $(ALLOBJS) $(LIBS) + +# BSDmake : +# $(CC) $(LDFLAGS) -o $@ $> $(LIBS) + +miniupnpdctl: config.h $(MINIUPNPDCTLOBJS) + $(CC) $(LDFLAGS) -o $@ $(MINIUPNPDCTLOBJS) + +testupnpdescgen: config.h $(TESTUPNPDESCGENOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTUPNPDESCGENOBJS) $(LIBS) + +testgetifstats: config.h $(TESTGETIFSTATSOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTGETIFSTATSOBJS) $(LIBS) + +testgetifaddr: config.h $(TESTGETIFADDROBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTGETIFADDROBJS) $(LIBS) + +testupnppermissions: config.h $(TESTUPNPPERMISSIONSOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTUPNPPERMISSIONSOBJS) $(LIBS) + +testgetroute: config.h $(TESTGETROUTEOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTGETROUTEOBJS) $(LIBS) + +testasyncsendto: config.h $(TESTASYNCSENDTOOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTASYNCSENDTOOBJS) + +testportinuse: config.h $(TESTPORTINUSEOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTPORTINUSEOBJS) $(LIBS) + +testminissdp: config.h $(TESTMINISSDPOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTMINISSDPOBJS) $(LIBS) + +# gmake : +# $(CC) $(CFLAGS) -o $@ $^ +# BSDmake : +# $(CC) $(CFLAGS) -o $@ $> + +config.h: genconfig.sh VERSION + ./genconfig.sh $(CONFIG_OPTIONS) + +.SUFFIXES: .o .c +.c.o: + $(CC) $(CFLAGS) -c -o $@ $< + +# $(CC) $(CFLAGS) -c -o $(.TARGET) $(.IMPSRC) + + diff --git a/src/contrib/miniupnp/miniupnpd/Makefile.linux b/src/contrib/miniupnp/miniupnpd/Makefile.linux new file mode 100644 index 0000000..e978fd2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/Makefile.linux @@ -0,0 +1,343 @@ +# $Id: Makefile.linux,v 1.95 2017/12/12 11:40:14 nanard Exp $ +# MiniUPnP project +# (c) 2006-2017 Thomas Bernard +# http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ +# Author : Thomas Bernard +# for use with GNU Make +# +# options can be passed to genconfig.sh through CONFIG_OPTIONS : +# $ CONFIG_OPTIONS="--ipv6 --igd2" make -f Makefile.linux +# +# To install use : +# $ DESTDIR=/dummyinstalldir make -f Makefile.linux install +# or : +# $ INSTALLPREFIX=/usr/local make -f Makefile.linux install +# or : +# $ make -f Makefile.linux install +# (default INSTALLPREFIX is /usr) +# +# if your system hasn't iptables libiptc headers and binary correctly +# installed, you need to get iptables sources from http://netfilter.org/ +# ./configure them and build them then miniupnpd will build using : +# $ IPTABLESPATH=/path/to/iptables-1.4.1 make -f Makefile.linux +# +#CFLAGS = -O -g -DDEBUG +CFLAGS ?= -Os +CFLAGS += -fno-strict-aliasing +CFLAGS += -fno-common +CPPFLAGS += -D_GNU_SOURCE +CFLAGS += -Wall +CFLAGS += -Wextra -Wstrict-prototypes -Wdeclaration-after-statement +#CFLAGS += -Wno-missing-field-initializers +#CFLAGS += -ansi # iptables headers does use typeof which is a gcc extension +CC ?= gcc +RM = rm -f +INSTALL = install +STRIP ?= strip +PKG_CONFIG ?= pkg-config +CP = cp + + +INSTALLPREFIX ?= $(PREFIX)/usr +SBININSTALLDIR = $(INSTALLPREFIX)/sbin +ETCINSTALLDIR = $(PREFIX)/etc/miniupnpd +MANINSTALLDIR = $(INSTALLPREFIX)/share/man/man8 + +BASEOBJS = miniupnpd.o upnphttp.o upnpdescgen.o upnpsoap.o \ + upnpreplyparse.o minixml.o portinuse.o \ + upnpredirect.o getifaddr.o daemonize.o upnpglobalvars.o \ + options.o upnppermissions.o minissdp.o natpmp.o pcpserver.o \ + upnpevents.o upnputils.o getconnstatus.o \ + upnppinhole.o pcplearndscp.o asyncsendto.o + +LNXOBJS = linux/getifstats.o linux/ifacewatcher.o linux/getroute.o +NETFILTEROBJS = netfilter/iptcrdr.o netfilter/iptpinhole.o netfilter/nfct_get.o + +ALLOBJS = $(BASEOBJS) $(LNXOBJS) $(NETFILTEROBJS) + +PCFILE_FOUND := $(shell $(PKG_CONFIG) --exists libiptc; echo $$?) + +ifeq (${PCFILE_FOUND},0) + +IPTABLESVERSION := $(shell $(PKG_CONFIG) --modversion libiptc) +IPTVER1 := $(shell echo $(IPTABLESVERSION) | cut -d. -f1 ) +IPTVER2 := $(shell echo $(IPTABLESVERSION) | cut -d. -f2 ) +IPTVER3 := $(shell echo $(IPTABLESVERSION) | cut -d. -f3 ) +# test if iptables version >= 1.4.3 +TEST := $(shell [ $(IPTVER1) -gt 1 ] || \ + [ \( $(IPTVER1) -eq 1 \) -a \ + \( \( $(IPTVER2) -gt 4 \) -o \ + \( \( $(IPTVER2) -eq 4 \) -a \( $(IPTVER3) -ge 3 \) \) \) ] && echo 1 ) +ifeq ($(TEST), 1) +CPPFLAGS += -DIPTABLES_143 +endif + +CFLAGS += $(shell $(PKG_CONFIG) --cflags libiptc) +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l libiptc) +LDFLAGS += $(shell $(PKG_CONFIG) --libs-only-L libiptc) +LDFLAGS += $(shell $(PKG_CONFIG) --libs-only-other libiptc) +else + +ifeq "$(wildcard /etc/gentoo-release )" "" +LDLIBS ?= -liptc +else # gentoo +# the following is better, at least on gentoo with iptables 1.4.6 +# see http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=1618 +# and http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=2183 +LDLIBS ?= -lip4tc +CPPFLAGS := -DIPTABLES_143 $(CPPFLAGS) +endif + +ARCH ?= $(shell uname -m | grep -q "x86_64" && echo 64) +ifdef IPTABLESPATH +CPPFLAGS := $(CPPFLAGS) -I$(IPTABLESPATH)/include/ +LDFLAGS := $(LDFLAFGS) -L$(IPTABLESPATH)/libiptc/ +# get iptables version and set IPTABLES_143 macro if needed +ifeq ($(TARGET_OPENWRT),) +IPTABLESVERSION := $(shell grep "\#define VERSION" $(IPTABLESPATH)/config.h | tr -d \" |cut -d" " -f3 ) +IPTVER1 := $(shell echo $(IPTABLESVERSION) | cut -d. -f1 ) +IPTVER2 := $(shell echo $(IPTABLESVERSION) | cut -d. -f2 ) +IPTVER3 := $(shell echo $(IPTABLESVERSION) | cut -d. -f3 ) +# test if iptables version >= 1.4.3 +TEST := $(shell [ $(IPTVER1) -gt 1 ] || \ + [ \( $(IPTVER1) -eq 1 \) -a \ + \( \( $(IPTVER2) -gt 4 \) -o \ + \( \( $(IPTVER2) -eq 4 \) -a \( $(IPTVER3) -ge 3 \) \) \) ] && echo 1 ) +ifeq ($(TEST), 1) +CPPFLAGS := $(CPPFLAGS) -DIPTABLES_143 +# the following sucks, but works +LDLIBS = $(IPTABLESPATH)/libiptc/.libs/libip4tc.o +#LDLIBS = $(IPTABLESPATH)/libiptc/.libs/libiptc.a +else # ifeq ($(TEST), 1) +LDLIBS = $(IPTABLESPATH)/libiptc/libiptc.a +endif # ifeq ($(TEST), 1) +else # ($(TARGET_OPENWRT),) +# openWRT : +# check for system-wide iptables files. Test if iptables version >= 1.4.3 +# the following test has to be verified : +TEST := $(shell test -f /usr/include/iptables/internal.h && grep -q "\#define IPTABLES_VERSION" /usr/include/iptables/internal.h && echo 1) +ifeq ($(TEST), 1) +CPPFLAGS := $(CPPFLAGS) -DIPTABLES_143 +LDLIBS = -liptc +endif # ($(TEST), 1) +TEST_LIB := $(shell test -f /usr/lib$(ARCH)/libiptc.a && echo 1) +ifeq ($(TEST_LIB), 1) +LDLIBS = -liptc /usr/lib$(ARCH)/libiptc.a +endif # ($(TEST_LIB), 1) +endif # ($(TARGET_OPENWRT),) +else # ifdef IPTABLESPATH +# IPTABLESPATH not defined +# the following test has to be verified : +TEST := $(shell test -f /usr/include/xtables.h && grep -q "XTABLES_VERSION_CODE" /usr/include/xtables.h && echo 1) +ifeq ($(TEST), 1) +CPPFLAGS := $(CPPFLAGS) -DIPTABLES_143 +LDLIBS = -liptc +TESTIP4TC := $(shell test -f /lib/libip4tc.so && echo 1) +ifeq ($(TESTIP4TC), 1) +LDLIBS := $(LDLIBS) -lip4tc +endif # ($(TESTIP4TC), 1) +TESTIP6TC := $(shell test -f /lib/libip6tc.so && echo 1) +ifeq ($(TESTIP6TC), 1) +LDLIBS := $(LDLIBS) -lip6tc +endif # ($(TESTIP6TC), 1) +endif # ($(TEST), 1) +endif # ifdef IPTABLESPATH +endif # ifdef PCFILE_FOUND + +#LDLIBS += -lnfnetlink + +TEST := $(shell $(PKG_CONFIG) --atleast-version=1.0.2 libnetfilter_conntrack && $(PKG_CONFIG) --atleast-version=1.0.3 libmnl && echo 1) +ifeq ($(TEST),1) +CPPFLAGS += -DUSE_NFCT +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l libmnl) +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l libnetfilter_conntrack) +endif # ($(TEST),1) + +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l libssl) + +TEST := $(shell $(PKG_CONFIG) --exists uuid && echo 1) +ifeq ($(TEST),1) +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l uuid) +else +$(info please install uuid-dev package / libuuid) +endif # ($(TEST),1) + +TESTUPNPDESCGENOBJS = testupnpdescgen.o upnpdescgen.o + +EXECUTABLES = miniupnpd testupnpdescgen testgetifstats \ + testupnppermissions miniupnpdctl testgetifaddr \ + testgetroute testasyncsendto testportinuse \ + testssdppktgen testminissdp + +.PHONY: all clean install depend genuuid + +all: $(EXECUTABLES) + +clean: + $(RM) $(ALLOBJS) + $(RM) $(EXECUTABLES) + $(RM) testupnpdescgen.o testgetifstats.o + $(RM) testupnppermissions.o testgetifaddr.o + $(RM) testgetroute.o testasyncsendto.o + $(RM) testportinuse.o + $(RM) miniupnpdctl.o + $(RM) validateupnppermissions validategetifaddr validatessdppktgen + +install: miniupnpd miniupnpd.8 miniupnpd.conf genuuid \ + netfilter/iptables_init.sh netfilter/iptables_removeall.sh \ + netfilter/ip6tables_init.sh netfilter/ip6tables_removeall.sh \ + netfilter/miniupnpd_functions.sh \ + linux/miniupnpd.init.d.script + $(STRIP) miniupnpd + $(INSTALL) -d $(DESTDIR)$(SBININSTALLDIR) + $(INSTALL) miniupnpd $(DESTDIR)$(SBININSTALLDIR) + $(INSTALL) -d $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/iptables_init.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/iptables_removeall.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/ip6tables_init.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/ip6tables_removeall.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/miniupnpd_functions.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) --mode=0644 -b miniupnpd.conf $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) -d $(DESTDIR)$(PREFIX)/etc/init.d + $(INSTALL) linux/miniupnpd.init.d.script $(DESTDIR)$(PREFIX)/etc/init.d/miniupnpd + $(INSTALL) -d $(DESTDIR)$(MANINSTALLDIR) + $(INSTALL) --mode=0644 miniupnpd.8 $(DESTDIR)$(MANINSTALLDIR) + gzip -f $(DESTDIR)$(MANINSTALLDIR)/miniupnpd.8 + +# genuuid is using the uuidgen CLI tool which is part of libuuid +# from the e2fsprogs +# 'cat /proc/sys/kernel/random/uuid' could be also used +genuuid: +ifeq ($(TARGET_OPENWRT),) + sed -i -e "s/^uuid=[-0-9a-f]*/uuid=`(genuuid||uuidgen||uuid) 2>/dev/null`/" miniupnpd.conf +else + sed -i -e "s/^uuid=[-0-9a-f]*/uuid=`($(STAGING_DIR_HOST)/bin/genuuid||$(STAGING_DIR_HOST)/bin/uuidgen||$(STAGING_DIR_HOST)/bin/uuid) 2>/dev/null`/" miniupnpd.conf +endif + +check: validateupnppermissions validategetifaddr validatessdppktgen + +validateupnppermissions: testupnppermissions testupnppermissions.sh + ./testupnppermissions.sh + touch $@ + +validategetifaddr: testgetifaddr testgetifaddr.sh + ./testgetifaddr.sh + touch $@ + +validatessdppktgen: testssdppktgen + ./testssdppktgen + touch $@ + +miniupnpd: $(BASEOBJS) $(LNXOBJS) $(NETFILTEROBJS) + +testupnpdescgen: $(TESTUPNPDESCGENOBJS) + +testgetifstats: testgetifstats.o linux/getifstats.o + +testupnppermissions: testupnppermissions.o upnppermissions.o + +testgetifaddr: testgetifaddr.o getifaddr.o + +testgetroute: testgetroute.o linux/getroute.o upnputils.o + +testssdppktgen: testssdppktgen.o + +testasyncsendto: testasyncsendto.o asyncsendto.o upnputils.o \ + linux/getroute.o + +testportinuse: testportinuse.o portinuse.o getifaddr.o \ + netfilter/iptcrdr.o + +testminissdp: testminissdp.o minissdp.o upnputils.o upnpglobalvars.o \ + asyncsendto.o linux/getroute.o + + +miniupnpdctl: miniupnpdctl.o + +config.h: genconfig.sh VERSION + ./genconfig.sh $(CONFIG_OPTIONS) + +depend: config.h + makedepend -f$(MAKEFILE_LIST) -Y \ + $(ALLOBJS:.o=.c) $(TESTUPNPDESCGENOBJS:.o=.c) \ + testgetifstats.c testupnppermissions.c testgetifaddr.c \ + testgetroute.c testasyncsendto.c testportinuse.c \ + miniupnpdctl.c testssdppktgen.c 2>/dev/null + +# DO NOT DELETE + +miniupnpd.o: config.h macros.h upnpglobalvars.h upnppermissions.h +miniupnpd.o: miniupnpdtypes.h upnphttp.h upnpdescgen.h miniupnpdpath.h +miniupnpd.o: getifaddr.h upnpsoap.h options.h minissdp.h upnpredirect.h +miniupnpd.o: upnppinhole.h daemonize.h upnpevents.h asyncsendto.h natpmp.h +miniupnpd.o: pcpserver.h commonrdr.h upnputils.h ifacewatcher.h +upnphttp.o: config.h upnphttp.h upnpdescgen.h miniupnpdpath.h upnpsoap.h +upnphttp.o: upnpevents.h upnputils.h upnpglobalvars.h upnppermissions.h +upnphttp.o: miniupnpdtypes.h +upnpdescgen.o: config.h getifaddr.h upnpredirect.h upnpdescgen.h +upnpdescgen.o: miniupnpdpath.h upnpglobalvars.h upnppermissions.h +upnpdescgen.o: miniupnpdtypes.h upnpdescstrings.h upnpurns.h getconnstatus.h +upnpsoap.o: macros.h config.h upnpglobalvars.h upnppermissions.h +upnpsoap.o: miniupnpdtypes.h upnphttp.h upnpsoap.h upnpreplyparse.h +upnpsoap.o: upnpredirect.h upnppinhole.h getifaddr.h getifstats.h +upnpsoap.o: getconnstatus.h upnpurns.h +upnpreplyparse.o: upnpreplyparse.h minixml.h +minixml.o: minixml.h +portinuse.o: macros.h config.h upnpglobalvars.h upnppermissions.h +portinuse.o: miniupnpdtypes.h getifaddr.h portinuse.h netfilter/iptcrdr.h +portinuse.o: commonrdr.h +upnpredirect.o: macros.h config.h upnpredirect.h upnpglobalvars.h +upnpredirect.o: upnppermissions.h miniupnpdtypes.h upnpevents.h portinuse.h +upnpredirect.o: netfilter/iptcrdr.h commonrdr.h +getifaddr.o: config.h getifaddr.h +daemonize.o: daemonize.h config.h +upnpglobalvars.o: config.h upnpglobalvars.h upnppermissions.h +upnpglobalvars.o: miniupnpdtypes.h upnpdescstrings.h +options.o: config.h options.h upnppermissions.h upnpglobalvars.h +options.o: miniupnpdtypes.h +upnppermissions.o: config.h upnppermissions.h +minissdp.o: config.h upnpdescstrings.h miniupnpdpath.h upnphttp.h +minissdp.o: upnpglobalvars.h upnppermissions.h miniupnpdtypes.h minissdp.h +minissdp.o: upnputils.h getroute.h asyncsendto.h codelength.h macros.h +natpmp.o: macros.h config.h natpmp.h upnpglobalvars.h upnppermissions.h +natpmp.o: miniupnpdtypes.h getifaddr.h upnpredirect.h commonrdr.h upnputils.h +natpmp.o: portinuse.h asyncsendto.h +pcpserver.o: config.h pcpserver.h macros.h upnpglobalvars.h upnppermissions.h +pcpserver.o: miniupnpdtypes.h pcplearndscp.h upnpredirect.h commonrdr.h +pcpserver.o: getifaddr.h asyncsendto.h pcp_msg_struct.h netfilter/iptcrdr.h +pcpserver.o: commonrdr.h +upnpevents.o: config.h upnpevents.h miniupnpdpath.h upnpglobalvars.h +upnpevents.o: upnppermissions.h miniupnpdtypes.h upnpdescgen.h upnputils.h +upnputils.o: config.h upnputils.h upnpglobalvars.h upnppermissions.h +upnputils.o: miniupnpdtypes.h getroute.h +getconnstatus.o: getconnstatus.h getifaddr.h +upnppinhole.o: macros.h config.h upnpredirect.h upnpglobalvars.h +upnppinhole.o: upnppermissions.h miniupnpdtypes.h upnpevents.h upnppinhole.h +upnppinhole.o: netfilter/iptpinhole.h +pcplearndscp.o: config.h upnpglobalvars.h upnppermissions.h miniupnpdtypes.h +pcplearndscp.o: pcplearndscp.h +asyncsendto.o: asyncsendto.h upnputils.h +linux/getifstats.o: config.h getifstats.h +linux/ifacewatcher.o: config.h ifacewatcher.h config.h minissdp.h +linux/ifacewatcher.o: miniupnpdtypes.h getifaddr.h upnpglobalvars.h +linux/ifacewatcher.o: upnppermissions.h natpmp.h +linux/getroute.o: getroute.h upnputils.h +netfilter/iptcrdr.o: macros.h config.h netfilter/iptcrdr.h commonrdr.h +netfilter/iptcrdr.o: config.h upnpglobalvars.h upnppermissions.h +netfilter/iptcrdr.o: miniupnpdtypes.h +netfilter/iptpinhole.o: config.h netfilter/iptpinhole.h upnpglobalvars.h +netfilter/iptpinhole.o: upnppermissions.h config.h miniupnpdtypes.h +testupnpdescgen.o: macros.h config.h upnpdescgen.h upnpdescstrings.h +testupnpdescgen.o: getifaddr.h +upnpdescgen.o: config.h getifaddr.h upnpredirect.h upnpdescgen.h +upnpdescgen.o: miniupnpdpath.h upnpglobalvars.h upnppermissions.h +upnpdescgen.o: miniupnpdtypes.h upnpdescstrings.h upnpurns.h getconnstatus.h +testgetifstats.o: getifstats.h +testupnppermissions.o: upnppermissions.h config.h +testgetifaddr.o: config.h getifaddr.h +testgetroute.o: getroute.h upnputils.h upnpglobalvars.h upnppermissions.h +testgetroute.o: config.h miniupnpdtypes.h +testasyncsendto.o: miniupnpdtypes.h config.h upnputils.h asyncsendto.h +testportinuse.o: macros.h config.h portinuse.h +miniupnpdctl.o: macros.h +testssdppktgen.o: macros.h config.h miniupnpdpath.h upnphttp.h diff --git a/src/contrib/miniupnp/miniupnpd/Makefile.linux_nft b/src/contrib/miniupnp/miniupnpd/Makefile.linux_nft new file mode 100644 index 0000000..0c03ca3 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/Makefile.linux_nft @@ -0,0 +1,231 @@ +# MiniUPnP project +# (c) 2015 Tomofumi Hayashi +# http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ +# Author : Tomofumi Hayashi +# for use with GNU Make +# +# options can be passed to genconfig.sh through CONFIG_OPTIONS : +# $ CONFIG_OPTIONS="--ipv6 --igd2" make -f Makefile.linux +# +# To install use : +# $ DESTDIR=/dummyinstalldir make -f Makefile.linux_nft install +# or : +# $ INSTALLPREFIX=/usr/local make -f Makefile.linux_nft install +# or : +# $ make -f Makefile.linux install +# (default INSTALLPREFIX is /usr) +# +# +CFLAGS = -O -g #-DDEBUG +CFLAGS ?= -Os +CFLAGS += -fno-strict-aliasing +CFLAGS += -fno-common +CPPFLAGS += -D_GNU_SOURCE +CFLAGS += -Wall +CFLAGS += -Wextra -Wstrict-prototypes -Wdeclaration-after-statement +#CFLAGS += -Wno-missing-field-initializers +CC ?= gcc +RM = rm -f +INSTALL = install +STRIP ?= strip +PKG_CONFIG ?= pkg-config +CP = cp + + +INSTALLPREFIX ?= $(PREFIX)/usr +SBININSTALLDIR = $(INSTALLPREFIX)/sbin +ETCINSTALLDIR = $(PREFIX)/etc/miniupnpd +MANINSTALLDIR = $(INSTALLPREFIX)/share/man/man8 + +BASEOBJS = miniupnpd.o upnphttp.o upnpdescgen.o upnpsoap.o \ + upnpreplyparse.o minixml.o portinuse.o \ + upnpredirect.o getifaddr.o daemonize.o upnpglobalvars.o \ + options.o upnppermissions.o minissdp.o natpmp.o pcpserver.o \ + upnpevents.o upnputils.o getconnstatus.o \ + upnppinhole.o pcplearndscp.o asyncsendto.o + +LNXOBJS = linux/getifstats.o linux/ifacewatcher.o linux/getroute.o +NETFILTEROBJS = netfilter_nft/nftnlrdr.o netfilter_nft/nfct_get.o netfilter_nft/nftnlrdr_misc.o + +ALLOBJS = $(BASEOBJS) $(LNXOBJS) $(NETFILTEROBJS) + +PCFILE_FOUND := $(shell $(PKG_CONFIG) --exists libnftnl; echo $$?) + +ifeq (${PCFILE_FOUND},0) + +PKG_CONFIG_LIBS = libnftnl libmnl +CFLAGS += $(shell $(PKG_CONFIG) --cflags $(PKG_CONFIG_LIBS)) +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l $(PKG_CONFIG_LIBS)) +LDFLAGS += $(shell $(PKG_CONFIG) --libs-only-L $(PKG_CONFIG_LIBS)) +LDFLAGS += $(shell $(PKG_CONFIG) --libs-only-other $(PKG_CONFIG_LIBS)) +else + +ARCH ?= $(shell uname -m | grep -q "x86_64" && echo 64) +endif # ifdef PCFILE_FOUND + +#LDLIBS += -lnfnetlink + +TEST := $(shell $(PKG_CONFIG) --atleast-version=1.0.2 libnetfilter_conntrack && $(PKG_CONFIG) --atleast-version=1.0.3 libmnl && echo 1) +ifeq ($(TEST),1) +CPPFLAGS += -DUSE_NFCT +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l libmnl) +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l libnetfilter_conntrack) +endif # ($(TEST),1) + +LDLIBS += $(shell $(PKG_CONFIG) --static --libs-only-l libssl) + +TESTUPNPDESCGENOBJS = testupnpdescgen.o upnpdescgen.o + +EXECUTABLES = miniupnpd testupnpdescgen testgetifstats \ + testupnppermissions miniupnpdctl testgetifaddr \ + testgetroute testasyncsendto testportinuse + +.PHONY: all clean install depend genuuid + +all: $(EXECUTABLES) + +clean: + $(RM) $(ALLOBJS) + $(RM) $(EXECUTABLES) + $(RM) testupnpdescgen.o testgetifstats.o + $(RM) testupnppermissions.o testgetifaddr.o + $(RM) testgetroute.o testasyncsendto.o + $(RM) testportinuse.o + $(RM) miniupnpdctl.o + +install: miniupnpd miniupnpd.8 miniupnpd.conf genuuid \ + netfilter/iptables_init.sh netfilter/iptables_removeall.sh \ + netfilter/ip6tables_init.sh netfilter/ip6tables_removeall.sh \ + netfilter/miniupnpd_functions.sh \ + linux/miniupnpd.init.d.script + $(STRIP) miniupnpd + $(INSTALL) -d $(DESTDIR)$(SBININSTALLDIR) + $(INSTALL) miniupnpd $(DESTDIR)$(SBININSTALLDIR) + $(INSTALL) -d $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/iptables_init.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/iptables_removeall.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/ip6tables_init.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/ip6tables_removeall.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) netfilter/miniupnpd_functions.sh $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) --mode=0644 -b miniupnpd.conf $(DESTDIR)$(ETCINSTALLDIR) + $(INSTALL) -d $(DESTDIR)$(PREFIX)/etc/init.d + $(INSTALL) linux/miniupnpd.init.d.script $(DESTDIR)$(PREFIX)/etc/init.d/miniupnpd + $(INSTALL) -d $(DESTDIR)$(MANINSTALLDIR) + $(INSTALL) --mode=0644 miniupnpd.8 $(DESTDIR)$(MANINSTALLDIR) + gzip -f $(DESTDIR)$(MANINSTALLDIR)/miniupnpd.8 + +# genuuid is using the uuidgen CLI tool which is part of libuuid +# from the e2fsprogs +# 'cat /proc/sys/kernel/random/uuid' could be also used +genuuid: +ifeq ($(TARGET_OPENWRT),) + sed -i -e "s/^uuid=[-0-9a-f]*/uuid=`(genuuid||uuidgen||uuid) 2>/dev/null`/" miniupnpd.conf +else + sed -i -e "s/^uuid=[-0-9a-f]*/uuid=`($(STAGING_DIR_HOST)/bin/genuuid||$(STAGING_DIR_HOST)/bin/uuidgen||$(STAGING_DIR_HOST)/bin/uuid) 2>/dev/null`/" miniupnpd.conf +endif + +miniupnpd: $(BASEOBJS) $(LNXOBJS) $(NETFILTEROBJS) + +testupnpdescgen: $(TESTUPNPDESCGENOBJS) + +testgetifstats: testgetifstats.o linux/getifstats.o + +testupnppermissions: testupnppermissions.o upnppermissions.o + +testgetifaddr: testgetifaddr.o getifaddr.o + +testgetroute: testgetroute.o linux/getroute.o upnputils.o + +testasyncsendto: testasyncsendto.o asyncsendto.o upnputils.o \ + linux/getroute.o + +testportinuse: testportinuse.o portinuse.o getifaddr.o \ + netfilter_nft/nftnlrdr.o netfilter_nft/nftnlrdr_misc.o + +miniupnpdctl: miniupnpdctl.o + +config.h: genconfig.sh VERSION + ./genconfig.sh $(CONFIG_OPTIONS) + +depend: config.h + makedepend -f$(MAKEFILE_LIST) -Y \ + $(ALLOBJS:.o=.c) $(TESTUPNPDESCGENOBJS:.o=.c) \ + testgetifstats.c testupnppermissions.c testgetifaddr.c \ + testgetroute.c testasyncsendto.c testportinuse.c \ + miniupnpdctl.c 2>/dev/null + +# DO NOT DELETE + +miniupnpd.o: config.h macros.h upnpglobalvars.h upnppermissions.h +miniupnpd.o: miniupnpdtypes.h upnphttp.h upnpdescgen.h miniupnpdpath.h +miniupnpd.o: getifaddr.h upnpsoap.h options.h minissdp.h upnpredirect.h +miniupnpd.o: upnppinhole.h daemonize.h upnpevents.h asyncsendto.h natpmp.h +miniupnpd.o: pcpserver.h commonrdr.h upnputils.h ifacewatcher.h +upnphttp.o: config.h upnphttp.h upnpdescgen.h miniupnpdpath.h upnpsoap.h +upnphttp.o: upnpevents.h upnputils.h +upnpdescgen.o: config.h getifaddr.h upnpredirect.h upnpdescgen.h +upnpdescgen.o: miniupnpdpath.h upnpglobalvars.h upnppermissions.h +upnpdescgen.o: miniupnpdtypes.h upnpdescstrings.h upnpurns.h getconnstatus.h +upnpsoap.o: macros.h config.h upnpglobalvars.h upnppermissions.h +upnpsoap.o: miniupnpdtypes.h upnphttp.h upnpsoap.h upnpreplyparse.h +upnpsoap.o: upnpredirect.h upnppinhole.h getifaddr.h getifstats.h +upnpsoap.o: getconnstatus.h upnpurns.h +upnpreplyparse.o: upnpreplyparse.h minixml.h +minixml.o: minixml.h +portinuse.o: macros.h config.h upnpglobalvars.h upnppermissions.h +portinuse.o: miniupnpdtypes.h getifaddr.h portinuse.h netfilter_nft/nftnlrdr.h +portinuse.o: commonrdr.h +upnpredirect.o: macros.h config.h upnpredirect.h upnpglobalvars.h +upnpredirect.o: upnppermissions.h miniupnpdtypes.h upnpevents.h portinuse.h +upnpredirect.o: netfilter_nft/nftnlrdr.h commonrdr.h +getifaddr.o: config.h getifaddr.h +daemonize.o: daemonize.h config.h +upnpglobalvars.o: config.h upnpglobalvars.h upnppermissions.h +upnpglobalvars.o: miniupnpdtypes.h upnpdescstrings.h +options.o: config.h options.h upnppermissions.h upnpglobalvars.h +options.o: miniupnpdtypes.h +upnppermissions.o: config.h upnppermissions.h +minissdp.o: config.h upnpdescstrings.h miniupnpdpath.h upnphttp.h +minissdp.o: upnpglobalvars.h upnppermissions.h miniupnpdtypes.h minissdp.h +minissdp.o: upnputils.h getroute.h asyncsendto.h codelength.h +natpmp.o: macros.h config.h natpmp.h upnpglobalvars.h upnppermissions.h +natpmp.o: miniupnpdtypes.h getifaddr.h upnpredirect.h commonrdr.h upnputils.h +natpmp.o: portinuse.h asyncsendto.h +pcpserver.o: config.h pcpserver.h macros.h upnpglobalvars.h upnppermissions.h +pcpserver.o: miniupnpdtypes.h pcplearndscp.h upnpredirect.h commonrdr.h +pcpserver.o: getifaddr.h asyncsendto.h pcp_msg_struct.h netfilter_nft/nftnlrdr.h +pcpserver.o: commonrdr.h +upnpevents.o: config.h upnpevents.h miniupnpdpath.h upnpglobalvars.h +upnpevents.o: upnppermissions.h miniupnpdtypes.h upnpdescgen.h upnputils.h +upnputils.o: config.h upnputils.h upnpglobalvars.h upnppermissions.h +upnputils.o: miniupnpdtypes.h getroute.h +getconnstatus.o: getconnstatus.h getifaddr.h +upnppinhole.o: macros.h config.h upnpredirect.h upnpglobalvars.h +upnppinhole.o: upnppermissions.h miniupnpdtypes.h upnpevents.h +#upnppinhole.o: netfilter/iptpinhole.h +pcplearndscp.o: config.h upnpglobalvars.h upnppermissions.h miniupnpdtypes.h +pcplearndscp.o: pcplearndscp.h +asyncsendto.o: asyncsendto.h +linux/getifstats.o: config.h getifstats.h +linux/ifacewatcher.o: config.h ifacewatcher.h config.h minissdp.h +linux/ifacewatcher.o: miniupnpdtypes.h getifaddr.h upnpglobalvars.h +linux/ifacewatcher.o: upnppermissions.h natpmp.h +linux/getroute.o: getroute.h upnputils.h +netfilter_nft/nftnlrdr.o: macros.h config.h netfilter_nft/nftnlrdr.h commonrdr.h +netfilter_nft/nftnlrdr.o: config.h upnpglobalvars.h upnppermissions.h +netfilter_nft/nftnlrdr.o: miniupnpdtypes.h +netfilter_nft/iptpinhole.o: config.h netfilter_nft/iptpinhole.h upnpglobalvars.h +netfilter_nft/iptpinhole.o: upnppermissions.h config.h miniupnpdtypes.h +testupnpdescgen.o: macros.h config.h upnpdescgen.h upnpdescstrings.h +testupnpdescgen.o: getifaddr.h +upnpdescgen.o: config.h getifaddr.h upnpredirect.h upnpdescgen.h +upnpdescgen.o: miniupnpdpath.h upnpglobalvars.h upnppermissions.h +upnpdescgen.o: miniupnpdtypes.h upnpdescstrings.h upnpurns.h getconnstatus.h +testgetifstats.o: getifstats.h +testupnppermissions.o: upnppermissions.h config.h +testgetifaddr.o: config.h getifaddr.h +testgetroute.o: getroute.h upnputils.h upnpglobalvars.h upnppermissions.h +testgetroute.o: config.h miniupnpdtypes.h +testasyncsendto.o: miniupnpdtypes.h config.h upnputils.h asyncsendto.h +testportinuse.o: macros.h config.h portinuse.h +miniupnpdctl.o: macros.h diff --git a/src/contrib/miniupnp/miniupnpd/Makefile.macosx b/src/contrib/miniupnp/miniupnpd/Makefile.macosx new file mode 100644 index 0000000..3a01f7f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/Makefile.macosx @@ -0,0 +1,132 @@ +# MiniUPnP project +# http://miniupnp.free.fr/ +# Author: Thomas Bernard +# This Makefile should work for MacOSX +# +# To compile with pf with OS X 10.7+, you need to specify +# path to XNU bsd sources : +# INCLUDES="-I.../xnu/bsd -I.../xnu/libkern" make -f Makefile.macosx +# +# To install use : +# $ PREFIX=/dummyinstalldir make -f Makefile.macosx install +# or : +# $ make -f Makefile.macosx install +# +CFLAGS = -Wall -O -g3 -DDEBUG +#CFLAGS = -Wall -Os +#CC = gcc #better use clang ! +RM = rm -f +MV = mv +INSTALL = install +STRIP = strip + +# OSNAME and FWNAME are used for building OS or FW dependent code. +OSNAME = $(shell uname) +ARCH = $(shell uname -p) +# Firewall is ipfw up to OS X 10.6 Snow Leopard +# and pf since OS X 10.7 Lion (Darwin 11.0) +FWNAME = $(shell [ `uname -r | cut -d. -f1` -ge 11 ] && echo "pf" || echo "ipfw" ) + +STD_OBJS = miniupnpd.o upnphttp.o upnpdescgen.o upnpsoap.o \ + upnpredirect.o getifaddr.o daemonize.o upnpglobalvars.o \ + options.o upnppermissions.o minissdp.o natpmp.o \ + upnpevents.o getconnstatus.o upnputils.o \ + upnppinhole.o asyncsendto.o portinuse.o pcpserver.o +MAC_OBJS = mac/getifstats.o bsd/ifacewatcher.o bsd/getroute.o +IPFW_OBJS = ipfw/ipfwrdr.o ipfw/ipfwaux.o +PF_OBJS = pf/obsdrdr.o +# pf/pfpinhole.o # SHOULD be used, but doesn't compile on e.g. OS X 10.9. +MISC_OBJS = upnpreplyparse.o minixml.o + +ALL_OBJS = $(STD_OBJS) $(MISC_OBJS) $(MAC_OBJS) +ifeq ($(FWNAME), ipfw) + ALL_OBJS += $(IPFW_OBJS) +else + ALL_OBJS += $(PF_OBJS) + CFLAGS += -DPF +endif + +TEST_UPNPDESCGEN_OBJS = testupnpdescgen.o upnpdescgen.o +TEST_GETIFSTATS_OBJS = testgetifstats.o mac/getifstats.o +TEST_UPNPPERMISSIONS_OBJS = testupnppermissions.o upnppermissions.o +TEST_GETIFADDR_OBJS = testgetifaddr.o getifaddr.o +TEST_PORTINUSE_OBJS = testportinuse.o portinuse.o getifaddr.o +TEST_ASYNCSENDTO_OBJS = testasyncsendto.o asyncsendto.o upnputils.o bsd/getroute.o +MINIUPNPDCTL_OBJS = miniupnpdctl.o + +EXECUTABLES = miniupnpd testupnpdescgen testgetifstats \ + testupnppermissions miniupnpdctl \ + testgetifaddr testportinuse testasyncsendto + +LIBS = + +# set PREFIX variable to install in the wanted place + +INSTALL_BINDIR = $(PREFIX)/sbin +INSTALL_ETCDIR = $(PREFIX)/etc/miniupnpd +INSTALL_MANDIR = $(PREFIX)/share/man/man8 + +all: $(EXECUTABLES) + +clean: + $(RM) $(ALL_OBJS) $(EXECUTABLES) \ + testupnpdescgen.o testgetifstats.o testupnppermissions.o \ + miniupnpdctl.o testgetifaddr.o config.h \ + mac/org.tuxfamily.miniupnpd.plist + +install: miniupnpd genuuid genlaunchd + $(STRIP) miniupnpd + $(INSTALL) -d $(INSTALL_BINDIR) + $(INSTALL) miniupnpd $(INSTALL_BINDIR) + $(INSTALL) -d $(INSTALL_ETCDIR) + $(INSTALL) -m 0644 -b miniupnpd.conf $(INSTALL_ETCDIR) + $(INSTALL) -d $(INSTALL_MANDIR) + $(INSTALL) miniupnpd.8 $(INSTALL_MANDIR) + $(INSTALL) -d $(PREFIX)/Library/LaunchDaemons + $(INSTALL) mac/org.tuxfamily.miniupnpd.plist $(PREFIX)/Library/LaunchDaemons + #$(INSTALL) ipfw/ipfw_init.sh $(INSTALL_ETCDIR) + #$(INSTALL) ipfw/ipfw_removeall.sh $(INSTALL_ETCDIR) + +genuuid: + $(MV) miniupnpd.conf miniupnpd.conf.before + sed -e "s/^uuid=[-0-9a-fA-F]*/uuid=`uuidgen 2>/dev/null`/" miniupnpd.conf.before > miniupnpd.conf + $(RM) miniupnpd.conf.before + +genlaunchd: + sed -e "s|INSTALLPREFIX|$(PREFIX)|g" mac/org.tuxfamily.miniupnpd.plist.before > mac/org.tuxfamily.miniupnpd.plist + +depend: config.h + mkdep $(ALL_OBJS:.o=.c) testupnpdescgen.c testgetifstats.c \ + testupnppermissions.c miniupnpdctl.c testgetifaddr.c + +miniupnpd: config.h $(ALL_OBJS) + $(CC) $(CFLAGS) -o $@ $(ALL_OBJS) $(LIBS) + +miniupnpdctl: config.h $(MINIUPNPDCTL_OBJS) + $(CC) $(CFLAGS) -o $@ $(MINIUPNPDCTL_OBJS) + +testupnpdescgen: config.h $(TEST_UPNPDESCGEN_OBJS) + $(CC) $(CFLAGS) -o $@ $(TEST_UPNPDESCGEN_OBJS) + +testgetifstats: config.h $(TEST_GETIFSTATS_OBJS) + $(CC) $(CFLAGS) -o $@ $(TEST_GETIFSTATS_OBJS) $(LIBS) + +testgetifaddr: config.h $(TEST_GETIFADDR_OBJS) + $(CC) $(CFLAGS) -o $@ $(TEST_GETIFADDR_OBJS) + +testupnppermissions: config.h $(TEST_UPNPPERMISSIONS_OBJS) + $(CC) $(CFLAGS) -o $@ $(TEST_UPNPPERMISSIONS_OBJS) + +testasyncsendto: config.h $(TEST_ASYNCSENDTO_OBJS) + $(CC) $(CFLAGS) -o $@ $(TEST_ASYNCSENDTO_OBJS) + +testportinuse: config.h $(TEST_PORTINUSE_OBJS) + $(CC) $(CFLAGS) -o $@ $(TEST_PORTINUSE_OBJS) + +config.h: genconfig.sh + ./genconfig.sh + +.SUFFIXES: .o .c +.c.o: + $(CC) $(CFLAGS) $(INCLUDES) -c -o $@ $< +# $(CC) $(CFLAGS) -c -o $(.TARGET) $(.IMPSRC) diff --git a/src/contrib/miniupnp/miniupnpd/Makefile.sunos b/src/contrib/miniupnp/miniupnpd/Makefile.sunos new file mode 100644 index 0000000..704d766 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/Makefile.sunos @@ -0,0 +1,202 @@ +# $Id: Makefile,v 1.88 2016/02/10 20:32:43 nanard Exp $ +# MiniUPnP project +# http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ +# Author: Thomas Bernard +# +# Makefile for miniupnpd (MiniUPnP daemon) +# +# This Makefile should work for SunOS/Solaris. +# This Makefile is NOT compatible with GNU Make. +# Linux users, please use Makefile.linux : +# make -f Makefile.linux +# +# options can be passed to genconfig.sh through CONFIG_OPTIONS : +# $ CONFIG_OPTIONS="--ipv6 --igd2" make +# + +CFLAGS = -pipe -Os +#CFLAGS = -pipe -O -g -DDEBUG +#CFLAGS += -ansi +CFLAGS += -Wall +CFLAGS += -W +CFLAGS += -Wstrict-prototypes +#CFLAGS += -Wdeclaration-after-statement +#CFLAGS += -Wno-missing-field-initializers +CFLAGS += -fno-common +CFLAGS += -D_XOPEN_SOURCE +CFLAGS += -D_XOPEN_SOURCE_EXTENDED=1 +CFLAGS += -D__EXTENSIONS__ +CC = gcc +RM = rm -f +MV = mv +INSTALL = install +STRIP = strip + +# OSNAME and FWNAME are used for building OS or FW dependent code. +OSNAME :sh = uname -s +ARCH :sh = uname -m + +FWNAME = ipf + +# Solaris specific CFLAGS +CFLAGS :sh += echo "-DSOLARIS2=`uname -r | cut -d. -f2`" +# "amd64" +CFLAGS += -m64 -mcmodel=kernel -mno-red-zone -ffreestanding +# "sparc64" +#CFLAGS += -m64 -mcmodel=medlow +LDFLAGS += -m64 + +STDOBJS = miniupnpd.o upnphttp.o upnpdescgen.o upnpsoap.o \ + upnpredirect.o getifaddr.o daemonize.o upnpglobalvars.o \ + options.o upnppermissions.o minissdp.o natpmp.o pcpserver.o \ + upnpevents.o upnputils.o getconnstatus.o \ + upnppinhole.o asyncsendto.o portinuse.o +BSDOBJS = bsd/getifstats.o bsd/ifacewatcher.o bsd/getroute.o +SUNOSOBJS = solaris/getifstats.o bsd/ifacewatcher.o bsd/getroute.o +MACOBJS = mac/getifstats.o bsd/ifacewatcher.o bsd/getroute.o +PFOBJS = pf/obsdrdr.o pf/pfpinhole.o +IPFOBJS = ipf/ipfrdr.o +IPFWOBJS = ipfw/ipfwrdr.o ipfw/ipfwaux.o +MISCOBJS = upnpreplyparse.o minixml.o + +ALLOBJS = $(STDOBJS) $(MISCOBJS) +ALLOBJS += $(SUNOSOBJS) +TESTGETIFSTATSOBJS = testgetifstats.o solaris/getifstats.o +TESTGETROUTEOBJS = testgetroute.o upnputils.o bsd/getroute.o + +#.if $(FWNAME) == "pf" +#ALLOBJS += $(PFOBJS) +#.elif $(FWNAME) == "ipfw" +#ALLOBJS += $(IPFWOBJS) +#.else +ALLOBJS += $(IPFOBJS) +#.endif + +TESTUPNPDESCGENOBJS = testupnpdescgen.o upnpdescgen.o +TESTUPNPPERMISSIONSOBJS = testupnppermissions.o upnppermissions.o +TESTGETIFADDROBJS = testgetifaddr.o getifaddr.o +MINIUPNPDCTLOBJS = miniupnpdctl.o +TESTASYNCSENDTOOBJS = testasyncsendto.o asyncsendto.o upnputils.o bsd/getroute.o +TESTPORTINUSEOBJS = testportinuse.o portinuse.o getifaddr.o + +EXECUTABLES = miniupnpd testupnpdescgen testgetifstats \ + testupnppermissions miniupnpdctl \ + testgetifaddr testgetroute testasyncsendto \ + testportinuse testssdppktgen + +LIBS += -lsocket -lnsl -lkstat -lresolv +LIBS += -luuid + +LIBS += -lssl -lcrypto + +# set PREFIX variable to install in the wanted place + +INSTALLBINDIR = $(PREFIX)/sbin +INSTALLETCDIR = $(PREFIX)/etc +# INSTALLMANDIR = $(PREFIX)/man +INSTALLMANDIR = /usr/share/man + +all: $(EXECUTABLES) + +clean: + $(RM) $(STDOBJS) $(BSDOBJS) $(SUNOSOBJS) $(MACOBJS) $(EXECUTABLES) \ + testupnpdescgen.o \ + $(MISCOBJS) config.h testgetifstats.o testupnppermissions.o \ + miniupnpdctl.o testgetifaddr.o testgetroute.o testasyncsendto.o \ + testportinuse.o \ + $(PFOBJS) $(IPFOBJS) $(IPFWOBJS) + $(RM) validateupnppermissions validategetifaddr validatessdppktgen + +install: miniupnpd genuuid + $(STRIP) miniupnpd + $(INSTALL) -d $(DESTDIR)$(INSTALLBINDIR) + $(INSTALL) -m 755 miniupnpd $(DESTDIR)$(INSTALLBINDIR) + $(INSTALL) -d $(DESTDIR)$(INSTALLETCDIR) + $(INSTALL) -b miniupnpd.conf $(DESTDIR)$(INSTALLETCDIR) + # TODO : install man page correctly +# $(INSTALL) -d $(INSTALLMANDIR) +# $(INSTALL) miniupnpd.8 $(INSTALLMANDIR)/cat8/miniupnpd.0 + +# genuuid is using the uuid cli tool available under OpenBSD 4.0 in +# the uuid-1.5.0 package +# any other cli tool returning a uuid on stdout should work. +UUID :sh = if which uuidgen 2>&1 > /dev/null; then \ + echo `uuidgen` ; \ + elif which uuid 2>&1 > /dev/null; then \ + echo `uuid` ; \ + else echo "00000000-0000-0000-0000-000000000000"; \ + fi + +# bash or ksh +SH :sh = which bash || which ksh + +genuuid: + $(MV) miniupnpd.conf miniupnpd.conf.before + sed -e "s/^uuid=[-0-9a-fA-F]*/uuid=$(UUID)/" miniupnpd.conf.before > miniupnpd.conf + $(RM) miniupnpd.conf.before + +check: validateupnppermissions validategetifaddr validatessdppktgen + +validateupnppermissions: testupnppermissions testupnppermissions.sh + $(SH) ./testupnppermissions.sh + touch $@ + +validategetifaddr: testgetifaddr testgetifaddr.sh + ./testgetifaddr.sh + touch $@ + +validatessdppktgen: testssdppktgen + ./testssdppktgen + touch $@ + +depend: config.h + mkdep $(ALLOBJS:.o=.c) testupnpdescgen.c testgetifstats.c \ + testupnppermissions.c miniupnpdctl.c testgetifaddr.c \ + testgetroute.c testportinuse.c testasyncsendto.c \ + testssdppktgen.c + +miniupnpd: config.h $(ALLOBJS) + $(CC) $(LDFLAGS) -o $@ $(ALLOBJS) $(LIBS) + +# BSDmake : +# $(CC) $(LDFLAGS) -o $@ $> $(LIBS) + +miniupnpdctl: config.h $(MINIUPNPDCTLOBJS) + $(CC) $(LDFLAGS) -o $@ $(MINIUPNPDCTLOBJS) -lsocket + +testupnpdescgen: config.h $(TESTUPNPDESCGENOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTUPNPDESCGENOBJS) $(LIBS) + +testgetifstats: config.h $(TESTGETIFSTATSOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTGETIFSTATSOBJS) $(LIBS) + +testgetifaddr: config.h $(TESTGETIFADDROBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTGETIFADDROBJS) $(LIBS) + +testupnppermissions: config.h $(TESTUPNPPERMISSIONSOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTUPNPPERMISSIONSOBJS) $(LIBS) + +testgetroute: config.h $(TESTGETROUTEOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTGETROUTEOBJS) $(LIBS) + +testasyncsendto: config.h $(TESTASYNCSENDTOOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTASYNCSENDTOOBJS) -lsocket -lnsl + +testportinuse: config.h $(TESTPORTINUSEOBJS) + $(CC) $(LDFLAGS) -o $@ $(TESTPORTINUSEOBJS) $(LIBS) + +# gmake : +# $(CC) $(CFLAGS) -o $@ $^ +# BSDmake : +# $(CC) $(CFLAGS) -o $@ $> + +config.h: genconfig.sh VERSION + ./genconfig.sh $(CONFIG_OPTIONS) + +.SUFFIXES: .o .c +.c.o: + $(CC) $(CFLAGS) -c -o $@ $< + +# $(CC) $(CFLAGS) -c -o $(.TARGET) $(.IMPSRC) + + diff --git a/src/contrib/miniupnp/miniupnpd/README b/src/contrib/miniupnp/miniupnpd/README new file mode 100644 index 0000000..92bfdee --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/README @@ -0,0 +1,37 @@ +MiniUPnP project +(c) 2006-2017 Thomas Bernard +webpage: http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ +github: https://github.com/miniupnp/miniupnp +freecode: http://freecode.com/projects/miniupnp +contact: miniupnp@free.fr + +This directory contain the miniUPnP daemon software, aka miniUPnPd. +This software is subject to the conditions detailed in +the LICENSE file provided with this distribution. + + +The miniUPnP daemon is an UPnP IGD (internet gateway device) +which provide NAT traversal services to any UPnP enabled client on +the network. +See http://www.upnp.org/ for more details on UPnP. +During the year 2011, support for IGD v2 has been added. +In 2012, IGD v2 WANIPv6FirewallControl has been implemented. + +IGD2 is still not enabled by default because of interoperability +issues. + +In addition to UPnP IGD, miniUPnPd supports NAT-PMP and PCP : + +See information about NAT Port Mapping Protocol (NAT-PMP) here : +http://miniupnp.free.fr/nat-pmp.html +NAT-PMP is the precursor of Port Control Protocol (PCP, RFC 6887). +In 2013, support for PCP has been added too. + +Read the INSTALL file for instructions to compile, install and +configure miniupnpd on your system. + +Report bugs to miniupnp@free.fr or on the web forum +https://miniupnp.tuxfamily.org/forum/ +or using https://github.com/miniupnp/miniupnp/issues + +Thomas Bernard diff --git a/src/contrib/miniupnp/miniupnpd/TODO b/src/contrib/miniupnp/miniupnpd/TODO new file mode 100644 index 0000000..3e12c63 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/TODO @@ -0,0 +1,24 @@ +- detect and inform of IP address changes or connectivity status change + => Done (ifacewatcher 2011/05) + To be improved. +- improve logging. +- improve NAT-PMP compliance + => to be checked + +- Tomato's version used to dynamically include the local LAN IP and WAN IP as the windows icon name - if it was displayed... + +- I just enabled my lease file to save the port mappings so I don't have to recreate them every time miniupnp gets restarted but instead what happens is the lease file gets recreated and the leases erased. + => Done (2009/02/14) + +- http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=1091 + +support IGD v2 : http://upnp.org/specs/gw/igd2/ + - Lease Duration support (mandatory in v2) => DONE + - WANIPv6FirewallControl + - pf : updatepinhole to do + - netfilter : ok ? + - ipfw/ipf : TODO + +implement port_in_use() for NetBSD + +- Do we need to TRIM arguments from SOAP ? diff --git a/src/contrib/miniupnp/miniupnpd/VERSION b/src/contrib/miniupnp/miniupnpd/VERSION new file mode 100644 index 0000000..cd5ac03 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/VERSION @@ -0,0 +1 @@ +2.0 diff --git a/src/contrib/miniupnp/miniupnpd/asyncsendto.c b/src/contrib/miniupnp/miniupnpd/asyncsendto.c new file mode 100644 index 0000000..8d47ca2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/asyncsendto.c @@ -0,0 +1,348 @@ +/* $Id: asyncsendto.c,v 1.8 2017/05/24 22:51:57 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "asyncsendto.h" +#include "upnputils.h" + +/* state diagram for a packet : + * + * | + * V + * -> ESCHEDULED -> ESENDNOW -> sent + * ^ | + * | V + * EWAITREADY -> sent + */ +struct scheduled_send { + LIST_ENTRY(scheduled_send) entries; + struct timeval ts; + enum {ESCHEDULED=1, EWAITREADY=2, ESENDNOW=3} state; + int sockfd; + const void * buf; + size_t len; + int flags; + const struct sockaddr *dest_addr; + socklen_t addrlen; + const struct sockaddr_in6 *src_addr; + char data[]; +}; + +static LIST_HEAD(listhead, scheduled_send) send_list = { NULL }; + +/* + * ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, + * const struct sockaddr *dest_addr, socklen_t addrlen); + */ +static ssize_t +send_from_to(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr_in6 *src_addr, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ +#ifdef IPV6_PKTINFO + if(src_addr) { + struct iovec iov; + struct in6_pktinfo ipi6; + uint8_t c[CMSG_SPACE(sizeof(ipi6))]; + struct msghdr msg; + struct cmsghdr* cmsg; + + iov.iov_base = (void *)buf; + iov.iov_len = len; + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + ipi6.ipi6_addr = src_addr->sin6_addr; + ipi6.ipi6_ifindex = src_addr->sin6_scope_id; + msg.msg_control = c; + msg.msg_controllen = sizeof(c); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = IPPROTO_IPV6; + cmsg->cmsg_type = IPV6_PKTINFO; + cmsg->cmsg_len = CMSG_LEN(sizeof(ipi6)); + memcpy(CMSG_DATA(cmsg), &ipi6, sizeof(ipi6)); + msg.msg_name = (void *)dest_addr; + msg.msg_namelen = addrlen; + return sendmsg(sockfd, &msg, flags); + } else { +#endif /* IPV6_PKTINFO */ + return sendto(sockfd, buf, len, flags, dest_addr, addrlen); +#ifdef IPV6_PKTINFO + } +#endif /* IPV6_PKTINFO */ +} + +/* delay = milli seconds */ +ssize_t +sendto_schedule2(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + const struct sockaddr_in6 *src_addr, + unsigned int delay) +{ + enum {ESCHEDULED, EWAITREADY, ESENDNOW} state; + ssize_t n; + size_t alloc_len; + struct timeval tv; + struct scheduled_send * elt; + + if(delay == 0) { + /* first try to send at once */ + n = send_from_to(sockfd, buf, len, flags, src_addr, dest_addr, addrlen); + if(n >= 0) + return n; + else if(errno == EAGAIN || errno == EWOULDBLOCK) { + /* use select() on this socket */ + state = EWAITREADY; + } else if(errno == EINTR) { + state = ESENDNOW; + } else { + /* uncatched error */ + return n; + } + } else { + state = ESCHEDULED; + } + + /* schedule */ + if(upnp_gettimeofday(&tv) < 0) { + return -1; + } + /* allocate enough space for structure + buffers */ + alloc_len = sizeof(struct scheduled_send) + len + addrlen; + if(src_addr) + alloc_len += sizeof(struct sockaddr_in6); + elt = malloc(alloc_len); + if(elt == NULL) { + syslog(LOG_ERR, "malloc failed to allocate %u bytes", + (unsigned)alloc_len); + return -1; + } + elt->state = state; + /* time the packet should be sent */ + elt->ts.tv_sec = tv.tv_sec + (delay / 1000); + elt->ts.tv_usec = tv.tv_usec + (delay % 1000) * 1000; + if(elt->ts.tv_usec > 1000000) { + elt->ts.tv_sec++; + elt->ts.tv_usec -= 1000000; + } + elt->sockfd = sockfd; + elt->flags = flags; + memcpy(elt->data, dest_addr, addrlen); + elt->dest_addr = (struct sockaddr *)elt->data; + elt->addrlen = addrlen; + if(src_addr) { + elt->src_addr = (struct sockaddr_in6 *)(elt->data + addrlen); + memcpy((void *)elt->src_addr, src_addr, sizeof(struct sockaddr_in6)); + elt->buf = (void *)(elt->data + addrlen + sizeof(struct sockaddr_in6)); + } else { + elt->src_addr = NULL; + elt->buf = (void *)(elt->data + addrlen); + } + elt->len = len; + memcpy((void *)elt->buf, buf, len); + /* insert */ + LIST_INSERT_HEAD( &send_list, elt, entries); + return 0; +} + +/* try to send at once, and queue the packet if needed */ +ssize_t +sendto_or_schedule(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ + return sendto_schedule2(sockfd, buf, len, flags, dest_addr, addrlen, NULL, 0); +} + +ssize_t +sendto_or_schedule2(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + const struct sockaddr_in6 *src_addr) +{ + return sendto_schedule2(sockfd, buf, len, flags, dest_addr, addrlen, src_addr, 0); +} + +/* get_next_scheduled_send() return number of scheduled send in list */ +int get_next_scheduled_send(struct timeval * next_send) +{ + int n = 0; + struct scheduled_send * elt; + if(next_send == NULL) + return -1; + for(elt = send_list.lh_first; elt != NULL; elt = elt->entries.le_next) { + if(n == 0 || (elt->ts.tv_sec < next_send->tv_sec) || + (elt->ts.tv_sec == next_send->tv_sec && elt->ts.tv_usec < next_send->tv_usec)) { + next_send->tv_sec = elt->ts.tv_sec; + next_send->tv_usec = elt->ts.tv_usec; + } + n++; + } + return n; +} + +/* update writefds for select() call + * return the number of packets to try to send at once */ +int get_sendto_fds(fd_set * writefds, int * max_fd, const struct timeval * now) +{ + int n = 0; + struct scheduled_send * elt; + for(elt = send_list.lh_first; elt != NULL; elt = elt->entries.le_next) { + if(elt->state == EWAITREADY) { + /* last sendto() call returned EAGAIN/EWOULDBLOCK */ + FD_SET(elt->sockfd, writefds); + if(elt->sockfd > *max_fd) + *max_fd = elt->sockfd; + n++; + } else if((elt->ts.tv_sec < now->tv_sec) || + (elt->ts.tv_sec == now->tv_sec && elt->ts.tv_usec <= now->tv_usec)) { + /* we waited long enough, now send ! */ + elt->state = ESENDNOW; + n++; + } + } + return n; +} + +/* executed sendto() when needed */ +int try_sendto(fd_set * writefds) +{ + int ret = 0; + ssize_t n; + struct scheduled_send * elt; + struct scheduled_send * next; + for(elt = send_list.lh_first; elt != NULL; elt = next) { + next = elt->entries.le_next; + if((elt->state == ESENDNOW) || + (elt->state == EWAITREADY && FD_ISSET(elt->sockfd, writefds))) { +#ifdef DEBUG + syslog(LOG_DEBUG, "%s: %d bytes on socket %d", + "try_sendto", (int)elt->len, elt->sockfd); +#endif + n = send_from_to(elt->sockfd, elt->buf, elt->len, elt->flags, + elt->src_addr, elt->dest_addr, elt->addrlen); + /*n = sendto(elt->sockfd, elt->buf, elt->len, elt->flags, + elt->dest_addr, elt->addrlen);*/ + if(n < 0) { + if(errno == EINTR) { + /* retry at once */ + elt->state = ESENDNOW; + continue; + } else if(errno == EAGAIN || errno == EWOULDBLOCK) { + /* retry once the socket is ready for writing */ + elt->state = EWAITREADY; + continue; + } else { + char addr_str[64]; + /* uncatched error */ + if(sockaddr_to_string(elt->dest_addr, addr_str, sizeof(addr_str)) <= 0) + addr_str[0] = '\0'; + syslog(LOG_ERR, "%s(sock=%d, len=%u, dest=%s): sendto: %m", + "try_sendto", elt->sockfd, (unsigned)elt->len, + addr_str); + ret--; + } + } else if((int)n != (int)elt->len) { + syslog(LOG_WARNING, "%s: %d bytes sent out of %d", + "try_sendto", (int)n, (int)elt->len); + } + /* remove from the list */ + LIST_REMOVE(elt, entries); + free(elt); + } + } + return ret; +} + +/* maximum execution time for finalize_sendto() in milliseconds */ +#define FINALIZE_SENDTO_DELAY (500) + +/* empty the list */ +void finalize_sendto(void) +{ + ssize_t n; + struct scheduled_send * elt; + struct scheduled_send * next; + fd_set writefds; + struct timeval deadline; + struct timeval now; + struct timeval timeout; + int max_fd; + + if(upnp_gettimeofday(&deadline) < 0) { + syslog(LOG_ERR, "gettimeofday: %m"); + return; + } + deadline.tv_usec += FINALIZE_SENDTO_DELAY*1000; + if(deadline.tv_usec > 1000000) { + deadline.tv_sec++; + deadline.tv_usec -= 1000000; + } + while(send_list.lh_first) { + FD_ZERO(&writefds); + max_fd = -1; + for(elt = send_list.lh_first; elt != NULL; elt = next) { + next = elt->entries.le_next; + syslog(LOG_DEBUG, "finalize_sendto(): %d bytes on socket %d", + (int)elt->len, elt->sockfd); + n = send_from_to(elt->sockfd, elt->buf, elt->len, elt->flags, + elt->src_addr, elt->dest_addr, elt->addrlen); + /*n = sendto(elt->sockfd, elt->buf, elt->len, elt->flags, + elt->dest_addr, elt->addrlen);*/ + if(n < 0) { + if(errno==EAGAIN || errno==EWOULDBLOCK) { + FD_SET(elt->sockfd, &writefds); + if(elt->sockfd > max_fd) + max_fd = elt->sockfd; + continue; + } + syslog(LOG_WARNING, "finalize_sendto(): socket=%d sendto: %m", elt->sockfd); + } + /* remove from the list */ + LIST_REMOVE(elt, entries); + free(elt); + } + /* check deadline */ + if(upnp_gettimeofday(&now) < 0) { + syslog(LOG_ERR, "gettimeofday: %m"); + return; + } + if(now.tv_sec > deadline.tv_sec || + (now.tv_sec == deadline.tv_sec && now.tv_usec > deadline.tv_usec)) { + /* deadline ! */ + while((elt = send_list.lh_first) != NULL) { + LIST_REMOVE(elt, entries); + free(elt); + } + return; + } + /* compute timeout value */ + timeout.tv_sec = deadline.tv_sec - now.tv_sec; + timeout.tv_usec = deadline.tv_usec - now.tv_usec; + if(timeout.tv_usec < 0) { + timeout.tv_sec--; + timeout.tv_usec += 1000000; + } + if(max_fd >= 0) { + if(select(max_fd + 1, NULL, &writefds, NULL, &timeout) < 0) { + syslog(LOG_ERR, "select: %m"); + return; + } + } + } +} + diff --git a/src/contrib/miniupnp/miniupnpd/asyncsendto.h b/src/contrib/miniupnp/miniupnpd/asyncsendto.h new file mode 100644 index 0000000..ef670c2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/asyncsendto.h @@ -0,0 +1,51 @@ +/* $Id: asyncsendto.h,v 1.3 2017/11/02 15:48:29 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef ASYNCSENDTO_H_INCLUDED +#define ASYNCSENDTO_H_INCLUDED +/* for fd_set */ +#include + +/* sendto_schedule() : see sendto(2) + * schedule sendto() call after delay (milliseconds) */ +ssize_t +sendto_schedule2(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + const struct sockaddr_in6 *src_addr, + unsigned int delay); + +#define sendto_schedule(sockfd, buf, len, flags, dest_addr, addrlen, delay) \ + sendto_schedule2(sockfd, buf, len, flags, dest_addr, addrlen, NULL, delay) + +/* sendto_schedule() : see sendto(2) + * try sendto() at once and schedule if EINTR/EAGAIN/EWOULDBLOCK */ +ssize_t +sendto_or_schedule(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen); + +/* same as sendto_schedule() except it will try to set source address + * (for IPV6 only) */ +ssize_t +sendto_or_schedule2(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + const struct sockaddr_in6 *src_addr); + +/* get_next_scheduled_send() + * return number of scheduled sendto + * set next_send to timestamp to send next packet */ +int get_next_scheduled_send(struct timeval * next_send); + +/* execute sendto() for needed packets */ +int try_sendto(fd_set * writefds); + +/* set writefds before select() */ +int get_sendto_fds(fd_set * writefds, int * max_fd, const struct timeval * now); + +/* empty the list */ +void finalize_sendto(void); + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/bsd/Makefile b/src/contrib/miniupnp/miniupnpd/bsd/Makefile new file mode 100644 index 0000000..e66d29a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/bsd/Makefile @@ -0,0 +1,20 @@ +# $Id: Makefile,v 1.2 2011/05/20 09:34:25 nanard Exp $ +# made for GNU Make +CFLAGS = -Wall -g +EXECUTABLES = testgetifstats testifacewatcher + +all: $(EXECUTABLES) + +clean: + rm -f *.o $(EXECUTABLES) + +testobsdrdr.o: testobsdrdr.c obsdrdr.h + +testgetifstats: testgetifstats.o getifstats.o + $(CC) $(CFLAGS) -o $@ $> -lkvm + +testifacewatcher: testifacewatcher.o ifacewatcher.o upnputils.o + $(CC) $(CFLAGS) -o $@ $> + +upnputils.o: ../upnputils.c + diff --git a/src/contrib/miniupnp/miniupnpd/bsd/getifstats.c b/src/contrib/miniupnp/miniupnpd/bsd/getifstats.c new file mode 100644 index 0000000..5443a99 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/bsd/getifstats.c @@ -0,0 +1,77 @@ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * author: Gleb Smirnoff + * (c) 2006 Ryan Wagoner + * (c) 2014 Gleb Smirnoff + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef ENABLE_GETIFSTATS_CACHING +#include +#endif + +#include "../getifstats.h" +#include "../config.h" + +int +getifstats(const char *ifname, struct ifdata *data) +{ + static struct ifaddrs *ifap, *ifa; +#ifdef ENABLE_GETIFSTATS_CACHING + static time_t cache_timestamp; + time_t current_time; +#endif + if(!data) + return -1; + data->baudrate = 4200000; + data->opackets = 0; + data->ipackets = 0; + data->obytes = 0; + data->ibytes = 0; + if(!ifname || ifname[0]=='\0') + return -1; + +#ifdef ENABLE_GETIFSTATS_CACHING + current_time = upnp_time(); + if (ifap != NULL && + current_time < cache_timestamp + GETIFSTATS_CACHING_DURATION) + goto copy; +#endif + + if (ifap != NULL) { + freeifaddrs(ifap); + ifap = NULL; + } + + if (getifaddrs(&ifap) != 0) { + syslog (LOG_ERR, "getifstats() : getifaddrs(): %s", + strerror(errno)); + return (-1); + } + + for (ifa = ifap; ifa; ifa = ifa->ifa_next) + if (ifa->ifa_addr->sa_family == AF_LINK && + strcmp(ifa->ifa_name, ifname) == 0) { +#ifdef ENABLE_GETIFSTATS_CACHING + cache_timestamp = current_time; +copy: +#endif +#define IFA_STAT(s) (((struct if_data *)ifa->ifa_data)->ifi_ ## s) + data->opackets = IFA_STAT(opackets); + data->ipackets = IFA_STAT(ipackets); + data->obytes = IFA_STAT(obytes); + data->ibytes = IFA_STAT(ibytes); + data->baudrate = IFA_STAT(baudrate); + return (0); + } + + return (-1); +} diff --git a/src/contrib/miniupnp/miniupnpd/bsd/getroute.c b/src/contrib/miniupnp/miniupnpd/bsd/getroute.c new file mode 100644 index 0000000..292504d --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/bsd/getroute.c @@ -0,0 +1,145 @@ +/* $Id: getroute.c,v 1.12 2015/11/19 11:46:30 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef AF_LINK +#include +#endif + +#include "../config.h" +#include "../upnputils.h" + +/* SA_SIZE() is a multiple of sizeof(long) with a minimum value of sizeof(long) */ +#ifndef SA_SIZE +#define SA_SIZE(sa) (((SA_LEN(sa)) == 0) ? sizeof(long) : (1 + (((SA_LEN(sa)) - 1) | (sizeof(long) - 1)))) +#endif /* SA_SIZE */ + +int +get_src_for_route_to(const struct sockaddr * dst, + void * src, size_t * src_len, + int * index) +{ + int found = 0; + int s; + int l, i; + char * p; + struct sockaddr * sa; + struct { + struct rt_msghdr m_rtm; + char m_space[512]; + } m_rtmsg; +#define rtm m_rtmsg.m_rtm + + if(dst == NULL) + return -1; + if(SA_LEN(dst) > 0) { + l = SA_LEN(dst); + } else { + if(dst->sa_family == AF_INET) + l = sizeof(struct sockaddr_in); + else if(dst->sa_family == AF_INET6) + l = sizeof(struct sockaddr_in6); + else { + syslog(LOG_ERR, "unknown address family %d", dst->sa_family); + return -1; + } + } + s = socket(PF_ROUTE, SOCK_RAW, dst->sa_family); + if(s < 0) { + syslog(LOG_ERR, "socket(PF_ROUTE) failed : %m"); + return -1; + } + memset(&rtm, 0, sizeof(rtm)); + rtm.rtm_type = RTM_GET; + rtm.rtm_flags = RTF_UP; + rtm.rtm_version = RTM_VERSION; + rtm.rtm_seq = 1; + rtm.rtm_addrs = RTA_DST | RTA_IFA | RTA_IFP; /* pass destination address, request source address & interface */ + memcpy(m_rtmsg.m_space, dst, l); +#if !defined(__sun) + ((struct sockaddr *)m_rtmsg.m_space)->sa_len = l; +#endif + rtm.rtm_msglen = sizeof(struct rt_msghdr) + l; + if(write(s, &m_rtmsg, rtm.rtm_msglen) < 0) { + syslog(LOG_ERR, "write: %m"); + close(s); + return -1; + } + + do { + l = read(s, &m_rtmsg, sizeof(m_rtmsg)); + if(l<0) { + syslog(LOG_ERR, "read: %m"); + close(s); + return -1; + } + syslog(LOG_DEBUG, "read l=%d seq=%d pid=%d sizeof(struct rt_msghdr)=%d", + l, rtm.rtm_seq, rtm.rtm_pid, (int)sizeof(struct rt_msghdr)); + } while(l > 0 && (rtm.rtm_pid != getpid() || rtm.rtm_seq != 1)); + close(s); + if(l <= 0) { + syslog(LOG_WARNING, "no matching ROUTE response message"); + return -1; + } + p = m_rtmsg.m_space; + if(rtm.rtm_addrs) { + for(i=1; i<0x8000; i <<= 1) { + if(i & rtm.rtm_addrs) { + char tmp[256] = { 0 }; + if(p >= (char *)&m_rtmsg + l) { + syslog(LOG_ERR, "error parsing ROUTE response message"); + break; + } + sa = (struct sockaddr *)p; + sockaddr_to_string(sa, tmp, sizeof(tmp)); + syslog(LOG_DEBUG, "offset=%3d type=%2d sa_len=%d sa_family=%d %s", + (int)(p - m_rtmsg.m_space), + i, SA_LEN(sa), sa->sa_family, tmp); + if(i == RTA_IFA) { + size_t len = 0; + void * paddr = NULL; + if(sa->sa_family == AF_INET) { + paddr = &((struct sockaddr_in *)sa)->sin_addr; + len = sizeof(struct in_addr); + } else if(sa->sa_family == AF_INET6) { + paddr = &((struct sockaddr_in6 *)sa)->sin6_addr; + len = sizeof(struct in6_addr); + } + if(paddr) { + if(src && src_len) { + if(*src_len < len) { + syslog(LOG_WARNING, "cannot copy src. %u<%u", + (unsigned)*src_len, (unsigned)len); + return -1; + } + memcpy(src, paddr, len); + *src_len = len; + } + found = 1; + } + } +#ifdef AF_LINK + else if((i == RTA_IFP) && (sa->sa_family == AF_LINK)) { + struct sockaddr_dl * sdl = (struct sockaddr_dl *)sa; + if(index) + *index = sdl->sdl_index; + } +#endif + p += SA_SIZE(sa); + } + } + } + return found ? 0 : -1; +} + diff --git a/src/contrib/miniupnp/miniupnpd/bsd/ifacewatcher.c b/src/contrib/miniupnp/miniupnpd/bsd/ifacewatcher.c new file mode 100644 index 0000000..c3949b3 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/bsd/ifacewatcher.c @@ -0,0 +1,125 @@ +/* $Id: ifacewatcher.c,v 1.8 2014/04/18 08:23:51 nanard Exp $ */ +/* Project MiniUPnP + * web : http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011 Thomas BERNARD + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include "../config.h" + +#include +#include +#include +#include +#include +#include +#include + +#define SALIGN (sizeof(long) - 1) +#define SA_RLEN(sa) (SA_LEN(sa) ? ((SA_LEN(sa) + SALIGN) & ~SALIGN) : (SALIGN + 1)) + +#include "../upnputils.h" +#include "../upnpglobalvars.h" + +extern volatile sig_atomic_t should_send_public_address_change_notif; + +int +OpenAndConfInterfaceWatchSocket(void) +{ + int s; + + /*s = socket(PF_ROUTE, SOCK_RAW, AF_INET);*/ + s = socket(PF_ROUTE, SOCK_RAW, AF_UNSPEC); +/* The family parameter may be AF_UNSPEC which will provide routing informa- + * tion for all address families, or can be restricted to a specific address + * family by specifying which one is desired. There can be more than one + * routing socket open per system. */ + if(s < 0) { + syslog(LOG_ERR, "OpenAndConfInterfaceWatchSocket socket: %m"); + } + return s; +} + +void +ProcessInterfaceWatchNotify(int s) +{ + char buf[4096]; + ssize_t len; + char tmp[64]; + struct rt_msghdr * rtm; + struct if_msghdr * ifm; + struct ifa_msghdr * ifam; +#ifdef RTM_IFANNOUNCE + struct if_announcemsghdr * ifanm; +#endif + char * p; + struct sockaddr * sa; + unsigned int ext_if_name_index = 0; + + len = recv(s, buf, sizeof(buf), 0); + if(len < 0) { + syslog(LOG_ERR, "ProcessInterfaceWatchNotify recv: %m"); + return; + } + if(ext_if_name) { + ext_if_name_index = if_nametoindex(ext_if_name); + } + rtm = (struct rt_msghdr *)buf; + syslog(LOG_DEBUG, "%u rt_msg : msglen=%d version=%d type=%d", (unsigned)len, + rtm->rtm_msglen, rtm->rtm_version, rtm->rtm_type); + switch(rtm->rtm_type) { + case RTM_IFINFO: /* iface going up/down etc. */ + ifm = (struct if_msghdr *)buf; + syslog(LOG_DEBUG, " RTM_IFINFO: addrs=%x flags=%x index=%hu", + ifm->ifm_addrs, ifm->ifm_flags, ifm->ifm_index); + break; + case RTM_ADD: /* Add Route */ + syslog(LOG_DEBUG, " RTM_ADD"); + break; + case RTM_DELETE: /* Delete Route */ + syslog(LOG_DEBUG, " RTM_DELETE"); + break; + case RTM_CHANGE: /* Change Metrics or flags */ + syslog(LOG_DEBUG, " RTM_CHANGE"); + break; + case RTM_GET: /* Report Metrics */ + syslog(LOG_DEBUG, " RTM_GET"); + break; +#ifdef RTM_IFANNOUNCE + case RTM_IFANNOUNCE: /* iface arrival/departure */ + ifanm = (struct if_announcemsghdr *)buf; + syslog(LOG_DEBUG, " RTM_IFANNOUNCE: index=%hu what=%hu ifname=%s", + ifanm->ifan_index, ifanm->ifan_what, ifanm->ifan_name); + break; +#endif +#ifdef RTM_IEEE80211 + case RTM_IEEE80211: /* IEEE80211 wireless event */ + syslog(LOG_DEBUG, " RTM_IEEE80211"); + break; +#endif + case RTM_NEWADDR: /* address being added to iface */ + ifam = (struct ifa_msghdr *)buf; + syslog(LOG_DEBUG, " RTM_NEWADDR: addrs=%x flags=%x index=%hu", + ifam->ifam_addrs, ifam->ifam_flags, ifam->ifam_index); + p = buf + sizeof(struct ifa_msghdr); + while(p < buf + len) { + sa = (struct sockaddr *)p; + sockaddr_to_string(sa, tmp, sizeof(tmp)); + syslog(LOG_DEBUG, " %s", tmp); + p += SA_RLEN(sa); + } + if(ifam->ifam_index == ext_if_name_index) { + should_send_public_address_change_notif = 1; + } + break; + case RTM_DELADDR: /* address being removed from iface */ + ifam = (struct ifa_msghdr *)buf; + if(ifam->ifam_index == ext_if_name_index) { + should_send_public_address_change_notif = 1; + } + break; + default: + syslog(LOG_DEBUG, "unprocessed RTM message type=%d", rtm->rtm_type); + } +} + diff --git a/src/contrib/miniupnp/miniupnpd/bsd/testgetifstats.c b/src/contrib/miniupnp/miniupnpd/bsd/testgetifstats.c new file mode 100644 index 0000000..4dff850 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/bsd/testgetifstats.c @@ -0,0 +1,32 @@ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include + +#include "../getifstats.h" + +int +main(int argc, char * * argv) +{ + int r; + struct ifdata data; + printf("usage: %s if_name\n", argv[0]); + if(argc<2) + return -1; + r = getifstats(argv[1], &data); + if(r<0) + printf("getifstats() failed\n"); + else + { + printf("ipackets = %10lu opackets = %10lu\n", + data.ipackets, data.opackets); + printf("ibytes = %10lu obytes = %10lu\n", + data.ibytes, data.obytes); + printf("baudrate = %10lu\n", data.baudrate); + } + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/bsd/testifacewatcher.c b/src/contrib/miniupnp/miniupnpd/bsd/testifacewatcher.c new file mode 100644 index 0000000..354b7e0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/bsd/testifacewatcher.c @@ -0,0 +1,32 @@ +/* $Id: testifacewatcher.c,v 1.2 2012/05/21 08:55:10 nanard Exp $ */ + +#include + +int +OpenAndConfInterfaceWatchSocket(void); + +void +ProcessInterfaceWatchNotify(int s); + +const char * ext_if_name; +volatile sig_atomic_t should_send_public_address_change_notif = 0; + +int main(int argc, char * * argv) +{ + int s; + + ext_if_name = "ep0"; + openlog("testifacewatcher", LOG_CONS|LOG_PERROR, LOG_USER); + + syslog(LOG_DEBUG, "test"); + s = OpenAndConfInterfaceWatchSocket(); + for(;;) { + if(should_send_public_address_change_notif) { + syslog(LOG_DEBUG, "should_send_public_address_change_notif !"); + should_send_public_address_change_notif = 0; + } + ProcessInterfaceWatchNotify(s); + } + closelog(); + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/codelength.h b/src/contrib/miniupnp/miniupnpd/codelength.h new file mode 100644 index 0000000..f5f8e30 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/codelength.h @@ -0,0 +1,54 @@ +/* $Id: codelength.h,v 1.5 2015/07/09 12:40:18 nanard Exp $ */ +/* Project : miniupnp + * Author : Thomas BERNARD + * copyright (c) 2005-2015 Thomas Bernard + * This software is subjet to the conditions detailed in the + * provided LICENCE file. */ +#ifndef CODELENGTH_H_INCLUDED +#define CODELENGTH_H_INCLUDED + +/* Encode length by using 7bit per Byte : + * Most significant bit of each byte specifies that the + * following byte is part of the code */ + +/* n : unsigned + * p : unsigned char * + */ +#define DECODELENGTH(n, p) n = 0; \ + do { n = (n << 7) | (*p & 0x7f); } \ + while((*(p++)&0x80) && (n<(1<<25))); + +/* n : unsigned + * READ : function/macro to read one byte (unsigned char) + */ +#define DECODELENGTH_READ(n, READ) \ + n = 0; \ + do { \ + unsigned char c; \ + READ(c); \ + n = (n << 7) | (c & 0x07f); \ + if(!(c&0x80)) break; \ + } while(n<(1<<25)); + +/* n : unsigned + * p : unsigned char * + * p_limit : unsigned char * + */ +#define DECODELENGTH_CHECKLIMIT(n, p, p_limit) \ + n = 0; \ + do { \ + if((p) >= (p_limit)) break; \ + n = (n << 7) | (*(p) & 0x7f); \ + } while((*((p)++)&0x80) && (n<(1<<25))); + + +/* n : unsigned + * p : unsigned char * + */ +#define CODELENGTH(n, p) if(n>=268435456) *(p++) = (n >> 28) | 0x80; \ + if(n>=2097152) *(p++) = (n >> 21) | 0x80; \ + if(n>=16384) *(p++) = (n >> 14) | 0x80; \ + if(n>=128) *(p++) = (n >> 7) | 0x80; \ + *(p++) = n & 0x7f; + +#endif /* CODELENGTH_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpd/commonrdr.h b/src/contrib/miniupnp/miniupnpd/commonrdr.h new file mode 100644 index 0000000..61352e4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/commonrdr.h @@ -0,0 +1,67 @@ +/* $Id: commonrdr.h,v 1.10 2016/02/12 12:34:39 nanard Exp $ */ +/* MiniUPnP project + * (c) 2006-2016 Thomas Bernard + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef COMMONRDR_H_INCLUDED +#define COMMONRDR_H_INCLUDED + +#include "config.h" + +/* init and shutdown functions */ +/* init_redirect() return values : + * 0 : OK + * -1 : error */ +int +init_redirect(void); + +void +shutdown_redirect(void); + +/* get_redirect_rule() gets internal IP and port from + * interface, external port and protocol + * return value : + * 0 success (rule found) + * -1 error or rule not found + */ +int +get_redirect_rule(const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); + +/* get_redirect_rule_by_index() + * return values : + * 0 success (rule found) + * -1 error or rule not found */ +int +get_redirect_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); + +/* return an (malloc'ed) array of "external" port for which there is + * a port mapping. number is the size of the array */ +unsigned short * +get_portmappings_in_range(unsigned short startport, unsigned short endport, + int proto, unsigned int * number); + +/* update the port mapping internal port, decription and timestamp */ +int +update_portmapping(const char * ifname, unsigned short eport, int proto, + unsigned short iport, const char * desc, + unsigned int timestamp); + +/* update the port mapping decription and timestamp */ +int +update_portmapping_desc_timestamp(const char * ifname, + unsigned short eport, int proto, + const char * desc, unsigned int timestamp); + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/daemonize.c b/src/contrib/miniupnp/miniupnpd/daemonize.c new file mode 100644 index 0000000..4c5909d --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/daemonize.c @@ -0,0 +1,129 @@ +/* $Id: daemonize.c,v 1.12 2011/05/27 09:35:02 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "daemonize.h" +#include "config.h" + +#ifndef USE_DAEMON + +int +daemonize(void) +{ + int pid, i; + + switch(fork()) + { + /* fork error */ + case -1: + perror("fork()"); + exit(1); + + /* child process */ + case 0: + /* obtain a new process group */ + if( (pid = setsid()) < 0) + { + perror("setsid()"); + exit(1); + } + + /* close all descriptors */ + for (i=getdtablesize();i>=0;--i) close(i); + + i = open("/dev/null", O_RDWR); /* open stdin */ + dup(i); /* stdout */ + dup(i); /* stderr */ + + umask(027); + chdir("/"); /* chdir to /tmp ? */ + + return pid; + + /* parent process */ + default: + exit(0); + } +} +#endif + +int +writepidfile(const char * fname, int pid) +{ + char pidstring[16]; + int pidstringlen; + int pidfile; + + if(!fname || fname[0] == '\0') + return -1; + + if( (pidfile = open(fname, O_WRONLY|O_CREAT, 0644)) < 0) + { + syslog(LOG_ERR, "Unable to open pidfile for writing %s: %m", fname); + return -1; + } + + pidstringlen = snprintf(pidstring, sizeof(pidstring), "%d\n", pid); + if(pidstringlen <= 0) + { + syslog(LOG_ERR, + "Unable to write to pidfile %s: snprintf(): FAILED", fname); + close(pidfile); + return -1; + } + else + { + if(write(pidfile, pidstring, pidstringlen) < 0) + syslog(LOG_ERR, "Unable to write to pidfile %s: %m", fname); + } + + close(pidfile); + + return 0; +} + +int +checkforrunning(const char * fname) +{ + char buffer[64]; + int pidfile; + pid_t pid; + + if(!fname || fname[0] == '\0') + return -1; + + if( (pidfile = open(fname, O_RDONLY)) < 0) + return 0; + + memset(buffer, 0, 64); + + if(read(pidfile, buffer, 63)) + { + if( (pid = atol(buffer)) > 0) + { + if(!kill(pid, 0)) + { + close(pidfile); + return -2; + } + } + } + + close(pidfile); + + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/daemonize.h b/src/contrib/miniupnp/miniupnpd/daemonize.h new file mode 100644 index 0000000..818ce73 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/daemonize.h @@ -0,0 +1,35 @@ +/* $Id: daemonize.h,v 1.6 2008/01/29 13:04:46 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef DAEMONIZE_H_INCLUDED +#define DAEMONIZE_H_INCLUDED + +#include "config.h" + +#ifndef USE_DAEMON +/* daemonize() + * "fork" to background, detach from terminal, etc... + * returns: pid of the daemon, exits upon failure */ +int +daemonize(void); +#endif + +/* writepidfile() + * write the pid to a file */ +int +writepidfile(const char * fname, int pid); + +/* checkforrunning() + * check for another instance running + * returns: 0 only instance + * -1 invalid filename + * -2 another instance running */ +int +checkforrunning(const char * fname); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/genconfig.sh b/src/contrib/miniupnp/miniupnpd/genconfig.sh new file mode 100644 index 0000000..d0bcf1f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/genconfig.sh @@ -0,0 +1,634 @@ +#! /bin/sh +# $Id: genconfig.sh,v 1.95 2018/01/16 00:50:46 nanard Exp $ +# vim: tabstop=4 shiftwidth=4 noexpandtab +# +# miniupnp daemon +# http://miniupnp.free.fr or https://miniupnp.tuxfamily.org/ +# (c) 2006-2018 Thomas Bernard +# This software is subject to the conditions detailed in the +# LICENCE file provided within the distribution + +# default to UPnP Device Architecture (UDA) v1.1 +# some control points do not like UDA v2.0 +UPNP_VERSION_MAJOR=1 +UPNP_VERSION_MINOR=1 + +for argv; do +case "$argv" in + --ipv6) IPV6=1 ;; + --igd2) IGD2=1 ;; + --strict) STRICT=1 ;; + --leasefile) LEASEFILE=1 ;; + --vendorcfg) VENDORCFG=1 ;; + --pcp-peer) PCP_PEER=1 ;; + --portinuse) PORTINUSE=1 ;; + --uda-version=*) + UPNP_VERSION=$(echo $argv | cut -d= -f2) + UPNP_VERSION_MAJOR=$(echo $UPNP_VERSION | cut -s -d. -f1) + UPNP_VERSION_MINOR=$(echo $UPNP_VERSION | cut -s -d. -f2) + echo "Setting UPnP version major=$UPNP_VERSION_MAJOR minor=$UPNP_VERSION_MINOR" + if [ -z "$UPNP_VERSION_MAJOR" ] || [ -z "$UPNP_VERSION_MINOR" ] ; then + echo "UPnP Version invalid in option $argv" + exit 1 + fi ;; + --disable-pppconn) DISABLEPPPCONN=1 ;; + --help|-h) + echo "Usage : $0 [options]" + echo " --ipv6 enable IPv6" + echo " --igd2 build an IGDv2 instead of an IGDv1" + echo " --strict be more strict regarding compliance with UPnP specifications" + echo " --leasefile enable lease file" + echo " --vendorcfg enable configuration of manufacturer info" + echo " --pcp-peer enable PCP PEER operation" + echo " --portinuse enable port in use check" + echo " --uda-version=x.x set advertised UPnP version (default to ${UPNP_VERSION_MAJOR}.${UPNP_VERSION_MINOR})" + echo " --disable-pppconn disable WANPPPConnection" + exit 1 + ;; + *) + echo "Option not recognized : $argv" + echo "use -h option to display help" + exit 1 + ;; +esac +done + +RM="rm -f" +MV="mv" +CONFIGFILE=`mktemp tmp.config.h.XXXXXXXXXX` +CONFIGFILE_FINAL="config.h" +CONFIGMACRO="CONFIG_H_INCLUDED" + +MINIUPNPD_DATE=`date +"%Y%m%d"` +if [ -n "$SOURCE_DATE_EPOCH" ]; then + MINIUPNPD_DATE=`date --utc --date="@$SOURCE_DATE_EPOCH" +"%Y%m%d"` +fi + +# Facility to syslog +LOG_MINIUPNPD="LOG_DAEMON" + +# detecting the OS name and version +OS_NAME=`uname -s` +OS_VERSION=`uname -r` + +# pfSense special case +if [ -f /etc/platform ]; then + if [ `cat /etc/platform` = "pfSense" ]; then + OS_NAME=pfSense + OS_VERSION=`cat /etc/version` + fi +fi + +# OpenWRT special case +if [ -f ./os.openwrt ]; then + OS_NAME=OpenWRT + OS_VERSION=$(cat ./os.openwrt) +fi + +# AstLinux special case +if [ -f ./os.astlinux ]; then + OS_NAME=AstLinux + OS_VERSION=$(cat ./os.astlinux) +fi + +# Tomato USB special case +if [ -f ../shared/tomato_version ]; then + OS_NAME=Tomato + TOMATO_VER=`cat ../shared/tomato_version | cut -d' ' -f2,3` + OS_VERSION="Tomato $TOMATO_VER" +fi + +${RM} ${CONFIGFILE} + +echo "/* MiniUPnP Project" >> ${CONFIGFILE} +echo " * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/" >> ${CONFIGFILE} +echo " * (c) 2006-2018 Thomas Bernard" >> ${CONFIGFILE} +echo " * generated by $0 on `date`" >> ${CONFIGFILE} +echo " * `uname -a`" >> ${CONFIGFILE} +if [ -z "$*" ] ; then + echo " * using no command line option */" >> ${CONFIGFILE} +else + echo " * using command line options $* */" >> ${CONFIGFILE} +fi +echo "#ifndef $CONFIGMACRO" >> ${CONFIGFILE} +echo "#define $CONFIGMACRO" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} +echo "#include " >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} +echo "#define MINIUPNPD_VERSION \"`cat VERSION`\"" >> ${CONFIGFILE} +echo "#define MINIUPNPD_DATE \"$MINIUPNPD_DATE\"" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +cat >> ${CONFIGFILE} <> ${CONFIGFILE} +cat >> ${CONFIGFILE} <> ${CONFIGFILE} + +# OS Specific stuff +case $OS_NAME in + OpenBSD) + MAJORVER=`echo $OS_VERSION | cut -d. -f1` + MINORVER=`echo $OS_VERSION | cut -d. -f2` + #echo "OpenBSD majorversion=$MAJORVER minorversion=$MINORVER" + # rtableid was introduced in OpenBSD 4.0 + if [ $MAJORVER -ge 4 ]; then + echo "#define PFRULE_HAS_RTABLEID" >> ${CONFIGFILE} + fi + # from the 3.8 version, packets and bytes counters are double : in/out + if [ \( $MAJORVER -ge 4 \) -o \( $MAJORVER -eq 3 -a $MINORVER -ge 8 \) ]; then + echo "#define PFRULE_INOUT_COUNTS" >> ${CONFIGFILE} + fi + # from the 4.7 version, new pf + if [ \( $MAJORVER -ge 5 \) -o \( $MAJORVER -eq 4 -a $MINORVER -ge 7 \) ]; then + echo "#define PF_NEWSTYLE" >> ${CONFIGFILE} + fi + # onrdomain was introduced in OpenBSD 5.0 + if [ $MAJORVER -ge 5 ]; then + echo "#define PFRULE_HAS_ONRDOMAIN" >> ${CONFIGFILE} + fi + FW=pf + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + OS_URL=http://www.openbsd.org/ + V6SOCKETS_ARE_V6ONLY=`sysctl -n net.inet6.ip6.v6only` + ;; + FreeBSD) + VER=`grep '#define __FreeBSD_version' /usr/include/sys/param.h | awk '{print $3}'` + if [ $VER -ge 700049 ]; then + echo "#define PFRULE_INOUT_COUNTS" >> ${CONFIGFILE} + fi + HAVE_IP_MREQN=1 + # new way to see which one to use PF or IPF. + # see http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=957 + if [ -f /etc/rc.subr ] && [ -f /etc/default/rc.conf ] ; then + # source file with handy subroutines like checkyesno + . /etc/rc.subr + # source config file so we can probe vars + . /etc/default/rc.conf + if checkyesno ipfilter_enable; then + echo "Using ipf" + FW=ipf + elif checkyesno pf_enable; then + echo "Using pf" + FW=pf + elif checkyesno firewall_enable; then + echo "Using ifpw" + FW=ipfw + fi + fi + if [ -z $FW ] ; then + echo "Could not detect usage of ipf, pf, ipfw. Compiling for pf by default" + FW=pf + fi + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + OS_URL=http://www.freebsd.org/ + V6SOCKETS_ARE_V6ONLY=`sysctl -n net.inet6.ip6.v6only` + ;; + pfSense) + # we need to detect if PFRULE_INOUT_COUNTS macro is needed + FW=pf + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + OS_URL=http://www.pfsense.com/ + V6SOCKETS_ARE_V6ONLY=`sysctl -n net.inet6.ip6.v6only` + ;; + NetBSD) + if [ -f /etc/rc.subr ] && [ -f /etc/rc.conf ] ; then + # source file with handy subroutines like checkyesno + . /etc/rc.subr + # source config file so we can probe vars + . /etc/rc.conf + if checkyesno pf; then + FW=pf + elif checkyesno ipfilter; then + FW=ipf + fi + fi + if [ -z $FW ] ; then + echo "Could not detect ipf nor pf, defaulting to pf." + FW=pf + fi + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + OS_URL=http://www.netbsd.org/ + ;; + DragonFly) + if [ -f /etc/rc.subr ] && [ -f /etc/rc.conf ] ; then + # source file with handy subroutines like checkyesno + . /etc/rc.subr + # source config file so we can probe vars + . /etc/rc.conf + if checkyesno pf; then + FW=pf + elif checkyesno ipfilter; then + FW=ipf + fi + fi + if [ -z $FW ] ; then + echo "Could not detect ipf nor pf, defaulting to pf." + FW=pf + fi + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + # PFRULE_INOUT_COUNTS should be set for DragonFly > 2.8 + # version detection is not yet added to this script. + echo "#define PFRULE_INOUT_COUNTS" >> ${CONFIGFILE} + # net.inet6.ip6.v6only has been on by default for many years + # and this sysctl node has been removed + V6SOCKETS_ARE_V6ONLY=1 + OS_URL=http://www.dragonflybsd.org/ + ;; + SunOS) + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + FW=ipf + echo "#define LOG_PERROR 0" >> ${CONFIGFILE} + echo "#define SOLARIS_KSTATS 1" >> ${CONFIGFILE} + # solaris 10 does not define u_int64_t ? + # but it does define uint64_t + echo "typedef uint64_t u_int64_t;" >> ${CONFIGFILE} + OS_URL=http://www.sun.com/solaris/ + ;; + Linux) + OS_URL=http://www.kernel.org/ + KERNVERA=`echo $OS_VERSION | awk -F. '{print $1}'` + KERNVERB=`echo $OS_VERSION | awk -F. '{print $2}'` + KERNVERC=`echo $OS_VERSION | awk -F. '{print $3}'` + KERNVERD=`echo $OS_VERSION | awk -F. '{print $4}'` + #echo "$KERNVERA.$KERNVERB.$KERNVERC.$KERNVERD" + # from the 2.4 version, struct ip_mreqn instead of struct ip_mreq + if [ \( $KERNVERA -ge 3 \) -o \( $KERNVERA -eq 2 -a $KERNVERB -ge 4 \) ]; then + HAVE_IP_MREQN=1 + fi + # Debian GNU/Linux special case + if [ -f /etc/debian_version ]; then + OS_NAME=Debian + OS_VERSION=`cat /etc/debian_version` + OS_URL=http://www.debian.org/ + fi + # same thing for Gentoo linux + if [ -f /etc/gentoo-release ]; then + OS_NAME=Gentoo + OS_VERSION=`cat /etc/gentoo-release` + OS_URL=http://www.gentoo.org/ + fi + # ClearOS special case + if [ -f /etc/clearos-release ]; then + OS_NAME=ClearOS + OS_VERSION=`grep ^base_version /etc/product | awk '{ print $3 }'` + OS_URL=https://www.clearos.com/ + fi + # use lsb_release (Linux Standard Base) when available + LSB_RELEASE=`which lsb_release` + if [ 0 -eq $? ]; then + OS_NAME=`${LSB_RELEASE} -i -s` + OS_VERSION=`${LSB_RELEASE} -r -s` + case $OS_NAME in + Debian) + OS_URL=http://www.debian.org/ + OS_VERSION=`${LSB_RELEASE} -c -s` + ;; + Ubuntu) + OS_URL=http://www.ubuntu.com/ + OS_VERSION=`${LSB_RELEASE} -c -s` + ;; + Gentoo) + OS_URL=http://www.gentoo.org/ + ;; + arch) + OS_URL=http://www.archlinux.org/ + OS_VERSION=`uname -r` + ;; + esac + fi + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + FW=netfilter + V6SOCKETS_ARE_V6ONLY=`/sbin/sysctl -n net.ipv6.bindv6only` + ;; + OpenWRT) + OS_URL=http://www.openwrt.org/ + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + FW=netfilter + ;; + AstLinux) + OS_URL=http://www.astlinux.org/ + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + FW=netfilter + ;; + Tomato) + OS_NAME=UPnP + OS_URL=http://tomatousb.org/ + echo "" >> ${CONFIGFILE} + echo "#ifndef TOMATO" >> ${CONFIGFILE} + echo "#define TOMATO" >> ${CONFIGFILE} + echo "#endif" >> ${CONFIGFILE} + echo "" >> ${CONFIGFILE} + echo "#ifdef LINUX26" >> ${CONFIGFILE} + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + echo "#endif" >> ${CONFIGFILE} + echo "#ifdef TCONFIG_IPV6" >> ${CONFIGFILE} + echo "#define ENABLE_IPV6" >> ${CONFIGFILE} + echo "#endif" >> ${CONFIGFILE} + FW=netfilter + ;; + Darwin) + MAJORVER=`echo $OS_VERSION | cut -d. -f1` + echo "#define USE_IFACEWATCHER 1" >> ${CONFIGFILE} + # OS X switched to pf since 10.7 Lion (Darwin 11.0) + if [ $MAJORVER -ge 11 ] ; then + FW=pf + echo "#define PFRULE_INOUT_COUNTS" >> ${CONFIGFILE} + else + FW=ipfw + fi + OS_URL=http://developer.apple.com/macosx + ;; + *) + echo "Unknown OS : $OS_NAME" + echo "Please contact the author at http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/." + exit 1 + ;; +esac + +case $FW in + pf) + echo "#define USE_PF 1" >> ${CONFIGFILE} + ;; + ipf) + echo "#define USE_IPF 1" >> ${CONFIGFILE} + ;; + ipfw) + echo "#define USE_IPFW 1" >> ${CONFIGFILE} + ;; + netfilter) + echo "#define USE_NETFILTER 1" >> ${CONFIGFILE} + ;; + *) + echo "Unknown Firewall/packet filtering software [$FW]" + echo "Please contact the author at http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/." + exit 1 + ;; +esac + +# UUID API +if grep uuid_create /usr/include/uuid.h > /dev/null 2>&1 ; then + echo "#define BSD_UUID" >> ${CONFIGFILE} +fi +if grep uuid_generate /usr/include/uuid/uuid.h > /dev/null 2>&1 ; then + echo "#define LIB_UUID" >> ${CONFIGFILE} +fi + +# set V6SOCKETS_ARE_V6ONLY to 0 if it was not set above +if [ -z "$V6SOCKETS_ARE_V6ONLY" ] ; then + V6SOCKETS_ARE_V6ONLY=0 +fi + +echo "Configuring compilation for [$OS_NAME] [$OS_VERSION] with [$FW] firewall software." +echo "Please edit config.h for more compilation options." + +# define SUPPORT_REMOTEHOST if the FW related code really supports setting +# a RemoteHost +if [ \( "$FW" = "netfilter" \) -o \( "$FW" = "pf" \) -o \( "$FW" = "ipfw" \) ] ; then + echo "#define SUPPORT_REMOTEHOST" >> ${CONFIGFILE} +fi + +echo "/* Enable IGD2 \"Port Triggering\" as defined in Section 2.5.16" >> ${CONFIGFILE} +echo " * figure 2.2 in UPnP-gw-WANIPConnection-v2-Service.pdf */" >> ${CONFIGFILE} +echo "#define ENABLE_PORT_TRIGGERING" >> ${CONFIGFILE} + +echo "" >> ${CONFIGFILE} +echo "#define OS_NAME \"$OS_NAME\"" >> ${CONFIGFILE} +echo "#define OS_VERSION \"$OS_NAME/$OS_VERSION\"" >> ${CONFIGFILE} +echo "#define OS_URL \"${OS_URL}\"" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* syslog facility to be used by miniupnpd */" >> ${CONFIGFILE} +echo "#define LOG_MINIUPNPD ${LOG_MINIUPNPD}" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Uncomment the following line to allow miniupnpd to be" >> ${CONFIGFILE} +echo " * controlled by miniupnpdctl */" >> ${CONFIGFILE} +echo "/*#define USE_MINIUPNPDCTL*/" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Comment the following line to disable NAT-PMP operations */" >> ${CONFIGFILE} +echo "#define ENABLE_NATPMP" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Comment the following line to disable PCP operations */" >> ${CONFIGFILE} +echo "#define ENABLE_PCP" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "#ifdef ENABLE_PCP" >> ${CONFIGFILE} +if [ -n "$PCP_PEER" ]; then +echo "/* Comment the following line to disable PCP PEER operation */" >> ${CONFIGFILE} +echo "#define PCP_PEER" >> ${CONFIGFILE} +else +echo "/* Uncomment the following line to enable PCP PEER operation */" >> ${CONFIGFILE} +echo "/*#define PCP_PEER*/" >> ${CONFIGFILE} +fi +echo "#ifdef PCP_PEER" >> ${CONFIGFILE} +echo "/*#define PCP_FLOWP*/" >> ${CONFIGFILE} +echo "#endif /*PCP_PEER*/" >> ${CONFIGFILE} +echo "/*#define PCP_SADSCP*/" >> ${CONFIGFILE} +echo "#endif /*ENABLE_PCP*/" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Uncomment the following line to enable generation of" >> ${CONFIGFILE} +echo " * filter rules with pf */" >> ${CONFIGFILE} +echo "/*#define PF_ENABLE_FILTER_RULES*/">> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Uncomment the following line to enable caching of results of" >> ${CONFIGFILE} +echo " * the getifstats() function */" >> ${CONFIGFILE} +echo "/*#define ENABLE_GETIFSTATS_CACHING*/" >> ${CONFIGFILE} +echo "/* The cache duration is indicated in seconds */" >> ${CONFIGFILE} +echo "#define GETIFSTATS_CACHING_DURATION 2" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Uncomment the following line to enable multiple external ip support */" >> ${CONFIGFILE} +echo "/* note : That is EXPERIMENTAL, do not use that unless you know perfectly what you are doing */" >> ${CONFIGFILE} +echo "/* Dynamic external ip adresses are not supported when this option is enabled." >> ${CONFIGFILE} +echo " * Also note that you would need to configure your .conf file accordingly. */" >> ${CONFIGFILE} +echo "/*#define MULTIPLE_EXTERNAL_IP*/" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Comment the following line to use home made daemonize() func instead" >> ${CONFIGFILE} +echo " * of BSD daemon() */" >> ${CONFIGFILE} +echo "#define USE_DAEMON" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Uncomment the following line to enable lease file support */" >> ${CONFIGFILE} +if [ -n "$LEASEFILE" ] ; then + echo "#define ENABLE_LEASEFILE" >> ${CONFIGFILE} +else + echo "/*#define ENABLE_LEASEFILE*/" >> ${CONFIGFILE} +fi +echo "/* Uncomment the following line to store remaining time in lease file */" >> ${CONFIGFILE} +echo "/*#define LEASEFILE_USE_REMAINING_TIME*/" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Uncomment the following line to enable port in use check */" >> ${CONFIGFILE} +if [ -n "$PORTINUSE" ]; then + echo "#define CHECK_PORTINUSE" >> ${CONFIGFILE} +else + echo "/*#define CHECK_PORTINUSE*/" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +echo "/* Define one or none of the two following macros in order to make some" >> ${CONFIGFILE} +echo " * clients happy. It will change the XML Root Description of the IGD." >> ${CONFIGFILE} +echo " * Enabling the Layer3Forwarding Service seems to be the more compatible" >> ${CONFIGFILE} +echo " * option. */" >> ${CONFIGFILE} +echo "/*#define HAS_DUMMY_SERVICE*/" >> ${CONFIGFILE} +echo "#define ENABLE_L3F_SERVICE" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* define ADVERTISE_WANPPPCONN to allow buggy Control Point to use" >> ${CONFIGFILE} +echo " * WANPPPConnection instead of WANIPConnection. */" >> ${CONFIGFILE} +if [ -n "$STRICT" ] || [ -n "$DISABLEPPPCONN" ] ; then + echo "/*#define ADVERTISE_WANPPPCONN*/" >> ${CONFIGFILE} +else + echo "#define ADVERTISE_WANPPPCONN" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +echo "/* Enable IP v6 support */" >> ${CONFIGFILE} +if [ -n "$IPV6" ]; then + echo "#define ENABLE_IPV6" >> ${CONFIGFILE} +else + echo "/*#define ENABLE_IPV6*/" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +echo "/* Define V6SOCKETS_ARE_V6ONLY if AF_INET6 sockets are restricted" >> ${CONFIGFILE} +echo " * to IPv6 communications only. */" >> ${CONFIGFILE} +if [ $V6SOCKETS_ARE_V6ONLY -eq 1 ] ; then + echo "#define V6SOCKETS_ARE_V6ONLY" >> ${CONFIGFILE} +else + echo "/*#define V6SOCKETS_ARE_V6ONLY*/" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +if [ -n "$HAVE_IP_MREQN" ]; then + echo "#define HAVE_IP_MREQN" >> ${CONFIGFILE} + echo "" >> ${CONFIGFILE} +fi + +echo "/* Enable the support of IGD v2 specification." >> ${CONFIGFILE} +echo " * This is not fully tested yet and can cause incompatibilities with some" >> ${CONFIGFILE} +echo " * control points, so enable with care. */" >> ${CONFIGFILE} +if [ -n "$IGD2" ]; then + echo "#define IGD_V2" >> ${CONFIGFILE} +else + echo "/*#define IGD_V2*/" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +echo "#ifdef IGD_V2" >> ${CONFIGFILE} +echo "/* Enable DeviceProtection service (IGDv2) */" >> ${CONFIGFILE} +echo "#define ENABLE_DP_SERVICE" >> ${CONFIGFILE} +echo "/*#define ENABLE_HTTPS*/" >> ${CONFIGFILE} +echo "/*#define HTTPS_CERTFILE \"/path/to/certificate.pem\"*/" >> ${CONFIGFILE} +echo "/*#define HTTPS_KEYFILE \"/path/to/private.key\"*/" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} +echo "/* Enable WANIPv6FirewallControl service (IGDv2). needs IPv6 */" >> ${CONFIGFILE} +echo "#ifdef ENABLE_IPV6" >> ${CONFIGFILE} +echo "#define ENABLE_6FC_SERVICE" >> ${CONFIGFILE} +echo "#endif /* ENABLE_IPV6 */" >> ${CONFIGFILE} +echo "#endif /* IGD_V2 */" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* UPnP Events support. Working well enough to be enabled by default." >> ${CONFIGFILE} +echo " * It can be disabled to save a few bytes. */" >> ${CONFIGFILE} +echo "#define ENABLE_EVENTS" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* include interface name in pf and ipf rules */" >> ${CONFIGFILE} +echo "#define USE_IFNAME_IN_RULES" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Experimental NFQUEUE support. */" >> ${CONFIGFILE} +echo "/*#define ENABLE_NFQUEUE*/" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Enable to make MiniUPnPd more strict about UPnP conformance" >> ${CONFIGFILE} +echo " * and the messages it receives from control points */" >> ${CONFIGFILE} +if [ -n "$STRICT" ] ; then + echo "#define UPNP_STRICT" >> ${CONFIGFILE} +else + echo "/*#define UPNP_STRICT*/" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +echo "/* If SSDP_RESPOND_SAME_VERSION is defined, the M-SEARCH response" >> ${CONFIGFILE} +echo " * include the same device version as was contained in the search" >> ${CONFIGFILE} +echo " * request. It conforms to UPnP DA v1.1 */" >> ${CONFIGFILE} +echo "#define SSDP_RESPOND_SAME_VERSION" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Add the optional Date: header in all HTTP responses */" >> ${CONFIGFILE} +if [ -n "$STRICT" ] ; then + echo "#define ENABLE_HTTP_DATE" >> ${CONFIGFILE} +else + echo "/*#define ENABLE_HTTP_DATE*/" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +echo "/* Wait a little before answering M-SEARCH request */" >> ${CONFIGFILE} +if [ -n "$STRICT" ] ; then + echo "#define DELAY_MSEARCH_RESPONSE" >> ${CONFIGFILE} +else + echo "/*#define DELAY_MSEARCH_RESPONSE*/" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +echo "/* disable reading and parsing of config file (miniupnpd.conf) */" >> ${CONFIGFILE} +echo "/*#define DISABLE_CONFIG_FILE*/" >> ${CONFIGFILE} +echo "" >> ${CONFIGFILE} + +echo "/* Uncomment the following line to configure all manufacturer infos through miniupnpd.conf */" >> ${CONFIGFILE} +if [ -n "$VENDORCFG" ] ; then + echo "#define ENABLE_MANUFACTURER_INFO_CONFIGURATION" >> ${CONFIGFILE} +else + echo "/*#define ENABLE_MANUFACTURER_INFO_CONFIGURATION*/" >> ${CONFIGFILE} +fi +echo "" >> ${CONFIGFILE} + +cat >> ${CONFIGFILE} <> ${CONFIGFILE} <> ${CONFIGFILE} <> ${CONFIGFILE} + +${MV} ${CONFIGFILE} ${CONFIGFILE_FINAL} + +exit 0 diff --git a/src/contrib/miniupnp/miniupnpd/getconnstatus.c b/src/contrib/miniupnp/miniupnpd/getconnstatus.c new file mode 100644 index 0000000..332784c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/getconnstatus.c @@ -0,0 +1,74 @@ +/* $Id: getconnstatus.c,v 1.6 2013/03/23 10:46:54 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include "getconnstatus.h" +#include "getifaddr.h" + +#define STATUS_UNCONFIGURED (0) +#define STATUS_CONNECTING (1) +#define STATUS_CONNECTED (2) +#define STATUS_PENDINGDISCONNECT (3) +#define STATUS_DISCONNECTING (4) +#define STATUS_DISCONNECTED (5) + +/** + * get the connection status + * return values : + * 0 - Unconfigured + * 1 - Connecting + * 2 - Connected + * 3 - PendingDisconnect + * 4 - Disconnecting + * 5 - Disconnected */ +int +get_wan_connection_status(const char * ifname) +{ + char addr[INET_ADDRSTRLEN]; + int r; + + /* we need a better implementation here. + * I'm afraid it should be device specific */ + r = getifaddr(ifname, addr, INET_ADDRSTRLEN, NULL, NULL); + return (r < 0) ? STATUS_DISCONNECTED : STATUS_CONNECTED; +} + +/** + * return the same value as get_wan_connection_status() + * as a C string */ +const char * +get_wan_connection_status_str(const char * ifname) +{ + int status; + const char * str = NULL; + + status = get_wan_connection_status(ifname); + switch(status) { + case 0: + str = "Unconfigured"; + break; + case 1: + str = "Connecting"; + break; + case 2: + str = "Connected"; + break; + case 3: + str = "PendingDisconnect"; + break; + case 4: + str = "Disconnecting"; + break; + case 5: + str = "Disconnected"; + break; + } + return str; +} + diff --git a/src/contrib/miniupnp/miniupnpd/getconnstatus.h b/src/contrib/miniupnp/miniupnpd/getconnstatus.h new file mode 100644 index 0000000..5b5da61 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/getconnstatus.h @@ -0,0 +1,30 @@ +/* $Id: getconnstatus.h,v 1.2 2011/05/23 20:22:41 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef GETCONNSTATUS_H_INCLUDED +#define GETCONNSTATUS_H_INCLUDED + +/** + * get the connection status + * return values : + * 0 - Unconfigured + * 1 - Connecting + * 2 - Connected + * 3 - PendingDisconnect + * 4 - Disconnecting + * 5 - Disconnected */ +int +get_wan_connection_status(const char * ifname); + +/** + * return the same value as get_wan_connection_status() + * as a C string */ +const char * +get_wan_connection_status_str(const char * ifname); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/getifaddr.c b/src/contrib/miniupnp/miniupnpd/getifaddr.c new file mode 100644 index 0000000..8016d63 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/getifaddr.c @@ -0,0 +1,261 @@ +/* $Id: getifaddr.c,v 1.19 2013/12/13 14:28:40 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(sun) +#include +#endif + +#include "config.h" +#include "getifaddr.h" +#if defined(USE_GETIFADDRS) || defined(ENABLE_IPV6) || defined(ENABLE_PCP) +#include +#endif + +int +getifaddr(const char * ifname, char * buf, int len, + struct in_addr * addr, struct in_addr * mask) +{ +#ifndef USE_GETIFADDRS + /* use ioctl SIOCGIFADDR. Works only for ip v4 */ + /* SIOCGIFADDR struct ifreq * */ + int s; + struct ifreq ifr; + int ifrlen; + struct sockaddr_in * ifaddr; + ifrlen = sizeof(ifr); + + if(!ifname || ifname[0]=='\0') + return -1; + s = socket(PF_INET, SOCK_DGRAM, 0); + if(s < 0) + { + syslog(LOG_ERR, "socket(PF_INET, SOCK_DGRAM): %m"); + return -1; + } + strncpy(ifr.ifr_name, ifname, IFNAMSIZ); + ifr.ifr_name[IFNAMSIZ-1] = '\0'; + if(ioctl(s, SIOCGIFFLAGS, &ifr, &ifrlen) < 0) + { + syslog(LOG_DEBUG, "ioctl(s, SIOCGIFFLAGS, ...): %m"); + close(s); + return -1; + } + if ((ifr.ifr_flags & IFF_UP) == 0) + { + syslog(LOG_DEBUG, "network interface %s is down", ifname); + close(s); + return -1; + } + strncpy(ifr.ifr_name, ifname, IFNAMSIZ); + if(ioctl(s, SIOCGIFADDR, &ifr, &ifrlen) < 0) + { + syslog(LOG_ERR, "ioctl(s, SIOCGIFADDR, ...): %m"); + close(s); + return -1; + } + ifaddr = (struct sockaddr_in *)&ifr.ifr_addr; + if(addr) *addr = ifaddr->sin_addr; + if(buf) + { + if(!inet_ntop(AF_INET, &ifaddr->sin_addr, buf, len)) + { + syslog(LOG_ERR, "inet_ntop(): %m"); + close(s); + return -1; + } + } + if(mask) + { + strncpy(ifr.ifr_name, ifname, IFNAMSIZ); + if(ioctl(s, SIOCGIFNETMASK, &ifr, &ifrlen) < 0) + { + syslog(LOG_ERR, "ioctl(s, SIOCGIFNETMASK, ...): %m"); + close(s); + return -1; + } +#ifdef ifr_netmask + *mask = ((struct sockaddr_in *)&ifr.ifr_netmask)->sin_addr; +#else + *mask = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr; +#endif + } + close(s); +#else /* ifndef USE_GETIFADDRS */ + /* Works for all address families (both ip v4 and ip v6) */ + struct ifaddrs * ifap; + struct ifaddrs * ife; + + if(!ifname || ifname[0]=='\0') + return -1; + if(getifaddrs(&ifap)<0) + { + syslog(LOG_ERR, "getifaddrs: %m"); + return -1; + } + for(ife = ifap; ife; ife = ife->ifa_next) + { + /* skip other interfaces if one was specified */ + if(ifname && (0 != strcmp(ifname, ife->ifa_name))) + continue; + if(ife->ifa_addr == NULL) + continue; + switch(ife->ifa_addr->sa_family) + { + case AF_INET: + if(buf) + { + inet_ntop(ife->ifa_addr->sa_family, + &((struct sockaddr_in *)ife->ifa_addr)->sin_addr, + buf, len); + } + if(addr) *addr = ((struct sockaddr_in *)ife->ifa_addr)->sin_addr; + if(mask) *mask = ((struct sockaddr_in *)ife->ifa_netmask)->sin_addr; + break; +/* + case AF_INET6: + inet_ntop(ife->ifa_addr->sa_family, + &((struct sockaddr_in6 *)ife->ifa_addr)->sin6_addr, + buf, len); +*/ + } + } + freeifaddrs(ifap); +#endif + return 0; +} + +#ifdef ENABLE_PCP + +int getifaddr_in6(const char * ifname, int af, struct in6_addr * addr) +{ +#if defined(ENABLE_IPV6) || defined(USE_GETIFADDRS) + struct ifaddrs * ifap; + struct ifaddrs * ife; +#ifdef ENABLE_IPV6 + const struct sockaddr_in6 * tmpaddr; +#endif /* ENABLE_IPV6 */ + int found = 0; + + if(!ifname || ifname[0]=='\0') + return -1; + if(getifaddrs(&ifap)<0) + { + syslog(LOG_ERR, "getifaddrs: %m"); + return -1; + } + for(ife = ifap; ife && !found; ife = ife->ifa_next) + { + /* skip other interfaces if one was specified */ + if(ifname && (0 != strcmp(ifname, ife->ifa_name))) + continue; + if(ife->ifa_addr == NULL) + continue; + if (ife->ifa_addr->sa_family != af) + continue; + switch(ife->ifa_addr->sa_family) + { + case AF_INET: + /* IPv4-mapped IPv6 address ::ffff:1.2.3.4 */ + memset(addr->s6_addr, 0, 10); + addr->s6_addr[10] = 0xff; + addr->s6_addr[11] = 0xff; + memcpy(addr->s6_addr + 12, + &(((struct sockaddr_in *)ife->ifa_addr)->sin_addr.s_addr), + 4); + found = 1; + break; + +#ifdef ENABLE_IPV6 + case AF_INET6: + tmpaddr = (const struct sockaddr_in6 *)ife->ifa_addr; + if(!IN6_IS_ADDR_LOOPBACK(&tmpaddr->sin6_addr) + && !IN6_IS_ADDR_LINKLOCAL(&tmpaddr->sin6_addr)) + { + memcpy(addr->s6_addr, + &tmpaddr->sin6_addr, + 16); + found = 1; + } + break; +#endif /* ENABLE_IPV6 */ + } + } + freeifaddrs(ifap); + return (found ? 0 : -1); +#else /* defined(ENABLE_IPV6) || defined(USE_GETIFADDRS) */ + /* IPv4 only */ + struct in_addr addr4; + if(af != AF_INET) + return -1; + if(getifaddr(ifname, NULL, 0, &addr4, NULL) < 0) + return -1; + /* IPv4-mapped IPv6 address ::ffff:1.2.3.4 */ + memset(addr->s6_addr, 0, 10); + addr->s6_addr[10] = 0xff; + addr->s6_addr[11] = 0xff; + memcpy(addr->s6_addr + 12, &addr4.s_addr, 4); + return 0; +#endif +} +#endif /* ENABLE_PCP */ + +#ifdef ENABLE_IPV6 +int +find_ipv6_addr(const char * ifname, + char * dst, int n) +{ + struct ifaddrs * ifap; + struct ifaddrs * ife; + const struct sockaddr_in6 * addr; + char buf[64]; + int r = 0; + + if(!dst) + return -1; + + if(getifaddrs(&ifap)<0) + { + syslog(LOG_ERR, "getifaddrs: %m"); + return -1; + } + for(ife = ifap; ife; ife = ife->ifa_next) + { + /* skip other interfaces if one was specified */ + if(ifname && (0 != strcmp(ifname, ife->ifa_name))) + continue; + if(ife->ifa_addr == NULL) + continue; + if(ife->ifa_addr->sa_family == AF_INET6) + { + addr = (const struct sockaddr_in6 *)ife->ifa_addr; + if(!IN6_IS_ADDR_LOOPBACK(&addr->sin6_addr) + && !IN6_IS_ADDR_LINKLOCAL(&addr->sin6_addr)) + { + inet_ntop(ife->ifa_addr->sa_family, + &addr->sin6_addr, + buf, sizeof(buf)); + /* add brackets */ + snprintf(dst, n, "[%s]", buf); + r = 1; + } + } + } + freeifaddrs(ifap); + return r; +} +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/getifaddr.h b/src/contrib/miniupnp/miniupnpd/getifaddr.h new file mode 100644 index 0000000..2be2f45 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/getifaddr.h @@ -0,0 +1,32 @@ +/* $Id: getifaddr.h,v 1.8 2013/03/23 10:46:54 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef GETIFADDR_H_INCLUDED +#define GETIFADDR_H_INCLUDED + +struct in_addr; +struct in6_addr; + +/* getifaddr() + * take a network interface name and write the + * ip v4 address as text in the buffer + * returns: 0 success, -1 failure */ +int +getifaddr(const char * ifname, char * buf, int len, + struct in_addr * addr, struct in_addr * mask); + +int +getifaddr_in6(const char * ifname, int af, struct in6_addr* addr); + +/* find a non link local IP v6 address for the interface. + * if ifname is NULL, look for all interfaces */ +int +find_ipv6_addr(const char * ifname, + char * dst, int n); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/getifstats.h b/src/contrib/miniupnp/miniupnpd/getifstats.h new file mode 100644 index 0000000..e14b853 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/getifstats.h @@ -0,0 +1,25 @@ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2008 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef GETIFSTATS_H_INCLUDED +#define GETIFSTATS_H_INCLUDED + +struct ifdata { + unsigned long opackets; + unsigned long ipackets; + unsigned long obytes; + unsigned long ibytes; + unsigned long baudrate; +}; + +/* getifstats() + * Fill the ifdata structure with statistics for network interface ifname. + * Return 0 in case of success, -1 for bad arguments or any error */ +int +getifstats(const char * ifname, struct ifdata * data); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/getroute.h b/src/contrib/miniupnp/miniupnpd/getroute.h new file mode 100644 index 0000000..86d0496 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/getroute.h @@ -0,0 +1,17 @@ +/* $Id: getroute.h,v 1.3 2013/02/06 10:50:04 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef GETROUTE_H_INCLUDED +#define GETROUTE_H_INCLUDED + +int +get_src_for_route_to(const struct sockaddr * dst, + void * src, size_t * src_len, + int * index); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/ifacewatcher.h b/src/contrib/miniupnp/miniupnpd/ifacewatcher.h new file mode 100644 index 0000000..4b2e15a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ifacewatcher.h @@ -0,0 +1,48 @@ +/* $Id: ifacewatcher.h,v 1.2 2011/05/20 09:42:49 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2009 Thomas Bernard + * + * ifacewatcher.h + * + * This file implements dynamic serving of new network interfaces + * which weren't available during daemon start. It also takes care + * of interfaces which become unavailable. + * + * Copyright (c) 2011, Alexey Osipov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ + +#ifndef IFACEWATCHER_H_INCLUDED +#define IFACEWATCHER_H_INCLUDED + +#include "config.h" + +#ifdef USE_IFACEWATCHER +int OpenAndConfInterfaceWatchSocket(void); +void ProcessInterfaceWatchNotify(int s); +#endif + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/ipf/Makefile b/src/contrib/miniupnp/miniupnpd/ipf/Makefile new file mode 100644 index 0000000..d46d1e9 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipf/Makefile @@ -0,0 +1,16 @@ +# $Id: Makefile,v 1.1 2007/09/25 19:44:43 nanard Exp $ +CC=gcc +CFLAGS=-Wall -g -I. + +all: testipfrdr + +clean: + rm *.o testipfrdr + +testipfrdr: testipfrdr.o ipfrdr.o + $(CC) -o $@ $^ + +ipfrdr.o: ipfrdr.c + +testipfrdr.o: testipfrdr.c + diff --git a/src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.c b/src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.c new file mode 100644 index 0000000..0a311b0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.c @@ -0,0 +1,831 @@ +/* $Id: ipfrdr.c,v 1.18 2016/02/12 14:12:25 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2007 Darren Reed + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +/* + * This is a workaround for troubles on FreeBSD, HPUX, OpenBSD. + * Needed here because on some systems gets included by things + * like + */ +#ifndef _KERNEL +# define ADD_KERNEL +# define _KERNEL +# define KERNEL +#endif +#ifdef __OpenBSD__ +struct file; +#endif +#include +#ifdef ADD_KERNEL +# undef _KERNEL +# undef KERNEL +#endif +#include +#include +#include +#include +#include +#if __FreeBSD_version >= 300000 +# include +#endif +#include +#include +#include +#include +#ifndef TCP_PAWS_IDLE /* IRIX */ +# include +#endif +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(__SVR4) && !defined(__svr4__) && defined(sun) +# include +#endif +#include +#include + +#include "../config.h" +#include "netinet/ipl.h" +#include "netinet/ip_compat.h" +#include "netinet/ip_fil.h" +#include "netinet/ip_nat.h" +#include "netinet/ip_state.h" + +#include "../macros.h" + +#ifndef __P +# ifdef __STDC__ +# define __P(x) x +# else +# define __P(x) () +# endif +#endif +#ifndef __STDC__ +# undef const +# define const +#endif + +#ifndef U_32_T +# define U_32_T 1 +# if defined(__NetBSD__) || defined(__OpenBSD__) || defined(__FreeBSD__) || \ + defined(__sgi) +typedef u_int32_t u_32_t; +# else +# if defined(__alpha__) || defined(__alpha) || defined(_LP64) +typedef unsigned int u_32_t; +# else +# if SOLARIS2 >= 6 +typedef uint32_t u_32_t; +# else +typedef unsigned int u_32_t; +# endif +# endif +# endif /* __NetBSD__ || __OpenBSD__ || __FreeBSD__ || __sgi */ +#endif /* U_32_T */ + + +#if defined(__NetBSD__) || defined(__OpenBSD__) || \ + (_BSDI_VERSION >= 199701) || (__FreeBSD_version >= 300000) || \ + SOLARIS || defined(__sgi) || defined(__osf__) || defined(linux) +# include +typedef int (* ioctlfunc_t) __P((int, ioctlcmd_t, ...)); +#else +typedef int (* ioctlfunc_t) __P((dev_t, ioctlcmd_t, void *)); +#endif +typedef void (* addfunc_t) __P((int, ioctlfunc_t, void *)); +typedef int (* copyfunc_t) __P((void *, void *, size_t)); + + +/* + * SunOS4 + */ +#if defined(sun) && !defined(__SVR4) && !defined(__svr4__) +extern int ioctl __P((int, int, void *)); +#endif + +#include "../upnpglobalvars.h" + +/* group name */ +static const char group_name[] = "miniupnpd"; + +static int dev = -1; +static int dev_ipl = -1; + +/* IPFilter cannot store redirection descriptions, so we use our + * own structure to store them */ +struct rdr_desc { + struct rdr_desc * next; + unsigned short eport; + int proto; + unsigned int timestamp; + char str[]; +}; + +/* pointer to the chained list where descriptions are stored */ +static struct rdr_desc * rdr_desc_list; + +static void +add_redirect_desc(unsigned short eport, int proto, + unsigned int timestamp, const char * desc) +{ + struct rdr_desc * p; + size_t l; + + if (desc != NULL) { + l = strlen(desc) + 1; + p = malloc(sizeof(struct rdr_desc) + l); + if (p) { + p->next = rdr_desc_list; + p->eport = eport; + p->proto = proto; + p->timestamp = timestamp; + memcpy(p->str, desc, l); + rdr_desc_list = p; + } + } +} + +static void +del_redirect_desc(unsigned short eport, int proto) +{ + struct rdr_desc * p, * last; + + last = NULL; + for (p = rdr_desc_list; p; p = p->next) { + if(p->eport == eport && p->proto == proto) { + if (last == NULL) + rdr_desc_list = p->next; + else + last->next = p->next; + free(p); + return; + } + } +} + +static void +get_redirect_desc(unsigned short eport, int proto, char * desc, int desclen, unsigned int * timestamp) +{ + struct rdr_desc * p; + + if (desc == NULL || desclen == 0) + return; + for (p = rdr_desc_list; p; p = p->next) { + if (p->eport == eport && p->proto == proto) + { + strncpy(desc, p->str, desclen); + *timestamp = p->timestamp; + return; + } + } + return; +} + +int init_redirect(void) +{ + + dev = open(IPNAT_NAME, O_RDWR); + if (dev < 0) { + syslog(LOG_ERR, "open(\"%s\"): %m", IPNAT_NAME); + return -1; + } + dev_ipl = open(IPL_NAME, O_RDWR); + if (dev_ipl < 0) { + syslog(LOG_ERR, "open(\"%s\"): %m", IPL_NAME); + return -1; + } + return 0; +} + +void shutdown_redirect(void) +{ + + if (dev >= 0) { + close(dev); + dev = -1; + } + if (dev_ipl >= 0) { + close(dev_ipl); + dev = -1; + } + return; +} + +int +add_redirect_rule2(const char * ifname, const char * rhost, + unsigned short eport, const char * iaddr, unsigned short iport, + int proto, const char * desc, unsigned int timestamp) +{ + struct ipnat ipnat; + struct ipfobj obj; + int r; + + if (dev < 0) { + syslog(LOG_ERR, "%s not open", IPNAT_NAME); + return -1; + } + + memset(&obj, 0, sizeof(obj)); + memset(&ipnat, 0, sizeof(ipnat)); + + ipnat.in_redir = NAT_REDIRECT; +#if IPFILTER_VERSION >= 5000000 + ipnat.in_pr[0] = proto; + ipnat.in_pr[1] = proto; +#else + ipnat.in_p = proto; +#endif + if (proto == IPPROTO_TCP) + ipnat.in_flags = IPN_TCP; + if (proto == IPPROTO_UDP) + ipnat.in_flags = IPN_UDP; + ipnat.in_dcmp = FR_EQUAL; +#if IPFILTER_VERSION >= 5000000 + ipnat.in_dpmin = htons(eport); + ipnat.in_dpmax = htons(eport); + ipnat.in_dpnext = htons(iport); + ipnat.in_v[0] = 4; + ipnat.in_v[1] = 4; +#else + ipnat.in_pmin = htons(eport); + ipnat.in_pmax = htons(eport); + ipnat.in_pnext = htons(iport); + ipnat.in_v = 4; +#endif + strlcpy(ipnat.in_tag.ipt_tag, group_name, IPFTAG_LEN); + +#ifdef USE_IFNAME_IN_RULES + if (ifname) { +#if IPFILTER_VERSION >= 5000000 + /* XXX check for stack overflow ! */ + ipnat.in_ifnames[0] = 0; + ipnat.in_ifnames[1] = 0; + strlcpy(ipnat.in_names, ifname, IFNAMSIZ); + ipnat.in_namelen = strlen(ipnat.in_names) + 1; +#else + strlcpy(ipnat.in_ifnames[0], ifname, IFNAMSIZ); + strlcpy(ipnat.in_ifnames[1], ifname, IFNAMSIZ); +#endif + } +#endif + + if(rhost && rhost[0] != '\0' && rhost[0] != '*') + { +#if IPFILTER_VERSION >= 5000000 + inet_pton(AF_INET, rhost, &ipnat.in_nsrc.na_addr[0].in4); /* in_nsrcip */ + ipnat.in_nsrc.na_addr[1].in4.s_addr = 0xffffffff; /* in_nsrcmsk */ +#else + inet_pton(AF_INET, rhost, &ipnat.in_src[0].in4); + ipnat.in_src[1].in4.s_addr = 0xffffffff; +#endif + } + +#if IPFILTER_VERSION >= 5000000 + inet_pton(AF_INET, iaddr, &ipnat.in_ndst.na_addr[0].in4); /* in_ndstip */ + ipnat.in_ndst.na_addr[1].in4.s_addr = 0xffffffff; /* in_ndstmsk */ +#else + inet_pton(AF_INET, iaddr, &ipnat.in_in[0].in4); + ipnat.in_in[1].in4.s_addr = 0xffffffff; +#endif + + obj.ipfo_rev = IPFILTER_VERSION; + obj.ipfo_size = sizeof(ipnat); + obj.ipfo_ptr = &ipnat; + obj.ipfo_type = IPFOBJ_IPNAT; + + r = ioctl(dev, SIOCADNAT, &obj); + if (r == -1) + syslog(LOG_ERR, "ioctl(SIOCADNAT): %m"); + else + add_redirect_desc(eport, proto, timestamp, desc); + return r; +} + +/* get_redirect_rule() + * return value : 0 success (found) + * -1 = error or rule not found */ +int +get_redirect_rule(const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + ipfgeniter_t iter; + ipfobj_t obj; + ipnat_t ipn; + int r; + UNUSED(ifname); + + memset(&obj, 0, sizeof(obj)); + obj.ipfo_rev = IPFILTER_VERSION; + obj.ipfo_type = IPFOBJ_GENITER; + obj.ipfo_size = sizeof(iter); + obj.ipfo_ptr = &iter; + + iter.igi_type = IPFGENITER_IPNAT; +#if IPFILTER_VERSION > 4011300 + iter.igi_nitems = 1; +#endif + iter.igi_data = &ipn; + + if (dev < 0) { + syslog(LOG_ERR, "%s not open", IPNAT_NAME); + return -1; + } + + r = -1; + do { + if (ioctl(dev, SIOCGENITER, &obj) == -1) { + syslog(LOG_ERR, "ioctl(dev, SIOCGENITER): %m"); + break; + } +#if IPFILTER_VERSION >= 5000000 + if (eport == ntohs(ipn.in_dpmin) && + eport == ntohs(ipn.in_dpmax) && + strcmp(ipn.in_tag.ipt_tag, group_name) == 0 && + ipn.in_pr[0] == proto) +#else + if (eport == ntohs(ipn.in_pmin) && + eport == ntohs(ipn.in_pmax) && + strcmp(ipn.in_tag.ipt_tag, group_name) == 0 && + ipn.in_p == proto) +#endif + { + strlcpy(desc, "", desclen); + if (packets != NULL) + *packets = 0; + if (bytes != NULL) + *bytes = 0; + if (iport != NULL) +#if IPFILTER_VERSION >= 5000000 + *iport = ntohs(ipn.in_dpnext); +#else + *iport = ntohs(ipn.in_pnext); +#endif + if ((desc != NULL) && (timestamp != NULL)) + get_redirect_desc(eport, proto, desc, desclen, timestamp); + if ((rhost != NULL) && (rhostlen > 0)) +#if IPFILTER_VERSION >= 5000000 + inet_ntop(AF_INET, &ipn.in_nsrc.na_addr[0].in4, rhost, rhostlen); /* in_nsrcip */ +#else + inet_ntop(AF_INET, &ipn.in_src[0].in4, rhost, rhostlen); +#endif +#if IPFILTER_VERSION >= 5000000 + inet_ntop(AF_INET, &ipn.in_ndst.na_addr[0].in4, iaddr, iaddrlen); /* in_ndstip */ +#else + inet_ntop(AF_INET, &ipn.in_in[0].in4, iaddr, iaddrlen); +#endif + r = 0; + } + } while (ipn.in_next != NULL); + return r; +} + + +int +get_redirect_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + ipfgeniter_t iter; + ipfobj_t obj; + ipnat_t ipn; + int n, r; + + if (index < 0) + return -1; + + if (dev < 0) { + syslog(LOG_ERR, "%s not open", IPNAT_NAME); + return -1; + } + + memset(&obj, 0, sizeof(obj)); + obj.ipfo_rev = IPFILTER_VERSION; + obj.ipfo_ptr = &iter; + obj.ipfo_size = sizeof(iter); + obj.ipfo_type = IPFOBJ_GENITER; + + iter.igi_type = IPFGENITER_IPNAT; +#if IPFILTER_VERSION > 4011300 + iter.igi_nitems = 1; +#endif + iter.igi_data = &ipn; + + n = 0; + r = -1; + do { + if (ioctl(dev, SIOCGENITER, &obj) == -1) { + syslog(LOG_ERR, "%s:ioctl(SIOCGENITER): %m", + "get_redirect_rule_by_index"); + break; + } + + if (strcmp(ipn.in_tag.ipt_tag, group_name) != 0) + continue; + + if (index == n++) { +#if IPFILTER_VERSION >= 5000000 + *proto = ipn.in_pr[0]; + *eport = ntohs(ipn.in_dpmax); + *iport = ntohs(ipn.in_dpnext); +#else + *proto = ipn.in_p; + *eport = ntohs(ipn.in_pmax); + *iport = ntohs(ipn.in_pnext); +#endif + + if (ifname) +#if IPFILTER_VERSION >= 5000000 + strlcpy(ifname, ipn.in_names + ipn.in_ifnames[0], IFNAMSIZ); +#else + strlcpy(ifname, ipn.in_ifnames[0], IFNAMSIZ); +#endif + if (packets != NULL) + *packets = 0; + if (bytes != NULL) + *bytes = 0; + if ((desc != NULL) && (timestamp != NULL)) + get_redirect_desc(*eport, *proto, desc, desclen, timestamp); + if ((rhost != NULL) && (rhostlen > 0)) +#if IPFILTER_VERSION >= 5000000 + inet_ntop(AF_INET, &ipn.in_nsrc.na_addr[0].in4, rhost, rhostlen); /* in_nsrcip */ +#else + inet_ntop(AF_INET, &ipn.in_src[0].in4, rhost, rhostlen); +#endif +#if IPFILTER_VERSION >= 5000000 + inet_ntop(AF_INET, &ipn.in_ndst.na_addr[0].in4, iaddr, iaddrlen); /* in_ndstip */ +#else + inet_ntop(AF_INET, &ipn.in_in[0].in4, iaddr, iaddrlen); +#endif + r = 0; + } + } while (ipn.in_next != NULL); + return r; +} + +static int +real_delete_redirect_rule(const char * ifname, unsigned short eport, int proto) +{ + ipfgeniter_t iter; + ipfobj_t obj; + ipnat_t ipn; + int r; + UNUSED(ifname); + + memset(&obj, 0, sizeof(obj)); + obj.ipfo_rev = IPFILTER_VERSION; + obj.ipfo_type = IPFOBJ_GENITER; + obj.ipfo_size = sizeof(iter); + obj.ipfo_ptr = &iter; + + iter.igi_type = IPFGENITER_IPNAT; +#if IPFILTER_VERSION > 4011300 + iter.igi_nitems = 1; +#endif + iter.igi_data = &ipn; + + if (dev < 0) { + syslog(LOG_ERR, "%s not open", IPNAT_NAME); + return -1; + } + + r = -1; + do { + if (ioctl(dev, SIOCGENITER, &obj) == -1) { + syslog(LOG_ERR, "%s:ioctl(SIOCGENITER): %m", + "delete_redirect_rule"); + break; + } +#if IPFILTER_VERSION >= 5000000 + if (eport == ntohs(ipn.in_dpmin) && + eport == ntohs(ipn.in_dpmax) && + strcmp(ipn.in_tag.ipt_tag, group_name) == 0 && + ipn.in_pr[0] == proto) +#else + if (eport == ntohs(ipn.in_pmin) && + eport == ntohs(ipn.in_pmax) && + strcmp(ipn.in_tag.ipt_tag, group_name) == 0 && + ipn.in_p == proto) +#endif + { + obj.ipfo_rev = IPFILTER_VERSION; + obj.ipfo_size = sizeof(ipn); + obj.ipfo_ptr = &ipn; + obj.ipfo_type = IPFOBJ_IPNAT; + r = ioctl(dev, SIOCRMNAT, &obj); + if (r == -1) + syslog(LOG_ERR, "%s:ioctl(SIOCRMNAT): %m", + "delete_redirect_rule"); + /* Delete the desc even if the above failed */ + del_redirect_desc(eport, proto); + break; + } + } while (ipn.in_next != NULL); + return r; +} + +/* FIXME: For some reason, the iter isn't reset every other delete, + * so we attempt 2 deletes. */ +int +delete_redirect_rule(const char * ifname, unsigned short eport, int proto) +{ + int r; + + r = real_delete_redirect_rule(ifname, eport, proto); + if (r == -1) + r = real_delete_redirect_rule(ifname, eport, proto); + return r; +} + +/* thanks to Seth Mos for this function */ +int +add_filter_rule2(const char * ifname, const char * rhost, + const char * iaddr, unsigned short eport, unsigned short iport, + int proto, const char * desc) +{ + ipfobj_t obj; + frentry_t fr; + fripf_t ipffr; + int r; + UNUSED(ifname); UNUSED(desc); + UNUSED(iport); + + if (dev_ipl < 0) { + syslog(LOG_ERR, "%s not open", IPL_NAME); + return -1; + } + + memset(&obj, 0, sizeof(obj)); + memset(&fr, 0, sizeof(fr)); + memset(&ipffr, 0, sizeof(ipffr)); + + fr.fr_flags = FR_PASS|FR_KEEPSTATE|FR_QUICK|FR_INQUE; + if (GETFLAG(LOGPACKETSMASK)) + fr.fr_flags |= FR_LOG|FR_LOGFIRST; +#if IPFILTER_VERSION >= 5000000 + fr.fr_family = PF_INET; +#else + fr.fr_v = 4; +#endif + + fr.fr_type = FR_T_IPF; + fr.fr_dun.fru_ipf = &ipffr; + fr.fr_dsize = sizeof(ipffr); + fr.fr_isc = (void *)-1; + + fr.fr_proto = proto; + fr.fr_mproto = 0xff; + fr.fr_dcmp = FR_EQUAL; + fr.fr_dport = eport; +#ifdef USE_IFNAME_IN_RULES + if (ifname) { +#if IPFILTER_VERSION >= 5000000 + /* XXX check for stack overflow ! */ + fr.fr_ifnames[0] = fr.fr_namelen; + strlcpy(fr.fr_names + fr.fr_ifnames[0], ifname, IFNAMSIZ); + fr.fr_namelen += strlen(ifname) + 1; +#else + strlcpy(fr.fr_ifnames[0], ifname, IFNAMSIZ); +#endif + } +#endif +#if IPFILTER_VERSION >= 5000000 + /* XXX check for stack overflow ! */ + fr.fr_group = fr.fr_namelen; + strlcpy(fr.fr_names + fr.fr_group, group_name, FR_GROUPLEN); + fr.fr_namelen += strlen(group_name) + 1; +#else + strlcpy(fr.fr_group, group_name, sizeof(fr.fr_group)); +#endif + + if (proto == IPPROTO_TCP) { + fr.fr_tcpf = TH_SYN; + fr.fr_tcpfm = TH_SYN|TH_ACK|TH_RST|TH_FIN|TH_URG|TH_PUSH; + } + + if(rhost && rhost[0] != '\0' && rhost[0] != '*') + { + inet_pton(AF_INET, rhost, &fr.fr_saddr); + fr.fr_smask = 0xffffffff; + } + + inet_pton(AF_INET, iaddr, &fr.fr_daddr); + fr.fr_dmask = 0xffffffff; + + obj.ipfo_rev = IPFILTER_VERSION; + obj.ipfo_ptr = &fr; + obj.ipfo_size = sizeof(fr); + + r = ioctl(dev_ipl, SIOCINAFR, &obj); + if (r == -1) { + if (errno == ESRCH) + syslog(LOG_ERR, + "SIOCINAFR(missing 'head %s' rule?):%m", + group_name); + else + syslog(LOG_ERR, "SIOCINAFR:%m"); + } + return r; +} + +int +delete_filter_rule(const char * ifname, unsigned short eport, int proto) +{ + ipfobj_t wobj, dobj; + ipfruleiter_t rule; + u_long darray[1000]; + u_long array[1000]; + friostat_t fio; + frentry_t *fp; + int r; + UNUSED(ifname); + + if (dev_ipl < 0) { + syslog(LOG_ERR, "%s not open", IPL_NAME); + return -1; + } + + wobj.ipfo_rev = IPFILTER_VERSION; + wobj.ipfo_type = IPFOBJ_IPFSTAT; + wobj.ipfo_size = sizeof(fio); + wobj.ipfo_ptr = &fio; + + if (ioctl(dev_ipl, SIOCGETFS, &wobj) == -1) { + syslog(LOG_ERR, "ioctl(SIOCGETFS): %m"); + return -1; + } + + wobj.ipfo_rev = IPFILTER_VERSION; + wobj.ipfo_ptr = &rule; + wobj.ipfo_size = sizeof(rule); + wobj.ipfo_type = IPFOBJ_IPFITER; + + fp = (frentry_t *)array; + fp->fr_dun.fru_data = darray; + fp->fr_dsize = sizeof(darray); + + rule.iri_inout = 0; + rule.iri_active = fio.f_active; +#if IPFILTER_VERSION > 4011300 + rule.iri_nrules = 1; + rule.iri_v = 4; +#endif + rule.iri_rule = fp; + strlcpy(rule.iri_group, group_name, sizeof(rule.iri_group)); + + dobj.ipfo_rev = IPFILTER_VERSION; + dobj.ipfo_size = sizeof(*fp); + dobj.ipfo_type = IPFOBJ_FRENTRY; + + r = -1; + do { + memset(array, 0xff, sizeof(array)); + + if (ioctl(dev_ipl, SIOCIPFITER, &wobj) == -1) { + syslog(LOG_ERR, "ioctl(SIOCIPFITER): %m"); + break; + } + + if (fp->fr_data != NULL) + fp->fr_data = (char *)fp + sizeof(*fp); + if ((fp->fr_type & ~FR_T_BUILTIN) == FR_T_IPF && + fp->fr_dport == eport && + fp->fr_proto == proto) + { + dobj.ipfo_ptr = fp; + + r = ioctl(dev_ipl, SIOCRMAFR, &dobj); + if (r == -1) + syslog(LOG_ERR, "ioctl(SIOCRMAFR): %m"); + break; + } + } while (fp->fr_next != NULL); + return r; +} + +unsigned short * +get_portmappings_in_range(unsigned short startport, unsigned short endport, + int proto, unsigned int * number) +{ + unsigned short *array, *array2; + unsigned int capacity; + unsigned short eport; + ipfgeniter_t iter; + ipfobj_t obj; + ipnat_t ipn; + + *number = 0; + if (dev < 0) { + syslog(LOG_ERR, "%s not open", IPNAT_NAME); + return NULL; + } + capacity = 128; + array = calloc(capacity, sizeof(unsigned short)); + if(!array) + { + syslog(LOG_ERR, "get_portmappings_in_range() : calloc error"); + return NULL; + } + + memset(&obj, 0, sizeof(obj)); + obj.ipfo_rev = IPFILTER_VERSION; + obj.ipfo_ptr = &iter; + obj.ipfo_size = sizeof(iter); + obj.ipfo_type = IPFOBJ_GENITER; + + iter.igi_type = IPFGENITER_IPNAT; +#if IPFILTER_VERSION > 4011300 + iter.igi_nitems = 1; +#endif + iter.igi_data = &ipn; + + do { + if (ioctl(dev, SIOCGENITER, &obj) == -1) { + syslog(LOG_ERR, "%s:ioctl(SIOCGENITER): %m", + "get_portmappings_in_range"); + break; + } + + if (strcmp(ipn.in_tag.ipt_tag, group_name) != 0) + continue; + +#if IPFILTER_VERSION >= 5000000 + eport = ntohs(ipn.in_dpmin); + if( (eport == ntohs(ipn.in_dpmax)) + && (ipn.in_pr[0] == proto) + && (startport <= eport) && (eport <= endport) ) +#else + eport = ntohs(ipn.in_pmin); + if( (eport == ntohs(ipn.in_pmax)) + && (ipn.in_p == proto) + && (startport <= eport) && (eport <= endport) ) +#endif + { + if(*number >= capacity) + { + /* need to increase the capacity of the array */ + capacity += 128; + array2 = realloc(array, sizeof(unsigned short)*capacity); + if(!array2) + { + syslog(LOG_ERR, "get_portmappings_in_range() : realloc(%lu) error", sizeof(unsigned short)*capacity); + *number = 0; + free(array); + return NULL; + } + array = array2; + } + array[*number] = eport; + (*number)++; + } + } while (ipn.in_next != NULL); + return array; +} + +/* update the port mapping internal port, decription and timestamp */ +int +update_portmapping(const char * ifname, unsigned short eport, int proto, + unsigned short iport, const char * desc, + unsigned int timestamp) +{ + UNUSED(ifname); UNUSED(eport); UNUSED(proto); + UNUSED(iport); UNUSED(desc); UNUSED(timestamp); + /* TODO: implement update_portmapping() */ + syslog(LOG_ERR, __FILE__ " update_portmapping() is not implemented"); + return -1; +} + +/* update the port mapping decription and timestamp */ +int +update_portmapping_desc_timestamp(const char * ifname, + unsigned short eport, int proto, + const char * desc, unsigned int timestamp) +{ + UNUSED(ifname); + del_redirect_desc(eport, proto); + add_redirect_desc(eport,proto, timestamp, desc); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.h b/src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.h new file mode 100644 index 0000000..f48b083 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipf/ipfrdr.h @@ -0,0 +1,55 @@ +/* $Id: ipfrdr.h,v 1.3 2007/11/02 22:54:01 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2007 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef IPFRDR_H_INCLUDED +#define IPFRDR_H_INCLUDED + +#include "../commonrdr.h" + +int +add_redirect_rule2(const char * ifname, const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp); + +int +add_filter_rule2(const char * ifname, const char * rhost, const char * iaddr, + unsigned short eport, unsigned short iport, + int proto, const char * desc); + + +/* get_redirect_rule() gets internal IP and port from + * interface, external port and protocl + */ +#if 0 +int +get_redirect_rule(const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + u_int64_t * packets, u_int64_t * bytes); + +int +get_redirect_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + u_int64_t * packets, u_int64_t * bytes); +#endif + +/* delete_redirect_rule() + */ +int +delete_redirect_rule(const char * ifname, unsigned short eport, int proto); + +/* delete_filter_rule() + */ +int +delete_filter_rule(const char * ifname, unsigned short eport, int proto); + +int +clear_redirect_rules(void); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/ipf/testipfrdr.c b/src/contrib/miniupnp/miniupnpd/ipf/testipfrdr.c new file mode 100644 index 0000000..61f8b91 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipf/testipfrdr.c @@ -0,0 +1,74 @@ +/* $Id: testipfrdr.c,v 1.3 2007/10/01 16:21:23 nanard Exp $ */ + +#include +#include +#include +#include +#include +#include "ipfrdr.h" + +/* test program for ipfrdr.c */ + +int runtime_flags = 0; + +void +list_eports_tcp(void) +{ + unsigned short * port_list; + unsigned int number = 0; + unsigned int i; + port_list = get_portmappings_in_range(0, 65535, IPPROTO_TCP, &number); + printf("%u ports redirected (TCP) :", number); + for(i = 0; i < number; i++) + { + printf(" %hu", port_list[i]); + } + printf("\n"); + free(port_list); + port_list = get_portmappings_in_range(0, 65535, IPPROTO_UDP, &number); + printf("%u ports redirected (UDP) :", number); + for(i = 0; i < number; i++) + { + printf(" %hu", port_list[i]); + } + printf("\n"); + free(port_list); +} + +int +main(int argc, char * * argv) +{ + char c; + + openlog("testipfrdrd", LOG_CONS|LOG_PERROR, LOG_USER); + if(init_redirect() < 0) + { + fprintf(stderr, "init_redirect() failed\n"); + return 1; + } + + printf("List rdr ports :\n"); + list_eports_tcp(); + + printf("Add redirection !\n"); + add_redirect_rule2("xennet0", "*", 12345, "192.168.1.100", 54321, IPPROTO_UDP, + "redirection description", 0); + add_redirect_rule2("xennet0", "8.8.8.8", 12345, "192.168.1.100", 54321, IPPROTO_TCP, + "redirection description", 0); + + printf("Check redirect rules with \"ipnat -l\" then press any key.\n"); + c = getchar(); + + printf("List rdr ports :\n"); + list_eports_tcp(); + + printf("Delete redirection !\n"); + delete_redirect_rule("xennet0", 12345, IPPROTO_UDP); + delete_redirect_rule("xennet0", 12345, IPPROTO_TCP); + + printf("List rdr ports :\n"); + list_eports_tcp(); + + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/ipfw/Makefile b/src/contrib/miniupnp/miniupnpd/ipfw/Makefile new file mode 100644 index 0000000..6a2467d --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipfw/Makefile @@ -0,0 +1,17 @@ +# $Id: Makefile,v 1.2 2009/08/20 09:31:10 nanard Exp $ +CC=gcc +CFLAGS=-Wall -g -I. +RM=rm -f + +all: testipfwrdr + +clean: + $(RM) *.o testipfwrdr + +testipfwrdr: testipfwrdr.o ipfwrdr.o ipfwaux.o + $(CC) -o $@ $^ + +ipfwrdr.o: ipfwrdr.c ipfwaux.c + +testipfwrdr.o: testipfwrdr.c + diff --git a/src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.c b/src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.c new file mode 100644 index 0000000..d5ecf35 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.c @@ -0,0 +1,107 @@ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2009-2012 Jardel Weyrich + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution + */ + +#include "ipfwaux.h" +#include +#include +#include +#include +#include + +int ipfw_exec(int optname, void * optval, uintptr_t optlen) { + static int sock = -1; + int result; + + switch (optname) { + case IP_FW_INIT: + if (sock == -1) + sock = socket(AF_INET, SOCK_RAW, IPPROTO_RAW); + if (sock < 0) { + syslog(LOG_ERR, "socket(SOCK_RAW): %m"); + return -1; + } + break; + case IP_FW_TERM: + if (sock != -1) + close(sock); + sock = -1; + break; + case IP_FW_ADD: + case IP_FW_DEL: + result = setsockopt(sock, IPPROTO_IP, optname, optval, optlen); + if (result == -1) { + syslog(LOG_ERR, "setsockopt(): %m"); + return -1; + } + break; + case IP_FW_GET: + result = getsockopt(sock, IPPROTO_IP, optname, optval, (socklen_t *)optlen); + if (result == -1) { + syslog(LOG_ERR, "getsockopt(): %m"); + return -1; + } + break; + default: + syslog(LOG_ERR, "unhandled option"); + return -1; + } + + return 0; +} + +void ipfw_free_ruleset(struct ip_fw ** rules) { + if (rules == NULL || *rules == NULL) + return; + free(*rules); + *rules = NULL; +} + +int ipfw_fetch_ruleset(struct ip_fw ** rules, int * total_fetched, int count) { + int fetched; + socklen_t size; + + if (rules == NULL || *total_fetched < 0 || count < 1) + return -1; + + size = sizeof(struct ip_fw) * (*total_fetched + count); + *rules = (struct ip_fw *)realloc(*rules, size); + if (*rules == NULL) { + syslog(LOG_ERR, "realloc(): %m"); + return -1; + } + + (*rules)->version = IP_FW_CURRENT_API_VERSION; + if (ipfw_exec(IP_FW_GET, *rules, (uintptr_t)&size) < 0) + return -1; + fetched = *total_fetched; + *total_fetched = size / sizeof(struct ip_fw); + + return *total_fetched - fetched; +} + +int ipfw_validate_protocol(int value) { + switch (value) { + case IPPROTO_TCP: + case IPPROTO_UDP: + break; + default: + syslog(LOG_ERR, "invalid protocol"); + return -1; + } + return 0; +} + +int ipfw_validate_ifname(const char * const value) { + int len = strlen(value); + if (len < 2 || len > FW_IFNLEN) { + syslog(LOG_ERR, "invalid interface name"); + return -1; + } + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.h b/src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.h new file mode 100644 index 0000000..44d4046 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipfw/ipfwaux.h @@ -0,0 +1,29 @@ +/* $Id: ipfwaux.h,v 1.6 2015/09/04 14:20:58 nanard Exp $ */ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2009-2012 Jardel Weyrich + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution + */ +#ifndef IPFWAUX_H +#define IPFWAUX_H + +#include +#include +#include +#include +#include + +#define IP_FW_BASE (IP_FW_ADD - 5) +#define IP_FW_INIT (IP_FW_BASE + 1) +#define IP_FW_TERM (IP_FW_BASE + 2) + +int ipfw_exec(int optname, void * optval, uintptr_t optlen); +void ipfw_free_ruleset(struct ip_fw ** rules); +int ipfw_fetch_ruleset(struct ip_fw ** rules, int * total_fetched, int count); +int ipfw_validate_protocol(int value); +int ipfw_validate_ifname(const char * const value); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.c b/src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.c new file mode 100644 index 0000000..ce37af3 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.c @@ -0,0 +1,548 @@ +/* $Id: ipfwrdr.c,v 1.16 2016/02/12 13:44:01 nanard Exp $ */ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2009 Jardel Weyrich + * (c) 2011-2016 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution + */ + +#include "../config.h" +#include "../macros.h" + +#include +#include +#include + +/* +This is a workaround for troubles on FreeBSD, HPUX, OpenBSD. +Needed here because on some systems gets included by things +like +*/ +#ifndef _KERNEL +# define ADD_KERNEL +# define _KERNEL +# define KERNEL +#endif +#ifdef __OpenBSD__ +struct file; +#endif +#include +#ifdef ADD_KERNEL +# undef _KERNEL +# undef KERNEL +#endif + +#include +#include +#include +#include +#include +#if __FreeBSD_version >= 300000 +# include +#endif +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ipfwaux.h" +#include "ipfwrdr.h" + +#include "../upnpglobalvars.h" + +/* init and shutdown functions */ + +int init_redirect(void) { + return ipfw_exec(IP_FW_INIT, NULL, 0); +} + +void shutdown_redirect(void) { + ipfw_exec(IP_FW_TERM, NULL, 0); +} + +/* ipfw cannot store descriptions and timestamp for port mappings so we keep + * our own list in memory */ +struct mapping_desc_time { + struct mapping_desc_time * next; + unsigned int timestamp; + unsigned short eport; + short proto; + char desc[]; +}; + +static struct mapping_desc_time * mappings_list = NULL; + +/* add an element to the port mappings descriptions & timestamp list */ +static void +add_desc_time(unsigned short eport, int proto, + const char * desc, unsigned int timestamp) +{ + struct mapping_desc_time * tmp; + size_t l; + if(!desc) + desc = "miniupnpd"; + l = strlen(desc) + 1; + tmp = malloc(sizeof(struct mapping_desc_time) + l); + if(tmp) { + /* fill the element and insert it as head of the list */ + tmp->next = mappings_list; + tmp->timestamp = timestamp; + tmp->eport = eport; + tmp->proto = (short)proto; + memcpy(tmp->desc, desc, l); + mappings_list = tmp; + } +} + +/* remove an element to the port mappings descriptions & timestamp list */ +static void +del_desc_time(unsigned short eport, int proto) +{ + struct mapping_desc_time * e; + struct mapping_desc_time * * p; + p = &mappings_list; + e = *p; + while(e) { + if(e->eport == eport && e->proto == (short)proto) { + *p = e->next; + free(e); + return; + } else { + p = &e->next; + e = *p; + } + } +} + +/* go through the list and find the description and timestamp */ +static void +get_desc_time(unsigned short eport, int proto, + char * desc, int desclen, + unsigned int * timestamp) +{ + struct mapping_desc_time * e; + + for(e = mappings_list; e; e = e->next) { + if(e->eport == eport && e->proto == (short)proto) { + if(desc) + strlcpy(desc, e->desc, desclen); + if(timestamp) + *timestamp = e->timestamp; + return; + } + } +} + +/* --- */ +int add_redirect_rule2( + const char * ifname, + const char * rhost, + unsigned short eport, + const char * iaddr, + unsigned short iport, + int proto, + const char * desc, + unsigned int timestamp) +{ + struct ip_fw rule; + int r; + + if (ipfw_validate_protocol(proto) < 0) + return -1; + if (ipfw_validate_ifname(ifname) < 0) + return -1; + + memset(&rule, 0, sizeof(struct ip_fw)); + rule.version = IP_FW_CURRENT_API_VERSION; +#if 0 + rule.fw_number = 1000; /* rule number */ + rule.context = (void *)desc; /* The description is kept in a separate list */ +#endif + rule.fw_prot = proto; /* protocol */ + rule.fw_flg |= IP_FW_F_IIFACE; /* interfaces to check */ + rule.fw_flg |= IP_FW_F_IIFNAME; /* interfaces to check by name */ + rule.fw_flg |= (IP_FW_F_IN | IP_FW_F_OUT); /* packet direction */ + rule.fw_flg |= IP_FW_F_FWD; /* forward action */ +#ifdef USE_IFNAME_IN_RULES + if (ifname != NULL) { + strlcpy(rule.fw_in_if.fu_via_if.name, ifname, IFNAMSIZ); /* src interface */ + rule.fw_in_if.fu_via_if.unit = -1; + } +#endif + if (inet_aton(iaddr, &rule.fw_out_if.fu_via_ip) == 0) { + syslog(LOG_ERR, "inet_aton(): %m"); + return -1; + } + memcpy(&rule.fw_dst, &rule.fw_out_if.fu_via_ip, sizeof(struct in_addr)); + memcpy(&rule.fw_fwd_ip.sin_addr, &rule.fw_out_if.fu_via_ip, sizeof(struct in_addr)); + rule.fw_dmsk.s_addr = INADDR_BROADCAST; /* TODO check this */ + IP_FW_SETNDSTP(&rule, 1); /* number of external ports */ + rule.fw_uar.fw_pts[0] = eport; /* external port */ + rule.fw_fwd_ip.sin_port = iport; /* internal port */ + if (rhost && rhost[0] != '\0') { + inet_aton(rhost, &rule.fw_src); + rule.fw_smsk.s_addr = htonl(INADDR_NONE); + } + + r = ipfw_exec(IP_FW_ADD, &rule, sizeof(rule)); + if(r >= 0) + add_desc_time(eport, proto, desc, timestamp); + return r; +} + +/* get_redirect_rule() + * return value : 0 success (found) + * -1 = error or rule not found */ +int get_redirect_rule( + const char * ifname, + unsigned short eport, + int proto, + char * iaddr, + int iaddrlen, + unsigned short * iport, + char * desc, + int desclen, + char * rhost, + int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, + u_int64_t * bytes) +{ + int i, count_rules, total_rules = 0; + struct ip_fw * rules = NULL; + + if (ipfw_validate_protocol(proto) < 0) + return -1; + if (ipfw_validate_ifname(ifname) < 0) + return -1; + if (timestamp) + *timestamp = 0; + + do { + count_rules = ipfw_fetch_ruleset(&rules, &total_rules, 10); + if (count_rules < 0) + goto error; + } while (count_rules == 10); + + for (i=0; ifw_prot && eport == ptr->fw_uar.fw_pts[0]) { + if (packets != NULL) + *packets = ptr->fw_pcnt; + if (bytes != NULL) + *bytes = ptr->fw_bcnt; + if (iport != NULL) + *iport = ptr->fw_fwd_ip.sin_port; + if (iaddr != NULL && iaddrlen > 0) { + /* looks like fw_out_if.fu_via_ip is zero */ + /*if (inet_ntop(AF_INET, &ptr->fw_out_if.fu_via_ip, iaddr, iaddrlen) == NULL) {*/ + if (inet_ntop(AF_INET, &ptr->fw_fwd_ip.sin_addr, iaddr, iaddrlen) == NULL) { + syslog(LOG_ERR, "inet_ntop(): %m"); + goto error; + } + } + if (rhost != NULL && rhostlen > 0) { + if (ptr->fw_src.s_addr == 0) + rhost[0] = '\0'; + else if (inet_ntop(AF_INET, &ptr->fw_src.s_addr, rhost, rhostlen) == NULL) { + syslog(LOG_ERR, "inet_ntop(): %m"); + goto error; + } + } + /* And what if we found more than 1 matching rule? */ + ipfw_free_ruleset(&rules); + get_desc_time(eport, proto, desc, desclen, timestamp); + return 0; + } + } + +error: + if (rules != NULL) + ipfw_free_ruleset(&rules); + return -1; +} + +int delete_redirect_rule( + const char * ifname, + unsigned short eport, + int proto) +{ + int i, count_rules, total_rules = 0; + struct ip_fw * rules = NULL; + + if (ipfw_validate_protocol(proto) < 0) + return -1; + if (ipfw_validate_ifname(ifname) < 0) + return -1; + + do { + count_rules = ipfw_fetch_ruleset(&rules, &total_rules, 10); + if (count_rules < 0) + goto error; + } while (count_rules == 10); + + for (i=0; ifw_prot && eport == ptr->fw_uar.fw_pts[0]) { + if (ipfw_exec(IP_FW_DEL, (struct ip_fw *)ptr, sizeof(*ptr)) < 0) + goto error; + /* And what if we found more than 1 matching rule? */ + ipfw_free_ruleset(&rules); + del_desc_time(eport, proto); + return 0; + } + } + +error: + if (rules != NULL) + ipfw_free_ruleset(&rules); + return -1; +} + +int add_filter_rule2( + const char * ifname, + const char * rhost, + const char * iaddr, + unsigned short eport, + unsigned short iport, + int proto, + const char * desc) +{ + UNUSED(ifname); + UNUSED(rhost); + UNUSED(iaddr); + UNUSED(eport); + UNUSED(iport); + UNUSED(proto); + UNUSED(desc); + return 0; /* nothing to do, always success */ +} + +int delete_filter_rule( + const char * ifname, + unsigned short eport, + int proto) +{ + UNUSED(ifname); + UNUSED(eport); + UNUSED(proto); + return 0; /* nothing to do, always success */ +} + +int get_redirect_rule_by_index( + int index, + char * ifname, + unsigned short * eport, + char * iaddr, + int iaddrlen, + unsigned short * iport, + int * proto, + char * desc, + int desclen, + char * rhost, + int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, + u_int64_t * bytes) +{ + int total_rules = 0; + struct ip_fw * rules = NULL; + + if (index < 0) /* TODO shouldn't we also validate the maximum? */ + return -1; + + if(timestamp) + *timestamp = 0; + + ipfw_fetch_ruleset(&rules, &total_rules, index + 1); + + if (total_rules > index) { + const struct ip_fw const * ptr = &rules[index]; + if (ptr->fw_prot == 0) /* invalid rule */ + goto error; + if (proto != NULL) + *proto = ptr->fw_prot; + if (eport != NULL) + *eport = ptr->fw_uar.fw_pts[0]; + if (iport != NULL) + *iport = ptr->fw_fwd_ip.sin_port; + if (ifname != NULL) + strlcpy(ifname, ptr->fw_in_if.fu_via_if.name, IFNAMSIZ); + if (packets != NULL) + *packets = ptr->fw_pcnt; + if (bytes != NULL) + *bytes = ptr->fw_bcnt; + if (iport != NULL) + *iport = ptr->fw_fwd_ip.sin_port; + if (iaddr != NULL && iaddrlen > 0) { + /* looks like fw_out_if.fu_via_ip is zero */ + /*if (inet_ntop(AF_INET, &ptr->fw_out_if.fu_via_ip, iaddr, iaddrlen) == NULL) {*/ + if (inet_ntop(AF_INET, &ptr->fw_fwd_ip.sin_addr, iaddr, iaddrlen) == NULL) { + syslog(LOG_ERR, "inet_ntop(): %m"); + goto error; + } + } + if (rhost != NULL && rhostlen > 0) { + if (ptr->fw_src.s_addr == 0) + rhost[0] = '\0'; + else if (inet_ntop(AF_INET, &ptr->fw_src.s_addr, rhost, rhostlen) == NULL) { + syslog(LOG_ERR, "inet_ntop(): %m"); + goto error; + } + } + ipfw_free_ruleset(&rules); + get_desc_time(*eport, *proto, desc, desclen, timestamp); + return 0; + } + +error: + if (rules != NULL) + ipfw_free_ruleset(&rules); + return -1; +} + +/* upnp_get_portmappings_in_range() + * return a list of all "external" ports for which a port + * mapping exists */ +unsigned short * +get_portmappings_in_range(unsigned short startport, + unsigned short endport, + int proto, + unsigned int * number) +{ + unsigned short *array = NULL, *array2 = NULL; + unsigned int capacity = 128; + int i, count_rules, total_rules = 0; + struct ip_fw * rules = NULL; + + if (ipfw_validate_protocol(proto) < 0) + return NULL; + + do { + count_rules = ipfw_fetch_ruleset(&rules, &total_rules, 10); + if (count_rules < 0) + goto error; + } while (count_rules == 10); + + array = calloc(capacity, sizeof(unsigned short)); + if(!array) { + syslog(LOG_ERR, "get_portmappings_in_range() : calloc error"); + goto error; + } + *number = 0; + + for (i=0; ifw_uar.fw_pts[0]; + if (proto == ptr->fw_prot + && startport <= eport + && eport <= endport) { + if(*number >= capacity) { + capacity += 128; + array2 = realloc(array, sizeof(unsigned short)*capacity); + if(!array2) { + syslog(LOG_ERR, "get_portmappings_in_range() : realloc(%lu) error", sizeof(unsigned short)*capacity); + *number = 0; + free(array); + goto error; + } + array = array2; + } + array[*number] = eport; + (*number)++; + } + } +error: + if (rules != NULL) + ipfw_free_ruleset(&rules); + return array; +} + +int +update_portmapping_desc_timestamp(const char * ifname, + unsigned short eport, int proto, + const char * desc, unsigned int timestamp) +{ + UNUSED(ifname); + del_desc_time(eport, proto); + add_desc_time(eport, proto, desc, timestamp); + return 0; +} + +int +update_portmapping(const char * ifname, unsigned short eport, int proto, + unsigned short iport, const char * desc, + unsigned int timestamp) +{ + int i, count_rules, total_rules = 0; + struct ip_fw * rules = NULL; + int r = -1; + char iaddr[16]; + char rhost[16]; + int found; + + if (ipfw_validate_protocol(proto) < 0) + return -1; + if (ipfw_validate_ifname(ifname) < 0) + return -1; + + do { + count_rules = ipfw_fetch_ruleset(&rules, &total_rules, 10); + if (count_rules < 0) + goto error; + } while (count_rules == 10); + + found = 0; + iaddr[0] = '\0'; + rhost[0] = '\0'; + + for (i=0; ifw_prot && eport == ptr->fw_uar.fw_pts[0]) { + if (inet_ntop(AF_INET, &ptr->fw_fwd_ip.sin_addr, iaddr, sizeof(iaddr)) == NULL) { + syslog(LOG_ERR, "inet_ntop(): %m"); + goto error; + } + if ((ptr->fw_src.s_addr != 0) && + (inet_ntop(AF_INET, &ptr->fw_src.s_addr, rhost, sizeof(rhost)) == NULL)) { + syslog(LOG_ERR, "inet_ntop(): %m"); + goto error; + } + found = 1; + if (ipfw_exec(IP_FW_DEL, (struct ip_fw *)ptr, sizeof(*ptr)) < 0) + goto error; + del_desc_time(eport, proto); + break; + } + } + ipfw_free_ruleset(&rules); + rules = NULL; + + if(found) + r = add_redirect_rule2(ifname, rhost, eport, iaddr, iport, proto, desc, timestamp); + +error: + if (rules != NULL) + ipfw_free_ruleset(&rules); + return r; +} diff --git a/src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.h b/src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.h new file mode 100644 index 0000000..9e55038 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipfw/ipfwrdr.h @@ -0,0 +1,79 @@ +/* $Id: ipfwrdr.h,v 1.6 2012/02/11 13:10:57 nanard Exp $ */ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2009 Jardel Weyrich + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution + */ + +#ifndef IPFWRDR_H_INCLUDED +#define IPFWRDR_H_INCLUDED + +#include "../commonrdr.h" + +int add_redirect_rule2( + const char * ifname, /* src interface (external) */ + const char * rhost, /* remote host (ip) */ + unsigned short eport, /* src port (external) */ + const char * iaddr, /* dst address (internal) */ + unsigned short iport, /* dst port (internal) */ + int proto, + const char * desc, + unsigned int timestamp); + +int add_filter_rule2( + const char * ifname, + const char * rhost, + const char * iaddr, + unsigned short eport, + unsigned short iport, + int proto, + const char * desc); + +#if 0 + +/* + * get_redirect_rule() gets internal IP and port from + * interface, external port and protocl +*/ +int get_redirect_rule( + const char * ifname, + unsigned short eport, + int proto, + char * iaddr, + int iaddrlen, + unsigned short * iport, + char * desc, + int desclen, + u_int64_t * packets, + u_int64_t * bytes); + +int get_redirect_rule_by_index( + int index, + char * ifname, + unsigned short * eport, + char * iaddr, + int iaddrlen, + unsigned short * iport, + int * proto, + char * desc, + int desclen, + u_int64_t * packets, + u_int64_t * bytes); + +#endif + +/* + * delete_redirect_rule() +*/ +int delete_redirect_rule(const char * ifname, unsigned short eport, int proto); + +/* + * delete_filter_rule() +*/ +int delete_filter_rule(const char * ifname, unsigned short eport, int proto); + +int clear_redirect_rules(void); + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/ipfw/testipfwrdr.c b/src/contrib/miniupnp/miniupnpd/ipfw/testipfwrdr.c new file mode 100644 index 0000000..d6791d2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/ipfw/testipfwrdr.c @@ -0,0 +1,87 @@ +/* $Id: testipfwrdr.c,v 1.7 2011/06/22 21:57:17 nanard Exp $ */ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2009-2011 Jardel Weyrich, Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution + */ + +#include +#include +#include +#include +#include "ipfwrdr.h" + +// test program for ipfwrdr.c +static const char * ifname = "lo0"; + +static void +list_port_mappings(void) +{ + int i; + unsigned short eport; + char iaddr[16]; + unsigned short iport; + int proto; + char desc[64]; + char rhost[32]; + unsigned int timestamp; + u_int64_t packets, bytes; + + printf("== Port Mapping List ==\n"); + for(i = 0;; i++) { + iaddr[0] = '\0'; + desc[0] = '\0'; + eport = iport = 0; + timestamp = 0; + packets = bytes = 0; + proto = -1; + if(get_redirect_rule_by_index(i, 0/*ifname*/, &eport, iaddr, sizeof(iaddr), + &iport, &proto, desc, sizeof(desc), + rhost, sizeof(rhost), + ×tamp, &packets, &bytes) < 0) + break; + printf("%2d - %5hu=>%15s:%5hu %d '%s' '%s' %u %" PRIu64 " %" PRIu64 "\n", + i, eport, iaddr, iport, proto, desc, rhost, timestamp, + packets, bytes); + } + printf("== %d Port Mapping%s ==\n", i, (i > 1)?"s":""); +} + +int main(int argc, char * * argv) { + unsigned int timestamp; + char desc[64]; + char addr[16]; + char rhost[40]; + unsigned short iport = 0; + const char * in_rhost = "8.8.8.8"; + + desc[0] = '\0'; + addr[0] = '\0'; + openlog("testipfwrdrd", LOG_CONS | LOG_PERROR, LOG_USER); + if(init_redirect() < 0) { + fprintf(stderr, "init_redirect() failed.\n"); + return 1; + } + list_port_mappings(); + delete_redirect_rule(ifname, 2222, IPPROTO_TCP); + delete_redirect_rule(ifname, 2223, IPPROTO_TCP); + add_redirect_rule2(ifname, "", 2223, + "10.1.1.17", 4445, IPPROTO_TCP, + "test miniupnpd", time(NULL) + 60); + add_redirect_rule2(ifname, in_rhost, 2222, + "10.1.1.16", 4444, IPPROTO_TCP, + "test miniupnpd", time(NULL) + 60); + get_redirect_rule(ifname, 2222, IPPROTO_TCP, addr, sizeof(addr), &iport, + desc, sizeof(desc), rhost, sizeof(rhost), + ×tamp, NULL, NULL); + printf("'%s' %s:%hu '%s' %u\n", rhost, addr, iport, desc, timestamp); + list_port_mappings(); + delete_redirect_rule(ifname, 2222, IPPROTO_TCP); + delete_redirect_rule(ifname, 2223, IPPROTO_TCP); + list_port_mappings(); + shutdown_redirect(); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/linux/getifstats.c b/src/contrib/miniupnp/miniupnpd/linux/getifstats.c new file mode 100644 index 0000000..5a4386c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/linux/getifstats.c @@ -0,0 +1,135 @@ +/* $Id: getifstats.c,v 1.14 2018/03/13 23:05:21 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include + +#include "../config.h" +#include "../getifstats.h" + +#ifdef GET_WIRELESS_STATS +#include +#include +#include +#include +#endif /* GET_WIRELESS_STATS */ + +/* that is the answer */ +#define BAUDRATE_DEFAULT 4200000 + +int +getifstats(const char * ifname, struct ifdata * data) +{ + FILE *f; + char line[512]; + char * p; + int i; + int r = -1; + char fname[64]; +#ifdef ENABLE_GETIFSTATS_CACHING + static time_t cache_timestamp = 0; + static struct ifdata cache_data; + time_t current_time; +#endif /* ENABLE_GETIFSTATS_CACHING */ + if(!data) + return -1; + data->baudrate = BAUDRATE_DEFAULT; + data->opackets = 0; + data->ipackets = 0; + data->obytes = 0; + data->ibytes = 0; + if(!ifname || ifname[0]=='\0') + return -1; +#ifdef ENABLE_GETIFSTATS_CACHING + current_time = upnp_time(); + if(current_time == ((time_t)-1)) { + syslog(LOG_ERR, "getifstats() : time() error : %m"); + } else { + if(current_time < cache_timestamp + GETIFSTATS_CACHING_DURATION) { + /* return cached data */ + memcpy(data, &cache_data, sizeof(struct ifdata)); + return 0; + } + } +#endif /* ENABLE_GETIFSTATS_CACHING */ + f = fopen("/proc/net/dev", "r"); + if(!f) { + syslog(LOG_ERR, "getifstats() : cannot open /proc/net/dev : %m"); + return -1; + } + /* discard the two header lines */ + if(!fgets(line, sizeof(line), f) || !fgets(line, sizeof(line), f)) { + syslog(LOG_ERR, "getifstats() : error reading /proc/net/dev : %m"); + } + while(fgets(line, sizeof(line), f)) { + p = line; + while(*p==' ') p++; + i = 0; + while(ifname[i] == *p) { + p++; i++; + } + /* TODO : how to handle aliases ? */ + if(ifname[i] || *p != ':') + continue; + p++; + while(*p==' ') p++; + data->ibytes = strtoul(p, &p, 0); + while(*p==' ') p++; + data->ipackets = strtoul(p, &p, 0); + /* skip 6 columns */ + for(i=6; i>0 && *p!='\0'; i--) { + while(*p==' ') p++; + while(*p!=' ' && *p) p++; + } + while(*p==' ') p++; + data->obytes = strtoul(p, &p, 0); + while(*p==' ') p++; + data->opackets = strtoul(p, &p, 0); + r = 0; + break; + } + fclose(f); + /* get interface speed */ + snprintf(fname, sizeof(fname), "/sys/class/net/%s/speed", ifname); + f = fopen(fname, "r"); + if(f) { + if(fgets(line, sizeof(line), f)) { + i = atoi(line); /* 65535 means unknown */ + if(i > 0 && i < 65535) + data->baudrate = 1000000*i; + } + fclose(f); + } else { + syslog(LOG_INFO, "cannot read %s file : %m", fname); + } +#ifdef GET_WIRELESS_STATS + if(data->baudrate == BAUDRATE_DEFAULT) { + struct iwreq iwr; + int s; + s = socket(AF_INET, SOCK_DGRAM, 0); + if(s >= 0) { + strncpy(iwr.ifr_name, ifname, IFNAMSIZ); + if(ioctl(s, SIOCGIWRATE, &iwr) >= 0) { + data->baudrate = iwr.u.bitrate.value; + } + close(s); + } + } +#endif /* GET_WIRELESS_STATS */ +#ifdef ENABLE_GETIFSTATS_CACHING + if(r==0 && current_time!=((time_t)-1)) { + /* cache the new data */ + cache_timestamp = current_time; + memcpy(&cache_data, data, sizeof(struct ifdata)); + } +#endif /* ENABLE_GETIFSTATS_CACHING */ + return r; +} + diff --git a/src/contrib/miniupnp/miniupnpd/linux/getroute.c b/src/contrib/miniupnp/miniupnpd/linux/getroute.c new file mode 100644 index 0000000..f9afea7 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/linux/getroute.c @@ -0,0 +1,197 @@ +/* $Id: getroute.c,v 1.4 2013/02/06 10:50:04 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2015 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +/*#include */ +#include +#include +#ifdef USE_LIBNFNETLINK +/* define USE_LIBNFNETLINK in order to use libnfnetlink + * instead of custom code + * see https://github.com/miniupnp/miniupnp/issues/110 */ +#include +#endif /* USE_LIBNFNETLINK */ + +#include "../getroute.h" +#include "../upnputils.h" + +int +get_src_for_route_to(const struct sockaddr * dst, + void * src, size_t * src_len, + int * index) +{ + int fd = -1; + struct nlmsghdr *h; + int status; + struct { + struct nlmsghdr n; + struct rtmsg r; + char buf[1024]; + } req; + struct sockaddr_nl nladdr; + struct iovec iov = { + .iov_base = (void*) &req.n, + }; + struct msghdr msg = { + .msg_name = &nladdr, + .msg_namelen = sizeof(nladdr), + .msg_iov = &iov, + .msg_iovlen = 1, + }; + const struct sockaddr_in * dst4; + const struct sockaddr_in6 * dst6; +#ifndef USE_LIBNFNETLINK + struct rtattr * rta; +#endif /* USE_LIBNFNETLINK */ + + memset(&req, 0, sizeof(req)); + req.n.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg)); + req.n.nlmsg_flags = NLM_F_REQUEST; + req.n.nlmsg_type = RTM_GETROUTE; + req.r.rtm_family = dst->sa_family; + req.r.rtm_table = 0; + req.r.rtm_protocol = 0; + req.r.rtm_scope = 0; + req.r.rtm_type = 0; + req.r.rtm_src_len = 0; + req.r.rtm_dst_len = 0; + req.r.rtm_tos = 0; + + { + char dst_str[128]; + sockaddr_to_string(dst, dst_str, sizeof(dst_str)); + syslog(LOG_DEBUG, "get_src_for_route_to (%s)", dst_str); + } + /* add address */ +#ifndef USE_LIBNFNETLINK + rta = (struct rtattr *)(((char*)&req) + NLMSG_ALIGN(req.n.nlmsg_len)); + rta->rta_type = RTA_DST; +#endif /* USE_LIBNFNETLINK */ + if(dst->sa_family == AF_INET) { + dst4 = (const struct sockaddr_in *)dst; +#ifdef USE_LIBNFNETLINK + nfnl_addattr_l(&req.n, sizeof(req), RTA_DST, &dst4->sin_addr, 4); +#else + rta->rta_len = RTA_SPACE(sizeof(dst4->sin_addr)); + memcpy(RTA_DATA(rta), &dst4->sin_addr, sizeof(dst4->sin_addr)); +#endif /* USE_LIBNFNETLINK */ + req.r.rtm_dst_len = 32; + } else { + dst6 = (const struct sockaddr_in6 *)dst; +#ifdef USE_LIBNFNETLINK + nfnl_addattr_l(&req.n, sizeof(req), RTA_DST, &dst6->sin6_addr, 16); +#else + rta->rta_len = RTA_SPACE(sizeof(dst6->sin6_addr)); + memcpy(RTA_DATA(rta), &dst6->sin6_addr, sizeof(dst6->sin6_addr)); +#endif /* USE_LIBNFNETLINK */ + req.r.rtm_dst_len = 128; + } +#ifndef USE_LIBNFNETLINK + req.n.nlmsg_len += rta->rta_len; +#endif /* USE_LIBNFNETLINK */ + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (fd < 0) { + syslog(LOG_ERR, "socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE) : %m"); + return -1; + } + + memset(&nladdr, 0, sizeof(nladdr)); + nladdr.nl_family = AF_NETLINK; + + req.n.nlmsg_seq = 1; + iov.iov_len = req.n.nlmsg_len; + + status = sendmsg(fd, &msg, 0); + + if (status < 0) { + syslog(LOG_ERR, "sendmsg(rtnetlink) : %m"); + goto error; + } + + memset(&req, 0, sizeof(req)); + + for(;;) { + iov.iov_len = sizeof(req); + status = recvmsg(fd, &msg, 0); + if(status < 0) { + if (errno == EINTR || errno == EAGAIN) + continue; + syslog(LOG_ERR, "recvmsg(rtnetlink) %m"); + goto error; + } + if(status == 0) { + syslog(LOG_ERR, "recvmsg(rtnetlink) EOF"); + goto error; + } + for (h = (struct nlmsghdr*)&req.n; status >= (int)sizeof(*h); ) { + int len = h->nlmsg_len; + int l = len - sizeof(*h); + + if (l<0 || len>status) { + if (msg.msg_flags & MSG_TRUNC) { + syslog(LOG_ERR, "Truncated message"); + } + syslog(LOG_ERR, "malformed message: len=%d", len); + goto error; + } + + if(nladdr.nl_pid != 0 || h->nlmsg_seq != 1/*seq*/) { + syslog(LOG_ERR, "wrong seq = %d\n", h->nlmsg_seq); + /* Don't forget to skip that message. */ + status -= NLMSG_ALIGN(len); + h = (struct nlmsghdr*)((char*)h + NLMSG_ALIGN(len)); + continue; + } + + if(h->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = (struct nlmsgerr*)NLMSG_DATA(h); + syslog(LOG_ERR, "NLMSG_ERROR %d : %s", err->error, strerror(-err->error)); + goto error; + } + if(h->nlmsg_type == RTM_NEWROUTE) { + struct rtattr * rta; + int len = h->nlmsg_len; + len -= NLMSG_LENGTH(sizeof(struct rtmsg)); + for(rta = RTM_RTA(NLMSG_DATA((h))); RTA_OK(rta, len); rta = RTA_NEXT(rta,len)) { + unsigned char * data = RTA_DATA(rta); + if(rta->rta_type == RTA_PREFSRC) { + if(src_len && src) { + if(*src_len < RTA_PAYLOAD(rta)) { + syslog(LOG_WARNING, "cannot copy src: %u<%lu", + (unsigned)*src_len, (unsigned long)RTA_PAYLOAD(rta)); + goto error; + } + *src_len = RTA_PAYLOAD(rta); + memcpy(src, data, RTA_PAYLOAD(rta)); + } + } else if(rta->rta_type == RTA_OIF) { + if(index) + memcpy(index, data, sizeof(int)); + } + } + close(fd); + return 0; + } + status -= NLMSG_ALIGN(len); + h = (struct nlmsghdr*)((char*)h + NLMSG_ALIGN(len)); + } + } + syslog(LOG_WARNING, "get_src_for_route_to() : src not found"); +error: + if(fd >= 0) + close(fd); + return -1; +} + diff --git a/src/contrib/miniupnp/miniupnpd/linux/ifacewatcher.c b/src/contrib/miniupnp/miniupnpd/linux/ifacewatcher.c new file mode 100644 index 0000000..7095a0f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/linux/ifacewatcher.c @@ -0,0 +1,350 @@ +/* $Id: ifacewatcher.c,v 1.9 2018/02/03 22:05:42 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * + * ifacewatcher.c + * + * This file implements dynamic serving of new network interfaces + * which weren't available during daemon start. It also takes care + * of interfaces which become unavailable. + * + * Copyright (c) 2011, Alexey Osipov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../config.h" + +#ifdef USE_IFACEWATCHER + +#include "../ifacewatcher.h" +#include "../minissdp.h" +#include "../getifaddr.h" +#include "../upnpglobalvars.h" +#include "../natpmp.h" + +extern volatile sig_atomic_t should_send_public_address_change_notif; + + +int +OpenAndConfInterfaceWatchSocket(void) +{ + int s; + struct sockaddr_nl addr; + + s = socket(PF_NETLINK, SOCK_RAW, NETLINK_ROUTE); + if (s == -1) + { + syslog(LOG_ERR, "socket(PF_NETLINK, SOCK_RAW, NETLINK_ROUTE): %m"); + return -1; + } + + memset(&addr, 0, sizeof(addr)); + addr.nl_family = AF_NETLINK; + addr.nl_groups = RTMGRP_LINK | RTMGRP_IPV4_IFADDR; + /*addr.nl_groups = RTMGRP_LINK | RTMGRP_IPV4_IFADDR | RTMGRP_IPV6_IFADDR;*/ + + if (bind(s, (struct sockaddr *)&addr, sizeof(addr)) < 0) + { + syslog(LOG_ERR, "bind(netlink): %m"); + close(s); + return -1; + } + + return s; +} + +#if 0 +/* disabled at the moment */ +int +ProcessInterfaceUp(struct ifinfomsg *ifi) +{ + struct lan_iface_s * lan_iface; + struct lan_iface_s * lan_iface2; + struct lan_addr_s * lan_addr; + char ifname[IFNAMSIZ]; + char ifstraddr[16]; + struct in_addr ifaddr; + + /* check if we already have this iface */ + for(lan_iface = lan_ifaces.lh_first; lan_iface != NULL; lan_iface = lan_iface->list.le_next) + if (lan_iface->iface.index == ifi->ifi_index) + break; + + if (lan_iface != NULL) + return 0; + + if (if_indextoname(ifi->ifi_index, ifname) == NULL) + { + syslog(LOG_ERR, "if_indextoname(%d, ifname) failed", ifi->ifi_index); + return -1; + } + + if (getifaddr(ifname, ifstraddr, 16) < 0) + { + syslog(LOG_DEBUG, "getifaddr(%s, ifaddr, 16) failed", ifname); + return 1; + } + + if (inet_pton(AF_INET, ifstraddr, &ifaddr) != 1) + { + syslog(LOG_ERR, "inet_pton(AF_INET, \"%s\", &ifaddr) failed", ifstraddr); + return -1; + } + + /* check if this new interface has address which we need to listen to */ + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) + { + if (lan_addr->addr.s_addr != ifaddr.s_addr) + continue; + + syslog(LOG_INFO, "Interface up: %s (%s)", ifname, ifstraddr); + + /* adding new lan_iface entry */ + lan_iface = (struct lan_iface_s *) malloc(sizeof(struct lan_iface_s)); + if (lan_iface == NULL) + { + syslog(LOG_ERR, "malloc(sizeof(struct lan_iface_s): %m"); + continue; + } + + lan_iface->lan_addr = lan_addr; + strncpy(lan_iface->iface.name, ifname, IFNAMSIZ); + lan_iface->iface.index = ifi->ifi_index; + lan_iface->iface.addr = ifaddr; + lan_iface->snotify = -1; +#ifdef ENABLE_NATPMP + lan_iface->snatpmp = -1; +#endif + + LIST_INSERT_HEAD(&lan_ifaces, lan_iface, list); + + /* adding multicast membership for SSDP */ + if(AddMulticastMembership(sudp, ifaddr.s_addr, ifi->ifi_index) < 0) + syslog(LOG_WARNING, "Failed to add multicast membership for interface %s (%s)", ifname, ifstraddr); + else + syslog(LOG_INFO, "Multicast membership added for %s (%s)", ifname, ifstraddr); + + /* create SSDP notify socket */ + if (OpenAndConfSSDPNotifySocket(lan_iface) < 0) + syslog(LOG_WARNING, "Failed to open SSDP notify socket for interface %s (%s)", ifname, ifstraddr); + +#ifdef ENABLE_NATPMP + /* create NAT-PMP socket */ + for(lan_iface2 = lan_ifaces.lh_first; lan_iface2 != NULL; lan_iface2 = lan_iface2->list.le_next) + if (lan_iface2->lan_addr->addr.s_addr == lan_iface->lan_addr->addr.s_addr && + lan_iface2->snatpmp >= 0) + lan_iface->snatpmp = lan_iface2->snatpmp; + + if (lan_iface->snatpmp < 0) + { + lan_iface->snatpmp = OpenAndConfNATPMPSocket(ifaddr.s_addr); + if (lan_iface->snatpmp < 0) + syslog(LOG_ERR, "OpenAndConfNATPMPSocket(ifaddr.s_addr) failed for %s (%s)", ifname, ifstraddr); + else + syslog(LOG_INFO, "Listening for NAT-PMP connection on %s:%d", ifstraddr, NATPMP_PORT); + } +#endif + } + + return 0; +} + +int +ProcessInterfaceDown(struct ifinfomsg *ifi) +{ + struct lan_iface_s * lan_iface; + struct lan_iface_s * lan_iface2; + + /* check if we have this iface */ + for(lan_iface = lan_ifaces.lh_first; lan_iface != NULL; lan_iface = lan_iface->list.le_next) + if (lan_iface->iface.index == ifi->ifi_index) + break; + + if (lan_iface == NULL) + return 0; + + /* one of our interfaces is going down, clean up */ + syslog(LOG_INFO, "Interface down: %s", lan_iface->iface.name); + + /* remove multicast membership for SSDP */ + if(DropMulticastMembership(sudp, lan_iface->lan_addr->addr.s_addr, lan_iface->iface.index) < 0) + syslog(LOG_WARNING, "Failed to drop multicast membership for interface %s (%s)", lan_iface->iface.name, lan_iface->lan_addr->str); + else + syslog(LOG_INFO, "Multicast membership dropped for %s (%s)", lan_iface->iface.name, lan_iface->lan_addr->str); + + /* closing SSDP notify socket */ + close(lan_iface->snotify); + +#ifdef ENABLE_NATPMP + /* closing NAT-PMP socket if it's not used anymore */ + for(lan_iface2 = lan_ifaces.lh_first; lan_iface2 != NULL; lan_iface2 = lan_iface2->list.le_next) + if (lan_iface2 != lan_iface && lan_iface2->snatpmp == lan_iface->snatpmp) + break; + + if (lan_iface2 == NULL) + close(lan_iface->snatpmp); +#endif + + /* del corresponding lan_iface entry */ + LIST_REMOVE(lan_iface, list); + free(lan_iface); + + return 0; +} +#endif + +void +ProcessInterfaceWatchNotify(int s) +{ + char buffer[4096]; + struct iovec iov; + struct msghdr hdr; + struct nlmsghdr *nlhdr; +#if 0 +/* disabled at the moment */ + struct ifinfomsg *ifi; +#endif + struct ifaddrmsg *ifa; + int len; + + struct rtattr *rth; + int rtl; + + unsigned int ext_if_name_index = 0; + + iov.iov_base = buffer; + iov.iov_len = sizeof(buffer); + + memset(&hdr, 0, sizeof(hdr)); + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + + len = recvmsg(s, &hdr, 0); + if (len < 0) + { + syslog(LOG_ERR, "recvmsg(s, &hdr, 0): %m"); + return; + } + + if(ext_if_name) { + ext_if_name_index = if_nametoindex(ext_if_name); + } + + for (nlhdr = (struct nlmsghdr *) buffer; + NLMSG_OK (nlhdr, (unsigned int)len); + nlhdr = NLMSG_NEXT (nlhdr, len)) + { + int is_del = 0; + char address[48]; + char ifname[IFNAMSIZ]; + address[0] = '\0'; + ifname[0] = '\0'; + if (nlhdr->nlmsg_type == NLMSG_DONE) + break; + switch(nlhdr->nlmsg_type) { + case RTM_DELLINK: + is_del = 1; + case RTM_NEWLINK: +#if 0 +/* disabled at the moment */ + ifi = (struct ifinfomsg *) NLMSG_DATA(nlhdr); + if(is_del) { + if(ProcessInterfaceDown(ifi) < 0) + syslog(LOG_ERR, "ProcessInterfaceDown(ifi) failed"); + } else { + if(ProcessInterfaceUp(ifi) < 0) + syslog(LOG_ERR, "ProcessInterfaceUp(ifi) failed"); + } +#endif + break; + case RTM_DELADDR: + is_del = 1; + case RTM_NEWADDR: + /* see /usr/include/linux/netlink.h + * and /usr/include/linux/rtnetlink.h */ + ifa = (struct ifaddrmsg *) NLMSG_DATA(nlhdr); + syslog(LOG_DEBUG, "%s %s index=%d fam=%d", "ProcessInterfaceWatchNotify", + is_del ? "RTM_DELADDR" : "RTM_NEWADDR", + ifa->ifa_index, ifa->ifa_family); + for(rth = IFA_RTA(ifa), rtl = IFA_PAYLOAD(nlhdr); + rtl && RTA_OK(rth, rtl); + rth = RTA_NEXT(rth, rtl)) { + char tmp[128]; + memset(tmp, 0, sizeof(tmp)); + switch(rth->rta_type) { + case IFA_ADDRESS: + case IFA_LOCAL: + case IFA_BROADCAST: + case IFA_ANYCAST: + inet_ntop(ifa->ifa_family, RTA_DATA(rth), tmp, sizeof(tmp)); + if(rth->rta_type == IFA_ADDRESS) + strncpy(address, tmp, sizeof(address)); + break; + case IFA_LABEL: + strncpy(tmp, RTA_DATA(rth), sizeof(tmp)); + strncpy(ifname, tmp, sizeof(ifname)); + break; + case IFA_CACHEINFO: + { + struct ifa_cacheinfo *cache_info; + cache_info = RTA_DATA(rth); + snprintf(tmp, sizeof(tmp), "valid=%u preferred=%u", + cache_info->ifa_valid, cache_info->ifa_prefered); + } + break; + default: + strncpy(tmp, "*unknown*", sizeof(tmp)); + } + syslog(LOG_DEBUG, " - %u - %s type=%d", + ifa->ifa_index, tmp, + rth->rta_type); + } + if(ifa->ifa_index == ext_if_name_index) { + should_send_public_address_change_notif = 1; + } + break; + default: + syslog(LOG_DEBUG, "%s type %d ignored", + "ProcessInterfaceWatchNotify", nlhdr->nlmsg_type); + } + } + +} + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/linux/miniupnpd.init.d.script b/src/contrib/miniupnp/miniupnpd/linux/miniupnpd.init.d.script new file mode 100644 index 0000000..f4ea7e3 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/linux/miniupnpd.init.d.script @@ -0,0 +1,57 @@ +#!/bin/sh +# $Id: miniupnpd.init.d.script,v 1.2 2007/09/23 16:11:12 nanard Exp $ +# MiniUPnP project +# author: Thomas Bernard +# website: http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + +### BEGIN INIT INFO +# Provides: miniupnpd +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: MiniUPnPd port-forwarding daemon +### END INIT INFO + +set -e + +MINIUPNPD=/usr/sbin/miniupnpd +ARGS='-f /etc/miniupnpd/miniupnpd.conf' + +IPTABLES_CREATE=/etc/miniupnpd/iptables_init.sh +IPTABLES_REMOVE=/etc/miniupnpd/iptables_removeall.sh + +test -f $MINIUPNPD || exit 0 + +. /lib/lsb/init-functions + +case "$1" in + start) + log_daemon_msg "Starting miniupnpd" "miniupnpd" + $IPTABLES_CREATE > /dev/null 2>&1 + start-stop-daemon --start --quiet --pidfile /var/run/miniupnpd.pid --startas $MINIUPNPD -- $ARGS $LSBNAMES + log_end_msg $? + ;; + stop) + log_daemon_msg "Stopping miniupnpd" "miniupnpd" + start-stop-daemon --stop --quiet --pidfile /var/run/miniupnpd.pid + log_end_msg $? + $IPTABLES_REMOVE > /dev/null 2>&1 + ;; + restart|reload|force-reload) + log_daemon_msg "Restarting miniupnpd" "miniupnpd" + start-stop-daemon --stop --retry 5 --quiet --pidfile /var/run/miniupnpd.pid + $IPTABLES_REMOVE > /dev/null 2>&1 + $IPTABLES_CREATE > /dev/null 2>&1 + start-stop-daemon --start --quiet --pidfile /var/run/miniupnpd.pid --startas $MINIUPNPD -- $ARGS $LSBNAMES + log_end_msg $? + ;; + status) + status_of_proc /usr/sbin/miniupnpd miniupnpd + ;; + *) + log_action_msg "Usage: /etc/init.d/miniupnpd {start|stop|restart|reload|force-reload}" + exit 2 + ;; +esac +exit 0 diff --git a/src/contrib/miniupnp/miniupnpd/mac/Makefile b/src/contrib/miniupnp/miniupnpd/mac/Makefile new file mode 100644 index 0000000..968562f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/mac/Makefile @@ -0,0 +1,14 @@ +# $Id: Makefile,v 1.2 2011/02/20 23:43:41 nanard Exp $ +# made for GNU Make +CFLAGS = -Wall -g +EXECUTABLES = testgetifstats + +all: $(EXECUTABLES) + +clean: + rm -f *.o $(EXECUTABLES) + +testmacrdr.o: testmacrdr.c macrdr.h + +testgetifstats: testgetifstats.o getifstats.o + diff --git a/src/contrib/miniupnp/miniupnpd/mac/getifstats.c b/src/contrib/miniupnp/miniupnpd/mac/getifstats.c new file mode 100644 index 0000000..83ab7f8 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/mac/getifstats.c @@ -0,0 +1,94 @@ +/* $Id: getifstats.c,v 1.8 2018/03/13 23:05:21 nanard Exp $ */ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2009 Jardel Weyrich + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../getifstats.h" +#include "../config.h" + +int getifstats(const char * ifname, struct ifdata * data) { + int mib[] = { CTL_NET, PF_ROUTE, 0, AF_INET, NET_RT_IFLIST, if_nametoindex(ifname) }; + const size_t mib_len = sizeof(mib) / sizeof(mib[0]); + size_t needed; + char *buf, *end, *p; + struct if_msghdr *ifm; + struct if_data ifdata; +#ifdef ENABLE_GETIFSTATS_CACHING + static time_t cache_timestamp = 0; + static struct ifdata cache_data; + time_t current_time; +#endif + + if (data == NULL || ifname == NULL || ifname[0] == '\0') + return -1; /* error */ + + data->baudrate = 4200000; + data->opackets = 0; + data->ipackets = 0; + data->obytes = 0; + data->ibytes = 0; + +#ifdef ENABLE_GETIFSTATS_CACHING + current_time = upnp_time(); + if (current_time == ((time_t)-1)) { + syslog(LOG_ERR, "getifstats() : time() error : %m"); + } else { + if (current_time < cache_timestamp + GETIFSTATS_CACHING_DURATION) { + memcpy(data, &cache_data, sizeof(struct ifdata)); + return 0; + } + } +#endif + + if (sysctl(mib, mib_len, NULL, &needed, NULL, 0) == -1) { + syslog(LOG_ERR, "sysctl(): %m"); + return -1; + } + buf = (char *) malloc(needed); + if (buf == NULL) + return -1; /* error */ + if (sysctl(mib, mib_len, buf, &needed, NULL, 0) == -1) { + syslog(LOG_ERR, "sysctl(): %m"); + free(buf); + return -1; /* error */ + } else { + for (end = buf + needed, p = buf; p < end; p += ifm->ifm_msglen) { + ifm = (struct if_msghdr *) p; + if (ifm->ifm_type == RTM_IFINFO && ifm->ifm_data.ifi_type == IFT_ETHER) { + ifdata = ifm->ifm_data; + data->opackets = ifdata.ifi_opackets; + data->ipackets = ifdata.ifi_ipackets; + data->obytes = ifdata.ifi_obytes; + data->ibytes = ifdata.ifi_ibytes; + data->baudrate = ifdata.ifi_baudrate; + free(buf); +#ifdef ENABLE_GETIFSTATS_CACHING + if (current_time!=((time_t)-1)) { + cache_timestamp = current_time; + memcpy(&cache_data, data, sizeof(struct ifdata)); + } +#endif + return 0; /* found, ok */ + } + } + } + free(buf); + return -1; /* not found or error */ +} + diff --git a/src/contrib/miniupnp/miniupnpd/mac/org.tuxfamily.miniupnpd.plist.before b/src/contrib/miniupnp/miniupnpd/mac/org.tuxfamily.miniupnpd.plist.before new file mode 100644 index 0000000..ed6e81f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/mac/org.tuxfamily.miniupnpd.plist.before @@ -0,0 +1,16 @@ + + + + + miniupnpd + org.tuxfamily.miniupnpd + ProgramArguments + + INSTALLPREFIX/sbin/miniupnpd + -f INSTALLPREFIX/etc/miniupnpd/miniupnpd.conf + + RunAtLoad + + + diff --git a/src/contrib/miniupnp/miniupnpd/mac/testgetifstats.c b/src/contrib/miniupnp/miniupnpd/mac/testgetifstats.c new file mode 100644 index 0000000..8e23f95 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/mac/testgetifstats.c @@ -0,0 +1,29 @@ +/* $Id: testgetifstats.c,v 1.3 2011/02/20 23:43:41 nanard Exp $ */ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2009 Jardel Weyrich + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution + */ + +#include +#include "../getifstats.h" + +int main(int argc, char * * argv) { + int r; + struct ifdata data; + printf("usage: %s if_name\n", argv[0]); + if (argc < 2) + return -1; + r = getifstats(argv[1], &data); + if (r < 0) + printf("getifstats() failed\n"); + else { + printf("ipackets = %10lu opackets = %10lu\n", data.ipackets, data.opackets); + printf("ibytes = %10lu obytes = %10lu\n", data.ibytes, data.obytes); + printf("baudrate = %10lu\n", data.baudrate); + } + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/macros.h b/src/contrib/miniupnp/miniupnpd/macros.h new file mode 100644 index 0000000..f1d93f6 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/macros.h @@ -0,0 +1,47 @@ +/* $Id: macros.h,v 1.1 2012/04/30 20:37:56 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2012-2015 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef MACROS_H_INCLUDED +#define MACROS_H_INCLUDED + +#define UNUSED(arg) (void)(arg) + +#include + +#ifndef INLINE +#define INLINE static inline +#endif +/* theses macros are designed to read/write unsigned short/long int + * from an unsigned char array in network order (big endian). + * Avoid pointer casting, so avoid accessing unaligned memory, which + * can crash with some cpu's */ +INLINE uint32_t readnu32(const uint8_t * p) +{ + return (p[0] << 24 | p[1] << 16 | p[2] << 8 | p[3]); +} +#define READNU32(p) readnu32(p) +INLINE uint16_t readnu16(const uint8_t * p) +{ + return (p[0] << 8 | p[1]); +} +#define READNU16(p) readnu16(p) +INLINE void writenu32(uint8_t * p, uint32_t n) +{ + p[0] = (n & 0xff000000) >> 24; + p[1] = (n & 0xff0000) >> 16; + p[2] = (n & 0xff00) >> 8; + p[3] = n & 0xff; +} +#define WRITENU32(p, n) writenu32(p, n) +INLINE void writenu16(uint8_t * p, uint16_t n) +{ + p[0] = (n & 0xff00) >> 8; + p[1] = n & 0xff; +} +#define WRITENU16(p, n) writenu16(p, n) + +#endif /* MACROS_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpd/minissdp.c b/src/contrib/miniupnp/miniupnpd/minissdp.c new file mode 100644 index 0000000..d345d64 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/minissdp.c @@ -0,0 +1,1566 @@ +/* $Id: minissdp.c,v 1.92 2018/03/13 10:52:39 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef IP_RECVIF +#include +#include +#include +#include +#endif + +#include "config.h" +#if defined(ENABLE_IPV6) && defined(UPNP_STRICT) +#include +#endif /* defined(ENABLE_IPV6) && defined(UPNP_STRICT) */ + +#include "upnpdescstrings.h" +#include "miniupnpdpath.h" +#include "upnphttp.h" +#include "upnpglobalvars.h" +#include "minissdp.h" +#include "upnputils.h" +#include "getroute.h" +#include "asyncsendto.h" +#include "codelength.h" +#include "macros.h" + +#ifndef MIN +#define MIN(x,y) (((x)<(y))?(x):(y)) +#endif /* MIN */ + +/* SSDP ip/port */ +#define SSDP_PORT (1900) +#define SSDP_MCAST_ADDR ("239.255.255.250") +#define LL_SSDP_MCAST_ADDR "FF02::C" +#define SL_SSDP_MCAST_ADDR "FF05::C" +#define GL_SSDP_MCAST_ADDR "FF0E::C" + +/* AddMulticastMembership() + * param s socket + * param lan_addr lan address + */ +static int +AddMulticastMembership(int s, struct lan_addr_s * lan_addr) +{ +#ifndef HAVE_IP_MREQN + /* The ip_mreqn structure appeared in Linux 2.4. */ + struct ip_mreq imr; /* Ip multicast membership */ +#else /* HAVE_IP_MREQN */ + struct ip_mreqn imr; /* Ip multicast membership */ +#endif /* HAVE_IP_MREQN */ + + /* setting up imr structure */ + imr.imr_multiaddr.s_addr = inet_addr(SSDP_MCAST_ADDR); + /*imr.imr_interface.s_addr = htonl(INADDR_ANY);*/ +#ifndef HAVE_IP_MREQN + imr.imr_interface.s_addr = lan_addr->addr.s_addr; +#else /* HAVE_IP_MREQN */ + imr.imr_address.s_addr = lan_addr->addr.s_addr; +#ifndef MULTIPLE_EXTERNAL_IP +#ifdef ENABLE_IPV6 + imr.imr_ifindex = lan_addr->index; +#else /* ENABLE_IPV6 */ + imr.imr_ifindex = if_nametoindex(lan_addr->ifname); +#endif /* ENABLE_IPV6 */ +#else /* MULTIPLE_EXTERNAL_IP */ + imr.imr_ifindex = 0; +#endif /* MULTIPLE_EXTERNAL_IP */ +#endif /* HAVE_IP_MREQN */ + +#ifndef HAVE_IP_MREQN + if (setsockopt(s, IPPROTO_IP, IP_ADD_MEMBERSHIP, (void *)&imr, sizeof(struct ip_mreq)) < 0) +#else /* HAVE_IP_MREQN */ + if (setsockopt(s, IPPROTO_IP, IP_ADD_MEMBERSHIP, (void *)&imr, sizeof(struct ip_mreqn)) < 0) +#endif /* HAVE_IP_MREQN */ + { + syslog(LOG_ERR, "setsockopt(udp, IP_ADD_MEMBERSHIP): %m"); + return -1; + } + + return 0; +} + +/* AddMulticastMembershipIPv6() + * param s socket (IPv6) + * param ifindex : interface index (0 : All interfaces) */ +#ifdef ENABLE_IPV6 +static int +AddMulticastMembershipIPv6(int s, unsigned int ifindex) +{ + struct ipv6_mreq mr; + + memset(&mr, 0, sizeof(mr)); + mr.ipv6mr_interface = ifindex; /* 0 : all interfaces */ +#ifndef IPV6_ADD_MEMBERSHIP +#define IPV6_ADD_MEMBERSHIP IPV6_JOIN_GROUP +#endif + inet_pton(AF_INET6, LL_SSDP_MCAST_ADDR, &mr.ipv6mr_multiaddr); + if(setsockopt(s, IPPROTO_IPV6, IPV6_ADD_MEMBERSHIP, &mr, sizeof(struct ipv6_mreq)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp, IPV6_ADD_MEMBERSHIP): %m"); + return -1; + } + inet_pton(AF_INET6, SL_SSDP_MCAST_ADDR, &mr.ipv6mr_multiaddr); + if(setsockopt(s, IPPROTO_IPV6, IPV6_ADD_MEMBERSHIP, &mr, sizeof(struct ipv6_mreq)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp, IPV6_ADD_MEMBERSHIP): %m"); + return -1; + } + inet_pton(AF_INET6, GL_SSDP_MCAST_ADDR, &mr.ipv6mr_multiaddr); + if(setsockopt(s, IPPROTO_IPV6, IPV6_ADD_MEMBERSHIP, &mr, sizeof(struct ipv6_mreq)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp, IPV6_ADD_MEMBERSHIP): %m"); + return -1; + } + return 0; +} +#endif + + +#if defined(ENABLE_IPV6) && defined(UPNP_STRICT) +static int get_link_local_addr(unsigned scope_id, struct in6_addr * addr6) +{ + struct ifaddrs * ifap; + struct ifaddrs * ife; + if(getifaddrs(&ifap)<0) { + syslog(LOG_ERR, "getifaddrs: %m"); + return -1; + } + for(ife = ifap; ife != NULL; ife = ife->ifa_next) { + if(ife->ifa_addr == NULL) continue; + if(ife->ifa_addr->sa_family != AF_INET6) continue; + if(!IN6_IS_ADDR_LINKLOCAL(&(((const struct sockaddr_in6 *)ife->ifa_addr)->sin6_addr))) continue; + if(scope_id != if_nametoindex(ife->ifa_name)) continue; + memcpy(addr6, &(((const struct sockaddr_in6 *)ife->ifa_addr)->sin6_addr), sizeof(struct in6_addr)); + break; + } + freeifaddrs(ifap); + return 0; +} +#endif /* defined(ENABLE_IPV6) && defined(UPNP_STRICT) */ + +/* Open and configure the socket listening for + * SSDP udp packets sent on 239.255.255.250 port 1900 + * SSDP v6 udp packets sent on FF02::C, or FF05::C, port 1900 */ +int +OpenAndConfSSDPReceiveSocket(int ipv6) +{ + int s; + struct sockaddr_storage sockname; + socklen_t sockname_len; + struct lan_addr_s * lan_addr; + const int on = 1; + + if( (s = socket(ipv6 ? PF_INET6 : PF_INET, SOCK_DGRAM, 0)) < 0) + { + syslog(LOG_ERR, "%s: socket(udp): %m", + "OpenAndConfSSDPReceiveSocket"); + return -1; + } + + memset(&sockname, 0, sizeof(struct sockaddr_storage)); +#ifdef ENABLE_IPV6 + if(ipv6) + { + struct sockaddr_in6 * saddr = (struct sockaddr_in6 *)&sockname; + saddr->sin6_family = AF_INET6; + saddr->sin6_port = htons(SSDP_PORT); + saddr->sin6_addr = ipv6_bind_addr; + sockname_len = sizeof(struct sockaddr_in6); + } + else +#endif /* ENABLE_IPV6 */ + { + struct sockaddr_in * saddr = (struct sockaddr_in *)&sockname; + saddr->sin_family = AF_INET; + saddr->sin_port = htons(SSDP_PORT); + /* NOTE : it seems it doesn't work when binding on the specific address */ + /*saddr->sin_addr.s_addr = inet_addr(UPNP_MCAST_ADDR);*/ + saddr->sin_addr.s_addr = htonl(INADDR_ANY); + /*saddr->sin_addr.s_addr = inet_addr(ifaddr);*/ + sockname_len = sizeof(struct sockaddr_in); + } + + if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0) + { + syslog(LOG_WARNING, "setsockopt(udp, SO_REUSEADDR): %m"); + } +#ifdef IP_RECVIF + /* BSD */ + if(!ipv6) { + if(setsockopt(s, IPPROTO_IP, IP_RECVIF, &on, sizeof(on)) < 0) + { + syslog(LOG_WARNING, "setsockopt(udp, IP_RECVIF): %m"); + } + } +#endif /* IP_RECVIF */ +#ifdef IP_PKTINFO + /* Linux */ + if(!ipv6) { + if(setsockopt(s, IPPROTO_IP, IP_PKTINFO, &on, sizeof(on)) < 0) + { + syslog(LOG_WARNING, "setsockopt(udp, IP_PKTINFO): %m"); + } + } +#endif /* IP_PKTINFO */ +#if defined(ENABLE_IPV6) && defined(IPV6_RECVPKTINFO) + if(ipv6) { + if(setsockopt(s, IPPROTO_IP, IPV6_RECVPKTINFO, &on, sizeof(on)) < 0) + { + syslog(LOG_WARNING, "setsockopt(udp, IPV6_RECVPKTINFO): %m"); + } + } +#endif /* defined(ENABLE_IPV6) && defined(IPV6_RECVPKTINFO) */ + + if(!set_non_blocking(s)) + { + syslog(LOG_WARNING, "%s: set_non_blocking(): %m", + "OpenAndConfSSDPReceiveSocket"); + } + +#if defined(SO_BINDTODEVICE) && !defined(MULTIPLE_EXTERNAL_IP) + /* One and only one LAN interface */ + if(lan_addrs.lh_first != NULL && lan_addrs.lh_first->list.le_next == NULL + && lan_addrs.lh_first->ifname[0] != '\0') + { + if(setsockopt(s, SOL_SOCKET, SO_BINDTODEVICE, + lan_addrs.lh_first->ifname, + strlen(lan_addrs.lh_first->ifname)) < 0) + syslog(LOG_WARNING, "%s: setsockopt(udp%s, SO_BINDTODEVICE, %s): %m", + "OpenAndConfSSDPReceiveSocket", ipv6 ? "6" : "", + lan_addrs.lh_first->ifname); + } +#endif /* defined(SO_BINDTODEVICE) && !defined(MULTIPLE_EXTERNAL_IP) */ + + if(bind(s, (struct sockaddr *)&sockname, sockname_len) < 0) + { + syslog(LOG_ERR, "%s: bind(udp%s): %m", + "OpenAndConfSSDPReceiveSocket", ipv6 ? "6" : ""); + close(s); + return -1; + } + +#ifdef ENABLE_IPV6 + if(ipv6) + { + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) + { + if(AddMulticastMembershipIPv6(s, lan_addr->index) < 0) + { + syslog(LOG_WARNING, + "Failed to add IPv6 multicast membership for interface %s", + strlen(lan_addr->str) ? lan_addr->str : "NULL"); + } + } + } + else +#endif + { + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) + { + if(AddMulticastMembership(s, lan_addr) < 0) + { + syslog(LOG_WARNING, + "Failed to add multicast membership for interface %s", + strlen(lan_addr->str) ? lan_addr->str : "NULL"); + } + } + } + + return s; +} + +/* open the UDP socket used to send SSDP notifications to + * the multicast group reserved for them */ +static int +OpenAndConfSSDPNotifySocket(in_addr_t addr) +{ + int s; + unsigned char loopchar = 0; + int bcast = 1; + unsigned char ttl = 2; /* UDA v1.1 says : + The TTL for the IP packet SHOULD default to 2 and + SHOULD be configurable. */ + /* TODO: Make TTL be configurable */ + struct in_addr mc_if; + struct sockaddr_in sockname; + + if( (s = socket(PF_INET, SOCK_DGRAM, 0)) < 0) + { + syslog(LOG_ERR, "socket(udp_notify): %m"); + return -1; + } + + mc_if.s_addr = addr; /*inet_addr(addr);*/ + + if(setsockopt(s, IPPROTO_IP, IP_MULTICAST_LOOP, (char *)&loopchar, sizeof(loopchar)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp_notify, IP_MULTICAST_LOOP): %m"); + close(s); + return -1; + } + + if(setsockopt(s, IPPROTO_IP, IP_MULTICAST_IF, (char *)&mc_if, sizeof(mc_if)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp_notify, IP_MULTICAST_IF): %m"); + close(s); + return -1; + } + + if(setsockopt(s, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) < 0) + { + syslog(LOG_WARNING, "setsockopt(udp_notify, IP_MULTICAST_TTL,): %m"); + } + + if(setsockopt(s, SOL_SOCKET, SO_BROADCAST, &bcast, sizeof(bcast)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp_notify, SO_BROADCAST): %m"); + close(s); + return -1; + } + + /* bind() socket before using sendto() is not mandatory + * (sendto() will implicitly bind the socket when called on + * an unbound socket) + * here it is used to se a specific sending address */ + memset(&sockname, 0, sizeof(struct sockaddr_in)); + sockname.sin_family = AF_INET; + sockname.sin_addr.s_addr = addr; /*inet_addr(addr);*/ + + if (bind(s, (struct sockaddr *)&sockname, sizeof(struct sockaddr_in)) < 0) + { + syslog(LOG_ERR, "bind(udp_notify): %m"); + close(s); + return -1; + } + + return s; +} + +#ifdef ENABLE_IPV6 +/* open the UDP socket used to send SSDP notifications to + * the multicast group reserved for them. IPv6 */ +static int +OpenAndConfSSDPNotifySocketIPv6(unsigned int if_index) +{ + int s; + unsigned int loop = 0; + /* UDA 2.0 : The hop limit of each IP packet for a Site-Local scope + * multicast message SHALL be configurable and SHOULD default to 10 */ + int hop_limit = 10; + struct sockaddr_in6 sockname; + + s = socket(PF_INET6, SOCK_DGRAM, 0); + if(s < 0) + { + syslog(LOG_ERR, "socket(udp_notify IPv6): %m"); + return -1; + } + if(setsockopt(s, IPPROTO_IPV6, IPV6_MULTICAST_IF, &if_index, sizeof(if_index)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp_notify IPv6, IPV6_MULTICAST_IF, %u): %m", if_index); + close(s); + return -1; + } + if(setsockopt(s, IPPROTO_IPV6, IPV6_MULTICAST_LOOP, &loop, sizeof(loop)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp_notify, IPV6_MULTICAST_LOOP): %m"); + close(s); + return -1; + } + if(setsockopt(s, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, &hop_limit, sizeof(hop_limit)) < 0) + { + syslog(LOG_ERR, "setsockopt(udp_notify, IPV6_MULTICAST_HOPS): %m"); + close(s); + return -1; + } + + /* bind() socket before using sendto() is not mandatory + * (sendto() will implicitly bind the socket when called on + * an unbound socket) + * but explicit bind permits to set port/scope_id/etc. */ + memset(&sockname, 0, sizeof(sockname)); + sockname.sin6_family = AF_INET6; + sockname.sin6_addr = in6addr_any; + /*sockname.sin6_port = htons(port);*/ + /*sockname.sin6_scope_id = if_index;*/ + if(bind(s, (struct sockaddr *)&sockname, sizeof(sockname)) < 0) + { + syslog(LOG_ERR, "bind(udp_notify IPv6): %m"); + close(s); + return -1; + } + + return s; +} +#endif + +int +OpenAndConfSSDPNotifySockets(int * sockets) +/*OpenAndConfSSDPNotifySockets(int * sockets, + struct lan_addr_s * lan_addr, int n_lan_addr)*/ +{ + int i; + struct lan_addr_s * lan_addr; + + for(i=0, lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + sockets[i] = OpenAndConfSSDPNotifySocket(lan_addr->addr.s_addr); + if(sockets[i] < 0) + goto error; + i++; +#ifdef ENABLE_IPV6 + if(GETFLAG(IPV6DISABLEDMASK)) + { + sockets[i] = -1; + } + else + { + sockets[i] = OpenAndConfSSDPNotifySocketIPv6(lan_addr->index); + if(sockets[i] < 0) + goto error; + } + i++; +#endif + } + return 0; +error: + while(--i >= 0) + { + close(sockets[i]); + sockets[i] = -1; + } + return -1; +} + +/* + * response from a LiveBox (Wanadoo) +HTTP/1.1 200 OK +CACHE-CONTROL: max-age=1800 +DATE: Thu, 01 Jan 1970 04:03:23 GMT +EXT: +LOCATION: http://192.168.0.1:49152/gatedesc.xml +SERVER: Linux/2.4.17, UPnP/1.0, Intel SDK for UPnP devices /1.2 +ST: upnp:rootdevice +USN: uuid:75802409-bccb-40e7-8e6c-fa095ecce13e::upnp:rootdevice + + * response from a Linksys 802.11b : +HTTP/1.1 200 OK +Cache-Control:max-age=120 +Location:http://192.168.5.1:5678/rootDesc.xml +Server:NT/5.0 UPnP/1.0 +ST:upnp:rootdevice +USN:uuid:upnp-InternetGatewayDevice-1_0-0090a2777777::upnp:rootdevice +EXT: + */ + +/* Responds to a SSDP "M-SEARCH" + * s : socket to use + * addr : peer + * st, st_len : ST: header + * suffix : suffix for USN: header + * host, port : our HTTP host, port + * delay : in milli-seconds + */ +static void +SendSSDPResponse(int s, const struct sockaddr * addr, + const char * st, int st_len, const char * suffix, + const char * host, unsigned short http_port, +#ifdef ENABLE_HTTPS + unsigned short https_port, +#endif + const char * uuidvalue, unsigned int delay) +{ + int l, n; + char buf[SSDP_PACKET_MAX_LEN]; + char addr_str[64]; + socklen_t addrlen; + int st_is_uuid; +#ifdef ENABLE_HTTP_DATE + char http_date[64]; + time_t t; + struct tm tm; + + time(&t); + gmtime_r(&t, &tm); + strftime(http_date, sizeof(http_date), + "%a, %d %b %Y %H:%M:%S GMT", &tm); +#endif + + st_is_uuid = (st_len == (int)strlen(uuidvalue)) && + (memcmp(uuidvalue, st, st_len) == 0); + /* + * follow guideline from document "UPnP Device Architecture 1.0" + * uppercase is recommended. + * DATE: is recommended + * SERVER: OS/ver UPnP/1.0 miniupnpd/1.0 + * - check what to put in the 'Cache-Control' header + * + * have a look at the document "UPnP Device Architecture v1.1 */ + l = snprintf(buf, sizeof(buf), "HTTP/1.1 200 OK\r\n" + "CACHE-CONTROL: max-age=120\r\n" +#ifdef ENABLE_HTTP_DATE + "DATE: %s\r\n" +#endif + "ST: %.*s%s\r\n" + "USN: %s%s%.*s%s\r\n" + "EXT:\r\n" + "SERVER: " MINIUPNPD_SERVER_STRING "\r\n" +#ifndef RANDOMIZE_URLS + "LOCATION: http://%s:%u" ROOTDESC_PATH "\r\n" +#ifdef ENABLE_HTTPS + "SECURELOCATION.UPNP.ORG: https://%s:%u" ROOTDESC_PATH "\r\n" +#endif /* ENABLE_HTTPS */ +#else /* RANDOMIZE_URLS */ + "LOCATION: http://%s:%u/%s" ROOTDESC_PATH "\r\n" +#ifdef ENABLE_HTTPS + "SECURELOCATION.UPNP.ORG: https://%s:%u/%s" ROOTDESC_PATH "\r\n" +#endif /* ENABLE_HTTPS */ +#endif /* RANDOMIZE_URLS */ + "OPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n" /* UDA v1.1 */ + "01-NLS: %u\r\n" /* same as BOOTID. UDA v1.1 */ + "BOOTID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "CONFIGID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "\r\n", +#ifdef ENABLE_HTTP_DATE + http_date, +#endif + st_len, st, suffix, + uuidvalue, st_is_uuid ? "" : "::", + st_is_uuid ? 0 : st_len, st, suffix, + host, (unsigned int)http_port, +#ifdef RANDOMIZE_URLS + random_url, +#endif /* RANDOMIZE_URLS */ +#ifdef ENABLE_HTTPS + host, (unsigned int)https_port, +#ifdef RANDOMIZE_URLS + random_url, +#endif /* RANDOMIZE_URLS */ +#endif /* ENABLE_HTTPS */ + upnp_bootid, upnp_bootid, upnp_configid); + if(l<0) + { + syslog(LOG_ERR, "%s: snprintf failed %m", + "SendSSDPResponse()"); + return; + } + else if((unsigned)l>=sizeof(buf)) + { + syslog(LOG_WARNING, "%s: truncated output (%u>=%u)", + "SendSSDPResponse()", (unsigned)l, (unsigned)sizeof(buf)); + l = sizeof(buf) - 1; + } + addrlen = (addr->sa_family == AF_INET6) + ? sizeof(struct sockaddr_in6) : sizeof(struct sockaddr_in); + n = sendto_schedule(s, buf, l, 0, + addr, addrlen, delay); + sockaddr_to_string(addr, addr_str, sizeof(addr_str)); + syslog(LOG_DEBUG, "%s: %d bytes to %s ST: %.*s", + "SendSSDPResponse()", + n, addr_str, l, buf); + if(n < 0) + { + syslog(LOG_ERR, "%s: sendto(udp): %m", + "SendSSDPResponse()"); + } +} + +static struct { + const char * s; + const int version; + const char * uuid; +} const known_service_types[] = +{ + {"upnp:rootdevice", 0, uuidvalue_igd}, +#ifdef IGD_V2 + {"urn:schemas-upnp-org:device:InternetGatewayDevice:", 2, uuidvalue_igd}, + {"urn:schemas-upnp-org:device:WANConnectionDevice:", 2, uuidvalue_wcd}, + {"urn:schemas-upnp-org:device:WANDevice:", 2, uuidvalue_wan}, + {"urn:schemas-upnp-org:service:WANIPConnection:", 2, uuidvalue_wcd}, + {"urn:schemas-upnp-org:service:DeviceProtection:", 1, uuidvalue_igd}, +#ifdef ENABLE_6FC_SERVICE + {"urn:schemas-upnp-org:service:WANIPv6FirewallControl:", 1, uuidvalue_wcd}, +#endif +#else /* IGD_V2 */ + /* IGD v1 */ + {"urn:schemas-upnp-org:device:InternetGatewayDevice:", 1, uuidvalue_igd}, + {"urn:schemas-upnp-org:device:WANConnectionDevice:", 1, uuidvalue_wcd}, + {"urn:schemas-upnp-org:device:WANDevice:", 1, uuidvalue_wan}, + {"urn:schemas-upnp-org:service:WANIPConnection:", 1, uuidvalue_wcd}, +#endif /* IGD_V2 */ + {"urn:schemas-upnp-org:service:WANCommonInterfaceConfig:", 1, uuidvalue_wan}, +#ifdef ADVERTISE_WANPPPCONN + /* We use WAN IP Connection, not PPP connection, + * but buggy control points may try to use WanPPPConnection + * anyway */ + {"urn:schemas-upnp-org:service:WANPPPConnection:", 1, uuidvalue_wcd}, +#endif /* ADVERTISE_WANPPPCONN */ +#ifdef ENABLE_L3F_SERVICE + {"urn:schemas-upnp-org:service:Layer3Forwarding:", 1, uuidvalue_igd}, +#endif /* ENABLE_L3F_SERVICE */ +/* we might want to support urn:schemas-wifialliance-org:device:WFADevice:1 + * urn:schemas-wifialliance-org:device:WFADevice:1 + * in the future */ + {0, 0, 0} +}; + +/* SendSSDPNotify() sends the SSDP NOTIFY to a specific + * destination, for a specific UPnP service or device */ +static void +SendSSDPNotify(int s, const struct sockaddr * dest, socklen_t dest_len, + const char * dest_str, + const char * host, unsigned short http_port, +#ifdef ENABLE_HTTPS + unsigned short https_port, +#endif + const char * nt, const char * suffix, + const char * usn1, const char * usn2, const char * usn3, + unsigned int lifetime) +{ + char bufr[SSDP_PACKET_MAX_LEN]; + int n, l; + + l = snprintf(bufr, sizeof(bufr), + "NOTIFY * HTTP/1.1\r\n" + "HOST: %s:%d\r\n" + "CACHE-CONTROL: max-age=%u\r\n" +#ifndef RANDOMIZE_URLS + "LOCATION: http://%s:%u" ROOTDESC_PATH "\r\n" +#ifdef ENABLE_HTTPS + "SECURELOCATION.UPNP.ORG: https://%s:%u" ROOTDESC_PATH "\r\n" +#endif /* ENABLE_HTTPS */ +#else /* RANDOMIZE_URLS */ + "LOCATION: http://%s:%u/%s" ROOTDESC_PATH "\r\n" +#ifdef ENABLE_HTTPS + "SECURELOCATION.UPNP.ORG: https://%s:%u/%s" ROOTDESC_PATH "\r\n" +#endif /* ENABLE_HTTPS */ +#endif /* RANDOMIZE_URLS */ + "SERVER: " MINIUPNPD_SERVER_STRING "\r\n" + "NT: %s%s\r\n" + "USN: %s%s%s%s\r\n" + "NTS: ssdp:alive\r\n" + "OPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n" /* UDA v1.1 */ + "01-NLS: %u\r\n" /* same as BOOTID field. UDA v1.1 */ + "BOOTID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "CONFIGID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "\r\n", + dest_str, SSDP_PORT, /* HOST: */ + lifetime, /* CACHE-CONTROL: */ + host, (unsigned int)http_port, /* LOCATION: */ +#ifdef RANDOMIZE_URLS + random_url, +#endif /* RANDOMIZE_URLS */ +#ifdef ENABLE_HTTPS + host, (unsigned int)https_port, /* SECURE-LOCATION: */ +#ifdef RANDOMIZE_URLS + random_url, +#endif /* RANDOMIZE_URLS */ +#endif /* ENABLE_HTTPS */ + nt, suffix, /* NT: */ + usn1, usn2, usn3, suffix, /* USN: */ + upnp_bootid, /* 01-NLS: */ + upnp_bootid, /* BOOTID.UPNP.ORG: */ + upnp_configid ); /* CONFIGID.UPNP.ORG: */ + if(l<0) { + syslog(LOG_ERR, "%s: snprintf error", "SendSSDPNotify()"); + return; + } else if((unsigned int)l >= sizeof(bufr)) { + syslog(LOG_WARNING, "%s: truncated output (%u>=%u)", + "SendSSDPNotify()", (unsigned)l, (unsigned)sizeof(bufr)); + l = sizeof(bufr) - 1; + } + n = sendto_or_schedule(s, bufr, l, 0, dest, dest_len); + if(n < 0) { + syslog(LOG_ERR, "sendto(udp_notify=%d, %s): %m", s, + host ? host : "NULL"); + } else if(n != l) { + syslog(LOG_NOTICE, "sendto() sent %d out of %d bytes", n, l); + } + /* Due to the unreliable nature of UDP, devices SHOULD send the entire + * set of discovery messages more than once with some delay between + * sets e.g. a few hundred milliseconds. To avoid network congestion + * discovery messages SHOULD NOT be sent more than three times. */ + n = sendto_schedule(s, bufr, l, 0, dest, dest_len, 250); + if(n < 0) { + syslog(LOG_ERR, "sendto(udp_notify=%d, %s): %m", s, + host ? host : "NULL"); + } +} + +/* SendSSDPNotifies() send SSPD NOTIFY for a specific + * LAN (network interface) for all devices / services */ +#ifdef ENABLE_HTTPS +static void +SendSSDPNotifies(int s, const char * host, unsigned short http_port, + unsigned short https_port, + unsigned int lifetime, int ipv6) +#else +static void +SendSSDPNotifies(int s, const char * host, unsigned short http_port, + unsigned int lifetime, int ipv6) +#endif +{ +#ifdef ENABLE_IPV6 + struct sockaddr_storage sockname; + /* UDA 1.1 AnnexA and UDA 2.0 only allow/define the use of + * Link-Local and Site-Local multicast scopes */ + static struct { const char * p1, * p2; } const mcast_addrs[] = + { { LL_SSDP_MCAST_ADDR, "[" LL_SSDP_MCAST_ADDR "]" }, /* Link Local */ + { SL_SSDP_MCAST_ADDR, "[" SL_SSDP_MCAST_ADDR "]" }, /* Site Local */ +#ifndef UPNP_STRICT + { GL_SSDP_MCAST_ADDR, "[" GL_SSDP_MCAST_ADDR "]" }, /* Global */ +#endif /* ! UPNP_STRICT */ + { NULL, NULL } }; + int j; +#else /* ENABLE_IPV6 */ + struct sockaddr_in sockname; +#endif /* ENABLE_IPV6 */ + socklen_t sockname_len; + const char * dest_str; + int i; + char ver_str[4]; +#ifndef ENABLE_IPV6 + UNUSED(ipv6); +#endif /* ENABLE_IPV6 */ + + memset(&sockname, 0, sizeof(sockname)); +#ifdef ENABLE_IPV6 + /* first iterate destinations for this LAN interface (only 1 for IPv4) */ + for(j = 0; (mcast_addrs[j].p1 != 0 && ipv6) || j < 1; j++) { + if(ipv6) { + struct sockaddr_in6 * p = (struct sockaddr_in6 *)&sockname; + sockname_len = sizeof(struct sockaddr_in6); + p->sin6_family = AF_INET6; + p->sin6_port = htons(SSDP_PORT); + inet_pton(AF_INET6, mcast_addrs[j].p1, &(p->sin6_addr)); + dest_str = mcast_addrs[j].p2; + /* UPnP Device Architecture 1.1 : + * Devices MUST multicast SSDP messages for each of the UPnP-enabled + * interfaces. The scope of multicast SSDP messages MUST be + * link local FF02::C if the message is sent from a link local address. + * If the message is sent from a global address it MUST be multicast + * using either global scope FF0E::C or site local scope FF05::C. + * In networks with complex topologies and overlapping sites, use of + * global scope is RECOMMENDED. */ + } else { +#else /* ENABLE_IPV6 */ + { +#endif /* ENABLE_IPV6 */ + /* IPv4 */ + struct sockaddr_in *p = (struct sockaddr_in *)&sockname; + sockname_len = sizeof(struct sockaddr_in); + p->sin_family = AF_INET; + p->sin_port = htons(SSDP_PORT); + p->sin_addr.s_addr = inet_addr(SSDP_MCAST_ADDR); + dest_str = SSDP_MCAST_ADDR; + } + + /* iterate all services / devices */ + for(i = 0; known_service_types[i].s; i++) { + if(i==0) + ver_str[0] = '\0'; + else + snprintf(ver_str, sizeof(ver_str), "%d", known_service_types[i].version); + SendSSDPNotify(s, (struct sockaddr *)&sockname, sockname_len, dest_str, + host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + known_service_types[i].s, ver_str, /* NT: */ + known_service_types[i].uuid, "::", + known_service_types[i].s, /* ver_str, USN: */ + lifetime); + /* for devices, also send NOTIFY on the uuid */ + if(0==memcmp(known_service_types[i].s, + "urn:schemas-upnp-org:device", sizeof("urn:schemas-upnp-org:device")-1)) { + SendSSDPNotify(s, (struct sockaddr *)&sockname, sockname_len, dest_str, + host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + known_service_types[i].uuid, "", /* NT: */ + known_service_types[i].uuid, "", "", /* ver_str, USN: */ + lifetime); + } + } /* for(i = 0; known_service_types[i].s; i++) */ +#ifdef ENABLE_IPV6 + } /* for(j = 0; (mcast_addrs[j].p1 != 0 && ipv6) || j < 1; j++) */ +#endif /* ENABLE_IPV6 */ +} + +/* SendSSDPNotifies2() sends SSDP NOTIFY packets on all interfaces + * for all destinations, all devices / services */ +void +SendSSDPNotifies2(int * sockets, + unsigned short http_port, +#ifdef ENABLE_HTTPS + unsigned short https_port, +#endif + unsigned int lifetime) +{ + int i; + struct lan_addr_s * lan_addr; + for(i = 0, lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) { + SendSSDPNotifies(sockets[i], lan_addr->str, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + lifetime, 0); + i++; +#ifdef ENABLE_IPV6 + if(sockets[i] >= 0) { + SendSSDPNotifies(sockets[i], ipv6_addr_for_http_with_brackets, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + lifetime, 1); + } + i++; +#endif /* ENABLE_IPV6 */ + } +} + +/* ProcessSSDPRequest() + * process SSDP M-SEARCH requests and responds to them */ +void +#ifdef ENABLE_HTTPS +ProcessSSDPRequest(int s, unsigned short http_port, unsigned short https_port) +#else +ProcessSSDPRequest(int s, unsigned short http_port) +#endif +{ + int n; + char bufr[1500]; +#ifdef ENABLE_IPV6 + struct sockaddr_storage sendername; +#else + struct sockaddr_in sendername; +#endif + int source_ifindex = -1; +#ifdef IP_PKTINFO + char cmbuf[CMSG_SPACE(sizeof(struct in_pktinfo))]; + struct iovec iovec = { + .iov_base = bufr, + .iov_len = sizeof(bufr) + }; + struct msghdr mh = { + .msg_name = &sendername, + .msg_namelen = sizeof(sendername), + .msg_iov = &iovec, + .msg_iovlen = 1, + .msg_control = cmbuf, + .msg_controllen = sizeof(cmbuf) + }; + struct cmsghdr *cmptr; +#endif /* IP_PKTINFO */ +#ifdef IP_RECVIF + char cmbuf[CMSG_SPACE(sizeof(struct sockaddr_dl))]; + struct iovec iovec = { + .iov_base = bufr, + .iov_len = sizeof(bufr) + }; + struct msghdr mh = { + .msg_name = &sendername, + .msg_namelen = sizeof(sendername), + .msg_iov = &iovec, + .msg_iovlen = 1, + .msg_control = cmbuf, + .msg_controllen = sizeof(cmbuf) + }; + struct cmsghdr *cmptr; +#endif /* IP_RECVIF */ + +#if defined(IP_RECVIF) || defined(IP_PKTINFO) + n = recvmsg(s, &mh, 0); +#else + socklen_t len_r; + len_r = sizeof(sendername); + n = recvfrom(s, bufr, sizeof(bufr), 0, + (struct sockaddr *)&sendername, &len_r); +#endif /* defined(IP_RECVIF) || defined(IP_PKTINFO) */ + if(n < 0) + { + /* EAGAIN, EWOULDBLOCK, EINTR : silently ignore (try again next time) + * other errors : log to LOG_ERR */ + if(errno != EAGAIN && + errno != EWOULDBLOCK && + errno != EINTR) + { + syslog(LOG_ERR, "recvfrom(udp): %m"); + } + return; + } + +#if defined(IP_RECVIF) || defined(IP_PKTINFO) + for(cmptr = CMSG_FIRSTHDR(&mh); cmptr != NULL; cmptr = CMSG_NXTHDR(&mh, cmptr)) + { + syslog(LOG_DEBUG, "level=%d type=%d", cmptr->cmsg_level, cmptr->cmsg_type); +#ifdef IP_PKTINFO + if(cmptr->cmsg_level == IPPROTO_IP && cmptr->cmsg_type == IP_PKTINFO) + { + struct in_pktinfo * pi; /* fields : ifindex, spec_dst, addr */ + pi = (struct in_pktinfo *)CMSG_DATA(cmptr); + syslog(LOG_DEBUG, "ifindex = %u %s", pi->ipi_ifindex, inet_ntoa(pi->ipi_spec_dst)); + source_ifindex = pi->ipi_ifindex; + } +#endif /* IP_PKTINFO */ +#if defined(ENABLE_IPV6) && defined(IPV6_RECVPKTINFO) + if(cmptr->cmsg_level == IPPROTO_IPV6 && cmptr->cmsg_type == IPV6_RECVPKTINFO) + { + struct in6_pktinfo * pi6; /* fields : ifindex, addr */ + pi6 = (struct in6_pktinfo *)CMSG_DATA(cmptr); + syslog(LOG_DEBUG, "ifindex = %u", pi6->ipi6_ifindex); + source_ifindex = pi6->ipi6_ifindex; + } +#endif /* defined(ENABLE_IPV6) && defined(IPV6_RECVPKTINFO) */ +#ifdef IP_RECVIF + if(cmptr->cmsg_level == IPPROTO_IP && cmptr->cmsg_type == IP_RECVIF) + { + struct sockaddr_dl *sdl; /* fields : len, family, index, type, nlen, alen, slen, data */ + sdl = (struct sockaddr_dl *)CMSG_DATA(cmptr); + syslog(LOG_DEBUG, "sdl_index = %d %s", sdl->sdl_index, link_ntoa(sdl)); + source_ifindex = sdl->sdl_index; + } +#endif /* IP_RECVIF */ + } +#endif /* defined(IP_RECVIF) || defined(IP_PKTINFO) */ +#ifdef ENABLE_HTTPS + ProcessSSDPData(s, bufr, n, (struct sockaddr *)&sendername, source_ifindex, + http_port, https_port); +#else + ProcessSSDPData(s, bufr, n, (struct sockaddr *)&sendername, source_ifindex, + http_port); +#endif + +} + +#ifdef ENABLE_HTTPS +void +ProcessSSDPData(int s, const char *bufr, int n, + const struct sockaddr * sender, int source_if, + unsigned short http_port, unsigned short https_port) +#else +void +ProcessSSDPData(int s, const char *bufr, int n, + const struct sockaddr * sender, int source_if, + unsigned short http_port) +#endif +{ + int i, l; + struct lan_addr_s * lan_addr = NULL; + const char * st = NULL; + int st_len = 0; + int st_ver = 0; + char sender_str[64]; + char ver_str[4]; + const char * announced_host = NULL; +#ifdef UPNP_STRICT +#ifdef ENABLE_IPV6 + char announced_host_buf[64]; +#endif +#endif +#if defined(UPNP_STRICT) || defined(DELAY_MSEARCH_RESPONSE) + int mx_value = -1; +#endif + unsigned int delay = 50; /* Non-zero default delay to prevent flooding */ + /* UPnP Device Architecture v1.1. 1.3.3 Search response : + * Devices responding to a multicast M-SEARCH SHOULD wait a random period + * of time between 0 seconds and the number of seconds specified in the + * MX field value of the search request before responding, in order to + * avoid flooding the requesting control point with search responses + * from multiple devices. If the search request results in the need for + * a multiple part response from the device, those multiple part + * responses SHOULD be spread at random intervals through the time period + * from 0 to the number of seconds specified in the MX header field. */ + char atoi_buffer[8]; + + /* get the string representation of the sender address */ + sockaddr_to_string(sender, sender_str, sizeof(sender_str)); + lan_addr = get_lan_for_peer(sender); + if(source_if >= 0) + { + if(lan_addr != NULL) + { + if(lan_addr->index != (unsigned)source_if && lan_addr->index != 0) + { + syslog(LOG_WARNING, "interface index not matching %u != %d", lan_addr->index, source_if); + } + } + else + { + /* use the interface index */ + for(lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + if(lan_addr->index == (unsigned)source_if) + break; + } + } + } + if(lan_addr == NULL) + { + syslog(LOG_WARNING, "SSDP packet sender %s (if_index=%d) not from a LAN, ignoring", + sender_str, source_if); + return; + } + + if(memcmp(bufr, "NOTIFY", 6) == 0) + { + /* ignore NOTIFY packets. We could log the sender and device type */ + return; + } + else if(memcmp(bufr, "M-SEARCH", 8) == 0) + { + i = 0; + while(i < n) + { + while((i < n - 1) && (bufr[i] != '\r' || bufr[i+1] != '\n')) + i++; + i += 2; + if((i < n - 3) && (strncasecmp(bufr+i, "st:", 3) == 0)) + { + st = bufr+i+3; + st_len = 0; + while((*st == ' ' || *st == '\t') && (st < bufr + n)) + st++; + while((st + st_len < bufr + n) + && (st[st_len]!='\r' && st[st_len]!='\n')) + st_len++; + l = st_len; + while(l > 0 && st[l-1] != ':') + l--; + memset(atoi_buffer, 0, sizeof(atoi_buffer)); + memcpy(atoi_buffer, st + l, MIN((int)(sizeof(atoi_buffer) - 1), st_len - l)); + st_ver = atoi(atoi_buffer); + syslog(LOG_DEBUG, "ST: %.*s (ver=%d)", st_len, st, st_ver); + /*j = 0;*/ + /*while(bufr[i+j]!='\r') j++;*/ + /*syslog(LOG_INFO, "%.*s", j, bufr+i);*/ + } +#if defined(UPNP_STRICT) || defined(DELAY_MSEARCH_RESPONSE) + else if((i < n - 3) && (strncasecmp(bufr+i, "mx:", 3) == 0)) + { + const char * mx; + int mx_len; + mx = bufr+i+3; + mx_len = 0; + while((mx < bufr + n) && (*mx == ' ' || *mx == '\t')) + mx++; + while((mx + mx_len < bufr + n) + && (mx[mx_len]!='\r' && mx[mx_len]!='\n')) + mx_len++; + memset(atoi_buffer, 0, sizeof(atoi_buffer)); + memcpy(atoi_buffer, mx, MIN((int)(sizeof(atoi_buffer) - 1), mx_len)); + mx_value = atoi(atoi_buffer); + syslog(LOG_DEBUG, "MX: %.*s (value=%d)", mx_len, mx, mx_value); + } +#endif /* defined(UPNP_STRICT) || defined(DELAY_MSEARCH_RESPONSE) */ +#if defined(UPNP_STRICT) + /* Fix UDA-1.2.10 Man header empty or invalid */ + else if((i < n - 4) && (strncasecmp(bufr+i, "man:", 3) == 0)) + { + const char * man; + int man_len; + man = bufr+i+4; + man_len = 0; + while((man < bufr + n) && (*man == ' ' || *man == '\t')) + man++; + while((man + man_len < bufr + n) + && (man[man_len]!='\r' && man[man_len]!='\n')) + man_len++; + if((man_len < 15) || (strncmp(man, "\"ssdp:discover\"", 15) != 0)) { + syslog(LOG_INFO, "ignoring SSDP packet MAN empty or invalid header"); + return; + } + } +#endif /* defined(UPNP_STRICT) */ + } +#ifdef UPNP_STRICT + /* For multicast M-SEARCH requests, if the search request does + * not contain an MX header field, the device MUST silently + * discard and ignore the search request. */ + if(mx_value < 0) { + syslog(LOG_INFO, "ignoring SSDP packet missing MX: header"); + return; + } else if(mx_value > 5) { + /* If the MX header field specifies a field value greater + * than 5, the device SHOULD assume that it contained the + * value 5 or less. */ + mx_value = 5; + } +#elif defined(DELAY_MSEARCH_RESPONSE) + if(mx_value < 0) { + mx_value = 1; + } else if(mx_value > 5) { + /* If the MX header field specifies a field value greater + * than 5, the device SHOULD assume that it contained the + * value 5 or less. */ + mx_value = 5; + } +#endif + /*syslog(LOG_INFO, "SSDP M-SEARCH packet received from %s", + sender_str );*/ + if(st && (st_len > 0)) + { + syslog(LOG_INFO, "SSDP M-SEARCH from %s ST: %.*s", + sender_str, st_len, st); + /* find in which sub network the client is */ + if(sender->sa_family == AF_INET) + { + if (lan_addr == NULL) + { + syslog(LOG_ERR, + "Can't find in which sub network the client %s is", + sender_str); + return; + } + announced_host = lan_addr->str; + } +#ifdef ENABLE_IPV6 + else + { + /* IPv6 address with brackets */ +#ifdef UPNP_STRICT + int index; + struct in6_addr addr6; + size_t addr6_len = sizeof(addr6); + /* retrieve the IPv6 address which + * will be used locally to reach sender */ + memset(&addr6, 0, sizeof(addr6)); + if(IN6_IS_ADDR_LINKLOCAL(&(((struct sockaddr_in6 *)sender)->sin6_addr))) { + get_link_local_addr(((struct sockaddr_in6 *)sender)->sin6_scope_id, &addr6); + } else if(get_src_for_route_to (sender, &addr6, &addr6_len, &index) < 0) { + syslog(LOG_WARNING, "get_src_for_route_to() failed, using %s", ipv6_addr_for_http_with_brackets); + announced_host = ipv6_addr_for_http_with_brackets; + } + if(announced_host == NULL) { + if(inet_ntop(AF_INET6, &addr6, + announced_host_buf+1, + sizeof(announced_host_buf) - 2)) { + announced_host_buf[0] = '['; + i = strlen(announced_host_buf); + if(i < (int)sizeof(announced_host_buf) - 1) { + announced_host_buf[i] = ']'; + announced_host_buf[i+1] = '\0'; + } else { + syslog(LOG_NOTICE, "cannot suffix %s with ']'", + announced_host_buf); + } + announced_host = announced_host_buf; + } else { + syslog(LOG_NOTICE, "inet_ntop() failed %m"); + announced_host = ipv6_addr_for_http_with_brackets; + } + } +#else + announced_host = ipv6_addr_for_http_with_brackets; +#endif + } +#endif + /* Responds to request with a device as ST header */ + for(i = 0; known_service_types[i].s; i++) + { + l = (int)strlen(known_service_types[i].s); + if(l<=st_len && (0 == memcmp(st, known_service_types[i].s, l)) +#ifdef UPNP_STRICT + && (st_ver <= known_service_types[i].version) + /* only answer for service version lower or equal of supported one */ +#endif + ) + { + /* SSDP_RESPOND_SAME_VERSION : + * response is urn:schemas-upnp-org:service:WANIPConnection:1 when + * M-SEARCH included urn:schemas-upnp-org:service:WANIPConnection:1 + * else the implemented versions is included in the response + * + * From UPnP Device Architecture v1.1 : + * 1.3.2 [...] Updated versions of device and service types + * are REQUIRED to be fully backward compatible with + * previous versions. Devices MUST respond to M-SEARCH + * requests for any supported version. For example, if a + * device implements “urn:schemas-upnporg:service:xyz:2â€, + * it MUST respond to search requests for both that type + * and “urn:schemas-upnp-org:service:xyz:1â€. The response + * MUST specify the same version as was contained in the + * search request. [...] */ +#ifndef SSDP_RESPOND_SAME_VERSION + if(i==0) + ver_str[0] = '\0'; + else + snprintf(ver_str, sizeof(ver_str), "%d", known_service_types[i].version); +#endif + syslog(LOG_INFO, "Single search found"); +#ifdef DELAY_MSEARCH_RESPONSE + delay = random() / (1 + RAND_MAX / (1000 * mx_value)); +#ifdef DEBUG + syslog(LOG_DEBUG, "mx=%dsec delay=%ums", mx_value, delay); +#endif +#endif + SendSSDPResponse(s, sender, +#ifdef SSDP_RESPOND_SAME_VERSION + st, st_len, "", +#else + known_service_types[i].s, l, ver_str, +#endif + announced_host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + known_service_types[i].uuid, + delay); + break; + } + } + /* Responds to request with ST: ssdp:all */ + /* strlen("ssdp:all") == 8 */ + if(st_len==8 && (0 == memcmp(st, "ssdp:all", 8))) + { +#ifdef DELAY_MSEARCH_RESPONSE + unsigned int delay_increment = (mx_value * 1000) / 15; +#endif + syslog(LOG_INFO, "ssdp:all found"); + for(i=0; known_service_types[i].s; i++) + { +#ifdef DELAY_MSEARCH_RESPONSE + delay += delay_increment; +#endif + if(i==0) + ver_str[0] = '\0'; + else + snprintf(ver_str, sizeof(ver_str), "%d", known_service_types[i].version); + l = (int)strlen(known_service_types[i].s); + SendSSDPResponse(s, sender, + known_service_types[i].s, l, ver_str, + announced_host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + known_service_types[i].uuid, + delay); + } + /* also answer for uuid */ +#ifdef DELAY_MSEARCH_RESPONSE + delay += delay_increment; +#endif + SendSSDPResponse(s, sender, uuidvalue_igd, strlen(uuidvalue_igd), "", + announced_host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + uuidvalue_igd, delay); +#ifdef DELAY_MSEARCH_RESPONSE + delay += delay_increment; +#endif + SendSSDPResponse(s, sender, uuidvalue_wan, strlen(uuidvalue_wan), "", + announced_host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + uuidvalue_wan, delay); +#ifdef DELAY_MSEARCH_RESPONSE + delay += delay_increment; +#endif + SendSSDPResponse(s, sender, uuidvalue_wcd, strlen(uuidvalue_wcd), "", + announced_host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + uuidvalue_wcd, delay); + } + /* responds to request by UUID value */ + l = (int)strlen(uuidvalue_igd); + if(l==st_len) + { +#ifdef DELAY_MSEARCH_RESPONSE + delay = random() / (1 + RAND_MAX / (1000 * mx_value)); +#endif + if(0 == memcmp(st, uuidvalue_igd, l)) + { + syslog(LOG_INFO, "ssdp:uuid (IGD) found"); + SendSSDPResponse(s, sender, st, st_len, "", + announced_host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + uuidvalue_igd, delay); + } + else if(0 == memcmp(st, uuidvalue_wan, l)) + { + syslog(LOG_INFO, "ssdp:uuid (WAN) found"); + SendSSDPResponse(s, sender, st, st_len, "", + announced_host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + uuidvalue_wan, delay); + } + else if(0 == memcmp(st, uuidvalue_wcd, l)) + { + syslog(LOG_INFO, "ssdp:uuid (WCD) found"); + SendSSDPResponse(s, sender, st, st_len, "", + announced_host, http_port, +#ifdef ENABLE_HTTPS + https_port, +#endif + uuidvalue_wcd, delay); + } + } + } + else + { + syslog(LOG_INFO, "Invalid SSDP M-SEARCH from %s", sender_str); + } + } + else + { + syslog(LOG_NOTICE, "Unknown udp packet received from %s", sender_str); + } +} + +static int +SendSSDPbyebye(int s, const struct sockaddr * dest, socklen_t destlen, + const char * dest_str, + const char * nt, const char * suffix, + const char * usn1, const char * usn2, const char * usn3) +{ + int n, l; + char bufr[SSDP_PACKET_MAX_LEN]; + + l = snprintf(bufr, sizeof(bufr), + "NOTIFY * HTTP/1.1\r\n" + "HOST: %s:%d\r\n" + "NT: %s%s\r\n" + "USN: %s%s%s%s\r\n" + "NTS: ssdp:byebye\r\n" + "OPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n" /* UDA v1.1 */ + "01-NLS: %u\r\n" /* same as BOOTID field. UDA v1.1 */ + "BOOTID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "CONFIGID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "\r\n", + dest_str, SSDP_PORT, /* HOST : */ + nt, suffix, /* NT: */ + usn1, usn2, usn3, suffix, /* USN: */ + upnp_bootid, upnp_bootid, upnp_configid); + if(l<0) + { + syslog(LOG_ERR, "%s: snprintf error", "SendSSDPbyebye()"); + return -1; + } + else if((unsigned int)l >= sizeof(bufr)) + { + syslog(LOG_WARNING, "%s: truncated output (%u>=%u)", + "SendSSDPbyebye()", (unsigned)l, (unsigned)sizeof(bufr)); + l = sizeof(bufr) - 1; + } + n = sendto_or_schedule(s, bufr, l, 0, dest, destlen); + if(n < 0) + { + syslog(LOG_ERR, "sendto(udp_shutdown=%d): %m", s); + return -1; + } + else if(n != l) + { + syslog(LOG_NOTICE, "sendto() sent %d out of %d bytes", n, l); + return -1; + } + return 0; +} + +/* This will broadcast ssdp:byebye notifications to inform + * the network that UPnP is going down. */ +int +SendSSDPGoodbye(int * sockets, int n_sockets) +{ + struct sockaddr_in sockname4; +#ifdef ENABLE_IPV6 + struct sockaddr_in6 sockname6; + struct sockaddr * sockname; + socklen_t socknamelen; + int ipv6 = 0; +#endif + int i, j; + char ver_str[4]; + int ret = 0; + const char * dest_str; + + memset(&sockname4, 0, sizeof(struct sockaddr_in)); + sockname4.sin_family = AF_INET; + sockname4.sin_port = htons(SSDP_PORT); + sockname4.sin_addr.s_addr = inet_addr(SSDP_MCAST_ADDR); +#ifdef ENABLE_IPV6 + memset(&sockname6, 0, sizeof(struct sockaddr_in6)); + sockname6.sin6_family = AF_INET6; + sockname6.sin6_port = htons(SSDP_PORT); + inet_pton(AF_INET6, LL_SSDP_MCAST_ADDR, &(sockname6.sin6_addr)); +#else + dest_str = SSDP_MCAST_ADDR; +#endif + + for(j=0; j=sizeof(strbuf)) { + l = sizeof(strbuf) - 1; + } + CODELENGTH(l, p); + memcpy(p, strbuf, l); + p += l; + l = snprintf(strbuf, sizeof(strbuf), "%s::%s%s", + known_service_types[i].uuid, known_service_types[i].s, ver_str); + if(l<0) { + syslog(LOG_WARNING, "SubmitServicesToMiniSSDPD: snprintf %m"); + continue; + } else if((unsigned)l>=sizeof(strbuf)) { + l = sizeof(strbuf) - 1; + } + CODELENGTH(l, p); + memcpy(p, strbuf, l); + p += l; + l = (int)strlen(MINIUPNPD_SERVER_STRING); + CODELENGTH(l, p); + memcpy(p, MINIUPNPD_SERVER_STRING, l); + p += l; + l = snprintf(strbuf, sizeof(strbuf), "http://%s:%u" ROOTDESC_PATH, + host, (unsigned int)port); + if(l<0) { + syslog(LOG_WARNING, "SubmitServicesToMiniSSDPD: snprintf %m"); + continue; + } else if((unsigned)l>=sizeof(strbuf)) { + l = sizeof(strbuf) - 1; + } + CODELENGTH(l, p); + memcpy(p, strbuf, l); + p += l; + /* now write the encoded data */ + n = p - buffer; /* bytes to send */ + p = buffer; /* start */ + while(n > 0) { + l = write(s, p, n); + if (l < 0) { + if(errno == EINTR) + continue; + syslog(LOG_ERR, "write(): %m"); + close(s); + return -1; + } else if (l == 0) { + syslog(LOG_ERR, "write() returned 0"); + close(s); + return -1; + } + p += l; + n -= l; + } + } + close(s); + syslog(LOG_DEBUG, "%d service submitted to MiniSSDPd", i); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/minissdp.h b/src/contrib/miniupnp/miniupnpd/minissdp.h new file mode 100644 index 0000000..a0e51a0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/minissdp.h @@ -0,0 +1,58 @@ +/* $Id: minissdp.h,v 1.12 2014/04/09 07:20:59 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef MINISSDP_H_INCLUDED +#define MINISSDP_H_INCLUDED + +#include "miniupnpdtypes.h" + +int +OpenAndConfSSDPReceiveSocket(int ipv6); + +int +OpenAndConfSSDPNotifySockets(int * sockets); + +#ifdef ENABLE_HTTPS +void +SendSSDPNotifies2(int * sockets, + unsigned short http_port, + unsigned short https_port, + unsigned int lifetime); +#else +void +SendSSDPNotifies2(int * sockets, + unsigned short http_port, + unsigned int lifetime); +#endif + +void +#ifdef ENABLE_HTTPS +ProcessSSDPRequest(int s, + unsigned short http_port, unsigned short https_port); +#else +ProcessSSDPRequest(int s, unsigned short http_port); +#endif + +#ifdef ENABLE_HTTPS +void +ProcessSSDPData(int s, const char *bufr, int n, + const struct sockaddr * sendername, int source_if, + unsigned short http_port, unsigned short https_port); +#else +void +ProcessSSDPData(int s, const char *bufr, int n, + const struct sockaddr * sendername, int source_if, + unsigned short http_port); +#endif + +int +SendSSDPGoodbye(int * sockets, int n); + +int +SubmitServicesToMiniSSDPD(const char * host, unsigned short port); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/miniupnpd.8 b/src/contrib/miniupnp/miniupnpd/miniupnpd.8 new file mode 100644 index 0000000..83123c4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/miniupnpd.8 @@ -0,0 +1,89 @@ +.TH MINIUPNPD 8 +.SH NAME +miniupnpd \- UPnP Internet Gateway Device Daemon +.SH SYNOPSIS +.B miniupnpd +.RB [ "\-f \fIconfig_file" "] [" "\-i \fIext_ifname" "] [" "\-o \fIext_ip" ] +.RB [ "\-a \fIlistening_ip" "] [" "\-p \fIport" "] [" \-d "] [" \-U "] [" \-S "] [" \-N ] +.RB [ "\-u \fIuuid" "] [" "\-s \fIserial" "] [" "\-m \fImodel_number" ] +.RB [ "\-t \fInotify_interval" "] [" "\-P \fIpid_filename" ] +.RB [ "\-B \fIdown up" "] [" "\-w \fIurl" "] [" "\-r \fIclean_ruleset_interval" ] +.RB [ "\-A \fIpermission rule" "] [" "\-b \fIBOOTID" "] [" \-1 ] +.SH DESCRIPTION +miniupnpd act as a UPnP Internet Gateway Device. It is designed +to run on the gateway between the internet and a NAT'ed LAN. It provides +an interface, as defined in the UPnP standard, for enabling +clients on the LAN to ask for port redirections. +.SH OPTIONS +.TP +.BI \-f " config_file" +load the config from file. default is /etc/miniupnpd.conf. +.TP +.BI \-i " ext_ifname" +interface used to connect to the internet. +.TP +.BI \-o " ext_ip" +address used to connect to the internet. +default address of the interface will be used if not specified. +.TP +.BI \-a " listening_ip" +address on the LAN. \-a option can by used multiple time if LAN is +subdivised in several subnetworks. +.TP +.BI \-p " port" +port used for HTTP. +.TP +.B \-d +debug mode : do not go to background, output messages on console +and do not filter out low priority messages. +.TP +.B \-U +report system uptime instead of daemon uptime to clients. +.TP +.B \-S +sets "secure" mode : clients can only add mappings to their own ip +.TP +.B \-N +enables NAT-PMP functionality. +.TP +.BI \-u " uuid" +set the uuid of the UPnP Internet Gateway Device. +.TP +.BI \-s " serial" +serial number for the UPnP Internet Gateway Device. +.TP +.BI \-m " model_number" +model number for the UPnP Internet Gateway Device. +.TP +.BI \-t " notify_interval" +SSDP notify interval in seconds : +SSDP announce messages will be broadcasted at this interval. +.TP +.BI \-P " pid_filename" +pid file. default is /var/run/miniupnpd.pid +.TP +.BI \-B " down up" +download and upload bitrates reported to clients. +.TP +.BI \-w " url" +presentation url. default is first address on LAN, port 80. +.TP +.BI \-r " clean_ruleset_interval" +(minimum) interval between unused rules cleaning checks. +.TP +.BI \-A " permission rule" +use following syntax for permission rules : + (allow|deny) (external port range) ip/mask (internal port range) +.br +examples : + "allow 1024-65535 192.168.1.0/24 1024-65535" + "deny 0-65535 0.0.0.0/0 0-65535" +.TP +.BI \-b " BOOTID" +sets the value of BOOTID.UPNP.ORG SSDP header +.TP +.B \-1 +force reporting IGDv1 in rootDesc when compiled as IGDv2 *use with care* +.SH "SEE ALSO" +minissdpd(1) miniupnpc(3) +.SH BUGS diff --git a/src/contrib/miniupnp/miniupnpd/miniupnpd.c b/src/contrib/miniupnp/miniupnpd/miniupnpd.c new file mode 100644 index 0000000..625b5e6 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/miniupnpd.c @@ -0,0 +1,2741 @@ +/* $Id: miniupnpd.c,v 1.226 2018/03/13 10:35:55 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include "config.h" + +/* Experimental support for NFQUEUE interfaces */ +#ifdef ENABLE_NFQUEUE +/* apt-get install libnetfilter-queue-dev */ +#include +#include +#if 0 +#include /* Defines verdicts (NF_ACCEPT, etc) */ +#endif +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(sun) +#include +#elif !defined(__linux__) +/* for BSD's sysctl */ +#include +#endif + +/* unix sockets */ +#ifdef USE_MINIUPNPDCTL +#include +#endif + +#ifdef TOMATO +#include +#endif /* TOMATO */ +#include "macros.h" +#include "upnpglobalvars.h" +#include "upnphttp.h" +#include "upnpdescgen.h" +#include "miniupnpdpath.h" +#include "getifaddr.h" +#include "upnpsoap.h" +#include "options.h" +#include "minissdp.h" +#include "upnpredirect.h" +#include "upnppinhole.h" +#include "miniupnpdtypes.h" +#include "daemonize.h" +#include "upnpevents.h" +#include "asyncsendto.h" +#ifdef ENABLE_NATPMP +#include "natpmp.h" +#ifdef ENABLE_PCP +#include "pcpserver.h" +#else +#define PCP_MAX_LEN 32 +#endif +#endif +#include "commonrdr.h" +#include "upnputils.h" +#ifdef USE_IFACEWATCHER +#include "ifacewatcher.h" +#endif +#ifdef ENABLE_UPNPPINHOLE +#ifdef USE_NETFILTER +void init_iptpinhole(void); +#endif +#endif + +#ifndef DEFAULT_CONFIG +#define DEFAULT_CONFIG "/etc/miniupnpd.conf" +#endif + +#ifdef USE_MINIUPNPDCTL +struct ctlelem { + int socket; + LIST_ENTRY(ctlelem) entries; +}; +#endif /* USE_MINIUPNPDCTL */ + +#ifdef ENABLE_NFQUEUE +/* globals */ +static struct nfq_handle *nfqHandle; +static struct sockaddr_in ssdp; + +/* prototypes */ +static int nfqueue_cb( struct nfq_q_handle *qh, struct nfgenmsg *nfmsg, struct nfq_data *nfa, void *data) ; +int identify_ip_protocol (char *payload); +int get_udp_dst_port (char *payload); +#endif /* ENABLE_NFQUEUE */ + +/* variables used by signals */ +static volatile sig_atomic_t quitting = 0; +volatile sig_atomic_t should_send_public_address_change_notif = 0; +#if !defined(TOMATO) && defined(ENABLE_LEASEFILE) && defined(LEASEFILE_USE_REMAINING_TIME) +volatile sig_atomic_t should_rewrite_leasefile = 0; +#endif /* !TOMATO && ENABLE_LEASEFILE && LEASEFILE_USE_REMAINING_TIME */ + +#ifdef TOMATO +#if 1 +/* Tomato specific code */ +static volatile sig_atomic_t gotusr2 = 0; + +static void +sigusr2(int sig) +{ + gotusr2 = 1; +} + +static void +tomato_save(const char *fname) +{ + unsigned short eport; + unsigned short iport; + unsigned int leaseduration; + unsigned int timestamp; + char proto[4]; + char iaddr[32]; + char desc[64]; + char rhost[32]; + int n; + FILE *f; + int t; + char tmp[128]; + + strcpy(tmp, "/etc/upnp/saveXXXXXX"); + if ((t = mkstemp(tmp)) != -1) + { + if ((f = fdopen(t, "w")) != NULL) + { + n = 0; + while (upnp_get_redirection_infos_by_index(n, &eport, proto, &iport, iaddr, sizeof(iaddr), desc, sizeof(desc), rhost, sizeof(rhost), &leaseduration) == 0) + { + timestamp = (leaseduration > 0) ? time(NULL) + leaseduration : 0; + fprintf(f, "%s %u %s %u [%s] %u\n", proto, eport, iaddr, iport, desc, timestamp); + ++n; + } + fclose(f); + rename(tmp, fname); + } + else + { + close(t); + } + unlink(tmp); + } +} + +static void +tomato_load(void) +{ + FILE *f; + char s[256]; + unsigned short eport; + unsigned short iport; + unsigned int leaseduration; + unsigned int timestamp; + time_t current_time; + char proto[4]; + char iaddr[32]; + char *rhost; + char *a, *b; + + if ((f = fopen("/etc/upnp/data", "r")) != NULL) + { + current_time = time(NULL); + s[sizeof(s) - 1] = 0; + while (fgets(s, sizeof(s) - 1, f)) { + if (sscanf(s, "%3s %hu %31s %hu [%*[^]]] %u", proto, &eport, iaddr, &iport, ×tamp) >= 4) + { + if (((a = strchr(s, '[')) != NULL) && ((b = strrchr(a, ']')) != NULL)) + { + if (timestamp > 0) + { + if (timestamp > current_time) + leaseduration = timestamp - current_time; + else + continue; + } + else + { + leaseduration = 0; /* default value */ + } + *b = 0; + rhost = NULL; + syslog(LOG_DEBUG, "Read redirection [%s] from file: %s port %hu to %s port %hu, timestamp: %u (%u)", + a + 1, proto, eport, iaddr, iport, timestamp, leaseduration); + upnp_redirect(rhost, eport, iaddr, iport, proto, a + 1, leaseduration); + } + } + } + fclose(f); + } +#ifdef ENABLE_NATPMP +#if 0 + ScanNATPMPforExpiration(); +#endif +#endif /* ENABLE_NATPMP */ + unlink("/etc/upnp/load"); +} + +static void +tomato_delete(void) +{ + FILE *f; + char s[128]; + unsigned short eport; + unsigned short iport; + unsigned int leaseduration; + char proto[4]; + char iaddr[32]; + char desc[64]; + char rhost[32]; + int n; + + if ((f = fopen("/etc/upnp/delete", "r")) != NULL) + { + s[sizeof(s) - 1] = 0; + while (fgets(s, sizeof(s) - 1, f)) + { + if (sscanf(s, "%3s %hu", proto, &eport) == 2) + { + if (proto[0] == '*') + { + n = upnp_get_portmapping_number_of_entries(); + while (--n >= 0) + { + if (upnp_get_redirection_infos_by_index(n, &eport, proto, &iport, iaddr, sizeof(iaddr), desc, sizeof(desc), rhost, sizeof(rhost), &leaseduration) == 0) + { + upnp_delete_redirection(eport, proto); + } + } + break; + } + else + { + upnp_delete_redirection(eport, proto); + } + } + } + fclose(f); + unlink("/etc/upnp/delete"); + } +} + +static void +tomato_helper(void) +{ + struct stat st; + + if (stat("/etc/upnp/delete", &st) == 0) + { + tomato_delete(); + } + + if (stat("/etc/upnp/load", &st) == 0) + { + tomato_load(); + } + + if (stat("/etc/upnp/save", &st) == 0) + { + tomato_save("/etc/upnp/data"); + unlink("/etc/upnp/save"); + } + + if (stat("/etc/upnp/info", &st) == 0) + { + tomato_save("/etc/upnp/data.info"); + unlink("/etc/upnp/info"); + } +} +#endif /* 1 (tomato) */ +#endif /* TOMATO */ + +/* OpenAndConfHTTPSocket() : + * setup the socket used to handle incoming HTTP connections. */ +static int +#ifdef ENABLE_IPV6 +OpenAndConfHTTPSocket(unsigned short * port, int ipv6) +#else +OpenAndConfHTTPSocket(unsigned short * port) +#endif +{ + int s; + int i = 1; +#ifdef ENABLE_IPV6 + struct sockaddr_in6 listenname6; + struct sockaddr_in listenname4; +#else + struct sockaddr_in listenname; +#endif + socklen_t listenname_len; + + s = socket( +#ifdef ENABLE_IPV6 + ipv6 ? PF_INET6 : PF_INET, +#else + PF_INET, +#endif + SOCK_STREAM, 0); +#ifdef ENABLE_IPV6 + if(s < 0 && ipv6 && errno == EAFNOSUPPORT) + { + /* the system doesn't support IPV6 */ + syslog(LOG_WARNING, "socket(PF_INET6, ...) failed with EAFNOSUPPORT, disabling IPv6"); + SETFLAG(IPV6DISABLEDMASK); + ipv6 = 0; + /* Try again with IPv4 */ + s = socket(PF_INET, SOCK_STREAM, 0); + } +#endif + if(s < 0) + { + syslog(LOG_ERR, "socket(http): %m"); + return -1; + } + + if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i)) < 0) + { + syslog(LOG_WARNING, "setsockopt(http, SO_REUSEADDR): %m"); + } +#if 0 + /* enable this to force IPV6 only for IPV6 socket. + * see http://www.ietf.org/rfc/rfc3493.txt section 5.3 */ + if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &i, sizeof(i)) < 0) + { + syslog(LOG_WARNING, "setsockopt(http, IPV6_V6ONLY): %m"); + } +#endif + + if(!set_non_blocking(s)) + { + syslog(LOG_WARNING, "set_non_blocking(http): %m"); + } + +#ifdef ENABLE_IPV6 + if(ipv6) + { + memset(&listenname6, 0, sizeof(struct sockaddr_in6)); + listenname6.sin6_family = AF_INET6; + listenname6.sin6_port = htons(*port); + listenname6.sin6_addr = ipv6_bind_addr; + listenname_len = sizeof(struct sockaddr_in6); + } else { + memset(&listenname4, 0, sizeof(struct sockaddr_in)); + listenname4.sin_family = AF_INET; + listenname4.sin_port = htons(*port); + listenname4.sin_addr.s_addr = htonl(INADDR_ANY); + listenname_len = sizeof(struct sockaddr_in); + } +#else + memset(&listenname, 0, sizeof(struct sockaddr_in)); + listenname.sin_family = AF_INET; + listenname.sin_port = htons(*port); + listenname.sin_addr.s_addr = htonl(INADDR_ANY); + listenname_len = sizeof(struct sockaddr_in); +#endif + +#if defined(SO_BINDTODEVICE) && !defined(MULTIPLE_EXTERNAL_IP) + /* One and only one LAN interface */ + if(lan_addrs.lh_first != NULL && lan_addrs.lh_first->list.le_next == NULL + && strlen(lan_addrs.lh_first->ifname) > 0) + { + if(setsockopt(s, SOL_SOCKET, SO_BINDTODEVICE, + lan_addrs.lh_first->ifname, + strlen(lan_addrs.lh_first->ifname)) < 0) + syslog(LOG_WARNING, "setsockopt(http, SO_BINDTODEVICE, %s): %m", + lan_addrs.lh_first->ifname); + } +#endif /* defined(SO_BINDTODEVICE) && !defined(MULTIPLE_EXTERNAL_IP) */ + +#ifdef ENABLE_IPV6 + if(bind(s, + ipv6 ? (struct sockaddr *)&listenname6 : (struct sockaddr *)&listenname4, + listenname_len) < 0) +#else + if(bind(s, (struct sockaddr *)&listenname, listenname_len) < 0) +#endif + { + syslog(LOG_ERR, "bind(http): %m"); + close(s); + return -1; + } + + if(listen(s, 5) < 0) + { + syslog(LOG_ERR, "listen(http): %m"); + close(s); + return -1; + } + + if(*port == 0) { +#ifdef ENABLE_IPV6 + if(ipv6) { + struct sockaddr_in6 sockinfo; + socklen_t len = sizeof(struct sockaddr_in6); + if (getsockname(s, (struct sockaddr *)&sockinfo, &len) < 0) { + syslog(LOG_ERR, "getsockname(): %m"); + } else { + *port = ntohs(sockinfo.sin6_port); + } + } else { +#endif /* ENABLE_IPV6 */ + struct sockaddr_in sockinfo; + socklen_t len = sizeof(struct sockaddr_in); + if (getsockname(s, (struct sockaddr *)&sockinfo, &len) < 0) { + syslog(LOG_ERR, "getsockname(): %m"); + } else { + *port = ntohs(sockinfo.sin_port); + } +#ifdef ENABLE_IPV6 + } +#endif /* ENABLE_IPV6 */ + } + return s; +} + +static struct upnphttp * +ProcessIncomingHTTP(int shttpl, const char * protocol) +{ + int shttp; + socklen_t clientnamelen; +#ifdef ENABLE_IPV6 + struct sockaddr_storage clientname; + clientnamelen = sizeof(struct sockaddr_storage); +#else + struct sockaddr_in clientname; + clientnamelen = sizeof(struct sockaddr_in); +#endif + shttp = accept(shttpl, (struct sockaddr *)&clientname, &clientnamelen); + if(shttp<0) + { + /* ignore EAGAIN, EWOULDBLOCK, EINTR, we just try again later */ + if(errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) + syslog(LOG_ERR, "accept(http): %m"); + } + else + { + struct upnphttp * tmp = 0; + char addr_str[64]; + + sockaddr_to_string((struct sockaddr *)&clientname, addr_str, sizeof(addr_str)); +#ifdef DEBUG + syslog(LOG_DEBUG, "%s connection from %s", protocol, addr_str); +#endif /* DEBUG */ + if(get_lan_for_peer((struct sockaddr *)&clientname) == NULL) + { + /* The peer is not a LAN ! */ + syslog(LOG_WARNING, + "%s peer %s is not from a LAN, closing the connection", + protocol, addr_str); + close(shttp); + } + else + { + /* Create a new upnphttp object and add it to + * the active upnphttp object list */ + tmp = New_upnphttp(shttp); + if(tmp) + { +#ifdef ENABLE_IPV6 + if(clientname.ss_family == AF_INET) + { + tmp->clientaddr = ((struct sockaddr_in *)&clientname)->sin_addr; + } + else if(clientname.ss_family == AF_INET6) + { + struct sockaddr_in6 * addr = (struct sockaddr_in6 *)&clientname; + if(IN6_IS_ADDR_V4MAPPED(&addr->sin6_addr)) + { + memcpy(&tmp->clientaddr, + &addr->sin6_addr.s6_addr[12], + 4); + } + else + { + tmp->ipv6 = 1; + memcpy(&tmp->clientaddr_v6, + &addr->sin6_addr, + sizeof(struct in6_addr)); + } + } +#else + tmp->clientaddr = clientname.sin_addr; +#endif + memcpy(tmp->clientaddr_str, addr_str, sizeof(tmp->clientaddr_str)); + return tmp; + } + else + { + syslog(LOG_ERR, "New_upnphttp() failed"); + close(shttp); + } + } + } + return NULL; +} + +#ifdef ENABLE_NFQUEUE + +int identify_ip_protocol(char *payload) { + return payload[9]; +} + + +/* + * This function returns the destination port of the captured packet UDP + */ +int get_udp_dst_port(char *payload) { + char *pkt_data_ptr = NULL; + pkt_data_ptr = payload + sizeof(struct ip); + + /* Cast the UDP Header from the raw packet */ + struct udphdr *udp = (struct udphdr *) pkt_data_ptr; + + /* get the dst port of the packet */ + return(ntohs(udp->dest)); + +} +static int +OpenAndConfNFqueue(){ + + struct nfq_q_handle *myQueue; + struct nfnl_handle *netlinkHandle; + + int fd = 0, e = 0; + + inet_pton(AF_INET, "239.255.255.250", &(ssdp.sin_addr)); + + /* Get a queue connection handle from the module */ + if (!(nfqHandle = nfq_open())) { + syslog(LOG_ERR, "Error in nfq_open(): %m"); + return -1; + } + + /* Unbind the handler from processing any IP packets + Not totally sure why this is done, or if it's necessary... */ + if ((e = nfq_unbind_pf(nfqHandle, AF_INET)) < 0) { + syslog(LOG_ERR, "Error in nfq_unbind_pf(): %m"); + return -1; + } + + /* Bind this handler to process IP packets... */ + if (nfq_bind_pf(nfqHandle, AF_INET) < 0) { + syslog(LOG_ERR, "Error in nfq_bind_pf(): %m"); + return -1; + } + + /* Install a callback on queue -Q */ + if (!(myQueue = nfq_create_queue(nfqHandle, nfqueue, &nfqueue_cb, NULL))) { + syslog(LOG_ERR, "Error in nfq_create_queue(): %m"); + return -1; + } + + /* Turn on packet copy mode */ + if (nfq_set_mode(myQueue, NFQNL_COPY_PACKET, 0xffff) < 0) { + syslog(LOG_ERR, "Error setting packet copy mode (): %m"); + return -1; + } + + netlinkHandle = nfq_nfnlh(nfqHandle); + fd = nfnl_fd(netlinkHandle); + + return fd; + +} + + +static int nfqueue_cb( + struct nfq_q_handle *qh, + struct nfgenmsg *nfmsg, + struct nfq_data *nfa, + void *data) { + + char *pkt; + struct nfqnl_msg_packet_hdr *ph; + ph = nfq_get_msg_packet_hdr(nfa); + + if ( ph ) { + + int id = 0, size = 0; + id = ntohl(ph->packet_id); + + size = nfq_get_payload(nfa, &pkt); + + struct ip *iph = (struct ip *) pkt; + + int id_protocol = identify_ip_protocol(pkt); + + int dport = get_udp_dst_port(pkt); + + int x = sizeof (struct ip) + sizeof (struct udphdr); + + /* packets we are interested in are UDP multicast to 239.255.255.250:1900 + * and start with a data string M-SEARCH + */ + if ( (dport == 1900) && (id_protocol == IPPROTO_UDP) + && (ssdp.sin_addr.s_addr == iph->ip_dst.s_addr) ) { + + /* get the index that the packet came in on */ + u_int32_t idx = nfq_get_indev(nfa); + int i = 0; + for ( ;i < n_nfqix ; i++) { + if ( nfqix[i] == idx ) { + + struct udphdr *udp = (struct udphdr *) (pkt + sizeof(struct ip)); + + char *dd = pkt + x; + + struct sockaddr_in sendername; + sendername.sin_family = AF_INET; + sendername.sin_port = udp->source; + sendername.sin_addr.s_addr = iph->ip_src.s_addr; + + /* printf("pkt found %s\n",dd);*/ + ProcessSSDPData (sudp, dd, size - x, + &sendername, -1, (unsigned short) 5555); + } + } + } + + nfq_set_verdict(qh, id, NF_ACCEPT, 0, NULL); + + } else { + syslog(LOG_ERR,"nfq_get_msg_packet_hdr failed"); + return 1; + /* from nfqueue source: 0 = ok, >0 = soft error, <0 hard error */ + } + + return 0; +} + +static void ProcessNFQUEUE(int fd){ + char buf[4096]; + + socklen_t len_r; + struct sockaddr_in sendername; + len_r = sizeof(struct sockaddr_in); + + int res = recvfrom(fd, buf, sizeof(buf), 0, + (struct sockaddr *)&sendername, &len_r); + + nfq_handle_packet(nfqHandle, buf, res); +} +#endif + +/* Functions used to communicate with miniupnpdctl */ +#ifdef USE_MINIUPNPDCTL +static int +OpenAndConfCtlUnixSocket(const char * path) +{ + struct sockaddr_un localun; + int s; + s = socket(AF_UNIX, SOCK_STREAM, 0); + localun.sun_family = AF_UNIX; + strncpy(localun.sun_path, path, + sizeof(localun.sun_path)); + if(bind(s, (struct sockaddr *)&localun, + sizeof(struct sockaddr_un)) < 0) + { + syslog(LOG_ERR, "bind(sctl): %m"); + close(s); + s = -1; + } + else if(listen(s, 5) < 0) + { + syslog(LOG_ERR, "listen(sctl): %m"); + close(s); + s = -1; + } + return s; +} + +static void +write_upnphttp_details(int fd, struct upnphttp * e) +{ + char buffer[256]; + int len; + write(fd, "HTTP :\n", 7); + while(e) + { + len = snprintf(buffer, sizeof(buffer), + "%d %d %s req_buf=%p(%dbytes) res_buf=%p(%dbytes alloc)\n", + e->socket, e->state, e->HttpVer, + e->req_buf, e->req_buflen, + e->res_buf, e->res_buf_alloclen); + write(fd, buffer, len); + e = e->entries.le_next; + } +} + +static void +write_ctlsockets_list(int fd, struct ctlelem * e) +{ + char buffer[256]; + int len; + write(fd, "CTL :\n", 6); + while(e) + { + len = snprintf(buffer, sizeof(buffer), + "struct ctlelem: socket=%d\n", e->socket); + write(fd, buffer, len); + e = e->entries.le_next; + } +} + +#ifndef DISABLE_CONFIG_FILE +static void +write_option_list(int fd) +{ + char buffer[256]; + int len; + unsigned int i; + write(fd, "Options :\n", 10); + for(i=0; iifname)) + { + /* not starting with a digit : suppose it is an interface name */ + memcpy(lan_addr->ifname, str, n); + lan_addr->ifname[n] = '\0'; + if(getifaddr(lan_addr->ifname, lan_addr->str, sizeof(lan_addr->str), + &lan_addr->addr, &lan_addr->mask) < 0) { +#ifdef ENABLE_IPV6 + fprintf(stderr, "interface \"%s\" has no IPv4 address\n", str); + lan_addr->str[0] = '\0'; + lan_addr->addr.s_addr = htonl(0x00000000u); + lan_addr->mask.s_addr = htonl(0xffffffffu); +#else /* ENABLE_IPV6 */ + goto parselan_error; +#endif /* ENABLE_IPV6 */ + } + /*printf("%s => %s\n", lan_addr->ifname, lan_addr->str);*/ + } + else + { + if(n>15) + goto parselan_error; + memcpy(lan_addr->str, str, n); + lan_addr->str[n] = '\0'; + if(!inet_aton(lan_addr->str, &lan_addr->addr)) + goto parselan_error; + } + if(*p == '/') + { + const char * q = ++p; + while(*p && isdigit(*p)) + p++; + if(*p=='.') + { + /* parse mask in /255.255.255.0 format */ + while(*p && (*p=='.' || isdigit(*p))) + p++; + n = p - q; + if(n>15) + goto parselan_error; + memcpy(tmp, q, n); + tmp[n] = '\0'; + if(!inet_aton(tmp, &lan_addr->mask)) + goto parselan_error; + } + else + { + /* it is a /24 format */ + int nbits = atoi(q); + if(nbits > 32 || nbits < 0) + goto parselan_error; + lan_addr->mask.s_addr = htonl(nbits ? (0xffffffffu << (32 - nbits)) : 0); + } + } + else if(lan_addr->mask.s_addr == 0) + { + /* by default, networks are /24 */ + lan_addr->mask.s_addr = htonl(0xffffff00u); + } +#ifdef MULTIPLE_EXTERNAL_IP + /* skip spaces */ + while(*p && isspace(*p)) + p++; + if(*p) { + /* parse the exteral ip address to associate with this subnet */ + n = 0; + while(p[n] && !isspace(*p)) + n++; + if(n<=15) { + memcpy(lan_addr->ext_ip_str, p, n); + lan_addr->ext_ip_str[n] = '\0'; + if(!inet_aton(lan_addr->ext_ip_str, &lan_addr->ext_ip_addr)) { + /* error */ + fprintf(stderr, "Error parsing address : %s\n", lan_addr->ext_ip_str); + } + } + } +#endif + if(lan_addr->ifname[0] != '\0') + { + lan_addr->index = if_nametoindex(lan_addr->ifname); + if(lan_addr->index == 0) + fprintf(stderr, "Cannot get index for network interface %s", + lan_addr->ifname); + } +#ifdef ENABLE_IPV6 + else + { + fprintf(stderr, + "Error: please specify LAN network interface by name instead of IPv4 address : %s\n", + str); + return -1; + } +#else + else + { + syslog(LOG_NOTICE, "it is advised to use network interface name instead of %s", str); + } +#endif + return 0; +parselan_error: + fprintf(stderr, "Error parsing address/mask (or interface name) : %s\n", + str); + return -1; +} + +/* fill uuidvalue_wan and uuidvalue_wcd based on uuidvalue_igd */ +void complete_uuidvalues(void) +{ + size_t len; + len = strlen(uuidvalue_igd); + memcpy(uuidvalue_wan, uuidvalue_igd, len+1); + switch(uuidvalue_wan[len-1]) { + case '9': + uuidvalue_wan[len-1] = 'a'; + break; + case 'f': + uuidvalue_wan[len-1] = '0'; + break; + default: + uuidvalue_wan[len-1]++; + } + memcpy(uuidvalue_wcd, uuidvalue_wan, len+1); + switch(uuidvalue_wcd[len-1]) { + case '9': + uuidvalue_wcd[len-1] = 'a'; + break; + case 'f': + uuidvalue_wcd[len-1] = '0'; + break; + default: + uuidvalue_wcd[len-1]++; + } +} + +/* init phase : + * 1) read configuration file + * 2) read command line arguments + * 3) daemonize + * 4) open syslog + * 5) check and write pid file + * 6) set startup time stamp + * 7) compute presentation URL + * 8) set signal handlers + * 9) init random generator (srandom()) + * 10) init redirection engine + * 11) reload mapping from leasefile */ +static int +init(int argc, char * * argv, struct runtime_vars * v) +{ + int i; + int pid; + int debug_flag = 0; + int openlog_option; + struct sigaction sa; + /*const char * logfilename = 0;*/ + const char * presurl = 0; +#ifndef DISABLE_CONFIG_FILE + int options_flag = 0; + const char * optionsfile = DEFAULT_CONFIG; +#endif /* DISABLE_CONFIG_FILE */ + struct lan_addr_s * lan_addr; + struct lan_addr_s * lan_addr2; + + /* only print usage if -h is used */ + for(i=1; iport = -1; +#ifdef ENABLE_HTTPS + v->https_port = -1; +#endif + v->notify_interval = 30; /* seconds between SSDP announces */ + v->clean_ruleset_threshold = 20; + v->clean_ruleset_interval = 0; /* interval between ruleset check. 0=disabled */ +#ifndef DISABLE_CONFIG_FILE + /* read options file first since + * command line arguments have final say */ + if(readoptionsfile(optionsfile) < 0) + { + /* only error if file exists or using -f */ + if(access(optionsfile, F_OK) == 0 || options_flag) + fprintf(stderr, "Error reading configuration file %s\n", optionsfile); + } + else + { + for(i=0; i<(int)num_options; i++) + { + switch(ary_options[i].id) + { + case UPNPEXT_IFNAME: + ext_if_name = ary_options[i].value; + break; + case UPNPEXT_IP: + use_ext_ip_addr = ary_options[i].value; + break; + case UPNPLISTENING_IP: + lan_addr = (struct lan_addr_s *) malloc(sizeof(struct lan_addr_s)); + if (lan_addr == NULL) + { + fprintf(stderr, "malloc(sizeof(struct lan_addr_s)): %m"); + break; + } + if(parselanaddr(lan_addr, ary_options[i].value) != 0) + { + fprintf(stderr, "can't parse \"%s\" as a valid " +#ifndef ENABLE_IPV6 + "LAN address or " +#endif + "interface name\n", ary_options[i].value); + free(lan_addr); + break; + } + LIST_INSERT_HEAD(&lan_addrs, lan_addr, list); + break; +#ifdef ENABLE_IPV6 + case UPNPIPV6_LISTENING_IP: + if (inet_pton(AF_INET6, ary_options[i].value, &ipv6_bind_addr) < 1) + { + fprintf(stderr, "can't parse \"%s\" as valid IPv6 listening address", ary_options[i].value); + } + break; +#endif /* ENABLE_IPV6 */ + case UPNPPORT: + v->port = atoi(ary_options[i].value); + break; +#ifdef ENABLE_HTTPS + case UPNPHTTPSPORT: + v->https_port = atoi(ary_options[i].value); + break; +#endif + case UPNPBITRATE_UP: + upstream_bitrate = strtoul(ary_options[i].value, 0, 0); + break; + case UPNPBITRATE_DOWN: + downstream_bitrate = strtoul(ary_options[i].value, 0, 0); + break; + case UPNPPRESENTATIONURL: + presurl = ary_options[i].value; + break; +#ifdef ENABLE_MANUFACTURER_INFO_CONFIGURATION + case UPNPFRIENDLY_NAME: + strncpy(friendly_name, ary_options[i].value, FRIENDLY_NAME_MAX_LEN); + friendly_name[FRIENDLY_NAME_MAX_LEN-1] = '\0'; + break; + case UPNPMANUFACTURER_NAME: + strncpy(manufacturer_name, ary_options[i].value, MANUFACTURER_NAME_MAX_LEN); + manufacturer_name[MANUFACTURER_NAME_MAX_LEN-1] = '\0'; + break; + case UPNPMANUFACTURER_URL: + strncpy(manufacturer_url, ary_options[i].value, MANUFACTURER_URL_MAX_LEN); + manufacturer_url[MANUFACTURER_URL_MAX_LEN-1] = '\0'; + break; + case UPNPMODEL_NAME: + strncpy(model_name, ary_options[i].value, MODEL_NAME_MAX_LEN); + model_name[MODEL_NAME_MAX_LEN-1] = '\0'; + break; + case UPNPMODEL_DESCRIPTION: + strncpy(model_description, ary_options[i].value, MODEL_DESCRIPTION_MAX_LEN); + model_description[MODEL_DESCRIPTION_MAX_LEN-1] = '\0'; + break; + case UPNPMODEL_URL: + strncpy(model_url, ary_options[i].value, MODEL_URL_MAX_LEN); + model_url[MODEL_URL_MAX_LEN-1] = '\0'; + break; +#endif /* ENABLE_MANUFACTURER_INFO_CONFIGURATION */ +#ifdef USE_NETFILTER + case UPNPFORWARDCHAIN: + miniupnpd_forward_chain = ary_options[i].value; + break; + case UPNPNATCHAIN: + miniupnpd_nat_chain = ary_options[i].value; + break; + case UPNPNATPOSTCHAIN: + miniupnpd_nat_postrouting_chain = ary_options[i].value; + break; +#endif /* USE_NETFILTER */ + case UPNPNOTIFY_INTERVAL: + v->notify_interval = atoi(ary_options[i].value); + break; + case UPNPSYSTEM_UPTIME: + if(strcmp(ary_options[i].value, "yes") == 0) + SETFLAG(SYSUPTIMEMASK); /*sysuptime = 1;*/ + break; +#if defined(USE_PF) || defined(USE_IPF) + case UPNPPACKET_LOG: + if(strcmp(ary_options[i].value, "yes") == 0) + SETFLAG(LOGPACKETSMASK); /*logpackets = 1;*/ + break; +#endif /* defined(USE_PF) || defined(USE_IPF) */ + case UPNPUUID: + strncpy(uuidvalue_igd+5, ary_options[i].value, + strlen(uuidvalue_igd+5) + 1); + complete_uuidvalues(); + break; + case UPNPSERIAL: + strncpy(serialnumber, ary_options[i].value, SERIALNUMBER_MAX_LEN); + serialnumber[SERIALNUMBER_MAX_LEN-1] = '\0'; + break; + case UPNPMODEL_NUMBER: + strncpy(modelnumber, ary_options[i].value, MODELNUMBER_MAX_LEN); + modelnumber[MODELNUMBER_MAX_LEN-1] = '\0'; + break; + case UPNPCLEANTHRESHOLD: + v->clean_ruleset_threshold = atoi(ary_options[i].value); + break; + case UPNPCLEANINTERVAL: + v->clean_ruleset_interval = atoi(ary_options[i].value); + break; +#ifdef USE_PF + case UPNPANCHOR: + anchor_name = ary_options[i].value; + break; + case UPNPQUEUE: + queue = ary_options[i].value; + break; + case UPNPTAG: + tag = ary_options[i].value; + break; +#endif /* USE_PF */ +#ifdef ENABLE_NATPMP + case UPNPENABLENATPMP: + if(strcmp(ary_options[i].value, "yes") == 0) + SETFLAG(ENABLENATPMPMASK); /*enablenatpmp = 1;*/ + else + if(atoi(ary_options[i].value)) + SETFLAG(ENABLENATPMPMASK); + /*enablenatpmp = atoi(ary_options[i].value);*/ + break; +#endif /* ENABLE_NATPMP */ +#ifdef ENABLE_PCP + case UPNPPCPMINLIFETIME: + min_lifetime = atoi(ary_options[i].value); + if (min_lifetime > 120 ) { + min_lifetime = 120; + } + break; + case UPNPPCPMAXLIFETIME: + max_lifetime = atoi(ary_options[i].value); + if (max_lifetime > 86400 ) { + max_lifetime = 86400; + } + break; + case UPNPPCPALLOWTHIRDPARTY: + if(strcmp(ary_options[i].value, "yes") == 0) + SETFLAG(PCP_ALLOWTHIRDPARTYMASK); + break; +#endif /* ENABLE_PCP */ +#ifdef PF_ENABLE_FILTER_RULES + case UPNPQUICKRULES: + if(strcmp(ary_options[i].value, "no") == 0) + SETFLAG(PFNOQUICKRULESMASK); + break; +#endif /* PF_ENABLE_FILTER_RULES */ + case UPNPENABLE: + if(strcmp(ary_options[i].value, "yes") != 0) + CLEARFLAG(ENABLEUPNPMASK); + break; + case UPNPSECUREMODE: + if(strcmp(ary_options[i].value, "yes") == 0) + SETFLAG(SECUREMODEMASK); + break; +#ifdef ENABLE_LEASEFILE + case UPNPLEASEFILE: + lease_file = ary_options[i].value; + break; +#endif /* ENABLE_LEASEFILE */ + case UPNPMINISSDPDSOCKET: + minissdpdsocketpath = ary_options[i].value; + break; +#ifdef IGD_V2 + case UPNPFORCEIGDDESCV1: + if (strcmp(ary_options[i].value, "yes") == 0) + SETFLAG(FORCEIGDDESCV1MASK); + else if (strcmp(ary_options[i].value, "no") != 0 ) { + fprintf(stderr, "force_igd_desc_v1 can only be yes or no\n"); + } + break; +#endif + default: + fprintf(stderr, "Unknown option in file %s\n", + optionsfile); + } + } +#ifdef ENABLE_PCP + /* if lifetimes are inverse */ + if (min_lifetime >= max_lifetime) { + fprintf(stderr, "Minimum lifetime (%lu) is greater than or equal to maximum lifetime (%lu).\n", min_lifetime, max_lifetime); + fprintf(stderr, "Check your configuration file.\n"); + return 1; + } +#endif /* ENABLE_PCP */ + } +#endif /* DISABLE_CONFIG_FILE */ + + /* command line arguments processing */ + for(i=1; inotify_interval = atoi(argv[++i]); + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; + case 'r': + if(i+1 < argc) + v->clean_ruleset_interval = atoi(argv[++i]); + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; + case 'u': + if(i+1 < argc) { + strncpy(uuidvalue_igd+5, argv[++i], strlen(uuidvalue_igd+5) + 1); + complete_uuidvalues(); + } else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; +#ifdef ENABLE_MANUFACTURER_INFO_CONFIGURATION + case 'z': + if(i+1 < argc) + strncpy(friendly_name, argv[++i], FRIENDLY_NAME_MAX_LEN); + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + friendly_name[FRIENDLY_NAME_MAX_LEN-1] = '\0'; + break; +#endif /* ENABLE_MANUFACTURER_INFO_CONFIGURATION */ + case 's': + if(i+1 < argc) + strncpy(serialnumber, argv[++i], SERIALNUMBER_MAX_LEN); + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + serialnumber[SERIALNUMBER_MAX_LEN-1] = '\0'; + break; + case 'm': + if(i+1 < argc) + strncpy(modelnumber, argv[++i], MODELNUMBER_MAX_LEN); + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + modelnumber[MODELNUMBER_MAX_LEN-1] = '\0'; + break; +#ifdef ENABLE_NATPMP + case 'N': + /*enablenatpmp = 1;*/ + SETFLAG(ENABLENATPMPMASK); + break; +#endif /* ENABLE_NATPMP */ + case 'U': + /*sysuptime = 1;*/ + SETFLAG(SYSUPTIMEMASK); + break; + /*case 'l': + logfilename = argv[++i]; + break;*/ +#if defined(USE_PF) || defined(USE_IPF) + case 'L': + /*logpackets = 1;*/ + SETFLAG(LOGPACKETSMASK); + break; +#endif /* defined(USE_PF) || defined(USE_IPF) */ + case 'S': + SETFLAG(SECUREMODEMASK); + break; + case 'i': + if(i+1 < argc) + ext_if_name = argv[++i]; + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; +#ifdef USE_PF + case 'q': + if(i+1 < argc) + queue = argv[++i]; + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; + case 'T': + if(i+1 < argc) + tag = argv[++i]; + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; +#endif /* USE_PF */ + case 'p': + if(i+1 < argc) + v->port = atoi(argv[++i]); + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; +#ifdef ENABLE_HTTPS + case 'H': + if(i+1 < argc) + v->https_port = atoi(argv[++i]); + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; +#endif /* ENABLE_HTTPS */ +#ifdef ENABLE_NFQUEUE + case 'Q': + if(i+1list.le_next) + { + if (0 == strncmp(lan_addr2->str, lan_addr->str, 15)) + break; + } + if (lan_addr2 == NULL) + LIST_INSERT_HEAD(&lan_addrs, lan_addr, list); + } + else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); +#else /* #ifndef MULTIPLE_EXTERNAL_IP */ + if(i+2 < argc) + { + char *val = calloc((strlen(argv[i+1]) + strlen(argv[i+2]) + 2), sizeof(char)); + if (val == NULL) + { + fprintf(stderr, "memory allocation error for listen address storage\n"); + break; + } + sprintf(val, "%s %s", argv[i+1], argv[i+2]); + + lan_addr = (struct lan_addr_s *) malloc(sizeof(struct lan_addr_s)); + if (lan_addr == NULL) + { + fprintf(stderr, "malloc(sizeof(struct lan_addr_s)): %m"); + free(val); + break; + } + if(parselanaddr(lan_addr, val) != 0) + { + fprintf(stderr, "can't parse \"%s\" as a valid LAN address or interface name\n", val); + free(lan_addr); + free(val); + break; + } + /* check if we already have this address */ + for(lan_addr2 = lan_addrs.lh_first; lan_addr2 != NULL; lan_addr2 = lan_addr2->list.le_next) + { + if (0 == strncmp(lan_addr2->str, lan_addr->str, 15)) + break; + } + if (lan_addr2 == NULL) + LIST_INSERT_HEAD(&lan_addrs, lan_addr, list); + + free(val); + i+=2; + } + else + fprintf(stderr, "Option -%c takes two arguments.\n", argv[i][1]); +#endif /* #ifndef MULTIPLE_EXTERNAL_IP */ + break; + case 'A': + if(i+1 < argc) { + void * tmp; + tmp = realloc(upnppermlist, sizeof(struct upnpperm) * (num_upnpperm+1)); + if(tmp == NULL) { + fprintf(stderr, "memory allocation error for permission\n"); + } else { + upnppermlist = tmp; + if(read_permission_line(upnppermlist + num_upnpperm, argv[++i]) >= 0) { + num_upnpperm++; + } else { + fprintf(stderr, "Permission rule parsing error :\n%s\n", argv[i]); + } + } + } else + fprintf(stderr, "Option -%c takes one argument.\n", argv[i][1]); + break; + case 'f': + i++; /* discarding, the config file is already read */ + break; + default: + fprintf(stderr, "Unknown option: %s\n", argv[i]); + } + } + if(!ext_if_name || !lan_addrs.lh_first) + { + /* bad configuration */ + goto print_usage; + } + + if(debug_flag) + { + pid = getpid(); + } + else + { +#ifdef USE_DAEMON + if(daemon(0, 0)<0) { + perror("daemon()"); + } + pid = getpid(); +#else + pid = daemonize(); +#endif + } + + openlog_option = LOG_PID|LOG_CONS; + if(debug_flag) + { + openlog_option |= LOG_PERROR; /* also log on stderr */ + } + + openlog("miniupnpd", openlog_option, LOG_MINIUPNPD); + + if(!debug_flag) + { + /* speed things up and ignore LOG_INFO and LOG_DEBUG */ + setlogmask(LOG_UPTO(LOG_NOTICE)); + } + + if(checkforrunning(pidfilename) < 0) + { + syslog(LOG_ERR, "MiniUPnPd is already running. EXITING"); + return 1; + } + +#ifdef TOMATO + syslog(LOG_NOTICE, "version " MINIUPNPD_VERSION " started"); +#endif /* TOMATO */ + + set_startup_time(); + + /* presentation url */ + if(presurl) + { + strncpy(presentationurl, presurl, PRESENTATIONURL_MAX_LEN); + presentationurl[PRESENTATIONURL_MAX_LEN-1] = '\0'; + } + else + { + snprintf(presentationurl, PRESENTATIONURL_MAX_LEN, + "http://%s/", lan_addrs.lh_first->str); + /*"http://%s:%d/", lan_addrs.lh_first->str, 80);*/ + } + + /* set signal handler */ + memset(&sa, 0, sizeof(struct sigaction)); + sa.sa_handler = sigterm; + + if(sigaction(SIGTERM, &sa, NULL) < 0) + { + syslog(LOG_ERR, "Failed to set %s handler. EXITING", "SIGTERM"); + return 1; + } + if(sigaction(SIGINT, &sa, NULL) < 0) + { + syslog(LOG_ERR, "Failed to set %s handler. EXITING", "SIGINT"); + return 1; + } +#ifdef TOMATO + sa.sa_handler = sigusr2; + sigaction(SIGUSR2, &sa, NULL); + if(signal(SIGPIPE, SIG_IGN) == SIG_ERR) +#else /* TOMATO */ + sa.sa_handler = SIG_IGN; + if(sigaction(SIGPIPE, &sa, NULL) < 0) +#endif /* TOMATO */ + { + syslog(LOG_ERR, "Failed to ignore SIGPIPE signals"); + } + sa.sa_handler = sigusr1; + if(sigaction(SIGUSR1, &sa, NULL) < 0) + { + syslog(LOG_NOTICE, "Failed to set %s handler", "SIGUSR1"); + } +#if !defined(TOMATO) && defined(ENABLE_LEASEFILE) && defined(LEASEFILE_USE_REMAINING_TIME) + sa.sa_handler = sigusr2; + if(sigaction(SIGUSR2, &sa, NULL) < 0) + { + syslog(LOG_NOTICE, "Failed to set %s handler", "SIGUSR2"); + } +#endif /* !TOMATO && ENABLE_LEASEFILE && LEASEFILE_USE_REMAINING_TIME */ + + /* initialize random number generator */ + srandom((unsigned int)time(NULL)); +#ifdef RANDOMIZE_URLS + snprintf(random_url, RANDOM_URL_MAX_LEN, "%08lx", random()); +#endif /* RANDOMIZE_URLS */ + + /* initialize redirection engine (and pinholes) */ + if(init_redirect() < 0) + { + syslog(LOG_ERR, "Failed to init redirection engine. EXITING"); + return 1; + } +#ifdef ENABLE_UPNPPINHOLE +#ifdef USE_NETFILTER + init_iptpinhole(); +#endif +#endif + + if(writepidfile(pidfilename, pid) < 0) + pidfilename = NULL; + +#ifdef ENABLE_LEASEFILE + /*remove(lease_file);*/ + syslog(LOG_INFO, "Reloading rules from lease file"); + reload_from_lease_file(); +#endif + +#ifdef TOMATO + tomato_load(); +#endif /* TOMATO */ + + return 0; +print_usage: + fprintf(stderr, "Usage:\n\t" + "%s " +#ifndef DISABLE_CONFIG_FILE + "[-f config_file] " +#endif + "[-i ext_ifname] [-o ext_ip]\n" +#ifndef MULTIPLE_EXTERNAL_IP + "\t\t[-a listening_ip]" +#else + "\t\t[-a listening_ip ext_ip]" +#endif +#ifdef ENABLE_HTTPS + " [-H https_port]" +#endif + " [-p port] [-d]" +#if defined(USE_PF) || defined(USE_IPF) + " [-L]" +#endif + " [-U] [-S]" +#ifdef ENABLE_NATPMP + " [-N]" +#endif + "\n" + /*"[-l logfile] " not functionnal */ + "\t\t[-u uuid] [-s serial] [-m model_number] \n" + "\t\t[-t notify_interval] [-P pid_filename] " +#ifdef ENABLE_MANUFACTURER_INFO_CONFIGURATION + "[-z fiendly_name]" +#endif + "\n\t\t[-B down up] [-w url] [-r clean_ruleset_interval]\n" +#ifdef USE_PF + "\t\t[-q queue] [-T tag]\n" +#endif +#ifdef ENABLE_NFQUEUE + "\t\t[-Q queue] [-n name]\n" +#endif + "\t\t[-A \"permission rule\"] [-b BOOTID]" +#ifdef IGD_V2 + " [-1]" +#endif + "\n" + "\nNotes:\n\tThere can be one or several listening_ips.\n" + "\tNotify interval is in seconds. Default is 30 seconds.\n" + "\tDefault pid file is '%s'.\n" + "\tDefault config file is '%s'.\n" + "\tWith -d miniupnpd will run as a standard program.\n" +#if defined(USE_PF) || defined(USE_IPF) + "\t-L sets packet log in pf and ipf on.\n" +#endif + "\t-S sets \"secure\" mode : clients can only add mappings to their own ip\n" + "\t-U causes miniupnpd to report system uptime instead " + "of daemon uptime.\n" +#ifdef ENABLE_NATPMP + "\t-N enables NAT-PMP functionality.\n" +#endif + "\t-B sets bitrates reported by daemon in bits per second.\n" + "\t-w sets the presentation url. Default is http address on port 80\n" +#ifdef USE_PF + "\t-q sets the ALTQ queue in pf.\n" + "\t-T sets the tag name in pf.\n" +#endif +#ifdef ENABLE_NFQUEUE + "\t-Q sets the queue number that is used by NFQUEUE.\n" + "\t-n sets the name of the interface(s) that packets will arrive on.\n" +#endif + "\t-A use following syntax for permission rules :\n" + "\t (allow|deny) (external port range) ip/mask (internal port range)\n" + "\texamples :\n" + "\t \"allow 1024-65535 192.168.1.0/24 1024-65535\"\n" + "\t \"deny 0-65535 0.0.0.0/0 0-65535\"\n" + "\t-b sets the value of BOOTID.UPNP.ORG SSDP header\n" +#ifdef IGD_V2 + "\t-1 force reporting IGDv1 in rootDesc *use with care*\n" +#endif + "\t-h prints this help and quits.\n" + "", argv[0], pidfilename, DEFAULT_CONFIG); + return 1; +} + +/* === main === */ +/* process HTTP or SSDP requests */ +int +main(int argc, char * * argv) +{ + int i; + int shttpl = -1; /* socket for HTTP */ +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + int shttpl_v4 = -1; /* socket for HTTP (ipv4 only) */ +#endif +#ifdef ENABLE_HTTPS + int shttpsl = -1; /* socket for HTTPS */ +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + int shttpsl_v4 = -1; /* socket for HTTPS (ipv4 only) */ +#endif +#endif /* ENABLE_HTTPS */ + int sudp = -1; /* IP v4 socket for receiving SSDP */ +#ifdef ENABLE_IPV6 + int sudpv6 = -1; /* IP v6 socket for receiving SSDP */ +#endif +#ifdef ENABLE_NATPMP + int * snatpmp = NULL; /* also used for PCP */ +#endif +#if defined(ENABLE_IPV6) && defined(ENABLE_PCP) + int spcp_v6 = -1; +#endif +#ifdef ENABLE_NFQUEUE + int nfqh = -1; +#endif +#ifdef USE_IFACEWATCHER + int sifacewatcher = -1; +#endif + + int * snotify = NULL; + int addr_count; + LIST_HEAD(httplisthead, upnphttp) upnphttphead; + struct upnphttp * e = 0; + struct upnphttp * next; + fd_set readset; /* for select() */ + fd_set writeset; + struct timeval timeout, timeofday, lasttimeofday = {0, 0}; + int max_fd = -1; +#ifdef USE_MINIUPNPDCTL + int sctl = -1; + LIST_HEAD(ctlstructhead, ctlelem) ctllisthead; + struct ctlelem * ectl; + struct ctlelem * ectlnext; +#endif + struct runtime_vars v; + /* variables used for the unused-rule cleanup process */ + struct rule_state * rule_list = 0; + struct timeval checktime = {0, 0}; + struct lan_addr_s * lan_addr; +#ifdef ENABLE_UPNPPINHOLE + unsigned int next_pinhole_ts; +#endif + + if(init(argc, argv, &v) != 0) + return 1; +#ifdef ENABLE_HTTPS + if(init_ssl() < 0) + return 1; +#endif /* ENABLE_HTTPS */ + /* count lan addrs */ + addr_count = 0; + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) + addr_count++; + if(addr_count > 0) { +#ifndef ENABLE_IPV6 + snotify = calloc(addr_count, sizeof(int)); +#else + /* one for IPv4, one for IPv6 */ + snotify = calloc(addr_count * 2, sizeof(int)); +#endif + } +#ifdef ENABLE_NATPMP + if(addr_count > 0) { + snatpmp = malloc(addr_count * sizeof(int)); + for(i = 0; i < addr_count; i++) + snatpmp[i] = -1; + } +#endif + + LIST_INIT(&upnphttphead); +#ifdef USE_MINIUPNPDCTL + LIST_INIT(&ctllisthead); +#endif + + if( +#ifdef ENABLE_NATPMP + !GETFLAG(ENABLENATPMPMASK) && !GETFLAG(ENABLEUPNPMASK) +#else + !GETFLAG(ENABLEUPNPMASK) +#endif + ) { + syslog(LOG_ERR, "Why did you run me anyway?"); + return 0; + } + + syslog(LOG_INFO, "version " MINIUPNPD_VERSION " starting%s%sext if %s BOOTID=%u", +#ifdef ENABLE_NATPMP +#ifdef ENABLE_PCP + GETFLAG(ENABLENATPMPMASK) ? " NAT-PMP/PCP " : " ", +#else + GETFLAG(ENABLENATPMPMASK) ? " NAT-PMP " : " ", +#endif +#else + " ", +#endif + GETFLAG(ENABLEUPNPMASK) ? "UPnP-IGD " : "", + ext_if_name, upnp_bootid); + + if(GETFLAG(ENABLEUPNPMASK)) + { + unsigned short listen_port; + listen_port = (v.port > 0) ? v.port : 0; + /* open socket for HTTP connections. Listen on the 1st LAN address */ +#ifdef ENABLE_IPV6 + shttpl = OpenAndConfHTTPSocket(&listen_port, 1); +#else /* ENABLE_IPV6 */ + shttpl = OpenAndConfHTTPSocket(&listen_port); +#endif /* ENABLE_IPV6 */ + if(shttpl < 0) + { + syslog(LOG_ERR, "Failed to open socket for HTTP. EXITING"); + return 1; + } + v.port = listen_port; + syslog(LOG_NOTICE, "HTTP listening on port %d", v.port); +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + if(!GETFLAG(IPV6DISABLEDMASK)) + { + shttpl_v4 = OpenAndConfHTTPSocket(&listen_port, 0); + if(shttpl_v4 < 0) + { + syslog(LOG_ERR, "Failed to open socket for HTTP on port %d (IPv4). EXITING", v.port); + return 1; + } + } +#endif /* V6SOCKETS_ARE_V6ONLY */ +#ifdef ENABLE_HTTPS + /* https */ + listen_port = (v.https_port > 0) ? v.https_port : 0; +#ifdef ENABLE_IPV6 + shttpsl = OpenAndConfHTTPSocket(&listen_port, 1); +#else /* ENABLE_IPV6 */ + shttpsl = OpenAndConfHTTPSocket(&listen_port); +#endif /* ENABLE_IPV6 */ + if(shttpl < 0) + { + syslog(LOG_ERR, "Failed to open socket for HTTPS. EXITING"); + return 1; + } + v.https_port = listen_port; + syslog(LOG_NOTICE, "HTTPS listening on port %d", v.https_port); +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + shttpsl_v4 = OpenAndConfHTTPSocket(&listen_port, 0); + if(shttpsl_v4 < 0) + { + syslog(LOG_ERR, "Failed to open socket for HTTPS on port %d (IPv4). EXITING", v.https_port); + return 1; + } +#endif /* V6SOCKETS_ARE_V6ONLY */ +#endif /* ENABLE_HTTPS */ +#ifdef ENABLE_IPV6 + if(!GETFLAG(IPV6DISABLEDMASK)) { + if(find_ipv6_addr(lan_addrs.lh_first ? lan_addrs.lh_first->ifname : NULL, + ipv6_addr_for_http_with_brackets, sizeof(ipv6_addr_for_http_with_brackets)) > 0) { + syslog(LOG_NOTICE, "HTTP IPv6 address given to control points : %s", + ipv6_addr_for_http_with_brackets); + } else { + memcpy(ipv6_addr_for_http_with_brackets, "[::1]", 6); + syslog(LOG_WARNING, "no HTTP IPv6 address, disabling IPv6"); + SETFLAG(IPV6DISABLEDMASK); + } + } +#endif /* ENABLE_IPV6 */ + + /* open socket for SSDP connections */ + sudp = OpenAndConfSSDPReceiveSocket(0); + if(sudp < 0) + { + syslog(LOG_NOTICE, "Failed to open socket for receiving SSDP. Trying to use MiniSSDPd"); + if(SubmitServicesToMiniSSDPD(lan_addrs.lh_first->str, v.port) < 0) { + syslog(LOG_ERR, "Failed to connect to MiniSSDPd. EXITING"); + return 1; + } + } +#ifdef ENABLE_IPV6 + if(!GETFLAG(IPV6DISABLEDMASK)) + { + sudpv6 = OpenAndConfSSDPReceiveSocket(1); + if(sudpv6 < 0) + { + syslog(LOG_WARNING, "Failed to open socket for receiving SSDP (IP v6)."); + } + } +#endif + + /* open socket for sending notifications */ + if(OpenAndConfSSDPNotifySockets(snotify) < 0) + { + syslog(LOG_ERR, "Failed to open sockets for sending SSDP notify " + "messages. EXITING"); + return 1; + } + +#ifdef USE_IFACEWATCHER + /* open socket for kernel notifications about new network interfaces */ + if (sudp >= 0) + { + sifacewatcher = OpenAndConfInterfaceWatchSocket(); + if (sifacewatcher < 0) + { + syslog(LOG_ERR, "Failed to open socket for receiving network interface notifications"); + } + } +#endif + } + +#ifdef ENABLE_NATPMP + /* open socket for NAT PMP traffic */ + if(GETFLAG(ENABLENATPMPMASK)) + { + if(OpenAndConfNATPMPSockets(snatpmp) < 0) +#ifdef ENABLE_PCP + { + syslog(LOG_ERR, "Failed to open sockets for NAT-PMP/PCP."); + } else { + syslog(LOG_NOTICE, "Listening for NAT-PMP/PCP traffic on port %u", + NATPMP_PORT); + } +#else + { + syslog(LOG_ERR, "Failed to open sockets for NAT PMP."); + } else { + syslog(LOG_NOTICE, "Listening for NAT-PMP traffic on port %u", + NATPMP_PORT); + } +#endif + } +#endif + +#if defined(ENABLE_IPV6) && defined(ENABLE_PCP) + if(!GETFLAG(IPV6DISABLEDMASK)) { + spcp_v6 = OpenAndConfPCPv6Socket(); + } +#endif + + /* for miniupnpdctl */ +#ifdef USE_MINIUPNPDCTL + sctl = OpenAndConfCtlUnixSocket("/var/run/miniupnpd.ctl"); +#endif + +#ifdef ENABLE_NFQUEUE + if ( nfqueue != -1 && n_nfqix > 0) { + nfqh = OpenAndConfNFqueue(); + if(nfqh < 0) { + syslog(LOG_ERR, "Failed to open fd for NFQUEUE."); + return 1; + } else { + syslog(LOG_NOTICE, "Opened NFQUEUE %d",nfqueue); + } + } +#endif + +#ifdef TOMATO + tomato_helper(); +#endif + + /* main loop */ + while(!quitting) + { +#ifdef USE_TIME_AS_BOOTID + /* Correct startup_time if it was set with a RTC close to 0 */ + if((upnp_bootid<60*60*24) && (time(NULL)>60*60*24)) + { + upnp_bootid = time(NULL); + } +#endif +#if !defined(TOMATO) && defined(ENABLE_LEASEFILE) && defined(LEASEFILE_USE_REMAINING_TIME) + if(should_rewrite_leasefile) + { + lease_file_rewrite(); + should_rewrite_leasefile = 0; + } +#endif /* !TOMATO && ENABLE_LEASEFILE && LEASEFILE_USE_REMAINING_TIME */ + /* send public address change notifications if needed */ + if(should_send_public_address_change_notif) + { + syslog(LOG_INFO, "should send external iface address change notification(s)"); +#ifdef ENABLE_NATPMP + if(GETFLAG(ENABLENATPMPMASK)) + SendNATPMPPublicAddressChangeNotification(snatpmp, addr_count); +#endif +#ifdef ENABLE_EVENTS + if(GETFLAG(ENABLEUPNPMASK)) + { + upnp_event_var_change_notify(EWanIPC); + } +#endif +#ifdef ENABLE_PCP + if(GETFLAG(ENABLENATPMPMASK)) + { +#ifdef ENABLE_IPV6 + PCPPublicAddressChanged(snatpmp, addr_count, spcp_v6); +#else /* IPv4 only */ + PCPPublicAddressChanged(snatpmp, addr_count); +#endif + } +#endif + should_send_public_address_change_notif = 0; + } + /* Check if we need to send SSDP NOTIFY messages and do it if + * needed */ + if(upnp_gettimeofday(&timeofday) < 0) + { + syslog(LOG_ERR, "gettimeofday(): %m"); + timeout.tv_sec = v.notify_interval; + timeout.tv_usec = 0; + } + else + { + /* the comparaison is not very precise but who cares ? */ + if(timeofday.tv_sec >= (lasttimeofday.tv_sec + v.notify_interval)) + { + if (GETFLAG(ENABLEUPNPMASK)) + SendSSDPNotifies2(snotify, + (unsigned short)v.port, +#ifdef ENABLE_HTTPS + (unsigned short)v.https_port, +#endif + v.notify_interval << 1); + memcpy(&lasttimeofday, &timeofday, sizeof(struct timeval)); + timeout.tv_sec = v.notify_interval; + timeout.tv_usec = 0; + } + else + { + timeout.tv_sec = lasttimeofday.tv_sec + v.notify_interval + - timeofday.tv_sec; + if(timeofday.tv_usec > lasttimeofday.tv_usec) + { + timeout.tv_usec = 1000000 + lasttimeofday.tv_usec + - timeofday.tv_usec; + timeout.tv_sec--; + } + else + { + timeout.tv_usec = lasttimeofday.tv_usec - timeofday.tv_usec; + } + } + } + /* remove unused rules */ + if( v.clean_ruleset_interval + && (timeofday.tv_sec >= checktime.tv_sec + v.clean_ruleset_interval)) + { + if(rule_list) + { + remove_unused_rules(rule_list); + rule_list = NULL; + } + else + { + rule_list = get_upnp_rules_state_list(v.clean_ruleset_threshold); + } + memcpy(&checktime, &timeofday, sizeof(struct timeval)); + } + /* Remove expired port mappings, based on UPnP IGD LeaseDuration + * or NAT-PMP lifetime) */ + if(nextruletoclean_timestamp + && ((unsigned int)timeofday.tv_sec >= nextruletoclean_timestamp)) + { + syslog(LOG_DEBUG, "cleaning expired Port Mappings"); + get_upnp_rules_state_list(0); + } + if(nextruletoclean_timestamp + && ((unsigned int)timeout.tv_sec >= (nextruletoclean_timestamp - timeofday.tv_sec))) + { + timeout.tv_sec = nextruletoclean_timestamp - timeofday.tv_sec; + timeout.tv_usec = 0; + syslog(LOG_DEBUG, "setting timeout to %u sec", + (unsigned)timeout.tv_sec); + } +#ifdef ENABLE_UPNPPINHOLE + /* Clean up expired IPv6 PinHoles */ + next_pinhole_ts = 0; + upnp_clean_expired_pinholes(&next_pinhole_ts); + if(next_pinhole_ts && + timeout.tv_sec >= (int)(next_pinhole_ts - timeofday.tv_sec)) { + timeout.tv_sec = next_pinhole_ts - timeofday.tv_sec; + timeout.tv_usec = 0; + } +#endif /* ENABLE_UPNPPINHOLE */ + + /* select open sockets (SSDP, HTTP listen, and all HTTP soap sockets) */ + FD_ZERO(&readset); + FD_ZERO(&writeset); + + if (sudp >= 0) + { + FD_SET(sudp, &readset); + max_fd = MAX( max_fd, sudp); +#ifdef USE_IFACEWATCHER + if (sifacewatcher >= 0) + { + FD_SET(sifacewatcher, &readset); + max_fd = MAX(max_fd, sifacewatcher); + } +#endif + } + if (shttpl >= 0) + { + FD_SET(shttpl, &readset); + max_fd = MAX( max_fd, shttpl); + } +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + if (shttpl_v4 >= 0) + { + FD_SET(shttpl_v4, &readset); + max_fd = MAX( max_fd, shttpl_v4); + } +#endif +#ifdef ENABLE_HTTPS + if (shttpsl >= 0) + { + FD_SET(shttpsl, &readset); + max_fd = MAX( max_fd, shttpsl); + } +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + if (shttpsl_v4 >= 0) + { + FD_SET(shttpsl_v4, &readset); + max_fd = MAX( max_fd, shttpsl_v4); + } +#endif +#endif /* ENABLE_HTTPS */ +#ifdef ENABLE_IPV6 + if (sudpv6 >= 0) + { + FD_SET(sudpv6, &readset); + max_fd = MAX( max_fd, sudpv6); + } +#endif + +#ifdef ENABLE_NFQUEUE + if (nfqh >= 0) + { + FD_SET(nfqh, &readset); + max_fd = MAX( max_fd, nfqh); + } +#endif + + i = 0; /* active HTTP connections count */ + for(e = upnphttphead.lh_first; e != NULL; e = e->entries.le_next) + { + if(e->socket >= 0) + { + if(e->state <= EWaitingForHttpContent) + FD_SET(e->socket, &readset); + else if(e->state == ESendingAndClosing) + FD_SET(e->socket, &writeset); + else + continue; + max_fd = MAX(max_fd, e->socket); + i++; + } + } + /* for debug */ +#ifdef DEBUG + if(i > 1) + { + syslog(LOG_DEBUG, "%d active incoming HTTP connections", i); + } +#endif +#ifdef ENABLE_NATPMP + for(i=0; i= 0) { + FD_SET(snatpmp[i], &readset); + max_fd = MAX( max_fd, snatpmp[i]); + } + } +#endif +#if defined(ENABLE_IPV6) && defined(ENABLE_PCP) + if(spcp_v6 >= 0) { + FD_SET(spcp_v6, &readset); + max_fd = MAX(max_fd, spcp_v6); + } +#endif +#ifdef USE_MINIUPNPDCTL + if(sctl >= 0) { + FD_SET(sctl, &readset); + max_fd = MAX( max_fd, sctl); + } + + for(ectl = ctllisthead.lh_first; ectl; ectl = ectl->entries.le_next) + { + if(ectl->socket >= 0) { + FD_SET(ectl->socket, &readset); + max_fd = MAX( max_fd, ectl->socket); + } + } +#endif + +#ifdef ENABLE_EVENTS + upnpevents_selectfds(&readset, &writeset, &max_fd); +#endif + + /* queued "sendto" */ + { + struct timeval next_send; + i = get_next_scheduled_send(&next_send); + if(i > 0) { +#ifdef DEBUG + syslog(LOG_DEBUG, "%d queued sendto", i); +#endif + i = get_sendto_fds(&writeset, &max_fd, &timeofday); + if(timeofday.tv_sec > next_send.tv_sec || + (timeofday.tv_sec == next_send.tv_sec && timeofday.tv_usec >= next_send.tv_usec)) { + if(i > 0) { + timeout.tv_sec = 0; + timeout.tv_usec = 0; + } + } else { + struct timeval tmp_timeout; + tmp_timeout.tv_sec = (next_send.tv_sec - timeofday.tv_sec); + tmp_timeout.tv_usec = (next_send.tv_usec - timeofday.tv_usec); + if(tmp_timeout.tv_usec < 0) { + tmp_timeout.tv_usec += 1000000; + tmp_timeout.tv_sec--; + } + if(timeout.tv_sec > tmp_timeout.tv_sec + || (timeout.tv_sec == tmp_timeout.tv_sec && timeout.tv_usec > tmp_timeout.tv_usec)) { + timeout.tv_sec = tmp_timeout.tv_sec; + timeout.tv_usec = tmp_timeout.tv_usec; + } + } + } + } + + if(select(max_fd+1, &readset, &writeset, 0, &timeout) < 0) + { + if(quitting) goto shutdown; +#ifdef TOMATO + if (gotusr2) + { + gotusr2 = 0; + tomato_helper(); + continue; + } +#endif /* TOMATO */ + if(errno == EINTR) continue; /* interrupted by a signal, start again */ + syslog(LOG_ERR, "select(all): %m"); + syslog(LOG_ERR, "Failed to select open sockets. EXITING"); + return 1; /* very serious cause of error */ + } + i = try_sendto(&writeset); + if(i < 0) { + syslog(LOG_ERR, "try_sendto failed to send %d packets", -i); + } +#ifdef USE_MINIUPNPDCTL + for(ectl = ctllisthead.lh_first; ectl;) + { + ectlnext = ectl->entries.le_next; + if((ectl->socket >= 0) && FD_ISSET(ectl->socket, &readset)) + { + char buf[256]; + int l; + l = read(ectl->socket, buf, sizeof(buf)); + if(l > 0) + { + /*write(ectl->socket, buf, l);*/ + write_command_line(ectl->socket, argc, argv); +#ifndef DISABLE_CONFIG_FILE + write_option_list(ectl->socket); +#endif + write_permlist(ectl->socket, upnppermlist, num_upnpperm); + write_upnphttp_details(ectl->socket, upnphttphead.lh_first); + write_ctlsockets_list(ectl->socket, ctllisthead.lh_first); + write_ruleset_details(ectl->socket); +#ifdef ENABLE_EVENTS + write_events_details(ectl->socket); +#endif + /* close the socket */ + close(ectl->socket); + ectl->socket = -1; + } + else + { + close(ectl->socket); + ectl->socket = -1; + } + } + if(ectl->socket < 0) + { + LIST_REMOVE(ectl, entries); + free(ectl); + } + ectl = ectlnext; + } + if((sctl >= 0) && FD_ISSET(sctl, &readset)) + { + int s; + struct sockaddr_un clientname; + struct ctlelem * tmp; + socklen_t clientnamelen = sizeof(struct sockaddr_un); + /*syslog(LOG_DEBUG, "sctl!");*/ + s = accept(sctl, (struct sockaddr *)&clientname, + &clientnamelen); + syslog(LOG_DEBUG, "sctl! : '%s'", clientname.sun_path); + tmp = malloc(sizeof(struct ctlelem)); + if (tmp == NULL) + { + syslog(LOG_ERR, "Unable to allocate memory for ctlelem in main()"); + close(s); + } + else + { + tmp->socket = s; + LIST_INSERT_HEAD(&ctllisthead, tmp, entries); + } + } +#endif +#ifdef ENABLE_EVENTS + upnpevents_processfds(&readset, &writeset); +#endif +#ifdef ENABLE_NATPMP + /* process NAT-PMP packets */ + for(i=0; i= 0) && FD_ISSET(snatpmp[i], &readset)) + { + unsigned char msg_buff[PCP_MAX_LEN]; + struct sockaddr_in senderaddr; + socklen_t senderaddrlen; + int len; + memset(msg_buff, 0, PCP_MAX_LEN); + senderaddrlen = sizeof(senderaddr); + len = ReceiveNATPMPOrPCPPacket(snatpmp[i], + (struct sockaddr *)&senderaddr, + &senderaddrlen, + NULL, + msg_buff, sizeof(msg_buff)); + if (len < 1) + continue; +#ifdef ENABLE_PCP + if (msg_buff[0]==0) { /* version equals to 0 -> means NAT-PMP */ + /* Check if the packet is coming from a LAN to enforce RFC6886 : + * The NAT gateway MUST NOT accept mapping requests destined to the NAT + * gateway's external IP address or received on its external network + * interface. Only packets received on the internal interface(s) with a + * destination address matching the internal address(es) of the NAT + * gateway should be allowed. */ + /* TODO : move to ProcessIncomingNATPMPPacket() ? */ + lan_addr = get_lan_for_peer((struct sockaddr *)&senderaddr); + if(lan_addr == NULL) { + char sender_str[64]; + sockaddr_to_string((struct sockaddr *)&senderaddr, sender_str, sizeof(sender_str)); + syslog(LOG_WARNING, "NAT-PMP packet sender %s not from a LAN, ignoring", + sender_str); + continue; + } + ProcessIncomingNATPMPPacket(snatpmp[i], msg_buff, len, + &senderaddr); + } else { /* everything else can be PCP */ + ProcessIncomingPCPPacket(snatpmp[i], msg_buff, len, + (struct sockaddr *)&senderaddr, NULL); + } + +#else + /* Check if the packet is coming from a LAN to enforce RFC6886 : + * The NAT gateway MUST NOT accept mapping requests destined to the NAT + * gateway's external IP address or received on its external network + * interface. Only packets received on the internal interface(s) with a + * destination address matching the internal address(es) of the NAT + * gateway should be allowed. */ + /* TODO : move to ProcessIncomingNATPMPPacket() ? */ + lan_addr = get_lan_for_peer((struct sockaddr *)&senderaddr); + if(lan_addr == NULL) { + char sender_str[64]; + sockaddr_to_string((struct sockaddr *)&senderaddr, sender_str, sizeof(sender_str)); + syslog(LOG_WARNING, "NAT-PMP packet sender %s not from a LAN, ignoring", + sender_str); + continue; + } + ProcessIncomingNATPMPPacket(snatpmp[i], msg_buff, len, &senderaddr); +#endif + } + } +#endif +#if defined(ENABLE_IPV6) && defined(ENABLE_PCP) + /* in IPv6, only PCP is supported, not NAT-PMP */ + if(spcp_v6 >= 0 && FD_ISSET(spcp_v6, &readset)) + { + unsigned char msg_buff[PCP_MAX_LEN]; + struct sockaddr_in6 senderaddr; + socklen_t senderaddrlen; + struct sockaddr_in6 receiveraddr; + int len; + memset(msg_buff, 0, PCP_MAX_LEN); + senderaddrlen = sizeof(senderaddr); + len = ReceiveNATPMPOrPCPPacket(spcp_v6, + (struct sockaddr *)&senderaddr, + &senderaddrlen, + &receiveraddr, + msg_buff, sizeof(msg_buff)); + if(len >= 1) + ProcessIncomingPCPPacket(spcp_v6, msg_buff, len, + (struct sockaddr *)&senderaddr, + &receiveraddr); + } +#endif + /* process SSDP packets */ + if(sudp >= 0 && FD_ISSET(sudp, &readset)) + { + /*syslog(LOG_INFO, "Received UDP Packet");*/ +#ifdef ENABLE_HTTPS + ProcessSSDPRequest(sudp, (unsigned short)v.port, (unsigned short)v.https_port); +#else + ProcessSSDPRequest(sudp, (unsigned short)v.port); +#endif + } +#ifdef ENABLE_IPV6 + if(sudpv6 >= 0 && FD_ISSET(sudpv6, &readset)) + { + syslog(LOG_INFO, "Received UDP Packet (IPv6)"); +#ifdef ENABLE_HTTPS + ProcessSSDPRequest(sudpv6, (unsigned short)v.port, (unsigned short)v.https_port); +#else + ProcessSSDPRequest(sudpv6, (unsigned short)v.port); +#endif + } +#endif +#ifdef USE_IFACEWATCHER + /* process kernel notifications */ + if (sifacewatcher >= 0 && FD_ISSET(sifacewatcher, &readset)) + ProcessInterfaceWatchNotify(sifacewatcher); +#endif + + /* process active HTTP connections */ + /* LIST_FOREACH macro is not available under linux */ + for(e = upnphttphead.lh_first; e != NULL; e = e->entries.le_next) + { + if(e->socket >= 0) + { + if(FD_ISSET(e->socket, &readset) || + FD_ISSET(e->socket, &writeset)) + { + Process_upnphttp(e); + } + } + } + /* process incoming HTTP connections */ + if(shttpl >= 0 && FD_ISSET(shttpl, &readset)) + { + struct upnphttp * tmp; + tmp = ProcessIncomingHTTP(shttpl, "HTTP"); + if(tmp) + { + LIST_INSERT_HEAD(&upnphttphead, tmp, entries); + } + } +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + if(shttpl_v4 >= 0 && FD_ISSET(shttpl_v4, &readset)) + { + struct upnphttp * tmp; + tmp = ProcessIncomingHTTP(shttpl_v4, "HTTP"); + if(tmp) + { + LIST_INSERT_HEAD(&upnphttphead, tmp, entries); + } + } +#endif +#ifdef ENABLE_HTTPS + if(shttpsl >= 0 && FD_ISSET(shttpsl, &readset)) + { + struct upnphttp * tmp; + tmp = ProcessIncomingHTTP(shttpsl, "HTTPS"); + if(tmp) + { + InitSSL_upnphttp(tmp); + LIST_INSERT_HEAD(&upnphttphead, tmp, entries); + } + } +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + if(shttpsl_v4 >= 0 && FD_ISSET(shttpsl_v4, &readset)) + { + struct upnphttp * tmp; + tmp = ProcessIncomingHTTP(shttpsl_v4, "HTTPS"); + if(tmp) + { + InitSSL_upnphttp(tmp); + LIST_INSERT_HEAD(&upnphttphead, tmp, entries); + } + } +#endif +#endif /* ENABLE_HTTPS */ +#ifdef ENABLE_NFQUEUE + /* process NFQ packets */ + if(nfqh >= 0 && FD_ISSET(nfqh, &readset)) + { + /* syslog(LOG_INFO, "Received NFQUEUE Packet");*/ + ProcessNFQUEUE(nfqh); + } +#endif + /* delete finished HTTP connections */ + for(e = upnphttphead.lh_first; e != NULL; ) + { + next = e->entries.le_next; + if(e->state >= EToDelete) + { + LIST_REMOVE(e, entries); + Delete_upnphttp(e); + } + e = next; + } + + } /* end of main loop */ + +shutdown: + syslog(LOG_NOTICE, "shutting down MiniUPnPd"); + /* send good-bye */ + if (GETFLAG(ENABLEUPNPMASK)) + { +#ifndef ENABLE_IPV6 + if(SendSSDPGoodbye(snotify, addr_count) < 0) +#else + if(SendSSDPGoodbye(snotify, addr_count * 2) < 0) +#endif + { + syslog(LOG_ERR, "Failed to broadcast good-bye notifications"); + } + } + /* try to send pending packets */ + finalize_sendto(); + +#ifdef TOMATO + tomato_save("/etc/upnp/data"); +#endif /* TOMATO */ +#if defined(ENABLE_LEASEFILE) && defined(LEASEFILE_USE_REMAINING_TIME) + lease_file_rewrite(); +#endif /* ENABLE_LEASEFILE && LEASEFILE_USE_REMAINING_TIME */ + /* close out open sockets */ + while(upnphttphead.lh_first != NULL) + { + e = upnphttphead.lh_first; + LIST_REMOVE(e, entries); + Delete_upnphttp(e); + } + + if (sudp >= 0) close(sudp); + if (shttpl >= 0) close(shttpl); +#if defined(V6SOCKETS_ARE_V6ONLY) && defined(ENABLE_IPV6) + if (shttpl_v4 >= 0) close(shttpl_v4); +#endif +#ifdef ENABLE_IPV6 + if (sudpv6 >= 0) close(sudpv6); +#endif +#ifdef USE_IFACEWATCHER + if(sifacewatcher >= 0) close(sifacewatcher); +#endif +#ifdef ENABLE_NATPMP + for(i=0; i=0) + { + close(snatpmp[i]); + snatpmp[i] = -1; + } + } +#endif +#if defined(ENABLE_IPV6) && defined(ENABLE_PCP) + if(spcp_v6 >= 0) + { + close(spcp_v6); + spcp_v6 = -1; + } +#endif +#ifdef USE_MINIUPNPDCTL + if(sctl>=0) + { + close(sctl); + sctl = -1; + if(unlink("/var/run/miniupnpd.ctl") < 0) + { + syslog(LOG_ERR, "unlink() %m"); + } + } +#endif + + if (GETFLAG(ENABLEUPNPMASK)) + { +#ifndef ENABLE_IPV6 + for(i = 0; i < addr_count; i++) +#else + for(i = 0; i < addr_count * 2; i++) +#endif + close(snotify[i]); + } + + /* remove pidfile */ + if(pidfilename && (unlink(pidfilename) < 0)) + { + syslog(LOG_ERR, "Failed to remove pidfile %s: %m", pidfilename); + } + + /* delete lists */ + while(lan_addrs.lh_first != NULL) + { + lan_addr = lan_addrs.lh_first; + LIST_REMOVE(lan_addrs.lh_first, list); + free(lan_addr); + } + +#ifdef ENABLE_HTTPS + free_ssl(); +#endif +#ifdef ENABLE_NATPMP + free(snatpmp); +#endif + free(snotify); + closelog(); +#ifndef DISABLE_CONFIG_FILE + freeoptions(); +#endif + + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/miniupnpd.conf b/src/contrib/miniupnp/miniupnpd/miniupnpd.conf new file mode 100644 index 0000000..133566f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/miniupnpd.conf @@ -0,0 +1,152 @@ +# WAN network interface +#ext_ifname=eth1 +#ext_ifname=xl1 +# If the WAN interface has several IP addresses, you +# can specify the one to use below +#ext_ip= + +# LAN network interfaces IPs / networks +# There can be multiple listening IPs for SSDP traffic, in that case +# use multiple 'listening_ip=...' lines, one for each network interface. +# It can be IP address or network interface name (ie. "eth0") +# It is mandatory to use the network interface name in order to enable IPv6 +# HTTP is available on all interfaces. +# When MULTIPLE_EXTERNAL_IP is enabled, the external IP +# address associated with the subnet follows. For example: +# listening_ip=192.168.0.1/24 88.22.44.13 +#listening_ip=192.168.0.1/24 +#listening_ip=10.5.0.0/16 +#listening_ip=eth0 +# CAUTION: mixing up WAN and LAN interfaces may introduce security risks! +# Be sure to assign the correct interfaces to LAN and WAN and consider +# implementing UPnP permission rules at the bottom of this configuration file + +# Port for HTTP (descriptions and SOAP) traffic. Set to 0 for autoselect. +#http_port=0 +# Port for HTTPS. Set to 0 for autoselect (default) +#https_port=0 + +# Path to the UNIX socket used to communicate with MiniSSDPd +# If running, MiniSSDPd will manage M-SEARCH answering. +# default is /var/run/minissdpd.sock +#minissdpdsocket=/var/run/minissdpd.sock + +# Enable NAT-PMP support (default is no) +#enable_natpmp=yes + +# Enable UPNP support (default is yes) +#enable_upnp=no + +# PCP +# Configure the minimum and maximum lifetime of a port mapping in seconds +# 120s and 86400s (24h) are suggested values from PCP-base +#min_lifetime=120 +#max_lifetime=86400 + +# Chain names for netfilter (not used for pf or ipf). +# default is MINIUPNPD for both +#upnp_forward_chain=forwardUPnP +#upnp_nat_chain=UPnP +#upnp_nat_postrouting_chain=UPnP-Postrouting + +# Lease file location +#lease_file=/var/log/upnp.leases + +# To enable the next few runtime options, see compile time +# ENABLE_MANUFACTURER_INFO_CONFIGURATION (config.h) + +# Name of this service, default is "`uname -s` router" +#friendly_name=MiniUPnPd router + +# Manufacturer name, default is "`uname -s`" +#manufacturer_name=Manufacturer corp + +# Manufacturer URL, default is URL of OS vendor +#manufacturer_url=http://miniupnp.free.fr/ + +# Model name, default is "`uname -s` router" +#model_name=Router Model + +# Model description, default is "`uname -s` router" +#model_description=Very Secure Router - Model + +# Model URL, default is URL of OS vendor +#model_url=http://miniupnp.free.fr/ + +# Bitrates reported by daemon in bits per second +# by default miniupnpd tries to get WAN interface speed +#bitrate_up=1000000 +#bitrate_down=10000000 + +# Secure Mode, UPnP clients can only add mappings to their own IP +#secure_mode=yes +secure_mode=no + +# Default presentation URL is HTTP address on port 80 +# If set to an empty string, no presentationURL element will appear +# in the XML description of the device, which prevents MS Windows +# from displaying an icon in the "Network Connections" panel. +#presentation_url=http://www.mylan/index.php + +# Report system uptime instead of daemon uptime +system_uptime=yes + +# Notify interval in seconds. default is 30 seconds. +#notify_interval=240 +notify_interval=60 + +# Unused rules cleaning. +# never remove any rule before this threshold for the number +# of redirections is exceeded. default to 20 +#clean_ruleset_threshold=10 +# Clean process work interval in seconds. default to 0 (disabled). +# a 600 seconds (10 minutes) interval makes sense +clean_ruleset_interval=600 + +# Log packets in pf (default is no) +#packet_log=no + +# Anchor name in pf (default is miniupnpd) +#anchor=miniupnpd + +# ALTQ queue in pf +# Filter rules must be used for this to be used. +# compile with PF_ENABLE_FILTER_RULES (see config.h file) +#queue=queue_name1 + +# Tag name in pf +#tag=tag_name1 + +# Make filter rules in pf quick or not. default is yes +# active when compiled with PF_ENABLE_FILTER_RULES (see config.h file) +#quickrules=no + +# UUID, generate your own UUID with "make genuuid" +uuid=00000000-0000-0000-0000-000000000000 + +# Daemon's serial and model number when reporting to clients +# (in XML description) +#serial=12345678 +#model_number=1 + +# If compiled with IGD_V2 defined, force reporting IGDv1 in rootDesc (default +# is no) +#force_igd_desc_v1=no + +# UPnP permission rules +# (allow|deny) (external port range) IP/mask (internal port range) +# A port range is - or if there is only +# one port in the range. +# IP/mask format must be nnn.nnn.nnn.nnn/nn +# It is advised to only allow redirection of port >= 1024 +# and end the rule set with "deny 0-65535 0.0.0.0/0 0-65535" +# The following default ruleset allows specific LAN side IP addresses +# to request only ephemeral ports. It is recommended that users +# modify the IP ranges to match their own internal networks, and +# also consider implementing network-specific restrictions +# CAUTION: failure to enforce any rules may permit insecure requests to be made! +allow 1024-65535 192.168.0.0/24 1024-65535 +allow 1024-65535 192.168.1.0/24 1024-65535 +allow 1024-65535 192.168.0.0/23 22 +allow 12345 192.168.7.113/32 54321 +deny 0-65535 0.0.0.0/0 0-65535 diff --git a/src/contrib/miniupnp/miniupnpd/miniupnpd.rc.once.d.script b/src/contrib/miniupnp/miniupnpd/miniupnpd.rc.once.d.script new file mode 100644 index 0000000..c1d5f99 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/miniupnpd.rc.once.d.script @@ -0,0 +1,11 @@ +#!/bin/sh +# +# generate a UUID on first system start +# + +UUID=`uuidgen` +CONFFILE=/etc/miniupnpd/miniupnpd.conf + +mv "${CONFFILE}" "${CONFFILE}.before" +sed -e "s/^uuid=[-0-9a-fA-F]*/uuid=$UUID/" "${CONFFILE}.before" > "${CONFFILE}" +rm "${CONFFILE}.before" diff --git a/src/contrib/miniupnp/miniupnpd/miniupnpdctl.c b/src/contrib/miniupnp/miniupnpd/miniupnpdctl.c new file mode 100644 index 0000000..5758e0b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/miniupnpdctl.c @@ -0,0 +1,83 @@ +/* $Id: miniupnpdctl.c,v 1.10 2012/04/30 21:08:00 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2012 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "macros.h" + +#if 0 +static void sighandler(int sig) +{ + printf("received signal %d\n", sig); +} +#endif + +int +main(int argc, char * * argv) +{ + /*char test[] = "test!";*/ + static const char command[] = "show all\n"; + char buf[256]; + int l; + int s; + struct sockaddr_un addr; + UNUSED(argc); + UNUSED(argv); + + /*signal(SIGINT, sighandler);*/ + s = socket(AF_UNIX, SOCK_STREAM, 0); + if(s<0) + { + perror("socket"); + return 1; + } + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, "/var/run/miniupnpd.ctl", + sizeof(addr.sun_path)); + if(connect(s, (struct sockaddr *)&addr, sizeof(struct sockaddr_un)) < 0) + { + perror("connect"); + close(s); + return 1; + } + + printf("Connected.\n"); + if(write(s, command, sizeof(command)) < 0) + { + perror("write"); + close(s); + return 1; + } + for(;;) + { + l = read(s, buf, sizeof(buf)); + if(l<0) + { + perror("read"); + break; + } + if(l==0) + break; + /*printf("%d bytes read\n", l);*/ + fflush(stdout); + if(write(fileno(stdout), buf, l) < 0) { + perror("error writing to stdout"); + } + /*printf("\n");*/ + } + + close(s); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/miniupnpdctl.txt b/src/contrib/miniupnp/miniupnpd/miniupnpdctl.txt new file mode 100644 index 0000000..40bedbe --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/miniupnpdctl.txt @@ -0,0 +1,18 @@ +miniupnpdctl + +the communication between a running miniupnpd and miniupnpdctl is +established through a unix socket. "/var/run/miniupnpd.ctl" + +miniupnpdctl can send the following commands : + +1 - print all open sockets +2 - print config +3 - print current redirections + +show options +show permissions/show perms? +show rdr +show opensockets + +show all ? + diff --git a/src/contrib/miniupnp/miniupnpd/miniupnpdpath.h b/src/contrib/miniupnp/miniupnpd/miniupnpdpath.h new file mode 100644 index 0000000..a728ec1 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/miniupnpdpath.h @@ -0,0 +1,49 @@ +/* $Id: miniupnpdpath.h,v 1.8 2011/05/20 17:51:23 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2011 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef MINIUPNPDPATH_H_INCLUDED +#define MINIUPNPDPATH_H_INCLUDED + +#include "config.h" + +/* Paths and other URLs in the miniupnpd http server */ + +#define ROOTDESC_PATH "/rootDesc.xml" + +#ifdef HAS_DUMMY_SERVICE +#define DUMMY_PATH "/dummy.xml" +#endif + +#define WANCFG_PATH "/WANCfg.xml" +#define WANCFG_CONTROLURL "/ctl/CmnIfCfg" +#define WANCFG_EVENTURL "/evt/CmnIfCfg" + +#define WANIPC_PATH "/WANIPCn.xml" +#define WANIPC_CONTROLURL "/ctl/IPConn" +#define WANIPC_EVENTURL "/evt/IPConn" + +#ifdef ENABLE_L3F_SERVICE +#define L3F_PATH "/L3F.xml" +#define L3F_CONTROLURL "/ctl/L3F" +#define L3F_EVENTURL "/evt/L3F" +#endif + +#ifdef ENABLE_6FC_SERVICE +#define WANIP6FC_PATH "/WANIP6FC.xml" +#define WANIP6FC_CONTROLURL "/ctl/IP6FCtl" +#define WANIP6FC_EVENTURL "/evt/IP6FCtl" +#endif + +#ifdef ENABLE_DP_SERVICE +/* For DeviceProtection introduced in IGD v2 */ +#define DP_PATH "/DP.xml" +#define DP_CONTROLURL "/ctl/DP" +#define DP_EVENTURL "/evt/DP" +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/miniupnpdtypes.h b/src/contrib/miniupnp/miniupnpd/miniupnpdtypes.h new file mode 100644 index 0000000..4e71c7e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/miniupnpdtypes.h @@ -0,0 +1,30 @@ +/* $Id: miniupnpdtypes.h,v 1.4 2012/04/06 15:27:21 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2012 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef MINIUPNPDTYPES_H_INCLUDED +#define MINIUPNPDTYPES_H_INCLUDED + +#include "config.h" +#include +#include +#include + +/* structure and list for storing lan addresses + * with ascii representation and mask */ +struct lan_addr_s { + char ifname[IFNAMSIZ]; /* example: eth0 */ + unsigned int index; /* use if_nametoindex() */ + char str[16]; /* example: 192.168.0.1 */ + struct in_addr addr, mask; /* ip/mask */ +#ifdef MULTIPLE_EXTERNAL_IP + char ext_ip_str[16]; + struct in_addr ext_ip_addr; +#endif + LIST_ENTRY(lan_addr_s) list; +}; +LIST_HEAD(lan_addr_list, lan_addr_s); + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/minixml.c b/src/contrib/miniupnp/miniupnpd/minixml.c new file mode 100644 index 0000000..ed2d3c7 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/minixml.c @@ -0,0 +1,231 @@ +/* $Id: minixml.c,v 1.10 2012/03/05 19:42:47 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * minixml.c : the minimum size a xml parser can be ! */ +/* Project : miniupnp + * webpage: http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author : Thomas Bernard + +Copyright (c) 2005-2017, Thomas BERNARD +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include "minixml.h" + +/* parseatt : used to parse the argument list + * return 0 (false) in case of success and -1 (true) if the end + * of the xmlbuffer is reached. */ +static int parseatt(struct xmlparser * p) +{ + const char * attname; + int attnamelen; + const char * attvalue; + int attvaluelen; + while(p->xml < p->xmlend) + { + if(*p->xml=='/' || *p->xml=='>') + return 0; + if( !IS_WHITE_SPACE(*p->xml) ) + { + char sep; + attname = p->xml; + attnamelen = 0; + while(*p->xml!='=' && !IS_WHITE_SPACE(*p->xml) ) + { + attnamelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + while(*(p->xml++) != '=') + { + if(p->xml >= p->xmlend) + return -1; + } + while(IS_WHITE_SPACE(*p->xml)) + { + p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + sep = *p->xml; + if(sep=='\'' || sep=='\"') + { + p->xml++; + if(p->xml >= p->xmlend) + return -1; + attvalue = p->xml; + attvaluelen = 0; + while(*p->xml != sep) + { + attvaluelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + } + else + { + attvalue = p->xml; + attvaluelen = 0; + while( !IS_WHITE_SPACE(*p->xml) + && *p->xml != '>' && *p->xml != '/') + { + attvaluelen++; p->xml++; + if(p->xml >= p->xmlend) + return -1; + } + } + /*printf("%.*s='%.*s'\n", + attnamelen, attname, attvaluelen, attvalue);*/ + if(p->attfunc) + p->attfunc(p->data, attname, attnamelen, attvalue, attvaluelen); + } + p->xml++; + } + return -1; +} + +/* parseelt parse the xml stream and + * call the callback functions when needed... */ +static void parseelt(struct xmlparser * p) +{ + int i; + const char * elementname; + while(p->xml < (p->xmlend - 1)) + { + if((p->xml + 4) <= p->xmlend && (0 == memcmp(p->xml, "", 3) != 0); + p->xml += 3; + } + else if((p->xml)[0]=='<' && (p->xml)[1]!='?') + { + i = 0; elementname = ++p->xml; + while( !IS_WHITE_SPACE(*p->xml) + && (*p->xml!='>') && (*p->xml!='/') + ) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + /* to ignore namespace : */ + if(*p->xml==':') + { + i = 0; + elementname = ++p->xml; + } + } + if(i>0) + { + if(p->starteltfunc) + p->starteltfunc(p->data, elementname, i); + if(parseatt(p)) + return; + if(*p->xml!='/') + { + const char * data; + i = 0; data = ++p->xml; + if (p->xml >= p->xmlend) + return; + while( IS_WHITE_SPACE(*p->xml) ) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + } + /* CDATA are at least 9 + 3 characters long : */ + if((p->xmlend >= (p->xml + (9 + 3))) && (memcmp(p->xml, "xml += 9; + data = p->xml; + i = 0; + while(memcmp(p->xml, "]]>", 3) != 0) + { + i++; p->xml++; + if ((p->xml + 3) >= p->xmlend) + return; + } + if(i>0 && p->datafunc) + p->datafunc(p->data, data, i); + while(*p->xml!='<') + { + p->xml++; + if (p->xml >= p->xmlend) + return; + } + } + else + { + while(*p->xml!='<') + { + i++; p->xml++; + if ((p->xml + 1) >= p->xmlend) + return; + } + if(i>0 && p->datafunc && *(p->xml + 1) == '/') + p->datafunc(p->data, data, i); + } + } + } + else if(*p->xml == '/') + { + i = 0; elementname = ++p->xml; + if (p->xml >= p->xmlend) + return; + while((*p->xml != '>')) + { + i++; p->xml++; + if (p->xml >= p->xmlend) + return; + } + if(p->endeltfunc) + p->endeltfunc(p->data, elementname, i); + p->xml++; + } + } + else + { + p->xml++; + } + } +} + +/* the parser must be initialized before calling this function */ +void parsexml(struct xmlparser * parser) +{ + parser->xml = parser->xmlstart; + parser->xmlend = parser->xmlstart + parser->xmlsize; + parseelt(parser); +} + + diff --git a/src/contrib/miniupnp/miniupnpd/minixml.h b/src/contrib/miniupnp/miniupnpd/minixml.h new file mode 100644 index 0000000..19e6f51 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/minixml.h @@ -0,0 +1,37 @@ +/* $Id: minixml.h,v 1.6 2006/11/30 11:47:21 nanard Exp $ */ +/* minimal xml parser + * + * Project : miniupnp + * Website : http://miniupnp.free.fr/ + * Author : Thomas Bernard + * Copyright (c) 2005 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file provided in this distribution. + * */ +#ifndef MINIXML_H_INCLUDED +#define MINIXML_H_INCLUDED +#define IS_WHITE_SPACE(c) ((c==' ') || (c=='\t') || (c=='\r') || (c=='\n')) + +/* if a callback function pointer is set to NULL, + * the function is not called */ +struct xmlparser { + const char *xmlstart; + const char *xmlend; + const char *xml; /* pointer to current character */ + int xmlsize; + void * data; + void (*starteltfunc) (void *, const char *, int); + void (*endeltfunc) (void *, const char *, int); + void (*datafunc) (void *, const char *, int); + void (*attfunc) (void *, const char *, int, const char *, int); +}; + +/* parsexml() + * the xmlparser structure must be initialized before the call + * the following structure members have to be initialized : + * xmlstart, xmlsize, data, *func + * xml is for internal usage, xmlend is computed automatically */ +void parsexml(struct xmlparser *); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/natpmp.c b/src/contrib/miniupnp/miniupnpd/natpmp.c new file mode 100644 index 0000000..0ed09e0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/natpmp.c @@ -0,0 +1,486 @@ +/* $Id: natpmp.c,v 1.55 2018/03/12 22:41:54 nanard Exp $ */ +/* MiniUPnP project + * (c) 2007-2017 Thomas Bernard + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "macros.h" +#include "config.h" +#include "natpmp.h" +#include "upnpglobalvars.h" +#include "getifaddr.h" +#include "upnpredirect.h" +#include "commonrdr.h" +#include "upnputils.h" +#include "portinuse.h" +#include "asyncsendto.h" + +#ifdef ENABLE_NATPMP + +int OpenAndConfNATPMPSocket(in_addr_t addr) +{ + int snatpmp; + int i = 1; + snatpmp = socket(PF_INET, SOCK_DGRAM, 0/*IPPROTO_UDP*/); + if(snatpmp<0) + { + syslog(LOG_ERR, "%s: socket(): %m", + "OpenAndConfNATPMPSocket"); + return -1; + } + if(setsockopt(snatpmp, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i)) < 0) + { + syslog(LOG_WARNING, "%s: setsockopt(SO_REUSEADDR): %m", + "OpenAndConfNATPMPSocket"); + } + if(!set_non_blocking(snatpmp)) + { + syslog(LOG_WARNING, "%s: set_non_blocking(): %m", + "OpenAndConfNATPMPSocket"); + } + { + struct sockaddr_in natpmp_addr; + memset(&natpmp_addr, 0, sizeof(natpmp_addr)); + natpmp_addr.sin_family = AF_INET; + natpmp_addr.sin_port = htons(NATPMP_PORT); + /*natpmp_addr.sin_addr.s_addr = INADDR_ANY; */ + natpmp_addr.sin_addr.s_addr = addr; + if(bind(snatpmp, (struct sockaddr *)&natpmp_addr, sizeof(natpmp_addr)) < 0) + { + syslog(LOG_ERR, "%s: bind(): %m", + "OpenAndConfNATPMPSocket"); + close(snatpmp); + return -1; + } + } + return snatpmp; +} + +int OpenAndConfNATPMPSockets(int * sockets) +{ + int i; + struct lan_addr_s * lan_addr; + for(i = 0, lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + sockets[i] = OpenAndConfNATPMPSocket(lan_addr->addr.s_addr); + if(sockets[i] < 0) + goto error; + i++; + } + return 0; +error: + while(--i >= 0) + { + close(sockets[i]); + sockets[i] = -1; + } + return -1; +} + +static void FillPublicAddressResponse(unsigned char * resp, in_addr_t senderaddr) +{ +#ifndef MULTIPLE_EXTERNAL_IP + char tmp[16]; + UNUSED(senderaddr); + + if(use_ext_ip_addr) { + inet_pton(AF_INET, use_ext_ip_addr, resp+8); + } else { + if(!ext_if_name || ext_if_name[0]=='\0') { + resp[3] = 3; /* Network Failure (e.g. NAT box itself + * has not obtained a DHCP lease) */ + } else if(getifaddr(ext_if_name, tmp, INET_ADDRSTRLEN, NULL, NULL) < 0) { + syslog(LOG_ERR, "Failed to get IP for interface %s", ext_if_name); + resp[3] = 3; /* Network Failure (e.g. NAT box itself + * has not obtained a DHCP lease) */ + } else { + inet_pton(AF_INET, tmp, resp+8); /* ok */ + } + } +#else + struct lan_addr_s * lan_addr; + + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) { + if( (senderaddr & lan_addr->mask.s_addr) + == (lan_addr->addr.s_addr & lan_addr->mask.s_addr)) { + memcpy(resp+8, &lan_addr->ext_ip_addr, + sizeof(lan_addr->ext_ip_addr)); + break; + } + } +#endif +} + +/* + * Receives NATPMP and PCP packets and stores them in msg_buff. + * The sender information is stored in senderaddr. + * Returns number of bytes recevied, even if number is negative. + */ +int ReceiveNATPMPOrPCPPacket(int s, struct sockaddr * senderaddr, + socklen_t * senderaddrlen, + struct sockaddr_in6 * receiveraddr, + unsigned char * msg_buff, size_t msg_buff_size) +{ +#ifdef IPV6_PKTINFO + struct iovec iov; + uint8_t c[1000]; + struct msghdr msg; + int n; + struct cmsghdr *h; + + iov.iov_base = msg_buff; + iov.iov_len = msg_buff_size; + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = senderaddr; + msg.msg_namelen = *senderaddrlen; + msg.msg_control = c; + msg.msg_controllen = sizeof(c); + + n = recvmsg(s, &msg, 0); + if(n < 0) { + /* EAGAIN, EWOULDBLOCK and EINTR : silently ignore (retry next time) + * other errors : log to LOG_ERR */ + if(errno != EAGAIN && + errno != EWOULDBLOCK && + errno != EINTR) { + syslog(LOG_ERR, "recvmsg(natpmp): %m"); + } + return n; + } + + if(receiveraddr) { + memset(receiveraddr, 0, sizeof(struct sockaddr_in6)); + } + if ((msg.msg_flags & MSG_TRUNC) || (msg.msg_flags & MSG_CTRUNC)) { + syslog(LOG_WARNING, "%s: truncated message", + "ReceiveNATPMPOrPCPPacket"); + } + for(h = CMSG_FIRSTHDR(&msg); h; + h = CMSG_NXTHDR(&msg, h)) { + if(h->cmsg_level == IPPROTO_IPV6 && h->cmsg_type == IPV6_PKTINFO) { + char tmp[INET6_ADDRSTRLEN]; + struct in6_pktinfo *ipi6 = (struct in6_pktinfo *)CMSG_DATA(h); + syslog(LOG_DEBUG, "%s: packet destination: %s scope_id=%u", + "ReceiveNATPMPOrPCPPacket", + inet_ntop(AF_INET6, &ipi6->ipi6_addr, tmp, sizeof(tmp)), + ipi6->ipi6_ifindex); + if(receiveraddr) { + receiveraddr->sin6_addr = ipi6->ipi6_addr; + receiveraddr->sin6_scope_id = ipi6->ipi6_ifindex; + receiveraddr->sin6_family = AF_INET6; + receiveraddr->sin6_port = htons(NATPMP_PORT); + } + } + } +#else /* IPV6_PKTINFO */ + int n; + + n = recvfrom(s, msg_buff, msg_buff_size, 0, + senderaddr, senderaddrlen); + + if(n<0) { + /* EAGAIN, EWOULDBLOCK and EINTR : silently ignore (retry next time) + * other errors : log to LOG_ERR */ + if(errno != EAGAIN && + errno != EWOULDBLOCK && + errno != EINTR) { + syslog(LOG_ERR, "recvfrom(natpmp): %m"); + } + return n; + } +#endif /* IPV6_PKTINFO */ + + return n; +} + +/** read the request from the socket, process it and then send the + * response back. + */ +void ProcessIncomingNATPMPPacket(int s, unsigned char *msg_buff, int len, + struct sockaddr_in *senderaddr) +{ + unsigned char *req=msg_buff; /* request udp packet */ + unsigned char resp[32]; /* response udp packet */ + int resplen; + int n = len; + char senderaddrstr[16]; + + if(!inet_ntop(AF_INET, &senderaddr->sin_addr, + senderaddrstr, sizeof(senderaddrstr))) { + syslog(LOG_ERR, "inet_ntop(natpmp): %m"); + } + + syslog(LOG_INFO, "NAT-PMP request received from %s:%hu %dbytes", + senderaddrstr, ntohs(senderaddr->sin_port), n); + + if(n<2 || ((((req[1]-1)&~1)==0) && n<12)) { + syslog(LOG_WARNING, "discarding NAT-PMP request (too short) %dBytes", + n); + return; + } + if(req[1] & 128) { + /* discarding NAT-PMP responses silently */ + return; + } + memset(resp, 0, sizeof(resp)); + resplen = 8; + resp[1] = 128 + req[1]; /* response OPCODE is request OPCODE + 128 */ + /* setting response TIME STAMP : + * time elapsed since its port mapping table was initialized on + * startup or reset for any other reason */ + if(epoch_origin == 0) { + epoch_origin = startup_time; + } + WRITENU32(resp+4, upnp_time() - epoch_origin); + if(req[0] > 0) { + /* invalid version */ + syslog(LOG_WARNING, "unsupported NAT-PMP version : %u", + (unsigned)req[0]); + resp[3] = 1; /* unsupported version */ + } else switch(req[1]) { + case 0: /* Public address request */ + syslog(LOG_INFO, "NAT-PMP public address request"); + FillPublicAddressResponse(resp, senderaddr->sin_addr.s_addr); + resplen = 12; + break; + case 1: /* UDP port mapping request */ + case 2: /* TCP port mapping request */ + { + unsigned short iport; /* private port */ + unsigned short eport; /* public port */ + uint32_t lifetime; /* lifetime=0 => remove port mapping */ + int r; + int proto; + char iaddr_old[16]; + unsigned short iport_old; + unsigned int timestamp; + + iport = READNU16(req+4); + eport = READNU16(req+6); + lifetime = READNU32(req+8); + proto = (req[1]==1)?IPPROTO_UDP:IPPROTO_TCP; + syslog(LOG_INFO, "NAT-PMP port mapping request : " + "%hu->%s:%hu %s lifetime=%us", + eport, senderaddrstr, iport, + (req[1]==1)?"udp":"tcp", lifetime); + /* TODO: accept port mapping if iport ok but eport not ok + * (and set eport correctly) */ + if(lifetime == 0) { + /* remove the mapping */ + /* RFC6886 : + * A client MAY also send an explicit packet to request deletion of a + * mapping that is no longer needed. A client requests explicit + * deletion of a mapping by sending a message to the NAT gateway + * requesting the mapping, with the Requested Lifetime in Seconds set to + * zero. The Suggested External Port MUST be set to zero by the client + * on sending, and MUST be ignored by the gateway on reception. */ + int index = 0; + unsigned short eport2, iport2; + char iaddr2[16]; + int proto2; + char desc[64]; + eport = 0; /* to indicate correct removing of port mapping */ + while(get_redirect_rule_by_index(index, 0, + &eport2, iaddr2, sizeof(iaddr2), + &iport2, &proto2, + desc, sizeof(desc), + 0, 0, ×tamp, 0, 0) >= 0) { + syslog(LOG_DEBUG, "%d %d %hu->'%s':%hu '%s'", + index, proto2, eport2, iaddr2, iport2, desc); + if(0 == strcmp(iaddr2, senderaddrstr) + && 0 == memcmp(desc, "NAT-PMP", 7)) { + /* (iport == 0) => remove all the mappings for this client */ + if((iport == 0) || ((iport == iport2) && (proto == proto2))) { + r = _upnp_delete_redir(eport2, proto2); + if(r < 0) { + syslog(LOG_ERR, "Failed to remove NAT-PMP mapping eport %hu, protocol %s", + eport2, (proto2==IPPROTO_TCP)?"TCP":"UDP"); + resp[3] = 2; /* Not Authorized/Refused */ + break; + } else { + syslog(LOG_INFO, "NAT-PMP %s port %hu mapping removed", + proto2==IPPROTO_TCP?"TCP":"UDP", eport2); + index--; + } + } + } + index++; + } + } else if(iport==0) { + resp[3] = 2; /* Not Authorized/Refused */ + } else { /* iport > 0 && lifetime > 0 */ + unsigned short eport_first = 0; + int any_eport_allowed = 0; + char desc[64]; + if(eport==0) /* if no suggested external port, use same a internal port */ + eport = iport; + while(resp[3] == 0) { + if(eport_first == 0) { /* first time in loop */ + eport_first = eport; + } else if(eport == eport_first) { /* no eport available */ + if(any_eport_allowed == 0) { /* all eports rejected by permissions */ + syslog(LOG_ERR, "No allowed eport for NAT-PMP %hu %s->%s:%hu", + eport, (proto==IPPROTO_TCP)?"tcp":"udp", senderaddrstr, iport); + resp[3] = 2; /* Not Authorized/Refused */ + } else { /* at least one eport allowed (but none available) */ + syslog(LOG_ERR, "Failed to find available eport for NAT-PMP %hu %s->%s:%hu", + eport, (proto==IPPROTO_TCP)?"tcp":"udp", senderaddrstr, iport); + resp[3] = 4; /* Out of resources */ + } + break; + } + if(!check_upnp_rule_against_permissions(upnppermlist, num_upnpperm, eport, senderaddr->sin_addr, iport)) { + eport++; + if(eport == 0) eport++; /* skip port zero */ + continue; + } + any_eport_allowed = 1; /* at lease one eport is allowed */ +#ifdef CHECK_PORTINUSE + if (port_in_use(ext_if_name, eport, proto, senderaddrstr, iport) > 0) { + syslog(LOG_INFO, "port %hu protocol %s already in use", + eport, (proto==IPPROTO_TCP)?"tcp":"udp"); + eport++; + if(eport == 0) eport++; /* skip port zero */ + continue; + } +#endif + r = get_redirect_rule(ext_if_name, eport, proto, + iaddr_old, sizeof(iaddr_old), + &iport_old, 0, 0, 0, 0, + ×tamp, 0, 0); + if(r==0) { + if(strcmp(senderaddrstr, iaddr_old)==0 + && iport==iport_old) { + /* redirection already existing */ + syslog(LOG_INFO, "port %hu %s already redirected to %s:%hu, replacing", + eport, (proto==IPPROTO_TCP)?"tcp":"udp", iaddr_old, iport_old); + /* remove and then add again */ + if(_upnp_delete_redir(eport, proto) < 0) { + syslog(LOG_ERR, "failed to remove port mapping"); + break; + } + } else { + eport++; + if(eport == 0) eport++; /* skip port zero */ + continue; + } + } + /* do the redirection */ + timestamp = upnp_time() + lifetime; + snprintf(desc, sizeof(desc), "NAT-PMP %hu %s", + eport, (proto==IPPROTO_TCP)?"tcp":"udp"); + /* TODO : check return code */ + if(upnp_redirect_internal(NULL, eport, senderaddrstr, + iport, proto, desc, + timestamp) < 0) { + syslog(LOG_ERR, "Failed to add NAT-PMP %hu %s->%s:%hu '%s'", + eport, (proto==IPPROTO_TCP)?"tcp":"udp", senderaddrstr, iport, desc); + resp[3] = 3; /* Failure */ + } + break; + } + } + WRITENU16(resp+8, iport); /* private port */ + WRITENU16(resp+10, eport); /* public port */ + WRITENU32(resp+12, lifetime); /* Port Mapping lifetime */ + } + resplen = 16; + break; + default: + resp[3] = 5; /* Unsupported OPCODE */ + } + n = sendto_or_schedule(s, resp, resplen, 0, + (struct sockaddr *)senderaddr, sizeof(*senderaddr)); + if(n<0) { + syslog(LOG_ERR, "sendto(natpmp): %m"); + } else if(nlist.le_next; + FillPublicAddressResponse(notif, lan_addr->addr.s_addr); + } +#endif + /* Port to use in 2006 version of the NAT-PMP specification */ + sockname.sin_port = htons(NATPMP_PORT); + n = sendto_or_schedule(sockets[j], notif, 12, 0, + (struct sockaddr *)&sockname, sizeof(struct sockaddr_in)); + if(n < 0) + { + syslog(LOG_ERR, "%s: sendto(s_udp=%d, port=%d): %m", + "SendNATPMPPublicAddressChangeNotification", sockets[j], NATPMP_PORT); + return; + } + /* Port to use in 2008 version of the NAT-PMP specification */ + sockname.sin_port = htons(NATPMP_NOTIF_PORT); + n = sendto_or_schedule(sockets[j], notif, 12, 0, + (struct sockaddr *)&sockname, sizeof(struct sockaddr_in)); + if(n < 0) + { + syslog(LOG_ERR, "%s: sendto(s_udp=%d, port=%d): %m", + "SendNATPMPPublicAddressChangeNotification", sockets[j], NATPMP_NOTIF_PORT); + return; + } + } +} + +#endif /* ENABLE_NATPMP */ diff --git a/src/contrib/miniupnp/miniupnpd/natpmp.h b/src/contrib/miniupnp/miniupnpd/natpmp.h new file mode 100644 index 0000000..d44bcad --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/natpmp.h @@ -0,0 +1,35 @@ +/* $Id: natpmp.h,v 1.12 2014/03/24 10:49:46 nanard Exp $ */ +/* MiniUPnP project + * author : Thomas Bernard + * website : http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + */ +#ifndef NATPMP_H_INCLUDED +#define NATPMP_H_INCLUDED + +/* The NAT-PMP specification which can be found at the url : + * http://files.dns-sd.org/draft-cheshire-nat-pmp.txt + * draft version 3 of April 2008 + * define 5351 as listening port for the gateway, + * and the 224.0.0.1 port 5350 as the local link + * multicast address for address change announces. + * Previous versions of the specification defined 5351 + * as the port for address change announces. */ +#define NATPMP_PORT (5351) +#define NATPMP_NOTIF_PORT (5350) +#define NATPMP_NOTIF_ADDR ("224.0.0.1") + +int OpenAndConfNATPMPSockets(int * sockets); + +/* receiveraddr is only used with IPV6 sockets */ +int ReceiveNATPMPOrPCPPacket(int s, struct sockaddr * senderaddr, + socklen_t * senderaddrlen, + struct sockaddr_in6 * receiveraddr, + unsigned char * msg_buff, size_t msg_buff_size); + +void ProcessIncomingNATPMPPacket(int s, unsigned char * msg_buff, int len, + struct sockaddr_in * senderaddr); + +void SendNATPMPPublicAddressChangeNotification(int * sockets, int n_sockets); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/Makefile b/src/contrib/miniupnp/miniupnpd/netfilter/Makefile new file mode 100644 index 0000000..7bc52a2 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/Makefile @@ -0,0 +1,79 @@ +# $Id: Makefile,v 1.6 2012/04/26 13:50:48 nanard Exp $ +CFLAGS?=-Wall -g -D_GNU_SOURCE -DDEBUG -Wstrict-prototypes -Wdeclaration-after-statement +CC = gcc + +#LIBS = -liptc +LIBS = -lip4tc + +ARCH := $(shell uname -m | grep -q "x86_64" && echo 64) +ifdef IPTABLESPATH +CFLAGS := $(CFLAGS) -I$(IPTABLESPATH)/include/ +LDFLAGS := $(LDFLAFGS) -L$(IPTABLESPATH)/libiptc/ +# get iptables version and set IPTABLES_143 macro if needed +IPTABLESVERSION := $(shell grep "\#define VERSION" $(IPTABLESPATH)/config.h | tr -d \" |cut -d" " -f3 ) +IPTABLESVERSION1 := $(shell echo $(IPTABLESVERSION) | cut -d. -f1 ) +IPTABLESVERSION2 := $(shell echo $(IPTABLESVERSION) | cut -d. -f2 ) +IPTABLESVERSION3 := $(shell echo $(IPTABLESVERSION) | cut -d. -f3 ) +# test if iptables version >= 1.4.3 +TEST := $(shell [ \( \( $(IPTABLESVERSION1) -ge 1 \) -a \( $(IPTABLESVERSION2) -ge 4 \) \) -a \( $(IPTABLESVERSION3) -ge 3 \) ] && echo 1 ) +ifeq ($(TEST), 1) +CFLAGS := $(CFLAGS) -DIPTABLES_143 +# the following sucks, but works +LIBS = $(IPTABLESPATH)/libiptc/.libs/libip4tc.o +#LIBS = $(IPTABLESPATH)/libiptc/.libs/libiptc.a +else +LIBS = $(IPTABLESPATH)/libiptc/libiptc.a +endif +else +# check for system-wide iptables files. Test if iptables version >= 1.4.3 +#TEST := $(shell test -f /usr/include/iptables/internal.h && grep -q "\#define IPTABLES_VERSION" /usr/include/iptables/internal.h && echo 1) +TEST := $(shell test -f /usr/include/xtables.h && grep -q "XTABLES_VERSION_CODE" /usr/include/xtables.h && echo 1) +ifeq ($(TEST), 1) +CFLAGS := $(CFLAGS) -DIPTABLES_143 +LIBS = -liptc +TEST_LIB := $(shell test -f /usr/lib$(ARCH)/libiptc.a && echo 1) +ifeq ($(TEST_LIB), 1) +LIBS = -liptc /usr/lib$(ARCH)/libiptc.a +endif +endif +endif + +LIBS += /lib/libip4tc.so /lib/libip6tc.so + +all: iptcrdr.o testiptcrdr iptpinhole.o \ + testiptcrdr_peer testiptcrdr_dscp test_nfct_get +# testiptpinhole + +clean: + $(RM) *.o testiptcrdr testiptpinhole testiptcrdr_peer test_nfct_get \ + testiptcrdr_dscp + +testiptcrdr: testiptcrdr.o upnpglobalvars.o $(LIBS) + +testiptcrdr_peer: testiptcrdr_peer.o upnpglobalvars.o $(LIBS) + +testiptcrdr_dscp: testiptcrdr_dscp.o upnpglobalvars.o $(LIBS) + +testiptpinhole: testiptpinhole.o iptpinhole.o upnpglobalvars.o $(LIBS) + +test_nfct_get: test_nfct_get.o test_nfct_get.o -lmnl -lnetfilter_conntrack + +test_nfct_get.o: test_nfct_get.c + +testiptcrdr_peer.o: testiptcrdr_peer.c + +testiptcrdr_dscp.o: testiptcrdr_dscp.c + +iptcrdr.o: iptcrdr.c iptcrdr.h + +iptpinhole.o: iptpinhole.c iptpinhole.h + +upnpglobalvars.o: ../upnpglobalvars.c ../upnpglobalvars.h + $(CC) -c -o $@ $< + + +#depends +testiptcrdr.o: testiptcrdr.c iptcrdr.c +testiptcrdr_dscp.o: testiptcrdr_dscp.c iptcrdr.c +testiptcrdr_peer.o: testiptcrdr_peer.c iptcrdr.c +test_nfct_get.o: test_nfct_get.c nfct_get.c diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_display.sh b/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_display.sh new file mode 100644 index 0000000..4ef5995 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_display.sh @@ -0,0 +1,9 @@ +#! /bin/sh +# $Id: ip6tables_display.sh,v 1.1 2012/04/24 22:13:41 nanard Exp $ + +IPV6=1 +. $(dirname "$0")/miniupnpd_functions.sh + +#display all chains relative to miniupnpd +$IPTABLES -v -n -t filter -L FORWARD +$IPTABLES -v -n -t filter -L $CHAIN diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_flush.sh b/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_flush.sh new file mode 100644 index 0000000..ceec1e4 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_flush.sh @@ -0,0 +1,8 @@ +#! /bin/sh +# $Id: ip6tables_flush.sh,v 1.1 2012/04/24 22:13:41 nanard Exp $ + +IPV6=1 +. $(dirname "$0")/miniupnpd_functions.sh + +#flush all rules owned by miniupnpd +$IPTABLES -t filter -F $CHAIN diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_init.sh b/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_init.sh new file mode 100644 index 0000000..162ef13 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_init.sh @@ -0,0 +1,22 @@ +#! /bin/sh +# $Id: ip6tables_init_and_clean.sh,v 1.1 2012/04/24 22:13:41 nanard Exp $ +# Improved Miniupnpd iptables init script. +# Checks for state of filter before doing anything.. + +IPV6=1 +EXT=1 +. $(dirname "$0")/miniupnpd_functions.sh + +if [ "$FDIRTY" = "${CHAIN}Chain" ]; then + echo "Filter table dirty; Cleaning..." +elif [ "$FDIRTY" = "Chain" ]; then + echo "Dirty filter chain but no reference..? Fixing..." + $IPTABLES -t filter -A FORWARD -i $EXTIF ! -o $EXTIF -j $CHAIN +else + echo "Filter table clean..initalizing.." + $IPTABLES -t filter -N $CHAIN + $IPTABLES -t filter -A FORWARD -i $EXTIF ! -o $EXTIF -j $CHAIN +fi +if [ "$CLEAN" = "yes" ]; then + $IPTABLES -t filter -F $CHAIN +fi diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_removeall.sh b/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_removeall.sh new file mode 100644 index 0000000..126ca58 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/ip6tables_removeall.sh @@ -0,0 +1,16 @@ +#! /bin/sh +# $Id: ip6tables_removeall.sh,v 1.1 2012/04/24 22:13:41 nanard Exp $ + +IPV6=1 +EXT=1 +. $(dirname "$0")/miniupnpd_functions.sh + +#removing the MINIUPNPD chain for filter +if [ "$FDIRTY" = "${CHAIN}Chain" ]; then + $IPTABLES -t filter -F $CHAIN + $IPTABLES -t filter -D FORWARD -i $EXTIF ! -o $EXTIF -j $CHAIN + $IPTABLES -t filter -X $CHAIN +elif [ "$FDIRTY" = "Chain" ]; then + $IPTABLES -t filter -F $CHAIN + $IPTABLES -t filter -X $CHAIN +fi diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptables_display.sh b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_display.sh new file mode 100644 index 0000000..956375f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_display.sh @@ -0,0 +1,14 @@ +#! /bin/sh +# $Id: iptables_display.sh,v 1.4 2011/05/16 12:11:37 nanard Exp $ + +. $(dirname "$0")/miniupnpd_functions.sh + +#display all chains relative to miniupnpd +$IPTABLES -v -n -t nat -L PREROUTING +$IPTABLES -v -n -t nat -L $CHAIN +$IPTABLES -v -n -t nat -L POSTROUTING +$IPTABLES -v -n -t nat -L $CHAIN-POSTROUTING +$IPTABLES -v -n -t mangle -L PREROUTING +$IPTABLES -v -n -t mangle -L $CHAIN +$IPTABLES -v -n -t filter -L FORWARD +$IPTABLES -v -n -t filter -L $CHAIN diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptables_display_miniupnpd.sh b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_display_miniupnpd.sh new file mode 100644 index 0000000..1d69457 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_display_miniupnpd.sh @@ -0,0 +1,10 @@ +#! /bin/sh +# $Id: iptables_display_miniupnpd.sh,v 1.1 2016/02/12 15:23:29 nanard Exp $ + +. $(dirname "$0")/miniupnpd_functions.sh + +#display miniupnpd chains +$IPTABLES -v -n -t nat -L $CHAIN +$IPTABLES -v -n -t nat -L $CHAIN-POSTROUTING +$IPTABLES -v -n -t mangle -L $CHAIN +$IPTABLES -v -n -t filter -L $CHAIN diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptables_flush.sh b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_flush.sh new file mode 100644 index 0000000..e147829 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_flush.sh @@ -0,0 +1,10 @@ +#! /bin/sh +# $Id: iptables_flush.sh,v 1.6 2017/04/21 11:16:09 nanard Exp $ + +. $(dirname "$0")/miniupnpd_functions.sh + +#flush all rules owned by miniupnpd +$IPTABLES -t nat -F $CHAIN +$IPTABLES -t nat -F $CHAIN-POSTROUTING +$IPTABLES -t filter -F $CHAIN +$IPTABLES -t mangle -F $CHAIN diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptables_init.sh b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_init.sh new file mode 100644 index 0000000..1983277 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_init.sh @@ -0,0 +1,37 @@ +#! /bin/sh +# $Id: iptables_init_and_clean.sh,v 1.7 2017/04/21 11:16:09 nanard Exp $ +# Improved Miniupnpd iptables init script. +# Checks for state of filter before doing anything.. + +EXT=1 +. $(dirname "$0")/miniupnpd_functions.sh + +if [ "$NDIRTY" = "${CHAIN}Chain" ]; then + echo "Nat table dirty; Cleaning..." +elif [ "$NDIRTY" = "Chain" ]; then + echo "Dirty NAT chain but no reference..? Fixing..." + #$IPTABLES -t nat -A PREROUTING -d $EXTIP -i $EXTIF -j $CHAIN + $IPTABLES -t nat -A PREROUTING -i $EXTIF -j $CHAIN +else + echo "NAT table clean..initalizing.." + $IPTABLES -t nat -N $CHAIN + #$IPTABLES -t nat -A PREROUTING -d $EXTIP -i $EXTIF -j $CHAIN + $IPTABLES -t nat -A PREROUTING -i $EXTIF -j $CHAIN +fi +if [ "$CLEAN" = "yes" ]; then + $IPTABLES -t nat -F $CHAIN +fi + +if [ "$FDIRTY" = "${CHAIN}Chain" ]; then + echo "Filter table dirty; Cleaning..." +elif [ "$FDIRTY" = "Chain" ]; then + echo "Dirty filter chain but no reference..? Fixing..." + $IPTABLES -t filter -A FORWARD -i $EXTIF ! -o $EXTIF -j $CHAIN +else + echo "Filter table clean..initalizing.." + $IPTABLES -t filter -N MINIUPNPD + $IPTABLES -t filter -A FORWARD -i $EXTIF ! -o $EXTIF -j $CHAIN +fi +if [ "$CLEAN" = "yes" ]; then + $IPTABLES -t filter -F $CHAIN +fi diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptables_removeall.sh b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_removeall.sh new file mode 100644 index 0000000..cd24596 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptables_removeall.sh @@ -0,0 +1,46 @@ +#! /bin/sh +# $Id: iptables_removeall.sh,v 1.10 2017/04/21 11:16:09 nanard Exp $ + +EXT=1 +. $(dirname "$0")/miniupnpd_functions.sh + +#removing the MINIUPNPD chain for nat +if [ "$NDIRTY" = "${CHAIN}Chain" ]; then + $IPTABLES -t nat -F $CHAIN + #$IPTABLES -t nat -D PREROUTING -d $EXTIP -i $EXTIF -j $CHAIN + $IPTABLES -t nat -D PREROUTING -i $EXTIF -j $CHAIN + $IPTABLES -t nat -X $CHAIN +elif [ "$NDIRTY" = "Chain" ]; then + $IPTABLES -t nat -F $CHAIN + $IPTABLES -t nat -X $CHAIN +fi + +#removing the MINIUPNPD chain for mangle +if [ "$MDIRTY" = "${CHAIN}Chain" ]; then + $IPTABLES -t mangle -F $CHAIN + $IPTABLES -t mangle -D FORWARD -i $EXTIF -j $CHAIN + $IPTABLES -t mangle -X $CHAIN +elif [ "$MDIRTY" = "Chain" ]; then + $IPTABLES -t mangle -F $CHAIN + $IPTABLES -t mangle -X $CHAIN +fi + +#removing the MINIUPNPD chain for filter +if [ "$FDIRTY" = "${CHAIN}Chain" ]; then + $IPTABLES -t filter -F $CHAIN + $IPTABLES -t filter -D FORWARD -i $EXTIF ! -o $EXTIF -j $CHAIN + $IPTABLES -t filter -X $CHAIN +elif [ "$FDIRTY" = "Chain" ]; then + $IPTABLES -t filter -F $CHAIN + $IPTABLES -t filter -X $CHAIN +fi + +#removing the MINIUPNPD-POSTROUTING chain for nat +if [ "$NPDIRTY" = "${CHAIN}-POSTROUTINGChain" ]; then + $IPTABLES -t nat -F $CHAIN-POSTROUTING + $IPTABLES -t nat -D POSTROUTING -o $EXTIF -j $CHAIN-POSTROUTING + $IPTABLES -t nat -X $CHAIN-POSTROUTING +elif [ "$NPDIRTY" = "Chain" ]; then + $IPTABLES -t nat -F $CHAIN-POSTROUTING + $IPTABLES -t nat -X $CHAIN-POSTROUTING +fi diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptcrdr.c b/src/contrib/miniupnp/miniupnpd/netfilter/iptcrdr.c new file mode 100644 index 0000000..48c6dbb --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptcrdr.c @@ -0,0 +1,1948 @@ +/* $Id: iptcrdr.c,v 1.59 2016/03/08 09:23:52 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2016 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if IPTABLES_143 +/* IPTABLES API version >= 1.4.3 */ + +/* added in order to compile on gentoo : + * http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=2183 */ +#define BUILD_BUG_ON_ZERO(e) (sizeof(struct { int:-!!(e); })) +#define __must_be_array(a) \ + BUILD_BUG_ON_ZERO(__builtin_types_compatible_p(typeof(a), typeof(&a[0]))) +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + __must_be_array(arr)) +#define LIST_POISON2 ((void *) 0x00200200 ) + +#if 0 +#include +#else +#include "tiny_nf_nat.h" +#endif +#define ip_nat_multi_range nf_nat_multi_range +#define ip_nat_range nf_nat_range +#define IPTC_HANDLE struct iptc_handle * +#else +/* IPTABLES API version < 1.4.3 */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#include +#else +#if 0 +#include +#else +#include "tiny_nf_nat.h" +#endif +#endif +#define IPTC_HANDLE iptc_handle_t +#endif + +/* IPT_ALIGN was renamed XT_ALIGN in iptables-1.4.11 */ +#ifndef IPT_ALIGN +#define IPT_ALIGN XT_ALIGN +#endif + +#include "../macros.h" +#include "../config.h" +#include "iptcrdr.h" +#include "../upnpglobalvars.h" + +/* local functions declarations */ +static int +addnatrule(int proto, unsigned short eport, + const char * iaddr, unsigned short iport, + const char * rhost); + +static int +add_filter_rule(int proto, const char * rhost, + const char * iaddr, unsigned short iport); + +#ifdef ENABLE_PORT_TRIGGERING +static int +addmasqueraderule(int proto, + unsigned short eport, + const char * iaddr, unsigned short iport, + const char * rhost/*, const char * extif*/); +#endif /* ENABLE_PORT_TRIGGERING */ + +static int +addpeernatrule(int proto, + const char * eaddr, unsigned short eport, + const char * iaddr, unsigned short iport, + const char * rhost, unsigned short rport); + +static int +addpeerdscprule(int proto, unsigned char dscp, + const char * iaddr, unsigned short iport, + const char * rhost, unsigned short rport); + +/* dummy init and shutdown functions + * Only test iptc_init() */ +int init_redirect(void) +{ + IPTC_HANDLE h; + + h = iptc_init("nat"); + if(!h) { + syslog(LOG_ERR, "iptc_init() failed : %s", + iptc_strerror(errno)); + return -1; + } else { +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + } + return 0; +} + +void shutdown_redirect(void) +{ + return; +} + +/* convert an ip address to string */ +static int snprintip(char * dst, size_t size, uint32_t ip) +{ + return snprintf(dst, size, + "%u.%u.%u.%u", ip >> 24, (ip >> 16) & 0xff, + (ip >> 8) & 0xff, ip & 0xff); +} + +/* netfilter cannot store redirection descriptions, so we use our + * own structure to store them */ +struct rdr_desc { + struct rdr_desc * next; + unsigned int timestamp; + unsigned short eport; + short proto; + char str[]; +}; + +/* pointer to the chained list where descriptions are stored */ +static struct rdr_desc * rdr_desc_list = 0; + +/* add a description to the list of redirection descriptions */ +static void +add_redirect_desc(unsigned short eport, int proto, + const char * desc, unsigned int timestamp) +{ + struct rdr_desc * p; + size_t l; + /* set a default description if none given */ + if(!desc) + desc = "miniupnpd"; + l = strlen(desc) + 1; + p = malloc(sizeof(struct rdr_desc) + l); + if(p) + { + p->next = rdr_desc_list; + p->timestamp = timestamp; + p->eport = eport; + p->proto = (short)proto; + memcpy(p->str, desc, l); + rdr_desc_list = p; + } +} + +/* delete a description from the list */ +static void +del_redirect_desc(unsigned short eport, int proto) +{ + struct rdr_desc * p, * last; + p = rdr_desc_list; + last = 0; + while(p) + { + if(p->eport == eport && p->proto == proto) + { + if(!last) + rdr_desc_list = p->next; + else + last->next = p->next; + free(p); + return; + } + last = p; + p = p->next; + } +} + +/* go through the list to find the description */ +static void +get_redirect_desc(unsigned short eport, int proto, + char * desc, int desclen, + unsigned int * timestamp) +{ + struct rdr_desc * p; + for(p = rdr_desc_list; p; p = p->next) + { + if(p->eport == eport && p->proto == (short)proto) + { + if(desc) + strncpy(desc, p->str, desclen); + if(timestamp) + *timestamp = p->timestamp; + return; + } + } + /* if no description was found, return miniupnpd as default */ + if(desc) + strncpy(desc, "miniupnpd", desclen); + if(timestamp) + *timestamp = 0; +} + +/* add_redirect_rule2() */ +int +add_redirect_rule2(const char * ifname, + const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp) +{ + int r; + UNUSED(ifname); + + r = addnatrule(proto, eport, iaddr, iport, rhost); + if(r >= 0) { + add_redirect_desc(eport, proto, desc, timestamp); +#ifdef ENABLE_PORT_TRIGGERING + /* http://www.netfilter.org/documentation/HOWTO/NAT-HOWTO-6.html#ss6.3 + * The default behavior is to alter the connection as little + * as possible, within the constraints of the rule given by + * the user. + * This means we won't remap ports unless we have to. */ + if(iport != eport) { + /* TODO : check if this should be done only with UDP */ + r = addmasqueraderule(proto, eport, iaddr, iport, rhost/*, ifname*/); + if(r < 0) { + syslog(LOG_NOTICE, "add_redirect_rule2(): addmasqueraderule returned %d", r); + } + } +#endif /* ENABLE_PORT_TRIGGERING */ + } + return r; +} + +/* add_peer_redirect_rule2() */ +int +add_peer_redirect_rule2(const char * ifname, + const char * rhost, unsigned short rport, + const char * eaddr, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp) +{ + int r; + UNUSED(ifname); + + r = addpeernatrule(proto, eaddr, eport, iaddr, iport, rhost, rport); + if(r >= 0) + add_redirect_desc(eport, proto, desc, timestamp); + return r; +} + +int +add_peer_dscp_rule2(const char * ifname, + const char * rhost, unsigned short rport, + unsigned char dscp, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp) +{ + int r; + UNUSED(ifname); + UNUSED(desc); + UNUSED(timestamp); + + r = addpeerdscprule(proto, dscp, iaddr, iport, rhost, rport); +/* if(r >= 0) + add_redirect_desc(dscp, proto, desc, timestamp); */ + return r; +} + +int +add_filter_rule2(const char * ifname, + const char * rhost, const char * iaddr, + unsigned short eport, unsigned short iport, + int proto, const char * desc) +{ + UNUSED(ifname); + UNUSED(eport); + UNUSED(desc); + + return add_filter_rule(proto, rhost, iaddr, iport); +} + +/* get_redirect_rule() + * returns -1 if the rule is not found */ +int +get_redirect_rule(const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + return get_nat_redirect_rule(miniupnpd_nat_chain, + ifname, eport, proto, + iaddr, iaddrlen, iport, + desc, desclen, + rhost, rhostlen, + timestamp, packets, bytes); +} + +int +get_nat_redirect_rule(const char * nat_chain_name, const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + int r = -1; + IPTC_HANDLE h; + const struct ipt_entry * e; + const struct ipt_entry_target * target; + const struct ip_nat_multi_range * mr; + const struct ipt_entry_match *match; + UNUSED(ifname); + + h = iptc_init("nat"); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() failed : %s", + "get_nat_redirect_rule", iptc_strerror(errno)); + return -1; + } + if(!iptc_is_chain(nat_chain_name, h)) + { + syslog(LOG_ERR, "chain %s not found", nat_chain_name); + } + else + { +#ifdef IPTABLES_143 + for(e = iptc_first_rule(nat_chain_name, h); + e; + e = iptc_next_rule(e, h)) +#else + for(e = iptc_first_rule(nat_chain_name, &h); + e; + e = iptc_next_rule(e, &h)) +#endif + { + if(proto==e->ip.proto) + { + match = (const struct ipt_entry_match *)&e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + if(eport != info->dpts[0]) + continue; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + if(eport != info->dpts[0]) + continue; + } + target = (void *)e + e->target_offset; + /* target = ipt_get_target(e); */ + mr = (const struct ip_nat_multi_range *)&target->data[0]; + snprintip(iaddr, iaddrlen, ntohl(mr->range[0].min_ip)); + *iport = ntohs(mr->range[0].min.all); + get_redirect_desc(eport, proto, desc, desclen, timestamp); + if(packets) + *packets = e->counters.pcnt; + if(bytes) + *bytes = e->counters.bcnt; + /* rhost */ + if(e->ip.src.s_addr && rhost) { + snprintip(rhost, rhostlen, ntohl(e->ip.src.s_addr)); + } + r = 0; + break; + } + } + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return r; +} + +/* get_redirect_rule_by_index() + * return -1 when the rule was not found */ +int +get_redirect_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + int r = -1; + int i = 0; + IPTC_HANDLE h; + const struct ipt_entry * e; + const struct ipt_entry_target * target; + const struct ip_nat_multi_range * mr; + const struct ipt_entry_match *match; + UNUSED(ifname); + + h = iptc_init("nat"); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() failed : %s", + "get_redirect_rule_by_index", iptc_strerror(errno)); + return -1; + } + if(!iptc_is_chain(miniupnpd_nat_chain, h)) + { + syslog(LOG_ERR, "chain %s not found", miniupnpd_nat_chain); + } + else + { +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_nat_chain, h); + e; + e = iptc_next_rule(e, h)) +#else + for(e = iptc_first_rule(miniupnpd_nat_chain, &h); + e; + e = iptc_next_rule(e, &h)) +#endif + { + if(i==index) + { + *proto = e->ip.proto; + match = (const struct ipt_entry_match *)&e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + *eport = info->dpts[0]; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + *eport = info->dpts[0]; + } + target = (void *)e + e->target_offset; + mr = (const struct ip_nat_multi_range *)&target->data[0]; + snprintip(iaddr, iaddrlen, ntohl(mr->range[0].min_ip)); + *iport = ntohs(mr->range[0].min.all); + get_redirect_desc(*eport, *proto, desc, desclen, timestamp); + if(packets) + *packets = e->counters.pcnt; + if(bytes) + *bytes = e->counters.bcnt; + /* rhost */ + if(rhost && rhostlen > 0) { + if(e->ip.src.s_addr) { + snprintip(rhost, rhostlen, ntohl(e->ip.src.s_addr)); + } else { + rhost[0] = '\0'; + } + } + r = 0; + break; + } + i++; + } + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return r; +} + +/* get_peer_rule_by_index() + * return -1 when the rule was not found */ +int +get_peer_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, unsigned short * rport, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + int r = -1; + int i = 0; + IPTC_HANDLE h; + const struct ipt_entry * e; + const struct ipt_entry_target * target; + const struct ip_nat_multi_range * mr; + const struct ipt_entry_match *match; + UNUSED(ifname); + + h = iptc_init("nat"); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() failed : %s", + "get_peer_rule_by_index", iptc_strerror(errno)); + return -1; + } + if(!iptc_is_chain(miniupnpd_nat_postrouting_chain, h)) + { + syslog(LOG_ERR, "chain %s not found", miniupnpd_nat_postrouting_chain); + } + else + { +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_nat_postrouting_chain, h); + e; + e = iptc_next_rule(e, h)) +#else + for(e = iptc_first_rule(miniupnpd_nat_postrouting_chain, &h); + e; + e = iptc_next_rule(e, &h)) +#endif + { + if(i==index) + { + *proto = e->ip.proto; + match = (const struct ipt_entry_match *)&e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + if (rport) + *rport = info->dpts[0]; + if (iport) + *iport = info->spts[0]; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + if (rport) + *rport = info->dpts[0]; + if (iport) + *iport = info->spts[0]; + } + target = (void *)e + e->target_offset; + mr = (const struct ip_nat_multi_range *)&target->data[0]; + *eport = ntohs(mr->range[0].min.all); + get_redirect_desc(*eport, *proto, desc, desclen, timestamp); + if(packets) + *packets = e->counters.pcnt; + if(bytes) + *bytes = e->counters.bcnt; + /* rhost */ + if(rhost && rhostlen > 0) { + if(e->ip.dst.s_addr) { + snprintip(rhost, rhostlen, ntohl(e->ip.dst.s_addr)); + } else { + rhost[0] = '\0'; + } + } + if(iaddr && iaddrlen > 0) { + if(e->ip.src.s_addr) { + snprintip(iaddr, iaddrlen, ntohl(e->ip.src.s_addr)); + } else { + rhost[0] = '\0'; + } + } + r = 0; + break; + } + i++; + } + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return r; +} + +/* delete_rule_and_commit() : + * subfunction used in delete_redirect_and_filter_rules() */ +static int +delete_rule_and_commit(unsigned int index, IPTC_HANDLE h, + const char * miniupnpd_chain, + const char * logcaller) +{ + int r = 0; +#ifdef IPTABLES_143 + if(!iptc_delete_num_entry(miniupnpd_chain, index, h)) +#else + if(!iptc_delete_num_entry(miniupnpd_chain, index, &h)) +#endif + { + syslog(LOG_ERR, "%s() : iptc_delete_num_entry(): %s\n", + logcaller, iptc_strerror(errno)); + r = -1; + } +#ifdef IPTABLES_143 + else if(!iptc_commit(h)) +#else + else if(!iptc_commit(&h)) +#endif + { + syslog(LOG_ERR, "%s() : iptc_commit(): %s\n", + logcaller, iptc_strerror(errno)); + r = -1; + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return r; +} + +/* delete_redirect_and_filter_rules() + */ +int +delete_redirect_and_filter_rules(unsigned short eport, int proto) +{ + int r = -1, r2 = -1; + unsigned index = 0; + unsigned i = 0; + IPTC_HANDLE h; + const struct ipt_entry * e; + const struct ipt_entry_target * target; + const struct ip_nat_multi_range * mr; + const struct ipt_entry_match *match; + unsigned short iport = 0; + uint32_t iaddr = 0; + + h = iptc_init("nat"); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() failed : %s", + "delete_redirect_and_filter_rules", iptc_strerror(errno)); + return -1; + } + /* First step : find the right nat rule */ + if(!iptc_is_chain(miniupnpd_nat_chain, h)) + { + syslog(LOG_ERR, "chain %s not found", miniupnpd_nat_chain); + } + else + { +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_nat_chain, h); + e; + e = iptc_next_rule(e, h), i++) +#else + for(e = iptc_first_rule(miniupnpd_nat_chain, &h); + e; + e = iptc_next_rule(e, &h), i++) +#endif + { + if(proto==e->ip.proto) + { + match = (const struct ipt_entry_match *)&e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + if(eport != info->dpts[0]) + continue; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + if(eport != info->dpts[0]) + continue; + } + /* get the index, the internal address and the internal port + * of the rule */ + index = i; + target = (void *)e + e->target_offset; + mr = (const struct ip_nat_multi_range *)&target->data[0]; + iaddr = mr->range[0].min_ip; + iport = ntohs(mr->range[0].min.all); + r = 0; + break; + } + } + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + if(r == 0) + { + syslog(LOG_INFO, "Trying to delete nat rule at index %u", index); + /* Now delete both rules */ + /* first delete the nat rule */ + h = iptc_init("nat"); + if(h) + { + r = delete_rule_and_commit(index, h, miniupnpd_nat_chain, "delete_redirect_rule"); + } + if((r == 0) && (h = iptc_init("filter"))) + { + i = 0; + /* we must find the right index for the filter rule */ +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_forward_chain, h); + e; + e = iptc_next_rule(e, h), i++) +#else + for(e = iptc_first_rule(miniupnpd_forward_chain, &h); + e; + e = iptc_next_rule(e, &h), i++) +#endif + { + if(proto==e->ip.proto) + { + match = (const struct ipt_entry_match *)&e->elems; + /*syslog(LOG_DEBUG, "filter rule #%u: %s %s", + i, match->u.user.name, inet_ntoa(e->ip.dst));*/ + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + if(iport != info->dpts[0]) + continue; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + if(iport != info->dpts[0]) + continue; + } + if(iaddr != e->ip.dst.s_addr) + continue; + index = i; + syslog(LOG_INFO, "Trying to delete filter rule at index %u", index); + r = delete_rule_and_commit(index, h, miniupnpd_forward_chain, "delete_filter_rule"); + h = NULL; + break; + } + } + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + } + + /*delete PEER rule*/ + if((h = iptc_init("nat"))) + { + i = 0; + /* we must find the right index for the filter rule */ +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_nat_postrouting_chain, h); + e; + e = iptc_next_rule(e, h), i++) +#else + for(e = iptc_first_rule(miniupnpd_nat_postrouting_chain, &h); + e; + e = iptc_next_rule(e, &h), i++) +#endif + { + if(proto==e->ip.proto) + { + target = (void *)e + e->target_offset; + mr = (const struct ip_nat_multi_range *)&target->data[0]; + syslog(LOG_DEBUG, "postrouting rule #%u: %s %s %hu", + i, target->u.user.name, inet_ntoa(e->ip.src), ntohs(mr->range[0].min.all)); + /* target->u.user.name SNAT / MASQUERADE */ + if (eport != ntohs(mr->range[0].min.all)) { + continue; + } + iaddr = e->ip.src.s_addr; + match = (const struct ipt_entry_match *)&e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + iport = info->spts[0]; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + iport = info->dpts[0]; + } + + index = i; + syslog(LOG_INFO, "Trying to delete peer rule at index %u", index); + r2 = delete_rule_and_commit(index, h, miniupnpd_nat_postrouting_chain, "delete_peer_rule"); + h = NULL; + break; + } + } + } + + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + /*delete DSCP rule*/ + if((r2==0)&&(h = iptc_init("mangle"))) + { + i = 0; + index = -1; + /* we must find the right index for the filter rule */ +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_nat_chain, h); + e; + e = iptc_next_rule(e, h), i++) +#else + for(e = iptc_first_rule(miniupnpd_nat_chain, &h); + e; + e = iptc_next_rule(e, &h), i++) +#endif + { + if(proto==e->ip.proto) + { + match = (const struct ipt_entry_match *)&e->elems; + /*syslog(LOG_DEBUG, "filter rule #%u: %s %s", + i, match->u.user.name, inet_ntoa(e->ip.dst));*/ + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + if(iport != info->spts[0]) + continue; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + if(iport != info->spts[0]) + continue; + } + if(iaddr != e->ip.src.s_addr) + continue; + index = i; + syslog(LOG_INFO, "Trying to delete dscp rule at index %u", index); + r2 = delete_rule_and_commit(index, h, miniupnpd_nat_chain, "delete_dscp_rule"); + h = NULL; + break; + } + } + if (h) + #ifdef IPTABLES_143 + iptc_free(h); + #else + iptc_free(&h); + #endif + } + + del_redirect_desc(eport, proto); + return r*r2; +} + +/* ==================================== */ +/* TODO : add the -m state --state NEW,ESTABLISHED,RELATED + * only for the filter rule */ +static struct ipt_entry_match * +get_tcp_match(unsigned short dport, unsigned short sport) +{ + struct ipt_entry_match *match; + struct ipt_tcp * tcpinfo; + size_t size; + size = IPT_ALIGN(sizeof(struct ipt_entry_match)) + + IPT_ALIGN(sizeof(struct ipt_tcp)); + match = calloc(1, size); + match->u.match_size = size; + strncpy(match->u.user.name, "tcp", sizeof(match->u.user.name)); + tcpinfo = (struct ipt_tcp *)match->data; + if (sport == 0) { + tcpinfo->spts[0] = 0; /* all source ports */ + tcpinfo->spts[1] = 0xFFFF; + } else { + tcpinfo->spts[0] = sport; /* specified source port */ + tcpinfo->spts[1] = sport; + } + if (dport == 0) { + tcpinfo->dpts[0] = 0; /* all destination ports */ + tcpinfo->dpts[1] = 0xFFFF; + } else { + tcpinfo->dpts[0] = dport; /* specified destination port */ + tcpinfo->dpts[1] = dport; + } + return match; +} + +static struct ipt_entry_match * +get_udp_match(unsigned short dport, unsigned short sport) +{ + struct ipt_entry_match *match; + struct ipt_udp * udpinfo; + size_t size; + size = IPT_ALIGN(sizeof(struct ipt_entry_match)) + + IPT_ALIGN(sizeof(struct ipt_udp)); + match = calloc(1, size); + match->u.match_size = size; + strncpy(match->u.user.name, "udp", sizeof(match->u.user.name)); + udpinfo = (struct ipt_udp *)match->data; + if (sport == 0) { + udpinfo->spts[0] = 0; /* all source ports */ + udpinfo->spts[1] = 0xFFFF; + } else { + udpinfo->spts[0] = sport; /* specified source port */ + udpinfo->spts[1] = sport; + } + if (dport == 0) { + udpinfo->dpts[0] = 0; /* all destination ports */ + udpinfo->dpts[1] = 0xFFFF; + } else { + udpinfo->dpts[0] = dport; /* specified destination port */ + udpinfo->dpts[1] = dport; + } + return match; +} + +static struct ipt_entry_target * +get_dnat_target(const char * daddr, unsigned short dport) +{ + struct ipt_entry_target * target; + struct ip_nat_multi_range * mr; + struct ip_nat_range * range; + size_t size; + + size = IPT_ALIGN(sizeof(struct ipt_entry_target)) + + IPT_ALIGN(sizeof(struct ip_nat_multi_range)); + target = calloc(1, size); + target->u.target_size = size; + strncpy(target->u.user.name, "DNAT", sizeof(target->u.user.name)); + /* one ip_nat_range already included in ip_nat_multi_range */ + mr = (struct ip_nat_multi_range *)&target->data[0]; + mr->rangesize = 1; + range = &mr->range[0]; + range->min_ip = range->max_ip = inet_addr(daddr); + range->flags |= IP_NAT_RANGE_MAP_IPS; + range->min.all = range->max.all = htons(dport); + range->flags |= IP_NAT_RANGE_PROTO_SPECIFIED; + return target; +} + +static struct ipt_entry_target * +get_snat_target(const char * saddr, unsigned short sport) +{ + struct ipt_entry_target * target; + struct ip_nat_multi_range * mr; + struct ip_nat_range * range; + size_t size; + + size = IPT_ALIGN(sizeof(struct ipt_entry_target)) + + IPT_ALIGN(sizeof(struct ip_nat_multi_range)); + target = calloc(1, size); + target->u.target_size = size; + strncpy(target->u.user.name, "SNAT", sizeof(target->u.user.name)); + /* one ip_nat_range already included in ip_nat_multi_range */ + mr = (struct ip_nat_multi_range *)&target->data[0]; + mr->rangesize = 1; + range = &mr->range[0]; + range->min_ip = range->max_ip = inet_addr(saddr); + range->flags |= IP_NAT_RANGE_MAP_IPS; + range->min.all = range->max.all = htons(sport); + range->flags |= IP_NAT_RANGE_PROTO_SPECIFIED; + return target; +} + +static struct ipt_entry_target * +get_dscp_target(unsigned char dscp) +{ + struct ipt_entry_target * target; + struct xt_DSCP_info * di; + size_t size; + + size = IPT_ALIGN(sizeof(struct ipt_entry_target)) + + IPT_ALIGN(sizeof(struct xt_DSCP_info)); + target = calloc(1, size); + target->u.target_size = size; + strncpy(target->u.user.name, "DSCP", sizeof(target->u.user.name)); + /* one ip_nat_range already included in ip_nat_multi_range */ + di = (struct xt_DSCP_info *)&target->data[0]; + di->dscp=dscp; + return target; +} + +#ifdef ENABLE_PORT_TRIGGERING +static struct ipt_entry_target * +get_masquerade_target(unsigned short port) +{ + struct ipt_entry_target * target; + struct ip_nat_multi_range * mr; + struct ip_nat_range * range; + size_t size; + + size = IPT_ALIGN(sizeof(struct ipt_entry_target)) + + IPT_ALIGN(sizeof(struct ip_nat_multi_range)); + target = calloc(1, size); + target->u.target_size = size; + strncpy(target->u.user.name, "MASQUERADE", sizeof(target->u.user.name)); + /* one ip_nat_range already included in ip_nat_multi_range */ + mr = (struct ip_nat_multi_range *)&target->data[0]; + mr->rangesize = 1; + range = &mr->range[0]; + range->min.tcp.port = range->max.tcp.port = htons(port); + /*range->min.all = range->max.all = htons(port);*/ + range->flags |= IP_NAT_RANGE_PROTO_SPECIFIED; + return target; +} +#endif /* ENABLE_PORT_TRIGGERING */ + +/* iptc_init_verify_and_append() + * return 0 on success, -1 on failure */ +static int +iptc_init_verify_and_append(const char * table, + const char * miniupnpd_chain, + struct ipt_entry * e, + const char * logcaller) +{ + IPTC_HANDLE h; + h = iptc_init(table); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() error : %s\n", + logcaller, iptc_strerror(errno)); + return -1; + } + if(!iptc_is_chain(miniupnpd_chain, h)) + { + syslog(LOG_ERR, "%s() : chain %s not found", + logcaller, miniupnpd_chain); + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return -1; + } + /* iptc_insert_entry(miniupnpd_chain, e, n, h/&h) could also be used */ +#ifdef IPTABLES_143 + if(!iptc_append_entry(miniupnpd_chain, e, h)) +#else + if(!iptc_append_entry(miniupnpd_chain, e, &h)) +#endif + { + syslog(LOG_ERR, "%s() : iptc_append_entry() error : %s\n", + logcaller, iptc_strerror(errno)); + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return -1; + } +#ifdef IPTABLES_143 + if(!iptc_commit(h)) +#else + if(!iptc_commit(&h)) +#endif + { + syslog(LOG_ERR, "%s() : iptc_commit() error : %s\n", + logcaller, iptc_strerror(errno)); + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return -1; + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return 0; +} + +/* add nat rule + * iptables -t nat -A MINIUPNPD -p [-s ] --dport -j DNAT --to : + * */ +static int +addnatrule(int proto, unsigned short eport, + const char * iaddr, unsigned short iport, + const char * rhost) +{ + int r = 0; + struct ipt_entry * e; + struct ipt_entry * tmp; + struct ipt_entry_match *match = NULL; + struct ipt_entry_target *target = NULL; + + e = calloc(1, sizeof(struct ipt_entry)); + if(!e) { + syslog(LOG_ERR, "%s: calloc(%d) error", "addnatrule", + (int)sizeof(struct ipt_entry)); + return -1; + } + e->ip.proto = proto; + if(proto == IPPROTO_TCP) { + match = get_tcp_match(eport, 0); + } else { + match = get_udp_match(eport, 0); + } + e->nfcache = NFC_IP_DST_PT; + target = get_dnat_target(iaddr, iport); + e->nfcache |= NFC_UNKNOWN; + tmp = realloc(e, sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size); + if(!tmp) { + syslog(LOG_ERR, "%s: realloc(%d) error", "addnatrule", + (int)(sizeof(struct ipt_entry) + match->u.match_size + target->u.target_size)); + free(e); + free(match); + free(target); + return -1; + } + e = tmp; + memcpy(e->elems, match, match->u.match_size); + memcpy(e->elems + match->u.match_size, target, target->u.target_size); + e->target_offset = sizeof(struct ipt_entry) + + match->u.match_size; + e->next_offset = sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size; + /* remote host */ + if(rhost && (rhost[0] != '\0') && (0 != strcmp(rhost, "*"))) { + e->ip.src.s_addr = inet_addr(rhost); + e->ip.smsk.s_addr = INADDR_NONE; + } + + r = iptc_init_verify_and_append("nat", miniupnpd_nat_chain, e, "addnatrule"); + free(target); + free(match); + free(e); + return r; +} + +/* for "Port Triggering" + * Section 2.5.16 figure 2.2 in UPnP-gw-WANIPConnection-v2-Service.pdf + * + * When a control point creates a port forwarding rule with AddPortMapping() + * action for inbound traffic , this rule MUST also be applied when NAT port + * triggering occurs for outbound traffic. + * + * iptables -t nat -A MINIUPNPD-POSTROUTING {-o } -s + * -p [-d ] --sport -j MASQUERADE --to-ports + */ +#ifdef ENABLE_PORT_TRIGGERING +static int +addmasqueraderule(int proto, + unsigned short eport, + const char * iaddr, unsigned short iport, + const char * rhost/*, const char * extif*/) +{ + int r = 0; + struct ipt_entry * e; + struct ipt_entry * tmp; + struct ipt_entry_match *match = NULL; + struct ipt_entry_target *target = NULL; + + e = calloc(1, sizeof(struct ipt_entry)); + if(!e) { + syslog(LOG_ERR, "%s: calloc(%d) error", "addmasqueraderule", + (int)sizeof(struct ipt_entry)); + return -1; + } + e->ip.proto = proto; + if(proto == IPPROTO_TCP) { + match = get_tcp_match(0, iport); + } else { + match = get_udp_match(0, iport); + } + e->nfcache = NFC_IP_DST_PT; + target = get_masquerade_target(eport); + e->nfcache |= NFC_UNKNOWN; + tmp = realloc(e, sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size); + if(!tmp) { + syslog(LOG_ERR, "%s: realloc(%d) error", "addmasqueraderule", + (int)(sizeof(struct ipt_entry) + match->u.match_size + target->u.target_size)); + free(e); + free(match); + free(target); + return -1; + } + e = tmp; + memcpy(e->elems, match, match->u.match_size); + memcpy(e->elems + match->u.match_size, target, target->u.target_size); + e->target_offset = sizeof(struct ipt_entry) + + match->u.match_size; + e->next_offset = sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size; +#if 0 + /* do not add outiface (-o) to rule, as the MINIUPNPD-POSTROUTING chain + * should already have matched it */ + if(extif != NULL) { + strncpy(e->ip.outiface, extif, sizeof(e->ip.outiface)); + memset(e->ip.outiface_mask, 0xff, strlen(e->ip.outiface) + 1);/* Include nul-terminator in match */ + } +#endif + /* internal host */ + if(iaddr && (iaddr[0] != '\0') && (0 != strcmp(iaddr, "*"))) + { + e->ip.src.s_addr = inet_addr(iaddr); + e->ip.smsk.s_addr = INADDR_NONE; + } + /* remote host */ + if(rhost && (rhost[0] != '\0') && (0 != strcmp(rhost, "*"))) { + e->ip.dst.s_addr = inet_addr(rhost); + e->ip.dmsk.s_addr = INADDR_NONE; + } + + r = iptc_init_verify_and_append("nat", miniupnpd_nat_postrouting_chain, e, "addmasqueraderule"); + free(target); + free(match); + free(e); + return r; +} +#endif /* ENABLE_PORT_TRIGGERING */ + +/* called by add_peer_redirect_rule2() + * + * iptables -t nat -A MINIUPNPD-POSTROUTING -s -d + * -p --sport --dport -j SNAT + * --to-source : */ +static int +addpeernatrule(int proto, + const char * eaddr, unsigned short eport, + const char * iaddr, unsigned short iport, + const char * rhost, unsigned short rport) +{ + int r = 0; + struct ipt_entry * e; + struct ipt_entry * tmp; + struct ipt_entry_match *match = NULL; + struct ipt_entry_target *target = NULL; + + e = calloc(1, sizeof(struct ipt_entry)); + if(!e) { + syslog(LOG_ERR, "%s: calloc(%d) error", "addpeernatrule", + (int)sizeof(struct ipt_entry)); + return -1; + } + e->ip.proto = proto; + /* TODO: Fill port matches and SNAT */ + if(proto == IPPROTO_TCP) { + match = get_tcp_match(rport, iport); + } else { + match = get_udp_match(rport, iport); + } + e->nfcache = NFC_IP_DST_PT | NFC_IP_SRC_PT; + target = get_snat_target(eaddr, eport); + e->nfcache |= NFC_UNKNOWN; + tmp = realloc(e, sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size); + if(!tmp) { + syslog(LOG_ERR, "%s: realloc(%d) error", "addpeernatrule", + (int)(sizeof(struct ipt_entry) + match->u.match_size + target->u.target_size)); + free(e); + free(match); + free(target); + return -1; + } + e = tmp; + memcpy(e->elems, match, match->u.match_size); + memcpy(e->elems + match->u.match_size, target, target->u.target_size); + e->target_offset = sizeof(struct ipt_entry) + + match->u.match_size; + e->next_offset = sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size; + + /* internal host */ + if(iaddr && (iaddr[0] != '\0') && (0 != strcmp(iaddr, "*"))) + { + e->ip.src.s_addr = inet_addr(iaddr); + e->ip.smsk.s_addr = INADDR_NONE; + } + /* remote host */ + if(rhost && (rhost[0] != '\0') && (0 != strcmp(rhost, "*"))) + { + e->ip.dst.s_addr = inet_addr(rhost); + e->ip.dmsk.s_addr = INADDR_NONE; + } + + r = iptc_init_verify_and_append("nat", miniupnpd_nat_postrouting_chain, e, "addpeernatrule"); + free(target); + free(match); + free(e); + return r; +} + +/* called by add_peer_dscp_rule2() + * iptables -t mangle -A MINIUPNPD -s -d + * -p --sport --dport -j DSCP + * --set-dscp 0xXXXX */ +static int +addpeerdscprule(int proto, unsigned char dscp, + const char * iaddr, unsigned short iport, + const char * rhost, unsigned short rport) +{ + int r = 0; + struct ipt_entry * e; + struct ipt_entry * tmp; + struct ipt_entry_match *match = NULL; + struct ipt_entry_target *target = NULL; + + e = calloc(1, sizeof(struct ipt_entry)); + if(!e) { + syslog(LOG_ERR, "%s: calloc(%d) error", "addpeerdscprule", + (int)sizeof(struct ipt_entry)); + return -1; + } + e->ip.proto = proto; + /* TODO: Fill port matches and SNAT */ + if(proto == IPPROTO_TCP) { + match = get_tcp_match(rport, iport); + } else { + match = get_udp_match(rport, iport); + } + e->nfcache = NFC_IP_DST_PT | NFC_IP_SRC_PT; + target = get_dscp_target(dscp); + e->nfcache |= NFC_UNKNOWN; + tmp = realloc(e, sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size); + if(!tmp) { + syslog(LOG_ERR, "%s: realloc(%d) error", "addpeerdscprule", + (int)(sizeof(struct ipt_entry) + match->u.match_size + target->u.target_size)); + free(e); + free(match); + free(target); + return -1; + } + e = tmp; + memcpy(e->elems, match, match->u.match_size); + memcpy(e->elems + match->u.match_size, target, target->u.target_size); + e->target_offset = sizeof(struct ipt_entry) + + match->u.match_size; + e->next_offset = sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size; + + /* internal host */ + if(iaddr && (iaddr[0] != '\0') && (0 != strcmp(iaddr, "*"))) + { + e->ip.src.s_addr = inet_addr(iaddr); + e->ip.smsk.s_addr = INADDR_NONE; + } + /* remote host */ + if(rhost && (rhost[0] != '\0') && (0 != strcmp(rhost, "*"))) + { + e->ip.dst.s_addr = inet_addr(rhost); + e->ip.dmsk.s_addr = INADDR_NONE; + } + + r = iptc_init_verify_and_append("mangle", miniupnpd_nat_chain, e, + "addpeerDSCPrule"); + free(target); + free(match); + free(e); + return r; +} + + +/* ================================= */ +static struct ipt_entry_target * +get_accept_target(void) +{ + struct ipt_entry_target * target = NULL; + size_t size; + size = IPT_ALIGN(sizeof(struct ipt_entry_target)) + + IPT_ALIGN(sizeof(int)); + target = calloc(1, size); + target->u.user.target_size = size; + strncpy(target->u.user.name, "ACCEPT", sizeof(target->u.user.name)); + return target; +} + +/* add_filter_rule() + * iptables -t filter -A MINIUPNPD [-s ] -p -d --dport -j ACCEPT */ +static int +add_filter_rule(int proto, const char * rhost, + const char * iaddr, unsigned short iport) +{ + int r = 0; + struct ipt_entry * e; + struct ipt_entry * tmp; + struct ipt_entry_match *match = NULL; + struct ipt_entry_target *target = NULL; + + e = calloc(1, sizeof(struct ipt_entry)); + if(!e) { + syslog(LOG_ERR, "%s: calloc(%d) error", "add_filter_rule", + (int)sizeof(struct ipt_entry)); + return -1; + } + e->ip.proto = proto; + if(proto == IPPROTO_TCP) { + match = get_tcp_match(iport,0); + } else { + match = get_udp_match(iport,0); + } + e->nfcache = NFC_IP_DST_PT; + e->ip.dst.s_addr = inet_addr(iaddr); + e->ip.dmsk.s_addr = INADDR_NONE; + target = get_accept_target(); + e->nfcache |= NFC_UNKNOWN; + tmp = realloc(e, sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size); + if(!tmp) { + syslog(LOG_ERR, "%s: realloc(%d) error", "add_filter_rule", + (int)(sizeof(struct ipt_entry) + match->u.match_size + target->u.target_size)); + free(e); + free(match); + free(target); + return -1; + } + e = tmp; + memcpy(e->elems, match, match->u.match_size); + memcpy(e->elems + match->u.match_size, target, target->u.target_size); + e->target_offset = sizeof(struct ipt_entry) + + match->u.match_size; + e->next_offset = sizeof(struct ipt_entry) + + match->u.match_size + + target->u.target_size; + /* remote host */ + if(rhost && (rhost[0] != '\0') && (0 != strcmp(rhost, "*"))) + { + e->ip.src.s_addr = inet_addr(rhost); + e->ip.smsk.s_addr = INADDR_NONE; + } + + r = iptc_init_verify_and_append("filter", miniupnpd_forward_chain, e, "add_filter_rule"); + free(target); + free(match); + free(e); + return r; +} + +/* return an (malloc'ed) array of "external" port for which there is + * a port mapping. number is the size of the array */ +unsigned short * +get_portmappings_in_range(unsigned short startport, unsigned short endport, + int proto, unsigned int * number) +{ + unsigned short * array; + unsigned int capacity; + unsigned short eport; + IPTC_HANDLE h; + const struct ipt_entry * e; + const struct ipt_entry_match *match; + + *number = 0; + capacity = 128; + array = calloc(capacity, sizeof(unsigned short)); + if(!array) + { + syslog(LOG_ERR, "%s() : calloc error", "get_portmappings_in_range"); + return NULL; + } + + h = iptc_init("nat"); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() failed : %s", + "get_portmappings_in_range", iptc_strerror(errno)); + free(array); + return NULL; + } + if(!iptc_is_chain(miniupnpd_nat_chain, h)) + { + syslog(LOG_ERR, "chain %s not found", miniupnpd_nat_chain); + free(array); + array = NULL; + } + else + { +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_nat_chain, h); + e; + e = iptc_next_rule(e, h)) +#else + for(e = iptc_first_rule(miniupnpd_nat_chain, &h); + e; + e = iptc_next_rule(e, &h)) +#endif + { + if(proto == e->ip.proto) + { + match = (const struct ipt_entry_match *)&e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + eport = info->dpts[0]; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + eport = info->dpts[0]; + } + if(startport <= eport && eport <= endport) + { + if(*number >= capacity) + { + unsigned short * tmp; + /* need to increase the capacity of the array */ + tmp = realloc(array, sizeof(unsigned short)*capacity); + if(!tmp) + { + syslog(LOG_ERR, "get_portmappings_in_range() : realloc(%u) error", + (unsigned)sizeof(unsigned short)*capacity); + *number = 0; + free(array); + array = NULL; + break; + } + array = tmp; + } + array[*number] = eport; + (*number)++; + } + } + } + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return array; +} + +int +update_portmapping_desc_timestamp(const char * ifname, + unsigned short eport, int proto, + const char * desc, unsigned int timestamp) +{ + UNUSED(ifname); + del_redirect_desc(eport, proto); + add_redirect_desc(eport, proto, desc, timestamp); + return 0; +} + +static int +update_rule_and_commit(const char * table, const char * chain, + unsigned index, const struct ipt_entry * e) +{ + IPTC_HANDLE h; + int r = 0; + + h = iptc_init(table); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() failed : %s", + "update_rule_and_commit", iptc_strerror(errno)); + return -1; + } +#ifdef IPTABLES_143 + if(!iptc_replace_entry(chain, e, index, h)) +#else + if(!iptc_replace_entry(chain, e, index, &h)) +#endif + { + syslog(LOG_ERR, "%s(): iptc_replace_entry: %s", + "update_rule_and_commit", iptc_strerror(errno)); + r = -1; + } +#ifdef IPTABLES_143 + else if(!iptc_commit(h)) +#else + else if(!iptc_commit(&h)) +#endif + { + syslog(LOG_ERR, "%s(): iptc_commit: %s", + "update_rule_and_commit", iptc_strerror(errno)); + r = -1; + } +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return r; +} + +int +update_portmapping(const char * ifname, unsigned short eport, int proto, + unsigned short iport, const char * desc, + unsigned int timestamp) +{ + int r = 0; + int found = 0; + unsigned index = 0; + unsigned i = 0; + IPTC_HANDLE h; + const struct ipt_entry * e; + struct ipt_entry * new_e = NULL; + size_t entry_len; + const struct ipt_entry_target * target; + struct ip_nat_multi_range * mr; + const struct ipt_entry_match *match; + uint32_t iaddr = 0; + unsigned short old_iport = 0; + + h = iptc_init("nat"); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() failed : %s", + "update_portmapping", iptc_strerror(errno)); + return -1; + } + /* First step : find the right nat rule */ + if(!iptc_is_chain(miniupnpd_nat_chain, h)) + { + syslog(LOG_ERR, "chain %s not found", miniupnpd_nat_chain); + r = -1; + } + else + { +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_nat_chain, h); + e; + e = iptc_next_rule(e, h), i++) +#else + for(e = iptc_first_rule(miniupnpd_nat_chain, &h); + e; + e = iptc_next_rule(e, &h), i++) +#endif + { + if(proto==e->ip.proto) + { + match = (const struct ipt_entry_match *)&e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + if(eport != info->dpts[0]) + continue; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + if(eport != info->dpts[0]) + continue; + } + /* we found the right rule */ + found = 1; + index = i; + target = (void *)e + e->target_offset; + mr = (struct ip_nat_multi_range *)&target->data[0]; + iaddr = mr->range[0].min_ip; + old_iport = ntohs(mr->range[0].min.all); + entry_len = sizeof(struct ipt_entry) + match->u.match_size + target->u.target_size; + new_e = malloc(entry_len); + if(new_e == NULL) { + syslog(LOG_ERR, "%s: malloc(%u) error", + "update_portmapping", (unsigned)entry_len); + r = -1; + } + else + { + memcpy(new_e, e, entry_len); + } + break; + } + } + } +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + if(!found || r < 0) + return -1; + syslog(LOG_INFO, "Trying to update nat rule at index %u", index); + target = (void *)new_e + new_e->target_offset; + mr = (struct ip_nat_multi_range *)&target->data[0]; + mr->range[0].min.all = mr->range[0].max.all = htons(iport); + /* first update the nat rule */ + r = update_rule_and_commit("nat", miniupnpd_nat_chain, index, new_e); + free(new_e); new_e = NULL; + if(r < 0) + return r; + + /* update filter rule */ + h = iptc_init("filter"); + if(!h) + { + syslog(LOG_ERR, "%s() : iptc_init() failed : %s", + "update_portmapping", iptc_strerror(errno)); + return -1; + } + i = 0; found = 0; + if(!iptc_is_chain(miniupnpd_forward_chain, h)) + { + syslog(LOG_ERR, "chain %s not found", miniupnpd_forward_chain); + } + else + { + /* we must find the right index for the filter rule */ +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_forward_chain, h); + e; + e = iptc_next_rule(e, h), i++) +#else + for(e = iptc_first_rule(miniupnpd_forward_chain, &h); + e; + e = iptc_next_rule(e, &h), i++) +#endif + { + if(proto!=e->ip.proto) + continue; + target = (void *)e + e->target_offset; + match = (const struct ipt_entry_match *)&e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + const struct ipt_tcp * info; + info = (const struct ipt_tcp *)match->data; + if(old_iport != info->dpts[0]) + continue; + } + else + { + const struct ipt_udp * info; + info = (const struct ipt_udp *)match->data; + if(old_iport != info->dpts[0]) + continue; + } + if(iaddr != e->ip.dst.s_addr) + continue; + index = i; + found = 1; + entry_len = sizeof(struct ipt_entry) + match->u.match_size + target->u.target_size; + new_e = malloc(entry_len); + if(new_e == NULL) { + syslog(LOG_ERR, "%s: malloc(%u) error", + "update_portmapping", (unsigned)entry_len); + r = -1; + } else { + memcpy(new_e, e, entry_len); + } + break; + } + } +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + if(!found || r < 0) + return -1; + + syslog(LOG_INFO, "Trying to update filter rule at index %u", index); + match = (struct ipt_entry_match *)&new_e->elems; + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + struct ipt_tcp * info; + info = (struct ipt_tcp *)match->data; + info->dpts[0] = info->dpts[1] = iport; + } + else + { + struct ipt_udp * info; + info = (struct ipt_udp *)match->data; + info->dpts[0] = info->dpts[1] = iport; + } + r = update_rule_and_commit("filter", miniupnpd_forward_chain, index, new_e); + free(new_e); new_e = NULL; + if(r < 0) + return r; + + return update_portmapping_desc_timestamp(ifname, eport, proto, desc, timestamp); +} + +/* ================================ */ +#ifdef DEBUG +static int +print_match(const struct ipt_entry_match *match) +{ + printf("match %s :\n", match->u.user.name); + if(0 == strncmp(match->u.user.name, "tcp", IPT_FUNCTION_MAXNAMELEN)) + { + struct ipt_tcp * tcpinfo; + tcpinfo = (struct ipt_tcp *)match->data; + printf(" srcport = %hu:%hu dstport = %hu:%hu\n", + tcpinfo->spts[0], tcpinfo->spts[1], + tcpinfo->dpts[0], tcpinfo->dpts[1]); + } + else if(0 == strncmp(match->u.user.name, "udp", IPT_FUNCTION_MAXNAMELEN)) + { + struct ipt_udp * udpinfo; + udpinfo = (struct ipt_udp *)match->data; + printf(" srcport = %hu:%hu dstport = %hu:%hu\n", + udpinfo->spts[0], udpinfo->spts[1], + udpinfo->dpts[0], udpinfo->dpts[1]); + } + return 0; +} + +static void +print_iface(const char * iface, const unsigned char * mask, int invert) +{ + unsigned i; + if(mask[0] == 0) + return; + if(invert) + printf("! "); + for(i=0; i> 24, (ip >> 16) & 0xff, + (ip >> 8) & 0xff, ip & 0xff); +} + +/* for debug */ +/* read the "filter" and "nat" tables */ +int +list_redirect_rule(const char * ifname) +{ + IPTC_HANDLE h; + const struct ipt_entry * e; + const struct ipt_entry_target * target; + const struct ip_nat_multi_range * mr; + const char * target_str; + char addr[16], mask[16]; + unsigned int index; + (void)ifname; + + h = iptc_init("nat"); + if(!h) + { + printf("iptc_init() error : %s\n", iptc_strerror(errno)); + return -1; + } + if(!iptc_is_chain(miniupnpd_nat_chain, h)) + { + printf("chain %s not found\n", miniupnpd_nat_chain); +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + return -1; + } + index = 0; +#ifdef IPTABLES_143 + for(e = iptc_first_rule(miniupnpd_nat_chain, h); + e; + e = iptc_next_rule(e, h)) + { + target_str = iptc_get_target(e, h); +#else + for(e = iptc_first_rule(miniupnpd_nat_chain, &h); + e; + e = iptc_next_rule(e, &h)) + { + target_str = iptc_get_target(e, &h); +#endif + printf("=== rule #%u ===\n", index); + inet_ntop(AF_INET, &e->ip.src, addr, sizeof(addr)); + inet_ntop(AF_INET, &e->ip.smsk, mask, sizeof(mask)); + printf("src = %s%s/%s\t", (e->ip.invflags & IPT_INV_SRCIP)?"! ":"", + /*inet_ntoa(e->ip.src), inet_ntoa(e->ip.smsk)*/ + addr, mask); + inet_ntop(AF_INET, &e->ip.dst, addr, sizeof(addr)); + inet_ntop(AF_INET, &e->ip.dmsk, mask, sizeof(mask)); + printf("dst = %s%s/%s\n", (e->ip.invflags & IPT_INV_DSTIP)?"! ":"", + /*inet_ntoa(e->ip.dst), inet_ntoa(e->ip.dmsk)*/ + addr, mask); + /*printf("in_if = %s out_if = %s\n", e->ip.iniface, e->ip.outiface);*/ + printf("in_if = "); + print_iface(e->ip.iniface, e->ip.iniface_mask, + e->ip.invflags & IPT_INV_VIA_IN); + printf("\tout_if = "); + print_iface(e->ip.outiface, e->ip.outiface_mask, + e->ip.invflags & IPT_INV_VIA_OUT); + printf("\n"); + printf("ip.proto = %s%d\n", (e->ip.invflags & IPT_INV_PROTO)?"! ":"", + e->ip.proto); + /* display matches stuff */ + if(e->target_offset) + { + IPT_MATCH_ITERATE(e, print_match); + /*printf("\n");*/ + } + printf("target = %s :\n", target_str); + target = (void *)e + e->target_offset; + mr = (const struct ip_nat_multi_range *)&target->data[0]; + printf(" ips "); + printip(ntohl(mr->range[0].min_ip)); + printf(" "); + printip(ntohl(mr->range[0].max_ip)); + printf("\n ports %hu %hu\n", ntohs(mr->range[0].min.all), + ntohs(mr->range[0].max.all)); + printf(" flags = %x\n", mr->range[0].flags); + index++; + } + if(h) +#ifdef IPTABLES_143 + iptc_free(h); +#else + iptc_free(&h); +#endif + printf("======\n"); + return 0; +} +#endif diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptcrdr.h b/src/contrib/miniupnp/miniupnpd/netfilter/iptcrdr.h new file mode 100644 index 0000000..21d5405 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptcrdr.h @@ -0,0 +1,65 @@ +/* $Id: iptcrdr.h,v 1.18 2012/03/05 20:36:20 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2011 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef IPTCRDR_H_INCLUDED +#define IPTCRDR_H_INCLUDED + +#include "../commonrdr.h" + +int +add_redirect_rule2(const char * ifname, + const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp); + +int +add_peer_redirect_rule2(const char * ifname, + const char * rhost, unsigned short rport, + const char * eaddr, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp); + +int +add_filter_rule2(const char * ifname, + const char * rhost, const char * iaddr, + unsigned short eport, unsigned short iport, + int proto, const char * desc); + +int +delete_redirect_and_filter_rules(unsigned short eport, int proto); + +int +add_peer_dscp_rule2(const char * ifname, + const char * rhost, unsigned short rport, + unsigned char dscp, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp); + +int get_nat_ext_addr(struct sockaddr* src, struct sockaddr *dst, uint8_t proto, + struct sockaddr* ret_ext); +int +get_peer_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, unsigned short * rport, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); +int +get_nat_redirect_rule(const char * nat_chain_name, const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); + +/* for debug */ +int +list_redirect_rule(const char * ifname); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptpinhole.c b/src/contrib/miniupnp/miniupnpd/netfilter/iptpinhole.c new file mode 100644 index 0000000..2f5fc08 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptpinhole.c @@ -0,0 +1,484 @@ +/* $Id: iptpinhole.c,v 1.18 2018/03/13 23:05:21 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2012-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include + +#include "../config.h" +#include "../macros.h" +#include "iptpinhole.h" +#include "../upnpglobalvars.h" +#include "../upnputils.h" + +#ifdef ENABLE_UPNPPINHOLE + +#include +#include +#include "tiny_nf_nat.h" + +#define IP6TC_HANDLE struct ip6tc_handle * + +static int next_uid = 1; + +static LIST_HEAD(pinhole_list_t, pinhole_t) pinhole_list; + +static struct pinhole_t * +get_pinhole(unsigned short uid); + +struct pinhole_t { + struct in6_addr saddr; + struct in6_addr daddr; + LIST_ENTRY(pinhole_t) entries; + unsigned int timestamp; + unsigned short sport; + unsigned short dport; + unsigned short uid; + unsigned char proto; + char desc[]; +}; + +void init_iptpinhole(void) +{ + LIST_INIT(&pinhole_list); +} + +void shutdown_iptpinhole(void) +{ + struct pinhole_t * p; + while(pinhole_list.lh_first != NULL) { + p = pinhole_list.lh_first; + LIST_REMOVE(p, entries); + free(p); + } +} + +/* return uid */ +static int +add_to_pinhole_list(struct in6_addr * saddr, unsigned short sport, + struct in6_addr * daddr, unsigned short dport, + int proto, const char *desc, unsigned int timestamp) +{ + struct pinhole_t * p; + + p = calloc(1, sizeof(struct pinhole_t) + strlen(desc) + 1); + if(!p) { + syslog(LOG_ERR, "add_to_pinhole_list calloc() error"); + return -1; + } + strcpy(p->desc, desc); + memcpy(&p->saddr, saddr, sizeof(struct in6_addr)); + p->sport = sport; + memcpy(&p->daddr, daddr, sizeof(struct in6_addr)); + p->dport = dport; + p->timestamp = timestamp; + p->proto = (unsigned char)proto; + LIST_INSERT_HEAD(&pinhole_list, p, entries); + while(get_pinhole(next_uid) != NULL) { + next_uid++; + if(next_uid > 65535) + next_uid = 1; + } + p->uid = next_uid; + next_uid++; + if(next_uid > 65535) + next_uid = 1; + return p->uid; +} + +static struct pinhole_t * +get_pinhole(unsigned short uid) +{ + struct pinhole_t * p; + + for(p = pinhole_list.lh_first; p != NULL; p = p->entries.le_next) { + if(p->uid == uid) + return p; + } + return NULL; /* not found */ +} + +/* new_match() + * Allocate and set a new ip6t_entry_match structure + * The caller must free() it after usage */ +static struct ip6t_entry_match * +new_match(int proto, unsigned short sport, unsigned short dport) +{ + struct ip6t_entry_match *match; + struct ip6t_tcp *info; /* TODO : use ip6t_udp if needed */ + size_t size; + const char * name; + size = XT_ALIGN(sizeof(struct ip6t_entry_match)) + + XT_ALIGN(sizeof(struct ip6t_tcp)); + match = calloc(1, size); + match->u.user.match_size = size; + switch(proto) { + case IPPROTO_TCP: + name = "tcp"; + break; + case IPPROTO_UDP: + name = "udp"; + break; + case IPPROTO_UDPLITE: + name = "udplite"; + break; + default: + name = NULL; + } + if(name) + strncpy(match->u.user.name, name, sizeof(match->u.user.name)); + else + syslog(LOG_WARNING, "no name for protocol %d", proto); + info = (struct ip6t_tcp *)match->data; + if(sport) { + info->spts[0] = sport; /* specified source port */ + info->spts[1] = sport; + } else { + info->spts[0] = 0; /* all source ports */ + info->spts[1] = 0xFFFF; + } + info->dpts[0] = dport; /* specified destination port */ + info->dpts[1] = dport; + return match; +} + +static struct ip6t_entry_target * +get_accept_target(void) +{ + struct ip6t_entry_target * target = NULL; + size_t size; + size = XT_ALIGN(sizeof(struct ip6t_entry_target)) + + XT_ALIGN(sizeof(int)); + target = calloc(1, size); + target->u.user.target_size = size; + strncpy(target->u.user.name, "ACCEPT", sizeof(target->u.user.name)); + return target; +} + +static int +ip6tc_init_verify_append(const char * table, + const char * chain, + struct ip6t_entry * e) +{ + IP6TC_HANDLE h; + + h = ip6tc_init(table); + if(!h) { + syslog(LOG_ERR, "ip6tc_init error : %s", ip6tc_strerror(errno)); + return -1; + } + if(!ip6tc_is_chain(chain, h)) { + syslog(LOG_ERR, "chain %s not found", chain); + goto error; + } + if(!ip6tc_append_entry(chain, e, h)) { + syslog(LOG_ERR, "ip6tc_append_entry() error : %s", ip6tc_strerror(errno)); + goto error; + } + if(!ip6tc_commit(h)) { + syslog(LOG_ERR, "ip6tc_commit() error : %s", ip6tc_strerror(errno)); + goto error; + } + return 0; /* ok */ +error: + ip6tc_free(h); + return -1; +} + +/* +ip6tables -I %s %d -p %s -i %s -s %s --sport %hu -d %s --dport %hu -j ACCEPT +ip6tables -I %s %d -p %s -i %s --sport %hu -d %s --dport %hu -j ACCEPT + +miniupnpd_forward_chain, line_number, proto, ext_if_name, raddr, rport, iaddr, iport + +ip6tables -t raw -I PREROUTING %d -p %s -i %s -s %s --sport %hu -d %s --dport %hu -j TRACE +ip6tables -t raw -I PREROUTING %d -p %s -i %s --sport %hu -d %s --dport %hu -j TRACE +*/ +int add_pinhole(const char * ifname, + const char * rem_host, unsigned short rem_port, + const char * int_client, unsigned short int_port, + int proto, const char * desc, unsigned int timestamp) +{ + int uid; + struct ip6t_entry * e; + struct ip6t_entry * tmp; + struct ip6t_entry_match *match = NULL; + struct ip6t_entry_target *target = NULL; + + e = calloc(1, sizeof(struct ip6t_entry)); + if(!e) { + syslog(LOG_ERR, "%s: calloc(%d) failed", + "add_pinhole", (int)sizeof(struct ip6t_entry)); + return -1; + } + e->ipv6.proto = proto; + if (proto) + e->ipv6.flags |= IP6T_F_PROTO; + + /* TODO: check if enforcing USE_IFNAME_IN_RULES is needed */ + if(ifname) + strncpy(e->ipv6.iniface, ifname, IFNAMSIZ); + if(rem_host && (rem_host[0] != '\0')) { + inet_pton(AF_INET6, rem_host, &e->ipv6.src); + memset(&e->ipv6.smsk, 0xff, sizeof(e->ipv6.smsk)); + } + inet_pton(AF_INET6, int_client, &e->ipv6.dst); + memset(&e->ipv6.dmsk, 0xff, sizeof(e->ipv6.dmsk)); + + /*e->nfcache = NFC_IP_DST_PT;*/ + /*e->nfcache |= NFC_UNKNOWN;*/ + + match = new_match(proto, rem_port, int_port); + target = get_accept_target(); + tmp = realloc(e, sizeof(struct ip6t_entry) + + match->u.match_size + + target->u.target_size); + if(!tmp) { + syslog(LOG_ERR, "%s: realloc(%d) failed", + "add_pinhole", (int)(sizeof(struct ip6t_entry) + match->u.match_size + target->u.target_size)); + free(e); + free(match); + free(target); + return -1; + } + e = tmp; + memcpy(e->elems, match, match->u.match_size); + memcpy(e->elems + match->u.match_size, target, target->u.target_size); + e->target_offset = sizeof(struct ip6t_entry) + + match->u.match_size; + e->next_offset = sizeof(struct ip6t_entry) + + match->u.match_size + + target->u.target_size; + free(match); + free(target); + + if(ip6tc_init_verify_append("filter", miniupnpd_v6_filter_chain, e) < 0) { + free(e); + return -1; + } + uid = add_to_pinhole_list(&e->ipv6.src, rem_port, + &e->ipv6.dst, int_port, + proto, desc, timestamp); + free(e); + return uid; +} + +int +find_pinhole(const char * ifname, + const char * rem_host, unsigned short rem_port, + const char * int_client, unsigned short int_port, + int proto, + char *desc, int desc_len, unsigned int * timestamp) +{ + struct pinhole_t * p; + struct in6_addr saddr; + struct in6_addr daddr; + UNUSED(ifname); + + if(rem_host && (rem_host[0] != '\0')) { + inet_pton(AF_INET6, rem_host, &saddr); + } else { + memset(&saddr, 0, sizeof(struct in6_addr)); + } + inet_pton(AF_INET6, int_client, &daddr); + for(p = pinhole_list.lh_first; p != NULL; p = p->entries.le_next) { + if((proto == p->proto) && (rem_port == p->sport) && + (0 == memcmp(&saddr, &p->saddr, sizeof(struct in6_addr))) && + (int_port == p->dport) && + (0 == memcmp(&daddr, &p->daddr, sizeof(struct in6_addr)))) { + if(desc) strncpy(desc, p->desc, desc_len); + if(timestamp) *timestamp = p->timestamp; + return (int)p->uid; + } + } + return -2; /* not found */ +} + +int +delete_pinhole(unsigned short uid) +{ + struct pinhole_t * p; + IP6TC_HANDLE h; + const struct ip6t_entry * e; + const struct ip6t_entry_match *match = NULL; + /*const struct ip6t_entry_target *target = NULL;*/ + unsigned int index; + + p = get_pinhole(uid); + if(!p) + return -2; /* not found */ + + h = ip6tc_init("filter"); + if(!h) { + syslog(LOG_ERR, "ip6tc_init error : %s", ip6tc_strerror(errno)); + return -1; + } + if(!ip6tc_is_chain(miniupnpd_v6_filter_chain, h)) { + syslog(LOG_ERR, "chain %s not found", miniupnpd_v6_filter_chain); + goto error; + } + index = 0; + for(e = ip6tc_first_rule(miniupnpd_v6_filter_chain, h); + e; + e = ip6tc_next_rule(e, h)) { + if((e->ipv6.proto == p->proto) && + (0 == memcmp(&e->ipv6.src, &p->saddr, sizeof(e->ipv6.src))) && + (0 == memcmp(&e->ipv6.dst, &p->daddr, sizeof(e->ipv6.dst)))) { + const struct ip6t_tcp * info; + match = (const struct ip6t_entry_match *)&e->elems; + info = (const struct ip6t_tcp *)&match->data; + if((info->spts[0] == p->sport) && (info->dpts[0] == p->dport)) { + if(!ip6tc_delete_num_entry(miniupnpd_v6_filter_chain, index, h)) { + syslog(LOG_ERR, "ip6tc_delete_num_entry(%s,%u,...): %s", + miniupnpd_v6_filter_chain, index, ip6tc_strerror(errno)); + goto error; + } + if(!ip6tc_commit(h)) { + syslog(LOG_ERR, "ip6tc_commit(): %s", + ip6tc_strerror(errno)); + goto error; + } + ip6tc_free(h); + LIST_REMOVE(p, entries); + return 0; /* ok */ + } + } + index++; + } + ip6tc_free(h); + syslog(LOG_WARNING, "delete_pinhole() rule with PID=%hu not found", uid); + LIST_REMOVE(p, entries); + return -2; /* not found */ +error: + ip6tc_free(h); + return -1; +} + +int +update_pinhole(unsigned short uid, unsigned int timestamp) +{ + struct pinhole_t * p; + + p = get_pinhole(uid); + if(p) { + p->timestamp = timestamp; + return 0; + } else { + return -2; /* Not found */ + } +} + +int +get_pinhole_info(unsigned short uid, + char * rem_host, int rem_hostlen, + unsigned short * rem_port, + char * int_client, int int_clientlen, + unsigned short * int_port, + int * proto, char * desc, int desclen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + struct pinhole_t * p; + + p = get_pinhole(uid); + if(!p) + return -2; /* Not found */ + if(rem_host && (rem_host[0] != '\0')) { + if(inet_ntop(AF_INET6, &p->saddr, rem_host, rem_hostlen) == NULL) + return -1; + } + if(rem_port) + *rem_port = p->sport; + if(int_client) { + if(inet_ntop(AF_INET6, &p->daddr, int_client, int_clientlen) == NULL) + return -1; + } + if(int_port) + *int_port = p->dport; + if(proto) + *proto = p->proto; + if(timestamp) + *timestamp = p->timestamp; + if (desc) + strncpy(desc, p->desc, desclen); + if(packets || bytes) { + /* theses informations need to be read from netfilter */ + IP6TC_HANDLE h; + const struct ip6t_entry * e; + const struct ip6t_entry_match * match; + h = ip6tc_init("filter"); + if(!h) { + syslog(LOG_ERR, "ip6tc_init error : %s", ip6tc_strerror(errno)); + return -1; + } + for(e = ip6tc_first_rule(miniupnpd_v6_filter_chain, h); + e; + e = ip6tc_next_rule(e, h)) { + if((e->ipv6.proto == p->proto) && + (0 == memcmp(&e->ipv6.src, &p->saddr, sizeof(e->ipv6.src))) && + (0 == memcmp(&e->ipv6.dst, &p->daddr, sizeof(e->ipv6.dst)))) { + const struct ip6t_tcp * info; + match = (const struct ip6t_entry_match *)&e->elems; + info = (const struct ip6t_tcp *)&match->data; + if((info->spts[0] == p->sport) && (info->dpts[0] == p->dport)) { + if(packets) + *packets = e->counters.pcnt; + if(bytes) + *bytes = e->counters.bcnt; + break; + } + } + } + ip6tc_free(h); + } + return 0; +} + +int get_pinhole_uid_by_index(int index) +{ + struct pinhole_t * p; + + for(p = pinhole_list.lh_first; p != NULL; p = p->entries.le_next) + if (!index--) + return p->uid; + return -1; +} + +int +clean_pinhole_list(unsigned int * next_timestamp) +{ + unsigned int min_ts = UINT_MAX; + struct pinhole_t * p; + time_t current_time; + int n = 0; + + current_time = upnp_time(); + p = pinhole_list.lh_first; + while(p != NULL) { + if(p->timestamp <= (unsigned int)current_time) { + unsigned short uid = p->uid; + syslog(LOG_INFO, "removing expired pinhole with uid=%hu", uid); + p = p->entries.le_next; + if(delete_pinhole(uid) == 0) + n++; + else + break; + } else { + if(p->timestamp < min_ts) + min_ts = p->timestamp; + p = p->entries.le_next; + } + } + if(next_timestamp && (min_ts != UINT_MAX)) + *next_timestamp = min_ts; + return n; +} + +#endif /* ENABLE_UPNPPINHOLE */ diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/iptpinhole.h b/src/contrib/miniupnp/miniupnpd/netfilter/iptpinhole.h new file mode 100644 index 0000000..55f91a6 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/iptpinhole.h @@ -0,0 +1,43 @@ +/* $Id: iptpinhole.h,v 1.5 2012/05/08 20:41:45 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2012-2016 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#ifndef IPTPINHOLE_H_INCLUDED +#define IPTPINHOLE_H_INCLUDED + +#ifdef ENABLE_UPNPPINHOLE +#include + +int find_pinhole(const char * ifname, + const char * rem_host, unsigned short rem_port, + const char * int_client, unsigned short int_port, + int proto, + char *desc, int desc_len, unsigned int * timestamp); + +int add_pinhole(const char * ifname, + const char * rem_host, unsigned short rem_port, + const char * int_client, unsigned short int_port, + int proto, const char *desc, unsigned int timestamp); + +int update_pinhole(unsigned short uid, unsigned int timestamp); + +int delete_pinhole(unsigned short uid); + +int +get_pinhole_info(unsigned short uid, + char * rem_host, int rem_hostlen, unsigned short * rem_port, + char * int_client, int int_clientlen, + unsigned short * int_port, + int * proto, char * desc, int desclen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); + +int get_pinhole_uid_by_index(int index); + +int clean_pinhole_list(unsigned int * next_timestamp); + +#endif /* ENABLE_UPNPPINHOLE */ + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/miniupnpd_functions.sh b/src/contrib/miniupnp/miniupnpd/netfilter/miniupnpd_functions.sh new file mode 100644 index 0000000..20c7a1c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/miniupnpd_functions.sh @@ -0,0 +1,64 @@ +#! /bin/sh +# $Id: miniupnpd_functions.sh,v 1.1 2018/02/24 12:15:19 nanard Exp $ + +IP=$(which ip) || { + echo "Can't find ip" >&2 + exit 1 +} +if [ -z "$IPV6" ]; then + IPTABLES=$(which iptables) || { + echo "Can't find iptables" >&2 + exit 1 + } + IP="$IP -4" +else + IPTABLES=$(which ip6tables) || { + echo "Can't find ip6tables" >&2 + exit 1 + } + IP="$IP -6" +fi + +CHAIN=MINIUPNPD +CLEAN= + +while getopts ":c:i:f" opt; do + case $opt in + c) + CHAIN=$OPTARG + ;; + i) + EXTIF=$OPTARG + ;; + f) + CLEAN=yes + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -n "$EXT" ]; then + if [ -z "$EXTIF" ]; then + EXTIF=$(LC_ALL=C $IP route | grep 'default' | sed -e 's/.*dev[[:space:]]*//' -e 's/[[:space:]].*//') || { + echo "Can't find default interface" >&2 + exit 1 + } + fi + #if [ -z "$IPV6" ]; then + # EXTIP=$(LC_ALL=C $IP addr show $EXTIF | awk '/inet/ { print $2 }' | cut -d "/" -f 1) + #fi +fi + +FDIRTY=$(LC_ALL=C $IPTABLES -t filter -L -n | awk "/$CHAIN/ {printf \$1}") +if [ -z "$IPV6" ]; then + NDIRTY=$(LC_ALL=C $IPTABLES -t nat -L -n | awk "/$CHAIN/ {printf \$1}") + MDIRTY=$(LC_ALL=C $IPTABLES -t mangle -L -n | awk "/$CHAIN/ {printf \$1}") + NPDIRTY=$(LC_ALL=C $IPTABLES -t nat -L -n | awk "/$CHAIN-POSTROUTING/ {printf \$1}") +fi diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/nfct_get.c b/src/contrib/miniupnp/miniupnpd/netfilter/nfct_get.c new file mode 100644 index 0000000..cbffd28 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/nfct_get.c @@ -0,0 +1,258 @@ +#include +#include +#include +#include +#include +#include + +#ifdef USE_NFCT +#include +#include + +#include + +struct data_cb_s +{ + struct sockaddr_storage * ext; + uint8_t found; +}; + +static int data_cb(const struct nlmsghdr *nlh, void *data) +{ + struct nf_conntrack *ct; + struct data_cb_s * d = (struct data_cb_s*) data; + struct sockaddr_in* ext4 = (struct sockaddr_in*) d->ext; + + ct = nfct_new(); + if (ct == NULL) + return MNL_CB_OK; + nfct_nlmsg_parse(nlh, ct); + + if (data) { + ext4->sin_addr.s_addr = nfct_get_attr_u32(ct, ATTR_REPL_IPV4_DST); + ext4->sin_port = nfct_get_attr_u16(ct, ATTR_REPL_PORT_DST); + } + d->found = 1; + nfct_destroy(ct); + + return MNL_CB_OK; +} + +int get_nat_ext_addr(struct sockaddr* src, struct sockaddr *dst, uint8_t proto, + struct sockaddr_storage* ret_ext) +{ + struct mnl_socket *nl; + struct nlmsghdr *nlh; + struct nfgenmsg *nfh; + char buf[MNL_SOCKET_BUFFER_SIZE]; + unsigned int seq, portid; + struct nf_conntrack *ct; + int ret; + struct data_cb_s data; + + if ((!src)&&(!dst)) { + return 0; + } + + if (src->sa_family != dst->sa_family) { + return 0; + } + + nl = mnl_socket_open(NETLINK_NETFILTER); + if (nl == NULL) { +// perror("mnl_socket_open"); + goto free_nl; + } + + if (mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { +// perror("mnl_socket_bind"); + goto free_nl; + } + portid = mnl_socket_get_portid(nl); + + memset(buf, 0, sizeof(buf)); + nlh = mnl_nlmsg_put_header(buf); + nlh->nlmsg_type = (NFNL_SUBSYS_CTNETLINK << 8) | IPCTNL_MSG_CT_GET; + nlh->nlmsg_flags = NLM_F_REQUEST|NLM_F_ACK; + nlh->nlmsg_seq = seq = time(NULL); + + nfh = mnl_nlmsg_put_extra_header(nlh, sizeof(struct nfgenmsg)); + nfh->nfgen_family = src->sa_family; + nfh->version = NFNETLINK_V0; + nfh->res_id = 0; + + ct = nfct_new(); + if (ct == NULL) { + goto free_nl; + } + + nfct_set_attr_u8(ct, ATTR_L3PROTO, src->sa_family); + if (src->sa_family == AF_INET) { + struct sockaddr_in *src4 = (struct sockaddr_in *)src; + struct sockaddr_in *dst4 = (struct sockaddr_in *)dst; + nfct_set_attr_u32(ct, ATTR_IPV4_SRC, src4->sin_addr.s_addr); + nfct_set_attr_u32(ct, ATTR_IPV4_DST, dst4->sin_addr.s_addr); + nfct_set_attr_u16(ct, ATTR_PORT_SRC, src4->sin_port); + nfct_set_attr_u16(ct, ATTR_PORT_DST, dst4->sin_port); + } else if (src->sa_family == AF_INET6) { + struct sockaddr_in6 *src6 = (struct sockaddr_in6 *)src; + struct sockaddr_in6 *dst6 = (struct sockaddr_in6 *)dst; + nfct_set_attr(ct, ATTR_IPV6_SRC, &src6->sin6_addr); + nfct_set_attr(ct, ATTR_IPV6_DST, &dst6->sin6_addr); + nfct_set_attr_u16(ct, ATTR_PORT_SRC, src6->sin6_port); + nfct_set_attr_u16(ct, ATTR_PORT_DST, dst6->sin6_port); + } + nfct_set_attr_u8(ct, ATTR_L4PROTO, proto); + + nfct_nlmsg_build(nlh, ct); + + ret = mnl_socket_sendto(nl, nlh, nlh->nlmsg_len); + if (ret == -1) { + goto free_ct; + } + + ret = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + data.ext = ret_ext; + data.found = 0; + while (ret > 0) { + ret = mnl_cb_run(buf, ret, seq, portid, data_cb, &data); + if (ret <= MNL_CB_STOP) + break; + ret = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + } + +free_ct: + nfct_destroy(ct); +free_nl: + mnl_socket_close(nl); + + return data.found; +} + +#else +#define DST "dst=" +#define DST_PORT "dport=" +#define SRC "src=" +#define SRC_PORT "sport=" +#define IP_CONNTRACK_LOCATION "/proc/net/ip_conntrack" +#define NF_CONNTRACK_LOCATION "/proc/net/nf_conntrack" + +int get_nat_ext_addr(struct sockaddr* src, struct sockaddr *dst, uint8_t proto, + struct sockaddr_storage* ret_ext) +{ + FILE *f; + int af; + + if (!src) + return -2; + + af = src->sa_family; + + if ((f = fopen(NF_CONNTRACK_LOCATION, "r")) == NULL) { + if ((f = fopen(IP_CONNTRACK_LOCATION, "r")) == NULL) { + printf("could not read info about connections from the kernel, " + "make sure netfilter is enabled in kernel or by modules.\n"); + return -1; + } + } + + while (!feof(f)) { + char line[256], *str; + memset(line, 0, sizeof(line)); + str = fgets(line, sizeof(line), f); + if (line[0] != 0) { + char *token, *saveptr; + int j; + uint8_t src_f, src_port_f, dst_f, dst_port_f; + src_f=src_port_f=dst_f=dst_port_f=0; + + for (j = 1; ; j++, str = NULL) { + token = strtok_r(str, " ", &saveptr); + if (token == NULL) + break; + + if ((j==2)&&(af!=atoi(token))) + break; + if ((j==4)&&(proto!=atoi(token))) + break; + if (j<=4) + continue; + + if (strncmp(token, SRC, sizeof(SRC) - 1) == 0) { + char *srcip = token + sizeof(SRC) - 1; + uint32_t buf[4]; + memset(buf,0,sizeof(buf)); + + if (inet_pton(af, srcip, buf)!=1) + break; + + if (af==AF_INET) { + struct sockaddr_in *src4=(struct sockaddr_in*)src; + if (!src_f) { + if (src4->sin_addr.s_addr != buf[0]) + break; + src_f = 1; + } + } + } + if (strncmp(token, SRC_PORT, sizeof(SRC_PORT) - 1) == 0) { + char *src_port = token + sizeof(SRC_PORT) - 1; + uint16_t port=atoi(src_port); + + if (af==AF_INET) { + struct sockaddr_in *src4=(struct sockaddr_in*)src; + if (!src_port_f) { + if (ntohs(src4->sin_port) != port) + break; + src_port_f = 1; + } + } + } + + if (strncmp(token, DST, sizeof(DST) - 1) == 0) { + char *dstip = token + sizeof(DST) - 1; + uint32_t buf[4]; + memset(buf,0,sizeof(buf)); + if (inet_pton(af, dstip, buf)!=1) + break; + if (af==AF_INET) { + struct sockaddr_in *dst4=(struct sockaddr_in*)dst; + if (!dst_f) { + if (dst4->sin_addr.s_addr != buf[0]) + break; + dst_f = 1; + } else { + struct sockaddr_in*ret4=(struct sockaddr_in*)ret_ext; + ret_ext->ss_family = AF_INET; + ret4->sin_addr.s_addr = buf[0]; + } + } + } + if (strncmp(token, DST_PORT, sizeof(DST_PORT)-1) == 0) { + char *dst_port = token + sizeof(DST_PORT) - 1; + uint16_t port=atoi(dst_port); + if (af==AF_INET) { + struct sockaddr_in *dst4=(struct sockaddr_in*)dst; + if (!dst_port_f) { + if (ntohs(dst4->sin_port) != port) + break; + dst_port_f = 1; + } else { + struct sockaddr_in*ret4=(struct sockaddr_in*)ret_ext; + ret_ext->ss_family = AF_INET; + ret4->sin_port = htons(port); + } + } + } + } + if (src_f && src_port_f && dst_f && dst_port_f) { + fclose(f); + return 1; + } + } + } + fclose(f); + + return 0; +} +#endif diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/test_nfct_get.c b/src/contrib/miniupnp/miniupnpd/netfilter/test_nfct_get.c new file mode 100644 index 0000000..af8c07d --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/test_nfct_get.c @@ -0,0 +1,50 @@ +#include "nfct_get.c" + +int main(int argc, char *argv[]) +{ + struct sockaddr_storage src, dst, ext; + char buff[INET6_ADDRSTRLEN]; + + if (argc!=5) + return 0; + + if (1 != inet_pton(AF_INET, argv[1], + &((struct sockaddr_in*)&src)->sin_addr)) { + if (1 != inet_pton(AF_INET6, argv[1], + &((struct sockaddr_in6*) &src)->sin6_addr)) { + perror("bad input param"); + } else { + ((struct sockaddr_in6*)(&src))->sin6_port = htons(atoi(argv[2])); + src.ss_family = AF_INET6; + } + } else { + ((struct sockaddr_in*)(&src))->sin_port = htons(atoi(argv[2])); + src.ss_family = AF_INET; + } + + if (1 != inet_pton(AF_INET, argv[3], + &((struct sockaddr_in*)&dst)->sin_addr)) { + if (1 != inet_pton(AF_INET6, argv[3], + &((struct sockaddr_in6*) &dst)->sin6_addr)) { + perror("bad input param"); + } else { + ((struct sockaddr_in6*)(&dst))->sin6_port = htons(atoi(argv[4])); + dst.ss_family = AF_INET6; + } + } else { + ((struct sockaddr_in*)(&dst))->sin_port = htons(atoi(argv[4])); + dst.ss_family = AF_INET; + } + + if (get_nat_ext_addr((struct sockaddr*)&src, (struct sockaddr*)&dst, + IPPROTO_TCP, &ext)) { + printf("Ext address %s:%d\n", + inet_ntop(src.ss_family, + &((struct sockaddr_in*)&ext)->sin_addr, + buff, sizeof(buff)), + ntohs(((struct sockaddr_in*)(&ext))->sin_port)); + } else { + printf("no entry\n"); + } + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr.c b/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr.c new file mode 100644 index 0000000..59663f7 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr.c @@ -0,0 +1,91 @@ +/* $Id: testiptcrdr.c,v 1.21 2016/02/12 12:35:50 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2016 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include + +#include "iptcrdr.c" +/*#include "../commonrdr.h"*/ + +#ifndef PRIu64 +#define PRIu64 "llu" +#endif + +int +main(int argc, char ** argv) +{ + unsigned short eport, iport; + const char * iaddr; + int r; + int proto = IPPROTO_TCP; + printf("Usage %s [TCP/UDP]\n", argv[0]); + + if(argc<4) + return -1; + openlog("testiptcrdr", LOG_PERROR|LOG_CONS, LOG_LOCAL0); + if(init_redirect() < 0) + return -1; + eport = (unsigned short)atoi(argv[1]); + iaddr = argv[2]; + iport = (unsigned short)atoi(argv[3]); + if(argc >= 4) { + if(strcasecmp(argv[4], "udp") == 0) + proto = IPPROTO_UDP; + } + printf("trying to redirect port %hu to %s:%hu proto %d\n", + eport, iaddr, iport, proto); + if(addnatrule(proto, eport, iaddr, iport, NULL) < 0) + return -1; + r = addmasqueraderule(proto, eport, iaddr, iport, NULL); + syslog(LOG_DEBUG, "addmasqueraderule() returned %d", r); + if(add_filter_rule(proto, NULL, iaddr, iport) < 0) + return -1; +#if 0 + /* TEST */ + if(proto == IPPROTO_UDP) { + if(addpeernatrule(proto, "8.8.8.8"/*eaddr*/, eport, iaddr, iport, NULL, 0) < 0) + fprintf(stderr, "addpeenatrule failed\n"); + } +#endif + /*update_portmapping_desc_timestamp(NULL, eport, proto, "updated desc", time(NULL)+42);*/ + update_portmapping(NULL, eport, proto, iport+1, "updated rule", time(NULL)+42); + /* test */ + { + unsigned short p1, p2; + char addr[16]; + int proto2; + char desc[256]; + char rhost[256]; + unsigned int timestamp; + u_int64_t packets, bytes; + + desc[0] = '\0'; + if(get_redirect_rule_by_index(0, "", &p1, + addr, sizeof(addr), &p2, + &proto2, desc, sizeof(desc), + rhost, sizeof(rhost), + ×tamp, + &packets, &bytes) < 0) + { + printf("rule not found\n"); + } + else + { + printf("redirected port %hu to %s:%hu proto %d packets=%" PRIu64 " bytes=%" PRIu64 " ts=%u desc='%s'\n", + p1, addr, p2, proto2, packets, bytes, timestamp, desc); + } + } + printf("trying to list nat rules :\n"); + list_redirect_rule(NULL); + printf("deleting\n"); + delete_redirect_and_filter_rules(eport, proto); + shutdown_redirect(); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr_dscp.c b/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr_dscp.c new file mode 100644 index 0000000..4ad56d7 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr_dscp.c @@ -0,0 +1,73 @@ +/* $Id: testiptcrdr.c,v 1.18 2012/04/24 22:41:53 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2012 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include + +#include "iptcrdr.h" +#include "../commonrdr.h" +#include "iptcrdr.c" + +#ifndef PRIu64 +#define PRIu64 "llu" +#endif + +int +main(int argc, char ** argv) +{ + unsigned char dscp; + unsigned short iport, rport; + const char * iaddr, *rhost; + printf("Usage %s \n", argv[0]); + + if(argc<6) + return -1; + openlog("testiptcrdr_peer", LOG_PERROR|LOG_CONS, LOG_LOCAL0); + dscp = (unsigned short)atoi(argv[1]); + iaddr = argv[2]; + iport = (unsigned short)atoi(argv[3]); + rhost = argv[4]; + rport = (unsigned short)atoi(argv[5]); +#if 1 + if(addpeerdscprule(IPPROTO_TCP, dscp, iaddr, iport, rhost, rport) < 0) + return -1; +#endif + /* test */ + { + unsigned short p1, p2; + char addr[16]; + int proto2; + char desc[256]; + char rhost[256]; + unsigned int timestamp; + u_int64_t packets, bytes; + + desc[0] = '\0'; + if(get_redirect_rule_by_index(0, "", &p1, + addr, sizeof(addr), &p2, + &proto2, desc, sizeof(desc), + rhost, sizeof(rhost), + ×tamp, + &packets, &bytes) < 0) + { + printf("rule not found\n"); + } + else + { + printf("redirected port %hu to %s:%hu proto %d packets=%" PRIu64 " bytes=%" PRIu64 "\n", + p1, addr, p2, proto2, packets, bytes); + } + } + printf("trying to list nat rules :\n"); + list_redirect_rule(argv[1]); + printf("deleting\n"); +// delete_redirect_and_filter_rules(eport, IPPROTO_TCP); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr_peer.c b/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr_peer.c new file mode 100644 index 0000000..6e1d175 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/testiptcrdr_peer.c @@ -0,0 +1,74 @@ +/* $Id: testiptcrdr.c,v 1.18 2012/04/24 22:41:53 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2012 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include + +#include "iptcrdr.h" +#include "../commonrdr.h" +#include "iptcrdr.c" + +#ifndef PRIu64 +#define PRIu64 "llu" +#endif + +int +main(int argc, char ** argv) +{ + unsigned short eport, iport, rport; + const char * eaddr, *iaddr, *rhost; + printf("Usage %s \n", argv[0]); + + if(argc<6) + return -1; + openlog("testiptcrdr_peer", LOG_PERROR|LOG_CONS, LOG_LOCAL0); + eaddr = argv[1]; + eport = (unsigned short)atoi(argv[2]); + iaddr = argv[3]; + iport = (unsigned short)atoi(argv[4]); + rhost = argv[5]; + rport = (unsigned short)atoi(argv[6]); +#if 1 + printf("trying to redirect port %hu to %s:%hu\n", eport, iaddr, iport); + if(addpeernatrule(IPPROTO_TCP, eaddr, eport, iaddr, iport, rhost, rport) < 0) + return -1; +#endif + /* test */ + { + unsigned short p1, p2; + char addr[16]; + int proto2; + char desc[256]; + char rhost[256]; + unsigned int timestamp; + u_int64_t packets, bytes; + + desc[0] = '\0'; + if(get_redirect_rule_by_index(0, "", &p1, + addr, sizeof(addr), &p2, + &proto2, desc, sizeof(desc), + rhost, sizeof(rhost), + ×tamp, + &packets, &bytes) < 0) + { + printf("rule not found\n"); + } + else + { + printf("redirected port %hu to %s:%hu proto %d packets=%" PRIu64 " bytes=%" PRIu64 "\n", + p1, addr, p2, proto2, packets, bytes); + } + } + printf("trying to list nat rules :\n"); + list_redirect_rule(argv[1]); + printf("deleting\n"); + delete_redirect_and_filter_rules(eport, IPPROTO_TCP); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/testiptpinhole.c b/src/contrib/miniupnp/miniupnpd/netfilter/testiptpinhole.c new file mode 100644 index 0000000..caa3832 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/testiptpinhole.c @@ -0,0 +1,27 @@ +/* $Id: testiptpinhole.c,v 1.1 2012/04/26 13:50:48 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2012 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include + +#include "../config.h" +#include "iptpinhole.h" +#include "../commonrdr.h" + + +int main(int argc, char * * argv) +{ + int uid; + + openlog("testiptpinhole", LOG_PERROR|LOG_CONS, LOG_LOCAL0); + + uid = add_pinhole("eth0", NULL, 0, "ff::123", 54321, IPPROTO_TCP); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/netfilter/tiny_nf_nat.h b/src/contrib/miniupnp/miniupnpd/netfilter/tiny_nf_nat.h new file mode 100644 index 0000000..f7f5b27 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter/tiny_nf_nat.h @@ -0,0 +1,37 @@ +/* $Id: tiny_nf_nat.h,v 1.1 2011/07/30 13:14:36 nanard Exp $ */ +/* Only what miniupnpd needs, until linux-libc-dev gains nf_nat.h */ + +#ifndef TINY_NF_NAT_H +#define TINY_NF_NAT_H + +#include + +#define IP_NAT_RANGE_MAP_IPS 1 +#define IP_NAT_RANGE_PROTO_SPECIFIED 2 +#define IP_NAT_RANGE_PROTO_RANDOM 4 +#define IP_NAT_RANGE_PERSISTENT 8 + +union nf_conntrack_man_proto { + __be16 all; + struct { __be16 port; } tcp; + struct { __be16 port; } udp; + struct { __be16 id; } icmp; + struct { __be16 port; } dccp; + struct { __be16 port; } sctp; + struct { __be16 key; } gre; +}; + +struct nf_nat_range { + unsigned int flags; + __be32 min_ip, max_ip; + union nf_conntrack_man_proto min, max; +}; + +struct nf_nat_multi_range_compat { + unsigned int rangesize; + struct nf_nat_range range[1]; +}; + +#define nf_nat_multi_range nf_nat_multi_range_compat + +#endif /*TINY_NF_NAT_H*/ diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/Makefile b/src/contrib/miniupnp/miniupnpd/netfilter_nft/Makefile new file mode 100644 index 0000000..19846cf --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/Makefile @@ -0,0 +1,33 @@ +CFLAGS?=-Wall -g -D_GNU_SOURCE -DDEBUG -Wstrict-prototypes -Wdeclaration-after-statement +CC = gcc + +LIBS = -lnftnl -lmnl + +ARCH := $(shell uname -m | grep -q "x86_64" && echo 64) + +all: test_nfct_get testnftnlrdr + +clean: + $(RM) *.o testnftnlcrdr testnftnlpinhole testnftnlrdr_peer \ + test_nfct_get testnftnlrdr + +testnftnlrdr: nftnlrdr.o nftnlrdr_misc.o testnftnlrdr.o upnpglobalvars.o $(LIBS) + +testiptpinhole: testiptpinhole.o iptpinhole.o upnpglobalvars.o $(LIBS) + +test_nfct_get: test_nfct_get.o test_nfct_get.o -lmnl -lnetfilter_conntrack + +test_nfct_get.o: test_nfct_get.c + +testnftnlrdr_peer.o: testnftnlrdr_peer.c + +testnftnlrdr_dscp.o: testnftnlrdr_dscp.c + +nftnlrdr.o: nftnlrdr.c nftnlrdr.h + +nftnlrdr_misc.o: nftnlrdr_misc.c + +iptpinhole.o: iptpinhole.c iptpinhole.h + +upnpglobalvars.o: ../upnpglobalvars.c ../upnpglobalvars.h + $(CC) -c -o $@ $< diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/README.md b/src/contrib/miniupnp/miniupnpd/netfilter_nft/README.md new file mode 100644 index 0000000..b37d482 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/README.md @@ -0,0 +1,21 @@ +Miniupnpd nftables support by Tomofumi Hayashi (s1061123@gmail.com). + +##Current Status +nftables support is 'alpha' version, not "so much" stable. + +##Supported Features +- IPv4 NAT/Filter add/del. + +##How to build miniupnpd with nftables: +Run 'make' command with 'Makefile.linux_nft', + +`make -f Makefile.linux_nft` + +##How to Run +Please run 'netfilter_nft/scripts/nft_init.sh' to add miniupnpd chain. + +`sudo ./netfilter_nft/scripts/nft_init.sh` + +##FAQ +I will add this section when I get question. +Comments and Questions are welcome ;) diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/nfct_get.c b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nfct_get.c new file mode 100644 index 0000000..cbffd28 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nfct_get.c @@ -0,0 +1,258 @@ +#include +#include +#include +#include +#include +#include + +#ifdef USE_NFCT +#include +#include + +#include + +struct data_cb_s +{ + struct sockaddr_storage * ext; + uint8_t found; +}; + +static int data_cb(const struct nlmsghdr *nlh, void *data) +{ + struct nf_conntrack *ct; + struct data_cb_s * d = (struct data_cb_s*) data; + struct sockaddr_in* ext4 = (struct sockaddr_in*) d->ext; + + ct = nfct_new(); + if (ct == NULL) + return MNL_CB_OK; + nfct_nlmsg_parse(nlh, ct); + + if (data) { + ext4->sin_addr.s_addr = nfct_get_attr_u32(ct, ATTR_REPL_IPV4_DST); + ext4->sin_port = nfct_get_attr_u16(ct, ATTR_REPL_PORT_DST); + } + d->found = 1; + nfct_destroy(ct); + + return MNL_CB_OK; +} + +int get_nat_ext_addr(struct sockaddr* src, struct sockaddr *dst, uint8_t proto, + struct sockaddr_storage* ret_ext) +{ + struct mnl_socket *nl; + struct nlmsghdr *nlh; + struct nfgenmsg *nfh; + char buf[MNL_SOCKET_BUFFER_SIZE]; + unsigned int seq, portid; + struct nf_conntrack *ct; + int ret; + struct data_cb_s data; + + if ((!src)&&(!dst)) { + return 0; + } + + if (src->sa_family != dst->sa_family) { + return 0; + } + + nl = mnl_socket_open(NETLINK_NETFILTER); + if (nl == NULL) { +// perror("mnl_socket_open"); + goto free_nl; + } + + if (mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { +// perror("mnl_socket_bind"); + goto free_nl; + } + portid = mnl_socket_get_portid(nl); + + memset(buf, 0, sizeof(buf)); + nlh = mnl_nlmsg_put_header(buf); + nlh->nlmsg_type = (NFNL_SUBSYS_CTNETLINK << 8) | IPCTNL_MSG_CT_GET; + nlh->nlmsg_flags = NLM_F_REQUEST|NLM_F_ACK; + nlh->nlmsg_seq = seq = time(NULL); + + nfh = mnl_nlmsg_put_extra_header(nlh, sizeof(struct nfgenmsg)); + nfh->nfgen_family = src->sa_family; + nfh->version = NFNETLINK_V0; + nfh->res_id = 0; + + ct = nfct_new(); + if (ct == NULL) { + goto free_nl; + } + + nfct_set_attr_u8(ct, ATTR_L3PROTO, src->sa_family); + if (src->sa_family == AF_INET) { + struct sockaddr_in *src4 = (struct sockaddr_in *)src; + struct sockaddr_in *dst4 = (struct sockaddr_in *)dst; + nfct_set_attr_u32(ct, ATTR_IPV4_SRC, src4->sin_addr.s_addr); + nfct_set_attr_u32(ct, ATTR_IPV4_DST, dst4->sin_addr.s_addr); + nfct_set_attr_u16(ct, ATTR_PORT_SRC, src4->sin_port); + nfct_set_attr_u16(ct, ATTR_PORT_DST, dst4->sin_port); + } else if (src->sa_family == AF_INET6) { + struct sockaddr_in6 *src6 = (struct sockaddr_in6 *)src; + struct sockaddr_in6 *dst6 = (struct sockaddr_in6 *)dst; + nfct_set_attr(ct, ATTR_IPV6_SRC, &src6->sin6_addr); + nfct_set_attr(ct, ATTR_IPV6_DST, &dst6->sin6_addr); + nfct_set_attr_u16(ct, ATTR_PORT_SRC, src6->sin6_port); + nfct_set_attr_u16(ct, ATTR_PORT_DST, dst6->sin6_port); + } + nfct_set_attr_u8(ct, ATTR_L4PROTO, proto); + + nfct_nlmsg_build(nlh, ct); + + ret = mnl_socket_sendto(nl, nlh, nlh->nlmsg_len); + if (ret == -1) { + goto free_ct; + } + + ret = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + data.ext = ret_ext; + data.found = 0; + while (ret > 0) { + ret = mnl_cb_run(buf, ret, seq, portid, data_cb, &data); + if (ret <= MNL_CB_STOP) + break; + ret = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + } + +free_ct: + nfct_destroy(ct); +free_nl: + mnl_socket_close(nl); + + return data.found; +} + +#else +#define DST "dst=" +#define DST_PORT "dport=" +#define SRC "src=" +#define SRC_PORT "sport=" +#define IP_CONNTRACK_LOCATION "/proc/net/ip_conntrack" +#define NF_CONNTRACK_LOCATION "/proc/net/nf_conntrack" + +int get_nat_ext_addr(struct sockaddr* src, struct sockaddr *dst, uint8_t proto, + struct sockaddr_storage* ret_ext) +{ + FILE *f; + int af; + + if (!src) + return -2; + + af = src->sa_family; + + if ((f = fopen(NF_CONNTRACK_LOCATION, "r")) == NULL) { + if ((f = fopen(IP_CONNTRACK_LOCATION, "r")) == NULL) { + printf("could not read info about connections from the kernel, " + "make sure netfilter is enabled in kernel or by modules.\n"); + return -1; + } + } + + while (!feof(f)) { + char line[256], *str; + memset(line, 0, sizeof(line)); + str = fgets(line, sizeof(line), f); + if (line[0] != 0) { + char *token, *saveptr; + int j; + uint8_t src_f, src_port_f, dst_f, dst_port_f; + src_f=src_port_f=dst_f=dst_port_f=0; + + for (j = 1; ; j++, str = NULL) { + token = strtok_r(str, " ", &saveptr); + if (token == NULL) + break; + + if ((j==2)&&(af!=atoi(token))) + break; + if ((j==4)&&(proto!=atoi(token))) + break; + if (j<=4) + continue; + + if (strncmp(token, SRC, sizeof(SRC) - 1) == 0) { + char *srcip = token + sizeof(SRC) - 1; + uint32_t buf[4]; + memset(buf,0,sizeof(buf)); + + if (inet_pton(af, srcip, buf)!=1) + break; + + if (af==AF_INET) { + struct sockaddr_in *src4=(struct sockaddr_in*)src; + if (!src_f) { + if (src4->sin_addr.s_addr != buf[0]) + break; + src_f = 1; + } + } + } + if (strncmp(token, SRC_PORT, sizeof(SRC_PORT) - 1) == 0) { + char *src_port = token + sizeof(SRC_PORT) - 1; + uint16_t port=atoi(src_port); + + if (af==AF_INET) { + struct sockaddr_in *src4=(struct sockaddr_in*)src; + if (!src_port_f) { + if (ntohs(src4->sin_port) != port) + break; + src_port_f = 1; + } + } + } + + if (strncmp(token, DST, sizeof(DST) - 1) == 0) { + char *dstip = token + sizeof(DST) - 1; + uint32_t buf[4]; + memset(buf,0,sizeof(buf)); + if (inet_pton(af, dstip, buf)!=1) + break; + if (af==AF_INET) { + struct sockaddr_in *dst4=(struct sockaddr_in*)dst; + if (!dst_f) { + if (dst4->sin_addr.s_addr != buf[0]) + break; + dst_f = 1; + } else { + struct sockaddr_in*ret4=(struct sockaddr_in*)ret_ext; + ret_ext->ss_family = AF_INET; + ret4->sin_addr.s_addr = buf[0]; + } + } + } + if (strncmp(token, DST_PORT, sizeof(DST_PORT)-1) == 0) { + char *dst_port = token + sizeof(DST_PORT) - 1; + uint16_t port=atoi(dst_port); + if (af==AF_INET) { + struct sockaddr_in *dst4=(struct sockaddr_in*)dst; + if (!dst_port_f) { + if (ntohs(dst4->sin_port) != port) + break; + dst_port_f = 1; + } else { + struct sockaddr_in*ret4=(struct sockaddr_in*)ret_ext; + ret_ext->ss_family = AF_INET; + ret4->sin_port = htons(port); + } + } + } + } + if (src_f && src_port_f && dst_f && dst_port_f) { + fclose(f); + return 1; + } + } + } + fclose(f); + + return 0; +} +#endif diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr.c b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr.c new file mode 100644 index 0000000..389273f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr.c @@ -0,0 +1,499 @@ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2015 Tomofumi Hayashi + * + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include + +#include "tiny_nf_nat.h" + +#include "../macros.h" +#include "../config.h" +#include "nftnlrdr.h" +#include "../upnpglobalvars.h" + +#include "nftnlrdr_misc.h" + +#ifdef DEBUG +#define d_printf(x) do { printf x; } while (0) +#else +#define d_printf(x) +#endif + +/* dummy init and shutdown functions */ +int init_redirect(void) +{ + return 0; +} + +void shutdown_redirect(void) +{ + return; +} + + +int +add_redirect_rule2(const char * ifname, + const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp) +{ + struct nft_rule *r; + UNUSED(rhost); + UNUSED(timestamp); + d_printf(("add redirect rule2(%s, %s, %u, %s, %u, %d, %s)!\n", + ifname, rhost, eport, iaddr, iport, proto, desc)); + r = rule_set_dnat(NFPROTO_IPV4, ifname, proto, + 0, eport, + inet_addr(iaddr), iport, desc, NULL); + return nft_send_request(r, NFT_MSG_NEWRULE); +} + +/* + * This function submit the rule as following: + * nft add rule nat miniupnpd-pcp-peer ip + * saddr ip daddr tcp sport + * tcp dport snat : + */ +int +add_peer_redirect_rule2(const char * ifname, + const char * rhost, unsigned short rport, + const char * eaddr, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp) +{ + struct nft_rule *r; + UNUSED(ifname); UNUSED(timestamp); + + d_printf(("add peer redirect rule2()!\n")); + r = rule_set_snat(NFPROTO_IPV4, proto, + inet_addr(rhost), rport, + inet_addr(eaddr), eport, + inet_addr(iaddr), iport, desc, NULL); + + return nft_send_request(r, NFT_MSG_NEWRULE); +} + +/* + * This function submit the rule as following: + * nft add rule filter miniupnpd + * ip daddr tcp dport accept + * + */ +int +add_filter_rule2(const char * ifname, + const char * rhost, const char * iaddr, + unsigned short eport, unsigned short iport, + int proto, const char * desc) +{ + struct nft_rule *r = NULL; + in_addr_t rhost_addr = 0; + + d_printf(("add_filter_rule2(%s, %s, %s, %d, %d, %d, %s)\n", + ifname, rhost, iaddr, eport, iport, proto, desc)); + if (rhost != NULL && strcmp(rhost, "") != 0) { + rhost_addr = inet_addr(rhost); + } + r = rule_set_filter(NFPROTO_IPV4, ifname, proto, + rhost_addr, inet_addr(iaddr), eport, iport, + desc, 0); + return nft_send_request(r, NFT_MSG_NEWRULE); +} + +/* + * add_peer_dscp_rule2() is not supported due to nft does not support + * dscp set. + */ +int +add_peer_dscp_rule2(const char * ifname, + const char * rhost, unsigned short rport, + unsigned char dscp, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp) +{ + UNUSED(ifname); UNUSED(rhost); UNUSED(rport); + UNUSED(dscp); UNUSED(iaddr); UNUSED(iport); UNUSED(proto); + UNUSED(desc); UNUSED(timestamp); + syslog(LOG_ERR, "add_peer_dscp_rule2: not supported"); + return 0; +} + +/* + * Clear all rules corresponding eport/proto + */ +int +delete_redirect_and_filter_rules(unsigned short eport, int proto) +{ + rule_t *p; + struct nft_rule *r = NULL; + in_addr_t iaddr = 0; + uint16_t iport = 0; + extern void print_rule(rule_t *r) ; + + d_printf(("delete_redirect_and_filter_rules(%d %d)\n", eport, proto)); + reflesh_nft_cache(NFPROTO_IPV4); + LIST_FOREACH(p, &head, entry) { + if (p->eport == eport && p->proto == proto && + (p->type == RULE_NAT || p->type == RULE_SNAT)) { + iaddr = p->iaddr; + iport = p->iport; + + r = rule_del_handle(p); + /* Todo: send bulk request */ + nft_send_request(r, NFT_MSG_DELRULE); + break; + } + } + + if (iaddr == 0 && iport == 0) { + return -1; + } + reflesh_nft_cache(NFPROTO_IPV4); + LIST_FOREACH(p, &head, entry) { + if (p->eport == iport && + p->iaddr == iaddr && p->type == RULE_FILTER) { + r = rule_del_handle(p); + /* Todo: send bulk request */ + nft_send_request(r, NFT_MSG_DELRULE); + break; + } + } + + return 0; +} + +/* + * get peer by index as array. + * return -1 when not found. + */ +int +get_peer_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, unsigned short * rport, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + int i; + struct in_addr addr; + char *addr_str; + rule_t *r; + UNUSED(timestamp); UNUSED(packets); UNUSED(bytes); + + d_printf(("get_peer_rule_by_index()\n")); + reflesh_nft_cache(NFPROTO_IPV4); + if (peer_cache == NULL) { + return -1; + } + + for (i = 0; peer_cache[i] != NULL; i++) { + if (index == i) { + r = peer_cache[i]; + if (ifname != NULL) { + if_indextoname(r->ingress_ifidx, ifname); + } + if (eport != NULL) { + *eport = r->eport; + } + if (iaddr != NULL) { + addr.s_addr = r->iaddr; + addr_str = inet_ntoa(addr); + strncpy(iaddr , addr_str, iaddrlen); + } + if (iport != NULL) { + *iport = r->iport; + } + if (proto != NULL) { + *proto = r->proto; + } + if (rhost != NULL) { + addr.s_addr = r->rhost; + addr_str = inet_ntoa(addr); + strncpy(iaddr , addr_str, rhostlen); + } + if (rport != NULL) { + *rport = r->rport; + } + if (desc != NULL) { + strncpy(desc, r->desc, desclen); + } + + /* + * TODO: Implement counter in case of add {nat,filter} + */ + return 0; + } + } + return -1; +} + +/* + * get_redirect_rule() + * returns -1 if the rule is not found + */ +int +get_redirect_rule(const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + return get_nat_redirect_rule(NFT_TABLE_NAT, + ifname, eport, proto, + iaddr, iaddrlen, iport, + desc, desclen, + rhost, rhostlen, + timestamp, packets, bytes); +} + +/* + * get_redirect_rule_by_index() + * return -1 when the rule was not found + */ +int +get_redirect_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + int i; + struct in_addr addr; + char *addr_str; + rule_t *r; + UNUSED(timestamp); UNUSED(packets); UNUSED(bytes); + + d_printf(("get_redirect_rule_by_index()\n")); + reflesh_nft_cache(NFPROTO_IPV4); + if (redirect_cache == NULL) { + return -1; + } + + for (i = 0; redirect_cache[i] != NULL; i++) { + if (index == i) { + r = redirect_cache[i]; + if (ifname != NULL) { + if_indextoname(r->ingress_ifidx, ifname); + } + if (eport != NULL) { + *eport = r->eport; + } + if (iaddr != NULL) { + addr.s_addr = r->iaddr; + addr_str = inet_ntoa(addr); + strncpy(iaddr , addr_str, iaddrlen); + } + if (iport != NULL) { + *iport = r->iport; + } + if (proto != NULL) { + *proto = r->proto; + } + if (rhost != NULL) { + addr.s_addr = r->rhost; + addr_str = inet_ntoa(addr); + strncpy(iaddr , addr_str, rhostlen); + } + if (desc != NULL && r->desc) { + strncpy(desc, r->desc, desclen); + } + + /* + * TODO: Implement counter in case of add {nat,filter} + */ + return 0; + } + } + return -1; +} + +/* + * return -1 not found. + * return 0 found + */ +int +get_nat_redirect_rule(const char * nat_chain_name, const char * ifname, + unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + rule_t *p; + struct in_addr addr; + char *addr_str; + UNUSED(nat_chain_name); + UNUSED(ifname); + UNUSED(iaddrlen); + UNUSED(timestamp); + UNUSED(packets); + UNUSED(bytes); + + d_printf(("get_nat_redirect_rule()\n")); + reflesh_nft_cache(NFPROTO_IPV4); + + LIST_FOREACH(p, &head, entry) { + if (p->proto == proto && + p->eport == eport) { + if (p->rhost && rhost) { + addr.s_addr = p->rhost; + addr_str = inet_ntoa(addr); + strncpy(iaddr , addr_str, rhostlen); + + } + if (desc != NULL && p->desc) { + strncpy(desc, p->desc, desclen); + } + *iport = p->iport; + return 0; + } + } + + return -1; +} + +/* + * return an (malloc'ed) array of "external" port for which there is + * a port mapping. number is the size of the array + */ +unsigned short * +get_portmappings_in_range(unsigned short startport, unsigned short endport, + int proto, unsigned int * number) +{ + uint32_t capacity; + rule_t *p; + unsigned short *array; + unsigned short *tmp; + + d_printf(("get_portmappings_in_range()\n")); + *number = 0; + capacity = 128; + array = calloc(capacity, sizeof(unsigned short)); + + if (array == NULL) { + syslog(LOG_ERR, "get_portmappings_in_range(): calloc error"); + return NULL; + } + + LIST_FOREACH(p, &head, entry) { + if (p->proto == proto && + startport <= p->eport && + p->eport <= endport) { + + if (*number >= capacity) { + tmp = realloc(array, + sizeof(unsigned short)*capacity); + if (tmp == NULL) { + syslog(LOG_ERR, + "get_portmappings_in_range(): " + "realloc(%u) error", + (unsigned)sizeof(unsigned short)*capacity); + *number = 0; + free(array); + return NULL; + } + array = tmp; + } + array[*number] = p->eport; + (*number)++; + } + } + return array; +} + +/* for debug */ +/* read the "filter" and "nat" tables */ +int +list_redirect_rule(const char * ifname) +{ + rule_t *p; + UNUSED(ifname); + + reflesh_nft_cache(NFPROTO_IPV4); + + LIST_FOREACH(p, &head, entry) { + print_rule(p); + } + + return -1; + return 0; +} + + +#if 0 +/* delete_rule_and_commit() : + * subfunction used in delete_redirect_and_filter_rules() */ +static int +delete_rule_and_commit(unsigned int index, IPTC_HANDLE h, + const char * miniupnpd_chain, + const char * logcaller) +{ +/* TODO: Implement it */ +} + +/* TODO: Implement it */ +static void +print_iface(const char * iface, const unsigned char * mask, int invert) +{ + unsigned i; + if(mask[0] == 0) + return; + if(invert) + printf("! "); + for(i=0; i> 24, (ip >> 16) & 0xff, + (ip >> 8) & 0xff, ip & 0xff); +} +#endif + +#endif /* if 0 */ diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr.h b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr.h new file mode 100644 index 0000000..1acf4b3 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr.h @@ -0,0 +1,84 @@ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2015 Tomofumi Hayashi + * + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution. + */ + +#ifndef NFTNLRDR_H_INCLUDED +#define NFTNLRDR_H_INCLUDED + +#include "../commonrdr.h" +int init_redirect(void); +void shutdown_redirect(void); + +int +add_redirect_rule2(const char * ifname, + const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp); + +int +add_peer_redirect_rule2(const char * ifname, + const char * rhost, unsigned short rport, + const char * eaddr, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp); + +int +add_filter_rule2(const char * ifname, + const char * rhost, const char * iaddr, + unsigned short eport, unsigned short iport, + int proto, const char * desc); + +int +delete_redirect_and_filter_rules(unsigned short eport, int proto); + +int +add_peer_dscp_rule2(const char * ifname, + const char * rhost, unsigned short rport, + unsigned char dscp, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp); + +int +get_peer_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, unsigned short * rport, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); +int +get_nat_redirect_rule(const char * nat_chain_name, const char * ifname, + unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); +int +get_redirect_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); + +unsigned short * +get_portmappings_in_range(unsigned short startport, unsigned short endport, + int proto, unsigned int * number); + +/* in nfct_get.c */ +int get_nat_ext_addr(struct sockaddr* src, struct sockaddr *dst, uint8_t proto, + struct sockaddr* ret_ext); + +/* for debug */ +int +list_redirect_rule(const char * ifname); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr_misc.c b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr_misc.c new file mode 100644 index 0000000..a35c45e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr_misc.c @@ -0,0 +1,1145 @@ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2015 Tomofumi Hayashi + * + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include "nftnlrdr_misc.h" +#include "../macros.h" +#include "../upnpglobalvars.h" + +#ifdef DEBUG +#define d_printf(x) do { printf x; } while (0) +#else +#define d_printf(x) +#endif + +#define RULE_CACHE_INVALID 0 +#define RULE_CACHE_VALID 1 + +const char * miniupnpd_nft_nat_chain = "miniupnpd"; +const char * miniupnpd_nft_peer_chain = "miniupnpd-pcp-peer"; +const char * miniupnpd_nft_forward_chain = "miniupnpd"; + +static struct mnl_socket *nl = NULL; +struct rule_list head = LIST_HEAD_INITIALIZER(head); +static uint32_t rule_list_validate = RULE_CACHE_INVALID; + +uint32_t rule_list_length = 0; +uint32_t rule_list_peer_length = 0; + +rule_t **redirect_cache; +rule_t **peer_cache; + + +static const char * +get_family_string(uint32_t family) +{ + switch (family) { + case NFPROTO_IPV4: + return "ipv4"; + } + return "unknown family"; +} + +static const char * +get_proto_string(uint32_t proto) +{ + switch (proto) { + case IPPROTO_TCP: + return "tcp"; + case IPPROTO_UDP: + return "udp"; + } + return "unknown proto"; +} + +static const char * +get_verdict_string(uint32_t val) +{ + switch (val) { + case NF_ACCEPT: + return "accept"; + case NF_DROP: + return "drop"; + default: + return "unknown verdict"; + } +} + +void +print_rule(rule_t *r) +{ + struct in_addr addr; + char *iaddr_str = NULL, *rhost_str = NULL, *eaddr_str = NULL; + char ifname_buf[IF_NAMESIZE]; + + switch (r->type) { + case RULE_NAT: + if (r->iaddr != 0) { + addr.s_addr = r->iaddr; + iaddr_str = strdupa(inet_ntoa(addr)); + } + if (r->rhost != 0) { + addr.s_addr = r->rhost; + rhost_str = strdupa(inet_ntoa(addr)); + } + if (r->eaddr != 0) { + addr.s_addr = r->eaddr; + eaddr_str = strdupa(inet_ntoa(addr)); + } + if (r->nat_type == NFT_NAT_DNAT) { + printf("%"PRIu64":[%s/%s] iif %s, %s/%s, %d -> " + "%s:%d (%s)\n", + r->handle, + r->table, r->chain, + if_indextoname(r->ingress_ifidx, ifname_buf), + get_family_string(r->family), + get_proto_string(r->proto), r->eport, + iaddr_str, r->iport, + r->desc); + } else if (r->nat_type == NFT_NAT_SNAT) { + printf("%"PRIu64":[%s/%s] " + "nat type:%d, family:%d, ifidx: %d, " + "eaddr: %s, eport:%d, " + "proto:%d, iaddr: %s, " + "iport:%d, rhost:%s rport:%d (%s)\n", + r->handle, r->table, r->chain, + r->nat_type, r->family, r->ingress_ifidx, + eaddr_str, r->eport, + r->proto, iaddr_str, r->iport, + rhost_str, r->rport, + r->desc); + } else { + printf("%"PRIu64":[%s/%s] " + "nat type:%d, family:%d, ifidx: %d, " + "eaddr: %s, eport:%d, " + "proto:%d, iaddr: %s, iport:%d, rhost:%s (%s)\n", + r->handle, r->table, r->chain, + r->nat_type, r->family, r->ingress_ifidx, + eaddr_str, r->eport, + r->proto, iaddr_str, r->iport, rhost_str, + r->desc); + } + break; + case RULE_FILTER: + if (r->iaddr != 0) { + addr.s_addr = r->iaddr; + iaddr_str = strdupa(inet_ntoa(addr)); + } + if (r->rhost != 0) { + addr.s_addr = r->rhost; + rhost_str = strdupa(inet_ntoa(addr)); + } + printf("%"PRIu64":[%s/%s] %s/%s, %s %s:%d: %s (%s)\n", + r->handle, r->table, r->chain, + get_family_string(r->family), get_proto_string(r->proto), + rhost_str, + iaddr_str, r->eport, + get_verdict_string(r->filter_action), + r->desc); + break; + case RULE_COUNTER: + if (r->iaddr != 0) { + addr.s_addr = r->iaddr; + iaddr_str = strdupa(inet_ntoa(addr)); + } + if (r->rhost != 0) { + addr.s_addr = r->iaddr; + rhost_str = strdupa(inet_ntoa(addr)); + } + printf("%"PRIu64":[%s/%s] %s/%s, %s:%d: " + "packets:%"PRIu64", bytes:%"PRIu64"\n", + r->handle, r->table, r->chain, + get_family_string(r->family), get_proto_string(r->proto), + iaddr_str, r->eport, r->packets, r->bytes); + break; + default: + printf("nftables: unknown type: %d\n", r->type); + } +} + +static enum rule_reg_type * +get_reg_type_ptr(rule_t *r, uint32_t dreg) +{ + switch (dreg) { + case NFT_REG_1: + return &r->reg1_type; + case NFT_REG_2: + return &r->reg2_type; + default: + return NULL; + } +} + +static uint32_t * +get_reg_val_ptr(rule_t *r, uint32_t dreg) +{ + switch (dreg) { + case NFT_REG_1: + return &r->reg1_val; + case NFT_REG_2: + return &r->reg2_val; + default: + return NULL; + } +} + +static void +set_reg (rule_t *r, uint32_t dreg, enum rule_reg_type type, uint32_t val) +{ + if (dreg == NFT_REG_1) { + r->reg1_type = type; + if (type == RULE_REG_IMM_VAL) { + r->reg1_val = val; + } + } else if (dreg == NFT_REG_2) { + r->reg2_type = type; + if (type == RULE_REG_IMM_VAL) { + r->reg2_val = val; + } + } else if (dreg == NFT_REG_VERDICT) { + if (r->type == RULE_FILTER) { + r->filter_action = val; + } + } else { + syslog(LOG_ERR, "%s: unknown reg:%d", "set_reg", dreg); + } + return ; +} + +static inline void +parse_rule_immediate(struct nft_rule_expr *e, rule_t *r) +{ + uint32_t dreg, reg_val, reg_len; + + dreg = nft_rule_expr_get_u32(e, NFT_EXPR_IMM_DREG); + + if (dreg == NFT_REG_VERDICT) { + reg_val = nft_rule_expr_get_u32(e, NFT_EXPR_IMM_VERDICT); + } else { + reg_val = *(uint32_t *)nft_rule_expr_get(e, + NFT_EXPR_IMM_DATA, + ®_len); + } + + set_reg(r, dreg, RULE_REG_IMM_VAL, reg_val); + return; +} + +static inline void +parse_rule_counter(struct nft_rule_expr *e, rule_t *r) +{ + r->type = RULE_COUNTER; + r->bytes = nft_rule_expr_get_u64(e, NFT_EXPR_CTR_BYTES); + r->packets = nft_rule_expr_get_u64(e, NFT_EXPR_CTR_PACKETS); + + return; +} + +static inline void +parse_rule_meta(struct nft_rule_expr *e, rule_t *r) +{ + uint32_t key = nft_rule_expr_get_u32(e, NFT_EXPR_META_KEY); + uint32_t dreg = nft_rule_expr_get_u32(e, NFT_EXPR_META_DREG); + enum rule_reg_type reg_type; + + switch (key) { + case NFT_META_IIF: + reg_type = RULE_REG_IIF; + set_reg(r, dreg, reg_type, 0); + return ; + + case NFT_META_OIF: + reg_type = RULE_REG_IIF; + set_reg(r, dreg, reg_type, 0); + return ; + + } + syslog(LOG_DEBUG, "parse_rule_meta :Not support key %d\n", key); + + return; +} + +static inline void +parse_rule_nat(struct nft_rule_expr *e, rule_t *r) +{ + uint32_t addr_min_reg, addr_max_reg, proto_min_reg, proto_max_reg; + uint16_t proto_min_val; + r->type = RULE_NAT; + + r->nat_type = nft_rule_expr_get_u32(e, NFT_EXPR_NAT_TYPE); + r->family = nft_rule_expr_get_u32(e, NFT_EXPR_NAT_FAMILY); + addr_min_reg = nft_rule_expr_get_u32(e, NFT_EXPR_NAT_REG_ADDR_MIN); + addr_max_reg = nft_rule_expr_get_u32(e, NFT_EXPR_NAT_REG_ADDR_MAX); + proto_min_reg = nft_rule_expr_get_u32(e, NFT_EXPR_NAT_REG_PROTO_MIN); + proto_max_reg = nft_rule_expr_get_u32(e, NFT_EXPR_NAT_REG_PROTO_MAX); + + if (addr_min_reg != addr_max_reg || + proto_min_reg != proto_max_reg) { + syslog(LOG_ERR, "Unsupport proto/addr range for NAT"); + } + + proto_min_val = htons((uint16_t)*get_reg_val_ptr(r, proto_min_reg)); + if (r->nat_type == NFT_NAT_DNAT) { + r->iaddr = (in_addr_t)*get_reg_val_ptr(r, addr_min_reg); + r->iport = proto_min_val; + } else if (r->nat_type == NFT_NAT_SNAT) { + r->eaddr = (in_addr_t)*get_reg_val_ptr(r, addr_min_reg); + if (proto_min_reg == NFT_REG_1) { + r->eport = proto_min_val; + } + } + + set_reg(r, NFT_REG_1, RULE_REG_NONE, 0); + set_reg(r, NFT_REG_2, RULE_REG_NONE, 0); + return; +} + +static inline void +parse_rule_payload(struct nft_rule_expr *e, rule_t *r) +{ + uint32_t base, dreg, offset, len; + uint32_t *regptr; + + dreg = nft_rule_expr_get_u32(e, NFT_EXPR_PAYLOAD_DREG); + base = nft_rule_expr_get_u32(e, NFT_EXPR_PAYLOAD_BASE); + offset = nft_rule_expr_get_u32(e, NFT_EXPR_PAYLOAD_OFFSET); + len = nft_rule_expr_get_u32(e, NFT_EXPR_PAYLOAD_LEN); + regptr = get_reg_type_ptr(r, dreg); + + switch (base) { + case NFT_PAYLOAD_NETWORK_HEADER: + if (offset == offsetof(struct iphdr, daddr) && + len == sizeof(in_addr_t)) { + *regptr = RULE_REG_IP_DEST_ADDR; + return; + } else if (offset == offsetof(struct iphdr, saddr) && + len == sizeof(in_addr_t)) { + *regptr = RULE_REG_IP_SRC_ADDR; + return; + } else if (offset == offsetof(struct iphdr, saddr) && + len == sizeof(in_addr_t) * 2) { + *regptr = RULE_REG_IP_SD_ADDR; + return; + } else if (offset == offsetof(struct iphdr, protocol) && + len == sizeof(uint8_t)) { + *regptr = RULE_REG_IP_PROTO; + return; + } + case NFT_PAYLOAD_TRANSPORT_HEADER: + if (offset == offsetof(struct tcphdr, dest) && + len == sizeof(uint16_t)) { + *regptr = RULE_REG_TCP_DPORT; + return; + } else if (offset == offsetof(struct tcphdr, source) && + len == sizeof(uint16_t) * 2) { + *regptr = RULE_REG_TCP_SD_PORT; + return; + } + } + syslog(LOG_DEBUG, + "Unsupport payload: (dreg:%d, base:%d, offset:%d, len:%d)", + dreg, base, offset, len); + return; +} + +/* + * + * Note: Currently support only NFT_REG_1 + */ +static inline void +parse_rule_cmp(struct nft_rule_expr *e, rule_t *r) { + uint32_t data_len; + void *data_val; + uint32_t op, sreg; + uint16_t *ports; + in_addr_t *addrp; + + data_val = (void *)nft_rule_expr_get(e, NFT_EXPR_CMP_DATA, &data_len); + sreg = nft_rule_expr_get_u32(e, NFT_EXPR_CMP_SREG); + op = nft_rule_expr_get_u32(e, NFT_EXPR_CMP_OP); + + if (sreg != NFT_REG_1) { + syslog(LOG_ERR, "parse_rule_cmp: Unsupport reg:%d", sreg); + return; + } + + switch (r->reg1_type) { + case RULE_REG_IIF: + if (data_len == sizeof(uint32_t) && op == NFT_CMP_EQ) { + r->ingress_ifidx = *(uint32_t *)data_val; + r->reg1_type = RULE_REG_NONE; + return; + } + case RULE_REG_IP_SRC_ADDR: + if (data_len == sizeof(in_addr_t) && op == NFT_CMP_EQ) { + r->rhost = *(in_addr_t *)data_val; + r->reg1_type = RULE_REG_NONE; + return; + } + case RULE_REG_IP_DEST_ADDR: + if (data_len == sizeof(in_addr_t) && op == NFT_CMP_EQ) { + if (r->type == RULE_FILTER) { + r->iaddr = *(in_addr_t *)data_val; + } else { + r->rhost = *(in_addr_t *)data_val; + } + r->reg1_type = RULE_REG_NONE; + return; + } + case RULE_REG_IP_SD_ADDR: + if (data_len == sizeof(in_addr_t) * 2 && op == NFT_CMP_EQ) { + addrp = (in_addr_t *)data_val; + r->iaddr = addrp[0]; + r->rhost = addrp[1]; + r->reg1_type = RULE_REG_NONE; + return; + } + case RULE_REG_IP_PROTO: + if (data_len == sizeof(uint8_t) && op == NFT_CMP_EQ) { + r->proto = *(uint8_t *)data_val; + r->reg1_type = RULE_REG_NONE; + return; + } + case RULE_REG_TCP_DPORT: + if (data_len == sizeof(uint16_t) && op == NFT_CMP_EQ) { + r->eport = ntohs(*(uint16_t *)data_val); + r->reg1_type = RULE_REG_NONE; + return; + } + case RULE_REG_TCP_SD_PORT: + if (data_len == sizeof(uint16_t) * 2 && op == NFT_CMP_EQ) { + ports = (uint16_t *)data_val; + r->iport = ntohs(ports[0]); + r->rport = ntohs(ports[1]); + r->reg1_type = RULE_REG_NONE; + return; + } + default: + break; + } + syslog(LOG_DEBUG, "Unknown cmp (r1type:%d, data_len:%d, op:%d)", + r->reg1_type, data_len, op); + return; +} + +static int +rule_expr_cb(struct nft_rule_expr *e, void *data) +{ + rule_t *r = data; + const char *attr_name = nft_rule_expr_get_str(e, + NFT_RULE_EXPR_ATTR_NAME); + + if (strncmp("cmp", attr_name, sizeof("cmp")) == 0) { + parse_rule_cmp(e, r); + } else if (strncmp("nat", attr_name, sizeof("nat")) == 0) { + parse_rule_nat(e, r); + } else if (strncmp("meta", attr_name, sizeof("meta")) == 0) { + parse_rule_meta(e, r); + } else if (strncmp("counter", attr_name, sizeof("counter")) == 0) { + parse_rule_counter(e, r); + } else if (strncmp("payload", attr_name, sizeof("payload")) == 0) { + parse_rule_payload(e, r); + } else if (strncmp("immediate", attr_name, sizeof("immediate")) == 0) { + parse_rule_immediate(e, r); + } else { + syslog(LOG_ERR, "unknown attr: %s\n", attr_name); + } + return MNL_CB_OK; +} + + +static int +table_cb(const struct nlmsghdr *nlh, void *data) +{ + struct nft_rule *t; + uint32_t len; + struct nft_rule_expr *expr; + struct nft_rule_expr_iter *itr; + rule_t *r; + char *chain; + UNUSED(data); + + r = malloc(sizeof(rule_t)); + + memset(r, 0, sizeof(rule_t)); + t = nft_rule_alloc(); + if (t == NULL) { + perror("OOM"); + goto err; + } + + if (nft_rule_nlmsg_parse(nlh, t) < 0) { + perror("nft_rule_nlmsg_parse"); + goto err_free; + } + + chain = (char *)nft_rule_attr_get_data(t, NFT_RULE_ATTR_CHAIN, &len); + if (strcmp(chain, miniupnpd_nft_nat_chain) != 0 && + strcmp(chain, miniupnpd_nft_peer_chain) != 0 && + strcmp(chain, miniupnpd_nft_forward_chain) != 0) { + goto rule_skip; + } + + r->table = strdup( + (char *)nft_rule_attr_get_data(t, NFT_RULE_ATTR_TABLE, &len)); + r->chain = strdup(chain); + r->family = *(uint32_t*)nft_rule_attr_get_data(t, NFT_RULE_ATTR_FAMILY, + &len); + r->desc = (char *)nft_rule_attr_get_data(t, NFT_RULE_ATTR_USERDATA, + &len); + r->handle = *(uint32_t*)nft_rule_attr_get_data(t, + NFT_RULE_ATTR_HANDLE, + &len); + if (strcmp(r->table, NFT_TABLE_NAT) == 0) { + r->type = RULE_NAT; + } else if (strcmp(r->table, NFT_TABLE_FILTER) == 0) { + r->type = RULE_FILTER; + } + if (strcmp(r->chain, miniupnpd_nft_peer_chain) == 0) { + rule_list_peer_length++; + } + + itr = nft_rule_expr_iter_create(t); + + while ((expr = nft_rule_expr_iter_next(itr)) != NULL) { + rule_expr_cb(expr, r); + } + + if (r->type == RULE_NONE) { + free(r); + } else { + LIST_INSERT_HEAD(&head, r, entry); + rule_list_length++; + } + +rule_skip: +err_free: + nft_rule_free(t); +err: + return MNL_CB_OK; +} + +void +reflesh_nft_redirect_cache(void) +{ + rule_t *p; + int i; + uint32_t len; + + if (redirect_cache != NULL) { + free(redirect_cache); + } + len = rule_list_length - rule_list_peer_length; + if (len == 0) { + redirect_cache = NULL; + return; + } + + redirect_cache = (rule_t **)malloc(sizeof(rule_t *) * len); + bzero(redirect_cache, sizeof(rule_t *) * len); + + i = 0; + LIST_FOREACH(p, &head, entry) { + if (strcmp(p->chain, miniupnpd_nft_nat_chain) == 0 && + (p->type == RULE_NAT || p->type == RULE_SNAT)) { + redirect_cache[i] = p; + i++; + } + } + + return; +} + +void +reflesh_nft_peer_cache(void) +{ + rule_t *p; + int i; + + if (peer_cache != NULL) { + free(peer_cache); + } + if (rule_list_peer_length == 0) { + peer_cache = NULL; + return; + } + peer_cache = (rule_t **)malloc( + sizeof(rule_t *) * rule_list_peer_length); + bzero(peer_cache, sizeof(rule_t *) * rule_list_peer_length); + + i = 0; + LIST_FOREACH(p, &head, entry) { + if (strcmp(p->chain, miniupnpd_nft_peer_chain) == 0) { + peer_cache[i] = p; + i++; + } + } + + return; +} + +void +reflesh_nft_cache(uint32_t family) +{ + char buf[MNL_SOCKET_BUFFER_SIZE]; + struct nlmsghdr *nlh; + uint32_t portid, seq, type = NFT_OUTPUT_DEFAULT; + struct nft_rule *t; + rule_t *p1, *p2; + int ret; + + if (rule_list_validate == RULE_CACHE_VALID) { + return; + } + + t = NULL; + p1 = LIST_FIRST(&head); + if (p1 != NULL) { + while(p1 != NULL) { + p2 = (rule_t *)LIST_NEXT(p1, entry); + if (p1->desc != NULL) { + free(p1->desc); + } + if (p1->table != NULL) { + free(p1->table); + } + if (p1->chain != NULL) { + free(p1->chain); + } + free(p1); + p1 = p2; + } + } + LIST_INIT(&head); + + t = nft_rule_alloc(); + if (t == NULL) { + perror("OOM"); + exit(EXIT_FAILURE); + } + + seq = time(NULL); + nlh = nft_rule_nlmsg_build_hdr(buf, NFT_MSG_GETRULE, family, + NLM_F_DUMP, seq); + nft_rule_nlmsg_build_payload(nlh, t); + nft_rule_free(t); + + if (nl == NULL) { + nl = mnl_socket_open(NETLINK_NETFILTER); + if (nl == NULL) { + perror("mnl_socket_open"); + exit(EXIT_FAILURE); + } + + if (mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { + perror("mnl_socket_bind"); + exit(EXIT_FAILURE); + } + } + portid = mnl_socket_get_portid(nl); + + if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) { + perror("mnl_socket_send"); + exit(EXIT_FAILURE); + } + + rule_list_peer_length = 0; + rule_list_length = 0; + ret = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + while (ret > 0) { + ret = mnl_cb_run(buf, ret, seq, portid, table_cb, &type); + if (ret <= 0) + break; + ret = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + } + if (ret == -1) { + perror("error"); + exit(EXIT_FAILURE); + } + /* mnl_socket_close(nl); */ + + reflesh_nft_peer_cache(); + reflesh_nft_redirect_cache(); + rule_list_validate = RULE_CACHE_VALID; + return; +} + +static void +expr_add_payload(struct nft_rule *r, uint32_t base, uint32_t dreg, + uint32_t offset, uint32_t len) +{ + struct nft_rule_expr *e; + + e = nft_rule_expr_alloc("payload"); + if (e == NULL) { + perror("expr payload oom"); + exit(EXIT_FAILURE); + } + + nft_rule_expr_set_u32(e, NFT_EXPR_PAYLOAD_BASE, base); + nft_rule_expr_set_u32(e, NFT_EXPR_PAYLOAD_DREG, dreg); + nft_rule_expr_set_u32(e, NFT_EXPR_PAYLOAD_OFFSET, offset); + nft_rule_expr_set_u32(e, NFT_EXPR_PAYLOAD_LEN, len); + + nft_rule_add_expr(r, e); +} + +#if 0 +static void +expr_add_bitwise(struct nft_rule *r, uint32_t sreg, uint32_t dreg, + uint32_t len, uint32_t mask, uint32_t xor) +{ + struct nft_rule_expr *e; + + e = nft_rule_expr_alloc("bitwise"); + if (e == NULL) { + perror("expr cmp bitwise"); + exit(EXIT_FAILURE); + } + + nft_rule_expr_set_u32(e, NFT_EXPR_BITWISE_SREG, sreg); + nft_rule_expr_set_u32(e, NFT_EXPR_BITWISE_DREG, dreg); + nft_rule_expr_set_u32(e, NFT_EXPR_BITWISE_LEN, len); + nft_rule_expr_set(e, NFT_EXPR_BITWISE_MASK, &mask, sizeof(mask)); + nft_rule_expr_set(e, NFT_EXPR_BITWISE_XOR, &xor, sizeof(xor)); + + nft_rule_add_expr(r, e); +} +#endif + +static void +expr_add_cmp(struct nft_rule *r, uint32_t sreg, uint32_t op, + const void *data, uint32_t data_len) +{ + struct nft_rule_expr *e; + + e = nft_rule_expr_alloc("cmp"); + if (e == NULL) { + perror("expr cmp oom"); + exit(EXIT_FAILURE); + } + + nft_rule_expr_set_u32(e, NFT_EXPR_CMP_SREG, sreg); + nft_rule_expr_set_u32(e, NFT_EXPR_CMP_OP, op); + nft_rule_expr_set(e, NFT_EXPR_CMP_DATA, data, data_len); + + nft_rule_add_expr(r, e); +} + +static void +expr_add_meta(struct nft_rule *r, uint32_t meta_key, uint32_t dreg) +{ + struct nft_rule_expr *e; + + e = nft_rule_expr_alloc("meta"); + if (e == NULL) { + perror("expr meta oom"); + exit(EXIT_FAILURE); + } + + nft_rule_expr_set_u32(e, NFT_EXPR_META_KEY, meta_key); + nft_rule_expr_set_u32(e, NFT_EXPR_META_DREG, dreg); + + nft_rule_add_expr(r, e); +} + +static void +expr_set_reg_val_u32(struct nft_rule *r, enum nft_registers dreg, uint32_t val) +{ + struct nft_rule_expr *e; + e = nft_rule_expr_alloc("immediate"); + if (e == NULL) { + perror("expr dreg oom"); + exit(EXIT_FAILURE); + } + nft_rule_expr_set_u32(e, NFT_EXPR_IMM_DREG, dreg); + nft_rule_expr_set_u32(e, NFT_EXPR_IMM_DATA, val); + nft_rule_add_expr(r, e); +} + +static void +expr_set_reg_val_u16(struct nft_rule *r, enum nft_registers dreg, uint32_t val) +{ + struct nft_rule_expr *e; + e = nft_rule_expr_alloc("immediate"); + if (e == NULL) { + perror("expr dreg oom"); + exit(EXIT_FAILURE); + } + nft_rule_expr_set_u32(e, NFT_EXPR_IMM_DREG, dreg); + nft_rule_expr_set_u16(e, NFT_EXPR_IMM_DATA, val); + nft_rule_add_expr(r, e); +} + +static void +expr_set_reg_verdict(struct nft_rule *r, uint32_t val) +{ + struct nft_rule_expr *e; + e = nft_rule_expr_alloc("immediate"); + if (e == NULL) { + perror("expr dreg oom"); + exit(EXIT_FAILURE); + } + nft_rule_expr_set_u32(e, NFT_EXPR_IMM_DREG, NFT_REG_VERDICT); + nft_rule_expr_set_u32(e, NFT_EXPR_IMM_VERDICT, val); + nft_rule_add_expr(r, e); +} + +static void +expr_add_nat(struct nft_rule *r, uint32_t t, uint32_t family, + in_addr_t addr_min, uint32_t proto_min, uint32_t flags) +{ + struct nft_rule_expr *e; + UNUSED(flags); + + e = nft_rule_expr_alloc("nat"); + if (e == NULL) { + perror("expr nat oom"); + exit(EXIT_FAILURE); + } + + nft_rule_expr_set_u32(e, NFT_EXPR_NAT_TYPE, t); + nft_rule_expr_set_u32(e, NFT_EXPR_NAT_FAMILY, family); + + expr_set_reg_val_u32(r, NFT_REG_1, (uint32_t)addr_min); + nft_rule_expr_set_u32(e, NFT_EXPR_NAT_REG_ADDR_MIN, NFT_REG_1); + nft_rule_expr_set_u32(e, NFT_EXPR_NAT_REG_ADDR_MAX, NFT_REG_1); + expr_set_reg_val_u16(r, NFT_REG_2, proto_min); + nft_rule_expr_set_u16(e, NFT_EXPR_NAT_REG_PROTO_MIN, NFT_REG_2); + nft_rule_expr_set_u16(e, NFT_EXPR_NAT_REG_PROTO_MAX, NFT_REG_2); + + nft_rule_add_expr(r, e); +} + + +/* + * Todo: add expr for rhost + */ +struct nft_rule * +rule_set_snat(uint8_t family, uint8_t proto, + in_addr_t rhost, unsigned short rport, + in_addr_t ehost, unsigned short eport, + in_addr_t ihost, unsigned short iport, + const char *descr, + const char *handle) +{ + struct nft_rule *r = NULL; + uint32_t destport; + in_addr_t addr[2]; + uint16_t port[2]; + uint32_t descr_len; + UNUSED(handle); + + r = nft_rule_alloc(); + if (r == NULL) { + perror("OOM"); + exit(EXIT_FAILURE); + } + + nft_rule_attr_set(r, NFT_RULE_ATTR_TABLE, NFT_TABLE_NAT); + nft_rule_attr_set(r, NFT_RULE_ATTR_CHAIN, miniupnpd_nft_peer_chain); + if (descr != NULL) { + descr_len = strlen(descr); + nft_rule_attr_set_data(r, NFT_RULE_ATTR_USERDATA, + descr, descr_len); + } + nft_rule_attr_set_u32(r, NFT_RULE_ATTR_FAMILY, family); + + addr[0] = ihost; + addr[1] = rhost; + expr_add_payload(r, NFT_PAYLOAD_NETWORK_HEADER, NFT_REG_1, + offsetof(struct iphdr, saddr), sizeof(uint32_t)*2); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, addr, sizeof(uint32_t)*2); + + expr_add_payload(r, NFT_PAYLOAD_NETWORK_HEADER, NFT_REG_1, + offsetof(struct iphdr, protocol), sizeof(uint8_t)); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &proto, sizeof(uint8_t)); + + port[0] = htons(iport); + port[1] = htons(rport); + if (proto == IPPROTO_TCP) { + expr_add_payload(r, NFT_PAYLOAD_TRANSPORT_HEADER, NFT_REG_1, + offsetof(struct tcphdr, source), + sizeof(uint32_t)); + } else if (proto == IPPROTO_UDP) { + expr_add_payload(r, NFT_PAYLOAD_TRANSPORT_HEADER, NFT_REG_1, + offsetof(struct udphdr, source), + sizeof(uint32_t)); + } + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, port, sizeof(uint32_t)); + + destport = htons(eport); + expr_add_nat(r, NFT_NAT_SNAT, AF_INET, ehost, destport, 0); + + return r; +} + +/* + * Todo: add expr for rhost + */ +struct nft_rule * +rule_set_dnat(uint8_t family, const char * ifname, uint8_t proto, + in_addr_t rhost, unsigned short eport, + in_addr_t ihost, uint32_t iport, + const char *descr, + const char *handle) +{ + struct nft_rule *r = NULL; + uint16_t dport; + uint64_t handle_num; + uint32_t if_idx; + uint32_t descr_len; + + UNUSED(handle); + UNUSED(rhost); + + r = nft_rule_alloc(); + if (r == NULL) { + perror("OOM"); + exit(EXIT_FAILURE); + } + + nft_rule_attr_set(r, NFT_RULE_ATTR_TABLE, NFT_TABLE_NAT); + nft_rule_attr_set(r, NFT_RULE_ATTR_CHAIN, miniupnpd_nft_nat_chain); + nft_rule_attr_set_u32(r, NFT_RULE_ATTR_FAMILY, family); + if (descr != NULL) { + descr_len = strlen(descr); + nft_rule_attr_set_data(r, NFT_RULE_ATTR_USERDATA, + descr, descr_len); + } + + if (handle != NULL) { + handle_num = atoll(handle); + nft_rule_attr_set_u64(r, NFT_RULE_ATTR_POSITION, handle_num); + } + + if (ifname != NULL) { + if_idx = (uint32_t)if_nametoindex(ifname); + expr_add_meta(r, NFT_META_IIF, NFT_REG_1); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &if_idx, + sizeof(uint32_t)); + } + + expr_add_payload(r, NFT_PAYLOAD_NETWORK_HEADER, NFT_REG_1, + offsetof(struct iphdr, protocol), sizeof(uint8_t)); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &proto, sizeof(uint8_t)); + + if (proto == IPPROTO_TCP) { + dport = htons(eport); + expr_add_payload(r, NFT_PAYLOAD_TRANSPORT_HEADER, NFT_REG_1, + offsetof(struct tcphdr, dest), + sizeof(uint16_t)); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &dport, + sizeof(uint16_t)); + } else if (proto == IPPROTO_UDP) { + dport = htons(eport); + expr_add_payload(r, NFT_PAYLOAD_TRANSPORT_HEADER, NFT_REG_1, + offsetof(struct udphdr, dest), + sizeof(uint16_t)); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &dport, + sizeof(uint16_t)); + } + + expr_add_nat(r, NFT_NAT_DNAT, AF_INET, ihost, htons(iport), 0); + + return r; +} + +struct nft_rule * +rule_set_filter(uint8_t family, const char * ifname, uint8_t proto, + in_addr_t rhost, in_addr_t iaddr, unsigned short eport, + unsigned short iport, const char *descr, const char *handle) +{ + struct nft_rule *r = NULL; + uint16_t dport; + uint64_t handle_num; + uint32_t if_idx; + uint32_t descr_len; + UNUSED(eport); + + r = nft_rule_alloc(); + if (r == NULL) { + perror("OOM"); + exit(EXIT_FAILURE); + } + + nft_rule_attr_set(r, NFT_RULE_ATTR_TABLE, NFT_TABLE_FILTER); + nft_rule_attr_set(r, NFT_RULE_ATTR_CHAIN, miniupnpd_nft_forward_chain); + nft_rule_attr_set_u32(r, NFT_RULE_ATTR_FAMILY, family); + if (descr != NULL) { + descr_len = strlen(descr); + nft_rule_attr_set_data(r, NFT_RULE_ATTR_USERDATA, + descr, descr_len); + } + + if (handle != NULL) { + handle_num = atoll(handle); + nft_rule_attr_set_u64(r, NFT_RULE_ATTR_POSITION, handle_num); + } + + if (ifname != NULL) { + if_idx = (uint32_t)if_nametoindex(ifname); + expr_add_meta(r, NFT_META_IIF, NFT_REG_1); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &if_idx, + sizeof(uint32_t)); + } + + expr_add_payload(r, NFT_PAYLOAD_NETWORK_HEADER, NFT_REG_1, + offsetof(struct iphdr, daddr), sizeof(uint32_t)); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &iaddr, sizeof(uint32_t)); + + expr_add_payload(r, NFT_PAYLOAD_NETWORK_HEADER, NFT_REG_1, + offsetof(struct iphdr, protocol), sizeof(uint8_t)); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &proto, sizeof(uint8_t)); + + dport = htons(iport); + expr_add_payload(r, NFT_PAYLOAD_TRANSPORT_HEADER, NFT_REG_1, + offsetof(struct tcphdr, dest), sizeof(uint16_t)); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &dport, sizeof(uint16_t)); + + if (rhost != 0) { + expr_add_payload(r, NFT_PAYLOAD_NETWORK_HEADER, NFT_REG_1, + offsetof(struct iphdr, saddr), + sizeof(in_addr_t)); + expr_add_cmp(r, NFT_REG_1, NFT_CMP_EQ, &rhost, + sizeof(in_addr_t)); + } + + expr_set_reg_verdict(r, NF_ACCEPT); + + return r; +} + +struct nft_rule * +rule_del_handle(rule_t *rule) +{ + struct nft_rule *r = NULL; + + r = nft_rule_alloc(); + if (r == NULL) { + perror("OOM"); + exit(EXIT_FAILURE); + } + + nft_rule_attr_set(r, NFT_RULE_ATTR_TABLE, rule->table); + nft_rule_attr_set(r, NFT_RULE_ATTR_CHAIN, rule->chain); + nft_rule_attr_set_u32(r, NFT_RULE_ATTR_FAMILY, rule->family); + nft_rule_attr_set_u64(r, NFT_RULE_ATTR_HANDLE, rule->handle); + + return r; +} + +static void +nft_mnl_batch_put(char *buf, uint16_t type, uint32_t seq) +{ + struct nlmsghdr *nlh; + struct nfgenmsg *nfg; + + nlh = mnl_nlmsg_put_header(buf); + nlh->nlmsg_type = type; + nlh->nlmsg_flags = NLM_F_REQUEST; + nlh->nlmsg_seq = seq; + + nfg = mnl_nlmsg_put_extra_header(nlh, sizeof(*nfg)); + nfg->nfgen_family = AF_INET; + nfg->version = NFNETLINK_V0; + nfg->res_id = NFNL_SUBSYS_NFTABLES; +} + +int +nft_send_request(struct nft_rule * rule, uint16_t cmd) +{ + struct nlmsghdr *nlh; + struct mnl_nlmsg_batch *batch; + char buf[MNL_SOCKET_BUFFER_SIZE]; + uint32_t seq = time(NULL); + int ret; + + rule_list_validate = RULE_CACHE_INVALID; + if (nl == NULL) { + nl = mnl_socket_open(NETLINK_NETFILTER); + if (nl == NULL) { + perror("mnl_socket_open"); + return -1; + } + + if (mnl_socket_bind(nl, 0, MNL_SOCKET_AUTOPID) < 0) { + perror("mnl_socket_bind"); + return -1; + } + } + + batch = mnl_nlmsg_batch_start(buf, sizeof(buf)); + + nft_mnl_batch_put(mnl_nlmsg_batch_current(batch), + NFNL_MSG_BATCH_BEGIN, seq++); + mnl_nlmsg_batch_next(batch); + + nlh = nft_rule_nlmsg_build_hdr(mnl_nlmsg_batch_current(batch), + cmd, + nft_rule_attr_get_u32(rule, NFT_RULE_ATTR_FAMILY), + NLM_F_APPEND|NLM_F_CREATE|NLM_F_ACK, + seq++); + + nft_rule_nlmsg_build_payload(nlh, rule); + nft_rule_free(rule); + mnl_nlmsg_batch_next(batch); + + nft_mnl_batch_put(mnl_nlmsg_batch_current(batch), NFNL_MSG_BATCH_END, + seq++); + mnl_nlmsg_batch_next(batch); + + ret = mnl_socket_sendto(nl, mnl_nlmsg_batch_head(batch), + mnl_nlmsg_batch_size(batch)); + if (ret == -1) { + perror("mnl_socket_sendto"); + return -1; + } + + mnl_nlmsg_batch_stop(batch); + + ret = mnl_socket_recvfrom(nl, buf, sizeof(buf)); + if (ret == -1) { + perror("mnl_socket_recvfrom"); + return -1; + } + + ret = mnl_cb_run(buf, ret, 0, mnl_socket_get_portid(nl), NULL, NULL); + if (ret < 0) { + perror("mnl_cb_run"); + return -1; + } + + /* mnl_socket_close(nl); */ + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr_misc.h b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr_misc.h new file mode 100644 index 0000000..e04403e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/nftnlrdr_misc.h @@ -0,0 +1,91 @@ +/* + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2015 Tomofumi Hayashi + * + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution. + */ +#include + +#define NFT_TABLE_NAT "nat" +#define NFT_TABLE_FILTER "filter" + +enum rule_reg_type { + RULE_REG_NONE, + RULE_REG_IIF, + RULE_REG_OIF, + RULE_REG_IP_SRC_ADDR, + RULE_REG_IP_DEST_ADDR, + RULE_REG_IP_SD_ADDR, /* source & dest */ + RULE_REG_IP_PROTO, + RULE_REG_TCP_DPORT, + RULE_REG_TCP_SD_PORT, /* source & dest */ + RULE_REG_IMM_VAL, + RULE_REG_MAX, +}; + +enum rule_type { + RULE_NONE, + RULE_NAT, + RULE_SNAT, + RULE_FILTER, + RULE_COUNTER, +}; + +typedef struct rule_ { + LIST_ENTRY(rule_t) entry; + char * table; + char * chain; + uint64_t handle; + enum rule_type type; + uint32_t nat_type; + uint32_t filter_action; + uint32_t family; + uint32_t ingress_ifidx; + uint32_t egress_ifidx; + in_addr_t eaddr; + in_addr_t iaddr; + in_addr_t rhost; + uint16_t eport; + uint16_t iport; + uint16_t rport; + uint8_t proto; + enum rule_reg_type reg1_type; + enum rule_reg_type reg2_type; + uint32_t reg1_val; + uint32_t reg2_val; + uint64_t packets; + uint64_t bytes; + char *desc; +} rule_t; + +LIST_HEAD(rule_list, rule_); +extern struct rule_list head; +extern rule_t **peer_cache; +extern rule_t **redirect_cache; + +int +nft_send_request(struct nft_rule * rule, uint16_t cmd); +struct nft_rule * +rule_set_dnat(uint8_t family, const char * ifname, uint8_t proto, + in_addr_t rhost, unsigned short eport, + in_addr_t ihost, uint32_t iport, + const char *descr, + const char *handle); +struct nft_rule * +rule_set_snat(uint8_t family, uint8_t proto, + in_addr_t rhost, unsigned short rport, + in_addr_t ehost, unsigned short eport, + in_addr_t ihost, unsigned short iport, + const char *descr, + const char *handle); +struct nft_rule * +rule_set_filter(uint8_t family, const char * ifname, uint8_t proto, + in_addr_t rhost, in_addr_t iaddr, unsigned short eport, + unsigned short iport, const char * descr, const char *handle); +struct nft_rule * +rule_del_handle(rule_t *r); +void +reflesh_nft_cache(uint32_t family); +void print_rule(rule_t *r); diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_delete_chain.sh b/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_delete_chain.sh new file mode 100644 index 0000000..97f9fbe --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_delete_chain.sh @@ -0,0 +1,5 @@ +#! /sbin/nft -f + +delete chain nat miniupnpd +delete chain nat miniupnpd-pcp-peer +delete chain filter miniupnpd diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_flush.sh b/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_flush.sh new file mode 100644 index 0000000..bd6c878 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_flush.sh @@ -0,0 +1,5 @@ +#! /sbin/nft -f + +flush chain ip nat miniupnpd +flush chain ip nat miniupnpd-pcp-peer +flush chain ip filter miniupnpd diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_init.sh b/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_init.sh new file mode 100644 index 0000000..45b42ea --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_init.sh @@ -0,0 +1,47 @@ +#! /bin/sh + +nft list table nat > /dev/null +nft_nat_exists=$? +nft list table filter > /dev/null +nft_filter_exists=$? +nft list table mangle > /dev/null +nft_mangle_exists=$? + +if [ $nft_nat_exists -eq "1" ]; then + echo "create nat" + nft "add table nat" +fi +if [ $nft_filter_exists -eq "1" ]; then + echo "create filter" + nft "add table filter" +fi +if [ $nft_mangle_exists -eq "1" ]; then + echo "create mangle" + nft "add table mangle" +fi + +nft list chain nat miniupnpd > /dev/null +nft_nat_miniupnpd_exists=$? +nft list chain nat miniupnpd-pcp-peer > /dev/null +nft_nat_miniupnpd_pcp_peer_exists=$? +nft list chain filter miniupnpd > /dev/null +nft_filter_miniupnpd_exists=$? +nft list chain mangle miniupnpd > /dev/null +nft_mangle_miniupnpd_exists=$? + +if [ $nft_nat_miniupnpd_exists -eq "1" ]; then + echo "create chain in nat" + nft "add chain nat miniupnpd" +fi +if [ $nft_nat_miniupnpd_pcp_peer_exists -eq "1" ]; then + echo "create pcp peer chain in nat" + nft "add chain nat miniupnpd-pcp-peer" +fi +if [ $nft_filter_miniupnpd_exists -eq "1" ]; then + echo "create chain in filter " + nft "add chain filter miniupnpd" +fi +if [ $nft_mangle_miniupnpd_exists -eq "1" ]; then + echo "create chain in mangle" + nft "add chain mangle miniupnpd" +fi diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_removeall.sh b/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_removeall.sh new file mode 100644 index 0000000..11c38e7 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/scripts/nft_removeall.sh @@ -0,0 +1,5 @@ +#! /sbin/nft -f + +delete rule nat miniupnpd +delete rule nat miniupnpd-pcp-peer +delete rule filter miniupnpd diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/test_nfct_get.c b/src/contrib/miniupnp/miniupnpd/netfilter_nft/test_nfct_get.c new file mode 100644 index 0000000..af8c07d --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/test_nfct_get.c @@ -0,0 +1,50 @@ +#include "nfct_get.c" + +int main(int argc, char *argv[]) +{ + struct sockaddr_storage src, dst, ext; + char buff[INET6_ADDRSTRLEN]; + + if (argc!=5) + return 0; + + if (1 != inet_pton(AF_INET, argv[1], + &((struct sockaddr_in*)&src)->sin_addr)) { + if (1 != inet_pton(AF_INET6, argv[1], + &((struct sockaddr_in6*) &src)->sin6_addr)) { + perror("bad input param"); + } else { + ((struct sockaddr_in6*)(&src))->sin6_port = htons(atoi(argv[2])); + src.ss_family = AF_INET6; + } + } else { + ((struct sockaddr_in*)(&src))->sin_port = htons(atoi(argv[2])); + src.ss_family = AF_INET; + } + + if (1 != inet_pton(AF_INET, argv[3], + &((struct sockaddr_in*)&dst)->sin_addr)) { + if (1 != inet_pton(AF_INET6, argv[3], + &((struct sockaddr_in6*) &dst)->sin6_addr)) { + perror("bad input param"); + } else { + ((struct sockaddr_in6*)(&dst))->sin6_port = htons(atoi(argv[4])); + dst.ss_family = AF_INET6; + } + } else { + ((struct sockaddr_in*)(&dst))->sin_port = htons(atoi(argv[4])); + dst.ss_family = AF_INET; + } + + if (get_nat_ext_addr((struct sockaddr*)&src, (struct sockaddr*)&dst, + IPPROTO_TCP, &ext)) { + printf("Ext address %s:%d\n", + inet_ntop(src.ss_family, + &((struct sockaddr_in*)&ext)->sin_addr, + buff, sizeof(buff)), + ntohs(((struct sockaddr_in*)(&ext))->sin_port)); + } else { + printf("no entry\n"); + } + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/testnftnlrdr.c b/src/contrib/miniupnp/miniupnpd/netfilter_nft/testnftnlrdr.c new file mode 100644 index 0000000..efeccb3 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/testnftnlrdr.c @@ -0,0 +1,91 @@ +/* $Id: testiptcrdr.c,v 1.18 2012/04/24 22:41:53 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2012 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include + +#include "nftnlrdr.h" +#include "../commonrdr.h" + +#ifndef PRIu64 +#define PRIu64 "llu" +#endif + +static int +add_filter_rule(int proto, const char * rhost, + const char * iaddr, unsigned short iport) +{ + return add_filter_rule2(NULL, rhost, iaddr, 0, iport, proto, NULL); +} + +static int +addnatrule(int proto, unsigned short eport, + const char * iaddr, unsigned short iport, + const char * rhost) +{ + return add_redirect_rule2(NULL, rhost, eport, iaddr, iport, proto, NULL, 0); +} + +int +main(int argc, char ** argv) +{ + unsigned short eport, iport; + const char * iaddr; + + if(argc<4) { + printf("Usage %s \n", argv[0]); + return -1; + } + openlog("testnftnlrdr", LOG_PERROR|LOG_CONS, LOG_LOCAL0); + eport = (unsigned short)atoi(argv[1]); + iaddr = argv[2]; + iport = (unsigned short)atoi(argv[3]); + printf("trying to redirect port %hu to %s:%hu\n", eport, iaddr, iport); + if(addnatrule(IPPROTO_TCP, eport, iaddr, iport, NULL) < 0) { + printf("addnatrule() failed!\n"); + return -1; + } + if(add_filter_rule(IPPROTO_TCP, NULL, iaddr, iport) < 0) { + printf("add_filter_rule() failed!\n"); + return -1; + } + /* test */ + { + unsigned short p1, p2; + char addr[16]; + int proto2; + char desc[256]; + char rhost[256]; + unsigned int timestamp; + u_int64_t packets, bytes; + + desc[0] = '\0'; + printf("test0\n"); + if(get_redirect_rule_by_index(0, "", &p1, + addr, sizeof(addr), &p2, + &proto2, desc, sizeof(desc), + rhost, sizeof(rhost), + ×tamp, + &packets, &bytes) < 0) + { + printf("rule not found\n"); + } + else + { + printf("redirected port %hu to %s:%hu proto %d packets=%" PRIu64 " bytes=%" PRIu64 "\n", + p1, addr, p2, proto2, packets, bytes); + } + printf("test\n"); + } + printf("trying to list nat rules :\n"); + list_redirect_rule(argv[1]); + printf("deleting\n"); + delete_redirect_and_filter_rules(eport, IPPROTO_TCP); + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/netfilter_nft/tiny_nf_nat.h b/src/contrib/miniupnp/miniupnpd/netfilter_nft/tiny_nf_nat.h new file mode 100644 index 0000000..f7f5b27 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/netfilter_nft/tiny_nf_nat.h @@ -0,0 +1,37 @@ +/* $Id: tiny_nf_nat.h,v 1.1 2011/07/30 13:14:36 nanard Exp $ */ +/* Only what miniupnpd needs, until linux-libc-dev gains nf_nat.h */ + +#ifndef TINY_NF_NAT_H +#define TINY_NF_NAT_H + +#include + +#define IP_NAT_RANGE_MAP_IPS 1 +#define IP_NAT_RANGE_PROTO_SPECIFIED 2 +#define IP_NAT_RANGE_PROTO_RANDOM 4 +#define IP_NAT_RANGE_PERSISTENT 8 + +union nf_conntrack_man_proto { + __be16 all; + struct { __be16 port; } tcp; + struct { __be16 port; } udp; + struct { __be16 id; } icmp; + struct { __be16 port; } dccp; + struct { __be16 port; } sctp; + struct { __be16 key; } gre; +}; + +struct nf_nat_range { + unsigned int flags; + __be32 min_ip, max_ip; + union nf_conntrack_man_proto min, max; +}; + +struct nf_nat_multi_range_compat { + unsigned int rangesize; + struct nf_nat_range range[1]; +}; + +#define nf_nat_multi_range nf_nat_multi_range_compat + +#endif /*TINY_NF_NAT_H*/ diff --git a/src/contrib/miniupnp/miniupnpd/options.c b/src/contrib/miniupnp/miniupnpd/options.c new file mode 100644 index 0000000..9448ba8 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/options.c @@ -0,0 +1,323 @@ +/* $Id: options.c,v 1.28 2013/12/13 14:07:08 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * author: Ryan Wagoner + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include "config.h" +#include "options.h" +#include "upnppermissions.h" +#ifdef PCP_SADSCP +#include "pcplearndscp.h" +#endif /* PCP_SADSPC */ +#include "upnpglobalvars.h" + +#ifndef DISABLE_CONFIG_FILE +struct option * ary_options = NULL; +static char * string_repo = NULL; +unsigned int num_options = 0; + +static const struct { + enum upnpconfigoptions id; + const char * name; +} optionids[] = { + { UPNPEXT_IFNAME, "ext_ifname" }, + { UPNPEXT_IP, "ext_ip" }, + { UPNPLISTENING_IP, "listening_ip" }, +#ifdef ENABLE_IPV6 + { UPNPIPV6_LISTENING_IP, "ipv6_listening_ip" }, +#endif /* ENABLE_IPV6 */ + { UPNPPORT, "port" }, + { UPNPPORT, "http_port" }, /* "port" and "http_port" are synonims */ +#ifdef ENABLE_HTTPS + { UPNPHTTPSPORT, "https_port" }, +#endif /* ENABLE_HTTPS */ + { UPNPBITRATE_UP, "bitrate_up" }, + { UPNPBITRATE_DOWN, "bitrate_down" }, + { UPNPPRESENTATIONURL, "presentation_url" }, +#ifdef ENABLE_MANUFACTURER_INFO_CONFIGURATION + { UPNPFRIENDLY_NAME, "friendly_name" }, + { UPNPMANUFACTURER_NAME, "manufacturer_name" }, + { UPNPMANUFACTURER_URL, "manufacturer_url" }, + { UPNPMODEL_NAME, "model_name" }, + { UPNPMODEL_DESCRIPTION, "model_description" }, + { UPNPMODEL_URL, "model_url" }, +#endif + { UPNPNOTIFY_INTERVAL, "notify_interval" }, + { UPNPSYSTEM_UPTIME, "system_uptime" }, + { UPNPPACKET_LOG, "packet_log" }, + { UPNPUUID, "uuid"}, + { UPNPSERIAL, "serial"}, + { UPNPMODEL_NUMBER, "model_number"}, + { UPNPCLEANTHRESHOLD, "clean_ruleset_threshold"}, + { UPNPCLEANINTERVAL, "clean_ruleset_interval"}, +#ifdef USE_NETFILTER + { UPNPFORWARDCHAIN, "upnp_forward_chain"}, + { UPNPNATCHAIN, "upnp_nat_chain"}, + { UPNPNATPOSTCHAIN, "upnp_nat_postrouting_chain"}, +#endif +#ifdef ENABLE_NATPMP + { UPNPENABLENATPMP, "enable_natpmp"}, +#endif +#ifdef ENABLE_PCP + { UPNPPCPMINLIFETIME, "min_lifetime"}, + { UPNPPCPMAXLIFETIME, "max_lifetime"}, + { UPNPPCPALLOWTHIRDPARTY, "pcp_allow_thirdparty"}, +#endif + { UPNPENABLE, "enable_upnp"}, +#ifdef USE_PF + { UPNPANCHOR, "anchor"}, + { UPNPQUEUE, "queue"}, + { UPNPTAG, "tag"}, +#endif +#ifdef PF_ENABLE_FILTER_RULES + { UPNPQUICKRULES, "quickrules" }, +#endif +#ifdef ENABLE_LEASEFILE + { UPNPLEASEFILE, "lease_file"}, +#endif +#ifdef IGD_V2 + { UPNPFORCEIGDDESCV1, "force_igd_desc_v1"}, +#endif + { UPNPMINISSDPDSOCKET, "minissdpdsocket"}, + { UPNPSECUREMODE, "secure_mode"} +}; + +int +readoptionsfile(const char * fname) +{ + FILE *hfile = NULL; + char buffer[1024]; + char *equals; + char *name; + char *value; + char *t; + int linenum = 0; + unsigned int i; + enum upnpconfigoptions id; + size_t string_repo_len = 0; + size_t len; + void *tmp; + + if(!fname || (fname[0] == '\0')) + return -1; + + memset(buffer, 0, sizeof(buffer)); + +#ifdef DEBUG + printf("Reading configuration from file %s\n", fname); +#endif + + if(!(hfile = fopen(fname, "r"))) + return -1; + + if(ary_options != NULL) + { + free(ary_options); + num_options = 0; + } + + while(fgets(buffer, sizeof(buffer), hfile)) + { + linenum++; + t = strchr(buffer, '\n'); + if(t) + { + *t = '\0'; + t--; + /* remove spaces at the end of the line */ + while((t >= buffer) && isspace(*t)) + { + *t = '\0'; + t--; + } + } + + /* skip leading whitespaces */ + name = buffer; + while(isspace(*name)) + name++; + + /* check for comments or empty lines */ + if(name[0] == '#' || name[0] == '\0') continue; + + len = strlen(name); /* length of the whole line excluding leading + * and ending white spaces */ + /* check for UPnP permissions rule */ + if((len > 6) && (0 == memcmp(name, "allow", 5) || 0 == memcmp(name, "deny", 4))) + { + tmp = realloc(upnppermlist, sizeof(struct upnpperm) * (num_upnpperm+1)); + if(tmp == NULL) + { + fprintf(stderr, "memory allocation error. Permission line in file %s line %d\n", + fname, linenum); + } + else + { + upnppermlist = tmp; + /* parse the rule */ + if(read_permission_line(upnppermlist + num_upnpperm, name) >= 0) + { + num_upnpperm++; + } + else + { + fprintf(stderr, "parsing error file %s line %d : %s\n", + fname, linenum, name); + } + } + continue; + } +#ifdef PCP_SADSCP + /* check for DSCP values configuration */ + if((len > 15) && 0 == memcmp(name, "set_learn_dscp", sizeof("set_learn_dscp")-1) ) + { + tmp = realloc(dscp_values_list, sizeof(struct dscp_values) * (num_dscp_values+1)); + if(tmp == NULL) + { + fprintf(stderr, "memory allocation error. DSCP line in file %s line %d\n", + fname, linenum); + } + else + { + dscp_values_list = tmp; + /* parse the rule */ + if(read_learn_dscp_line(dscp_values_list + num_dscp_values, name) >= 0) + { + num_dscp_values++; + } + else + { + fprintf(stderr, "parsing error file %s line %d : %s\n", + fname, linenum, name); + } + } + continue; + } +#endif /* PCP_SADSCP */ + if(!(equals = strchr(name, '='))) + { + fprintf(stderr, "parsing error file %s line %d : %s\n", + fname, linenum, name); + continue; + } + + /* remove ending whitespaces */ + for(t=equals-1; t>name && isspace(*t); t--) + *t = '\0'; + + *equals = '\0'; + value = equals+1; + + /* skip leading whitespaces */ + while(isspace(*value)) + value++; + + id = UPNP_INVALID; + for(i=0; i +#include +#include +#include +#include + +#include "upnpglobalvars.h" +#include "pcplearndscp.h" + +#ifdef PCP_SADSCP + +void +print_dscp(void) { + unsigned int i; + + for (i=0; i < num_dscp_values; i++){ + syslog(LOG_DEBUG, "Appname %*.s, del %d, loss %d, jitter %d, dscp %d", + dscp_values_list[i].app_name_len, + dscp_values_list[i].app_name, dscp_values_list[i].delay, + dscp_values_list[i].loss, dscp_values_list[i].jitter, + dscp_values_list[i].dscp_value + ); + } +} + +int +read_learn_dscp_line(struct dscp_values *dscpvalues, char *p) +{ + char * q; + size_t len; + unsigned int sizeof_first_token = sizeof("set_learn_dscp") - 1; + int af_value; + int cs_value; + + /* first token: (set_learn_dscp) skip it */ + while(isspace(*p)) + p++; + if(0 == memcmp(p, "set_learn_dscp", sizeof_first_token)) + { + p += sizeof_first_token; + } + else + { + return -1; + } + while(isspace(*p)) + p++; + + /* second token: name of the application */ + // if + if(!(*p == '"')) + return -1; + p++; + for(q = p; !(*q == '"'); q++); + len = q - p; + if (len != 0) { + dscpvalues->app_name = strndup(p, len); + } else { + dscpvalues->app_name = NULL; + } + dscpvalues->app_name_len = len; + p = q + 1; + + /* third token: delay */ + while(isspace(*p)) + p++; + if(!isdigit(*p)) + goto exit_err_and_cleanup; + for(q = p; isdigit(*q); q++); + if(isspace(*q)) + { + *q = '\0'; + dscpvalues->delay = (unsigned char)atoi(p); + if (dscpvalues->delay >= 3) { + fprintf(stderr, "Wrong delay value %d in \n", dscpvalues->delay); + fprintf(stderr, "Delay can be from set {0,1,2} 0=low delay, 1=medium delay, 2=high delay\n"); + goto exit_err_and_cleanup; + } + } + else + { + goto exit_err_and_cleanup; + } + p = q + 1; + + /* fourth token: loss */ + while(isspace(*p)) + p++; + if(!isdigit(*p)) + goto exit_err_and_cleanup; + + for(q = p; isdigit(*q); q++); + if(isspace(*q)) + { + *q = '\0'; + dscpvalues->loss = (unsigned char)atoi(p); + if (dscpvalues->loss >= 3) { + fprintf(stderr, "Wrong loss value %d \n", dscpvalues->loss); + fprintf(stderr, "Delay can be from set {0,1,2} 0=low loss, 1=medium loss, 2=high loss\n"); + goto exit_err_and_cleanup; + } + } + else + { + goto exit_err_and_cleanup; + } + p = q + 1; + + /* fifth token: jitter */ + while(isspace(*p)) + p++; + if(!isdigit(*p)) + goto exit_err_and_cleanup; + for(q = p; isdigit(*q); q++); + if(isspace(*q)) + { + *q = '\0'; + dscpvalues->jitter = (unsigned char)atoi(p); + if (dscpvalues->jitter >= 3) { + fprintf(stderr, "Wrong jitter value %d \n", dscpvalues->jitter); + fprintf(stderr, "Delay can be from set {0,1,2} 0=low jitter, 1=medium jitter, 2=high jitter \n"); + goto exit_err_and_cleanup; + } + } + else + { + goto exit_err_and_cleanup; + } + p = q + 1; + while(isspace(*p)) + p++; + /*{ + }*/ + p = q + 1; + + /* sixth token: DSCP value */ + while(isspace(*p)) + p++; + if(!isdigit(*p) && !( toupper(*p) == 'A' && toupper(*(p+1)) == 'F') && + !( toupper(*p) == 'C' && toupper(*(p+1)) == 'S') && + !( toupper(*p) == 'E' && toupper(*(p+1)) == 'F') + ) + goto exit_err_and_cleanup; +// for(q = p; isdigit(*q) || (toupper(*q) == 'A') || (toupper(*q) == 'F'); q++); + for(q = p; isdigit(*q) || isalpha(*q); q++); + if(isspace(*q) || *q == '\0') + { + *q = '\0'; + if (toupper(*p) == 'A' && toupper(*(p+1)) == 'F'){ + p = p+2; + if (*p == '\0') { + dscpvalues->dscp_value = 0; + } + else if (!isdigit(*p)) { + goto exit_err_and_cleanup; + } + else { + af_value = atoi(p); + switch(af_value) { + case 11: + dscpvalues->dscp_value = 10; + break; + case 12: + dscpvalues->dscp_value = 12; + break; + case 13: + dscpvalues->dscp_value = 14; + break; + case 21: + dscpvalues->dscp_value = 18; + break; + case 22: + dscpvalues->dscp_value = 20; + break; + case 23: + dscpvalues->dscp_value = 22; + break; + case 31: + dscpvalues->dscp_value = 26; + break; + case 32: + dscpvalues->dscp_value = 28; + break; + case 33: + dscpvalues->dscp_value = 30; + break; + case 41: + dscpvalues->dscp_value = 34; + break; + case 42: + dscpvalues->dscp_value = 36; + break; + case 43: + dscpvalues->dscp_value = 38; + break; + default: + fprintf(stderr, "Unknown AF value %d \n", af_value); + goto exit_err_and_cleanup; + } + } + } + else if (toupper(*p) == 'C' && toupper(*(p+1)) == 'S'){ + p=p+2; + if (*p == '\0') { + dscpvalues->dscp_value = 0; + } + else if (!isdigit(*p)) { + fprintf(stderr, "Not digit after CS but %c \n", *p); + goto exit_err_and_cleanup; + } + else { + cs_value = atoi(p); + switch(cs_value) { + case 1: + dscpvalues->dscp_value = 8; + break; + case 2: + dscpvalues->dscp_value = 16; + break; + case 3: + dscpvalues->dscp_value = 24; + break; + case 4: + dscpvalues->dscp_value = 32; + break; + case 5: + dscpvalues->dscp_value = 40; + break; + case 6: + dscpvalues->dscp_value = 48; + break; + case 7: + dscpvalues->dscp_value = 56; + break; + default: + fprintf(stderr, "Unknown CS value %d \n", cs_value); + goto exit_err_and_cleanup; + } + } + } + else if (toupper(*p) == 'E' && toupper(*(p+1)) == 'F'){ + dscpvalues->dscp_value = 46; + } + else { + dscpvalues->dscp_value = (unsigned char)atoi(p); + } + } + else + { + goto exit_err_and_cleanup; + } + + return 0; + +exit_err_and_cleanup: + free(dscpvalues->app_name); + dscpvalues->app_name = NULL; + dscpvalues->app_name_len = 0; + return -1; + + +} + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/pcplearndscp.h b/src/contrib/miniupnp/miniupnpd/pcplearndscp.h new file mode 100644 index 0000000..93fee33 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pcplearndscp.h @@ -0,0 +1,51 @@ +/* $Id: pcplearndscp.h,v 1.2 2013/12/13 15:47:23 nanard Exp $ */ +/* MiniUPnP project + * Website : http://miniupnp.free.fr/ + * Author : Miroslav Bagljas + +Copyright (c) 2013 by Cisco Systems, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef PCPLEARNDSCP_H_INCLUDED +#define PCPLEARNDSCP_H_INCLUDED + +struct dscp_values { + char *app_name; + unsigned int app_name_len; + unsigned char delay; + unsigned char loss; + unsigned char jitter; + unsigned char dscp_value; +}; + + + +/* #set_learn_dscp "Webex" 1 1 1 34 */ +int +read_learn_dscp_line(struct dscp_values *dscpvalues, char *p); + +#endif /* PCPLEARNDSCP_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpd/pcpserver.c b/src/contrib/miniupnp/miniupnpd/pcpserver.c new file mode 100644 index 0000000..7930fa5 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pcpserver.c @@ -0,0 +1,1700 @@ +/* $Id: pcpserver.c,v 1.47 2018/03/13 10:21:19 nanard Exp $ */ +/* MiniUPnP project + * Website : http://miniupnp.free.fr/ + * Author : Peter Tatrai + +Copyright (c) 2013 by Cisco Systems, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +/* Current assumptions: + - IPv4 is always NATted (internal -> external) + - IPv6 is always firewalled (this may need some work, NAT6* do exist) + + - we make the judgement based on (in order, picking first one available): + - third party address + - internal client address + + TODO : handle NAT46, NAT64, NPT66. In addition, beyond FW/NAT + choice, should also add for proxy (=as much as possible transparent + pass-through to one or more servers). + + TODO: IPv6 permission handling (for the time being, we just assume + anyone on IPv6 is a good guy, but fixing that would include + upnppermissions rewrite to be AF neutral). +*/ + +#include "config.h" + +#ifdef ENABLE_PCP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pcpserver.h" +#include "natpmp.h" +#include "macros.h" +#include "upnpglobalvars.h" +#include "pcplearndscp.h" +#include "upnpredirect.h" +#include "commonrdr.h" +#include "getifaddr.h" +#include "asyncsendto.h" +#include "upnputils.h" +#include "portinuse.h" +#include "pcp_msg_struct.h" +#ifdef ENABLE_UPNPPINHOLE +#include "upnppinhole.h" +#endif /* ENABLE_UPNPPINHOLE */ + + +#ifdef PCP_PEER +/* TODO make this platform independent */ +#ifdef USE_NETFILTER +#include "netfilter/iptcrdr.h" +#else +#error "PCP Peer is only supported with NETFILTER" +#endif /* USE_NETFILTER */ +#endif /* PCP_PEER */ + +/* server specific information */ +struct pcp_server_info { + uint8_t server_version; +}; + +/* default server settings, highest version supported is the default */ +static const struct pcp_server_info this_server_info = {2}; + +/* structure holding information from PCP msg*/ +/* all variables are in host byte order except IP addresses */ +typedef struct pcp_info { + uint8_t version; + uint8_t opcode; + uint8_t result_code; + uint32_t lifetime; /* lifetime of the mapping */ + uint32_t epochtime; + /* both MAP and PEER opcode specific information */ + uint32_t nonce[3]; /* random value generated by client */ + uint8_t protocol; + uint16_t int_port; + const struct in6_addr *int_ip; /* in network order */ + uint16_t ext_port; + const struct in6_addr *ext_ip; /* Suggested external IP in network order*/ + /* PEER specific information */ +#ifdef PCP_PEER + uint16_t peer_port; + const struct in6_addr *peer_ip; /* Destination IP in network order */ +#endif /* PCP_PEER */ + +#ifdef PCP_SADSCP + /* SADSCP specific information */ + uint8_t delay_tolerance; + uint8_t loss_tolerance; + uint8_t jitter_tolerance; + uint8_t app_name_len; + const char* app_name; + uint8_t sadscp_dscp; + uint8_t matched_name; + int8_t is_sadscp_op; +#endif + +#ifdef PCP_FLOWP + uint8_t dscp_up; + uint8_t dscp_down; + int flowp_present; +#endif + uint8_t is_map_op; + uint8_t is_peer_op; + const struct in6_addr *thirdp_ip; + const struct in6_addr *mapped_ip; + char mapped_str[INET6_ADDRSTRLEN]; + int pfailure_present; + struct in6_addr sender_ip; + int is_fw; /* is this firewall operation? if not, nat. */ + char desc[64]; +} pcp_info_t; + +/* getPCPOpCodeStr() + * return a string representation of the PCP OpCode + * can be used for debug output */ +static const char * getPCPOpCodeStr(uint8_t opcode) +{ + switch(opcode) { + case PCP_OPCODE_ANNOUNCE: + return "ANNOUNCE"; + case PCP_OPCODE_MAP: + return "MAP"; + case PCP_OPCODE_PEER: + return "PEER"; +#ifdef PCP_SADSCP + case PCP_OPCODE_SADSCP: + return "SADSCP"; +#endif /* PCP_SADSCP */ + default: + return "UNKNOWN"; + } +} + +/* useful to copy ext_ip only if needed, as request and response + * buffers are same */ +static void copyIPv6IfDifferent(void * dest, const void * src) +{ + if(dest != src) { + memcpy(dest, src, sizeof(struct in6_addr)); + } +} + +#ifdef PCP_SADSCP +int get_dscp_value(pcp_info_t *pcp_msg_info) { + + unsigned int ind; + + for (ind = 0; ind < num_dscp_values; ind++) { + + if ((dscp_values_list[ind].app_name) && + (!strcmp(dscp_values_list[ind].app_name, + pcp_msg_info->app_name)) && + (pcp_msg_info->delay_tolerance == dscp_values_list[ind].delay) && + (pcp_msg_info->loss_tolerance == dscp_values_list[ind].loss) && + (pcp_msg_info->jitter_tolerance == dscp_values_list[ind].jitter) + ) + { + pcp_msg_info->sadscp_dscp = dscp_values_list[ind].dscp_value; + pcp_msg_info->matched_name = 1; + return 0; + } else + if ((pcp_msg_info->app_name_len==0) && + (dscp_values_list[ind].app_name_len==0) && + (pcp_msg_info->delay_tolerance == dscp_values_list[ind].delay) && + (pcp_msg_info->loss_tolerance == dscp_values_list[ind].loss) && + (pcp_msg_info->jitter_tolerance == dscp_values_list[ind].jitter) + ) + { + pcp_msg_info->sadscp_dscp = dscp_values_list[ind].dscp_value; + pcp_msg_info->matched_name = 0; + return 0; + } else + if ((dscp_values_list[ind].app_name_len==0) && + (pcp_msg_info->delay_tolerance == dscp_values_list[ind].delay) && + (pcp_msg_info->loss_tolerance == dscp_values_list[ind].loss) && + (pcp_msg_info->jitter_tolerance == dscp_values_list[ind].jitter) + ) + { + pcp_msg_info->sadscp_dscp = dscp_values_list[ind].dscp_value; + pcp_msg_info->matched_name = 0; + return 0; + } + } + //if nothing matched return Default value i.e. 0 + pcp_msg_info->sadscp_dscp = 0; + pcp_msg_info->matched_name = 0; + return 0; +} +#endif +/* + * Function extracting information from common_req (common request header) + * into pcp_msg_info. + * @return : when no problem occurred 0 is returned, 1 otherwise and appropriate + * result code is assigned to pcp_msg_info->result_code to indicate + * what kind of error occurred + */ +static int parseCommonRequestHeader(const uint8_t *common_req, pcp_info_t *pcp_msg_info) +{ + pcp_msg_info->version = common_req[0] ; + pcp_msg_info->opcode = common_req[1] & 0x7f; + pcp_msg_info->lifetime = READNU32(common_req + 4); + pcp_msg_info->int_ip = (struct in6_addr *)(common_req + 8); + pcp_msg_info->mapped_ip = (struct in6_addr *)(common_req + 8); + + + if ( (pcp_msg_info->version > this_server_info.server_version) ) { + pcp_msg_info->result_code = PCP_ERR_UNSUPP_VERSION; + return 1; + } + + if (pcp_msg_info->lifetime > max_lifetime ) { + pcp_msg_info->lifetime = max_lifetime; + } + + if ( (pcp_msg_info->lifetime < min_lifetime) && (pcp_msg_info->lifetime != 0) ) { + pcp_msg_info->lifetime = min_lifetime; + } + + return 0; +} + +#ifdef DEBUG +static void printMAPOpcodeVersion1(const uint8_t *buf) +{ + char map_addr[INET6_ADDRSTRLEN]; + syslog(LOG_DEBUG, "PCP MAP: v1 Opcode specific information. \n"); + syslog(LOG_DEBUG, "MAP protocol: \t\t %d\n", (int)buf[0] ); + syslog(LOG_DEBUG, "MAP int port: \t\t %d\n", (int)READNU16(buf+4)); + syslog(LOG_DEBUG, "MAP ext port: \t\t %d\n", (int)READNU16(buf+6)); + syslog(LOG_DEBUG, "MAP Ext IP: \t\t %s\n", inet_ntop(AF_INET6, + buf+8, map_addr, INET6_ADDRSTRLEN)); +} + +static void printMAPOpcodeVersion2(const uint8_t *buf) +{ + char map_addr[INET6_ADDRSTRLEN]; + syslog(LOG_DEBUG, "PCP MAP: v2 Opcode specific information."); + syslog(LOG_DEBUG, "MAP nonce: \t%08x%08x%08x", + READNU32(buf), READNU32(buf+4), READNU32(buf+8)); + syslog(LOG_DEBUG, "MAP protocol:\t%d", (int)buf[12]); + syslog(LOG_DEBUG, "MAP int port:\t%d", (int)READNU16(buf+16)); + syslog(LOG_DEBUG, "MAP ext port:\t%d", (int)READNU16(buf+18)); + syslog(LOG_DEBUG, "MAP Ext IP: \t%s", inet_ntop(AF_INET6, + buf+20, map_addr, INET6_ADDRSTRLEN)); +} +#endif /* DEBUG */ + +static void parsePCPMAP_version1(const uint8_t *map_v1, + pcp_info_t *pcp_msg_info) +{ + pcp_msg_info->is_map_op = 1; + pcp_msg_info->protocol = map_v1[0]; + pcp_msg_info->int_port = READNU16(map_v1 + 4); + pcp_msg_info->ext_port = READNU16(map_v1 + 6); + + pcp_msg_info->ext_ip = (struct in6_addr *)(map_v1 + 8); +} + +static void parsePCPMAP_version2(const uint8_t *map_v2, + pcp_info_t *pcp_msg_info) +{ + pcp_msg_info->is_map_op = 1; + memcpy(pcp_msg_info->nonce, map_v2, 12); + pcp_msg_info->protocol = map_v2[12]; + pcp_msg_info->int_port = READNU16(map_v2 + 16); + pcp_msg_info->ext_port = READNU16(map_v2 + 18); + + pcp_msg_info->ext_ip = (struct in6_addr *)(map_v2 + 20); +} + +#ifdef PCP_PEER +#ifdef DEBUG +static void printPEEROpcodeVersion1(const uint8_t *buf) +{ + char ext_addr[INET6_ADDRSTRLEN]; + char peer_addr[INET6_ADDRSTRLEN]; + syslog(LOG_DEBUG, "PCP PEER: v1 Opcode specific information. \n"); + syslog(LOG_DEBUG, "Protocol: \t\t %d\n", (int)buf[0]); + syslog(LOG_DEBUG, "Internal port: \t\t %d\n", READNU16(buf + 4)); + syslog(LOG_DEBUG, "External IP: \t\t %s\n", inet_ntop(AF_INET6, buf + 8, + ext_addr,INET6_ADDRSTRLEN)); + syslog(LOG_DEBUG, "External port port: \t\t %d\n", READNU16(buf + 6)); + syslog(LOG_DEBUG, "PEER IP: \t\t %s\n", inet_ntop(AF_INET6, buf + 28, + peer_addr,INET6_ADDRSTRLEN)); + syslog(LOG_DEBUG, "PEER port port: \t\t %d\n", READNU16(buf + 24)); +} + +static void printPEEROpcodeVersion2(const uint8_t *buf) +{ + char ext_addr[INET6_ADDRSTRLEN]; + char peer_addr[INET6_ADDRSTRLEN]; + + syslog(LOG_DEBUG, "PCP PEER: v2 Opcode specific information."); + syslog(LOG_DEBUG, "nonce: \t%08x%08x%08x", + READNU32(buf), READNU32(buf+4), READNU32(buf+8)); + syslog(LOG_DEBUG, "Protocol: \t%d", buf[12]); + syslog(LOG_DEBUG, "Internal port:\t%d", READNU16(buf + 16)); + syslog(LOG_DEBUG, "External IP: \t%s", inet_ntop(AF_INET6, buf + 20, + ext_addr, INET6_ADDRSTRLEN)); + syslog(LOG_DEBUG, "External port:\t%d", READNU16(buf + 18)); + syslog(LOG_DEBUG, "PEER IP: \t%s", inet_ntop(AF_INET6, buf + 40, + peer_addr, INET6_ADDRSTRLEN)); + syslog(LOG_DEBUG, "PEER port: \t%d", READNU16(buf + 36)); +} +#endif /* DEBUG */ + +/* + * Function extracting information from peer_buf to pcp_msg_info + * @return : when no problem occurred 0 is returned, 1 otherwise + */ +static void parsePCPPEER_version1(const uint8_t *buf, + pcp_info_t *pcp_msg_info) +{ + pcp_msg_info->is_peer_op = 1; + pcp_msg_info->protocol = buf[0]; + pcp_msg_info->int_port = READNU16(buf + 4); + pcp_msg_info->ext_port = READNU16(buf + 6); + pcp_msg_info->peer_port = READNU16(buf + 24); + + pcp_msg_info->ext_ip = (struct in6_addr *)(buf + 8); + pcp_msg_info->peer_ip = (struct in6_addr *)(buf + 28); +} + +/* + * Function extracting information from peer_buf to pcp_msg_info + * @return : when no problem occurred 0 is returned, 1 otherwise + */ +static void parsePCPPEER_version2(const uint8_t *buf, pcp_info_t *pcp_msg_info) +{ + pcp_msg_info->is_peer_op = 1; + memcpy(pcp_msg_info->nonce, buf, 12); + pcp_msg_info->protocol = buf[12]; + pcp_msg_info->int_port = READNU16(buf + 16); + pcp_msg_info->ext_port = READNU16(buf + 18); + pcp_msg_info->peer_port = READNU16(buf + 36); + + pcp_msg_info->ext_ip = (struct in6_addr *)(buf + 20); + pcp_msg_info->peer_ip = (struct in6_addr *)(buf + 40); +} +#endif /* PCP_PEER */ + +#ifdef PCP_SADSCP +#ifdef DEBUG +static void printSADSCPOpcode(const uint8_t *buf) +{ + unsigned char sadscp_tol; + sadscp_tol = buf[12]; /* tolerance_fields */ + + syslog(LOG_DEBUG, "PCP SADSCP: Opcode specific information.\n"); + syslog(LOG_DEBUG, "Delay tolerance %d \n", (sadscp_tol>>6)&3); + syslog(LOG_DEBUG, "Loss tolerance %d \n", (sadscp_tol>>4)&3); + syslog(LOG_DEBUG, "Jitter tolerance %d \n", (sadscp_tol>>2)&3); + syslog(LOG_DEBUG, "RRR %d \n", sadscp_tol&3); + syslog(LOG_DEBUG, "AppName Length %d \n", buf[13]); + syslog(LOG_DEBUG, "Application name %.*s \n", buf[13], buf + 14); +} +#endif //DEBUG + +static int parseSADSCP(const uint8_t *buf, pcp_info_t *pcp_msg_info) +{ + pcp_msg_info->delay_tolerance = (buf[12]>>6)&3; + pcp_msg_info->loss_tolerance = (buf[12]>>4)&3; + pcp_msg_info->jitter_tolerance = (buf[12]>>2)&3; + + if (pcp_msg_info->delay_tolerance == 3 || + pcp_msg_info->loss_tolerance == 3 || + pcp_msg_info->jitter_tolerance == 3 ) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return 1; + } + + pcp_msg_info->app_name = (const char *)(buf + 14); + pcp_msg_info->app_name_len = buf[13]; + + return 0; +} +#endif /* PCP_SADSCP */ + + +static int parsePCPOption(uint8_t* pcp_buf, int remain, pcp_info_t *pcp_msg_info) +{ +#ifdef DEBUG + char third_addr[INET6_ADDRSTRLEN]; +#endif /* DEBUG */ + unsigned short option_length; + + /* Do centralized option sanity checks here. */ + + if (remain < (int)PCP_OPTION_HDR_SIZE) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return 0; + } + + option_length = READNU16(pcp_buf + 2) + 4; /* len */ + + if (remain < option_length) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return 0; + } + + switch (pcp_buf[0]) { /* code */ + + case PCP_OPTION_3RD_PARTY: + + if (option_length != PCP_3RD_PARTY_OPTION_SIZE) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return 0; + } +#ifdef DEBUG + syslog(LOG_DEBUG, "PCP OPTION: \t Third party\n"); + syslog(LOG_DEBUG, "Third PARTY IP: \t %s\n", inet_ntop(AF_INET6, + pcp_buf + 4, third_addr, INET6_ADDRSTRLEN)); +#endif + if (pcp_msg_info->thirdp_ip ) { + syslog(LOG_ERR, "PCP: THIRD PARTY OPTION was already present. \n"); + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return 0; + } else { + pcp_msg_info->thirdp_ip = (struct in6_addr *)(pcp_buf + 4); + pcp_msg_info->mapped_ip = (struct in6_addr *)(pcp_buf + 4); + } + break; + + case PCP_OPTION_PREF_FAIL: + + if (option_length != PCP_PREFER_FAIL_OPTION_SIZE) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return 0; + } +#ifdef DEBUG + syslog(LOG_DEBUG, "PCP OPTION: \t Prefer failure \n"); +#endif + if (pcp_msg_info->opcode != PCP_OPCODE_MAP) { + syslog(LOG_DEBUG, "PCP: Unsupported OPTION for given OPCODE.\n"); + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + } + if (pcp_msg_info->pfailure_present != 0 ) { + syslog(LOG_DEBUG, "PCP: PREFER FAILURE OPTION was already present.\n"); + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + } else { + pcp_msg_info->pfailure_present = 1; + } + break; + + case PCP_OPTION_FILTER: + /* TODO fully implement filter */ + if (option_length != PCP_FILTER_OPTION_SIZE) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return 0; + } +#ifdef DEBUG + syslog(LOG_DEBUG, "PCP OPTION: \t Filter\n"); +#endif + if (pcp_msg_info->opcode != PCP_OPCODE_MAP) { + syslog(LOG_ERR, "PCP: Unsupported OPTION for given OPCODE.\n"); + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return 0; + } + break; + +#ifdef PCP_FLOWP + case PCP_OPTION_FLOW_PRIORITY: + +#ifdef DEBUG + syslog(LOG_DEBUG, "PCP OPTION: \t Flow priority\n"); +#endif + if (option_length != PCP_FLOW_PRIORITY_OPTION_SIZE) { + syslog(LOG_ERR, "PCP: Error processing DSCP. sizeof %d and remaining %d. flow len %d \n", + PCP_FLOW_PRIORITY_OPTION_SIZE, remain, READNU16(pcp_buf + 2)); + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return 0; + } + +#ifdef DEBUG + syslog(LOG_DEBUG, "DSCP UP: \t %d\n", pcp_buf[4]); + syslog(LOG_DEBUG, "DSCP DOWN: \t %d\n", pcp_buf[5]); +#endif + pcp_msg_info->dscp_up = pcp_buf[4]; + pcp_msg_info->dscp_down = pcp_buf[5]; + pcp_msg_info->flowp_present = 1; + + break; +#endif + default: + if (pcp_buf[0] < 128) { + syslog(LOG_ERR, "PCP: Unrecognized mandatory PCP OPTION: %d \n", (int)pcp_buf[0]); + /* Mandatory to understand */ + pcp_msg_info->result_code = PCP_ERR_UNSUPP_OPTION; + remain = 0; + break; + } + /* TODO - log optional not understood options? */ + break; + } + return option_length; +} + + +static void parsePCPOptions(void* pcp_buf, int remain, pcp_info_t *pcp_msg_info) +{ + int option_length; + + while (remain > 0) { + option_length = parsePCPOption(pcp_buf, remain, pcp_msg_info); + if (!option_length) + break; + remain -= option_length; + pcp_buf += option_length; + } + if (remain > 0) { + syslog(LOG_WARNING, "%s: remain=%d", "parsePCPOptions", remain); + } +} + + +/* CheckExternalAddress() + * Check that suggested external address in request match a real external + * IP address. + * Suggested address can also be 0 IPv4 or IPv6 address. + * (see http://tools.ietf.org/html/rfc6887#section-10 ) + * return values : + * 0 : check is OK + * -1 : check failed */ +static int CheckExternalAddress(pcp_info_t* pcp_msg_info) +{ + /* can contain a IPv4-mapped IPv6 address */ + static struct in6_addr external_addr; + int af; + + af = IN6_IS_ADDR_V4MAPPED(pcp_msg_info->mapped_ip) + ? AF_INET : AF_INET6; + + pcp_msg_info->is_fw = af == AF_INET6; + + if (pcp_msg_info->is_fw) { + external_addr = *pcp_msg_info->mapped_ip; + } else { + /* TODO : be able to handle case with multiple + * external addresses */ + if(use_ext_ip_addr) { + if (inet_pton(AF_INET, use_ext_ip_addr, + ((uint32_t*)external_addr.s6_addr)+3) == 1) { + ((uint32_t*)external_addr.s6_addr)[0] = 0; + ((uint32_t*)external_addr.s6_addr)[1] = 0; + ((uint32_t*)external_addr.s6_addr)[2] = htonl(0xFFFF); + } else if (inet_pton(AF_INET6, use_ext_ip_addr, external_addr.s6_addr) + != 1) { + pcp_msg_info->result_code = PCP_ERR_NETWORK_FAILURE; + return -1; + } + } else { + if(!ext_if_name || ext_if_name[0]=='\0') { + pcp_msg_info->result_code = PCP_ERR_NETWORK_FAILURE; + return -1; + } + if(getifaddr_in6(ext_if_name, af, &external_addr) < 0) { + pcp_msg_info->result_code = PCP_ERR_NETWORK_FAILURE; + return -1; + } + } + } + if (pcp_msg_info->ext_ip == NULL || + IN6_IS_ADDR_UNSPECIFIED(pcp_msg_info->ext_ip) || + (IN6_IS_ADDR_V4MAPPED(pcp_msg_info->ext_ip) + && ((uint32_t *)pcp_msg_info->ext_ip->s6_addr)[3] == INADDR_ANY)) { + /* no suggested external address : use real external address */ + pcp_msg_info->ext_ip = &external_addr; + return 0; + } + + if (!IN6_ARE_ADDR_EQUAL(pcp_msg_info->ext_ip, &external_addr)) { + syslog(LOG_ERR, + "PCP: External IP in request didn't match interface IP \n"); +#ifdef DEBUG + { + char s[INET6_ADDRSTRLEN]; + syslog(LOG_DEBUG, "Interface IP %s \n", + inet_ntop(AF_INET6, &external_addr.s6_addr, s, sizeof(s))); + syslog(LOG_DEBUG, "IP in the PCP request %s \n", + inet_ntop(AF_INET6, pcp_msg_info->ext_ip, s, sizeof(s))); + } +#endif + + if (pcp_msg_info->pfailure_present) { + pcp_msg_info->result_code = PCP_ERR_CANNOT_PROVIDE_EXTERNAL; + return -1; + } else { + pcp_msg_info->ext_ip = &external_addr; + } + + } + + return 0; +} + + +static const char* inet_n46top(const struct in6_addr* in, + char* buf, size_t buf_len) +{ + if (IN6_IS_ADDR_V4MAPPED(in)) { + return inet_ntop(AF_INET, ((uint32_t*)(in->s6_addr))+3, buf, buf_len); + } else { + return inet_ntop(AF_INET6, in, buf, buf_len); + } +} + +#ifdef PCP_PEER +static void FillSA(struct sockaddr *sa, const struct in6_addr *in6, + uint16_t port) +{ + if (IN6_IS_ADDR_V4MAPPED(in6)) { + struct sockaddr_in *sa4 = (struct sockaddr_in *)sa; + sa4->sin_family = AF_INET; + sa4->sin_addr.s_addr = ((uint32_t*)(in6)->s6_addr)[3]; + sa4->sin_port = htons(port); + } else { + struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *)sa; + sa6->sin6_family = AF_INET6; + sa6->sin6_addr = *in6; + sa6->sin6_port = htons(port); + } +} + +static const char* inet_satop(struct sockaddr* sa, char* buf, size_t buf_len) +{ + if (sa->sa_family == AF_INET) { + return inet_ntop(AF_INET, &(((struct sockaddr_in*)sa)->sin_addr), buf, buf_len); + } else { + return inet_ntop(AF_INET6, &(((struct sockaddr_in6*)sa)->sin6_addr), buf, buf_len); + } +} + +static int CreatePCPPeer_NAT(pcp_info_t *pcp_msg_info) +{ + struct sockaddr_storage intip; + struct sockaddr_storage peerip; + struct sockaddr_storage extip; + struct sockaddr_storage ret_extip; + + uint8_t proto = pcp_msg_info->protocol; + + uint16_t eport = pcp_msg_info->ext_port; /* public port */ + + char peerip_s[INET6_ADDRSTRLEN], extip_s[INET6_ADDRSTRLEN]; + time_t timestamp = upnp_time() + pcp_msg_info->lifetime; + int r; + + FillSA((struct sockaddr*)&intip, pcp_msg_info->mapped_ip, + pcp_msg_info->int_port); + FillSA((struct sockaddr*)&peerip, pcp_msg_info->peer_ip, + pcp_msg_info->peer_port); + FillSA((struct sockaddr*)&extip, pcp_msg_info->ext_ip, + eport); + + inet_satop((struct sockaddr*)&peerip, peerip_s, sizeof(peerip_s)); + inet_satop((struct sockaddr*)&extip, extip_s, sizeof(extip_s)); + + /* check if connection with given peer exists, if it was */ + /* already established use this external port */ + if (get_nat_ext_addr( (struct sockaddr*)&intip, (struct sockaddr*)&peerip, + proto, (struct sockaddr*)&ret_extip) == 1) { + if (ret_extip.ss_family == AF_INET) { + struct sockaddr_in* ret_ext4 = (struct sockaddr_in*)&ret_extip; + uint16_t ret_eport = ntohs(ret_ext4->sin_port); + eport = ret_eport; + } else if (ret_extip.ss_family == AF_INET6) { + struct sockaddr_in6* ret_ext6 = (struct sockaddr_in6*)&ret_extip; + uint16_t ret_eport = ntohs(ret_ext6->sin6_port); + eport = ret_eport; + } else { + return PCP_ERR_CANNOT_PROVIDE_EXTERNAL; + } + } + /* Create Peer Mapping */ + if (eport == 0) { + eport = pcp_msg_info->int_port; + } + +#ifdef PCP_FLOWP + if (pcp_msg_info->flowp_present && pcp_msg_info->dscp_up) { + if (add_peer_dscp_rule2(ext_if_name, peerip_s, + pcp_msg_info->peer_port, pcp_msg_info->dscp_up, + pcp_msg_info->mapped_str, pcp_msg_info->int_port, + proto, pcp_msg_info->desc, timestamp) < 0 ) { + syslog(LOG_ERR, "PCP: failed to add flowp upstream mapping %s:%hu->%s:%hu '%s'", + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + peerip_s, + pcp_msg_info->peer_port, + pcp_msg_info->desc); + return PCP_ERR_NO_RESOURCES; + } + } + + if (pcp_msg_info->flowp_present && pcp_msg_info->dscp_down) { + if (add_peer_dscp_rule2(ext_if_name, pcp_msg_info->mapped_str, + pcp_msg_info->int_port, pcp_msg_info->dscp_down, + peerip_s, pcp_msg_info->peer_port, proto, pcp_msg_info->desc, timestamp) + < 0 ) { + syslog(LOG_ERR, "PCP: failed to add flowp downstream mapping %s:%hu->%s:%hu '%s'", + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + peerip_s, + pcp_msg_info->peer_port, + pcp_msg_info->desc); + pcp_msg_info->result_code = PCP_ERR_NO_RESOURCES; + return PCP_ERR_NO_RESOURCES; + } + } +#endif + + r = add_peer_redirect_rule2(ext_if_name, + peerip_s, + pcp_msg_info->peer_port, + extip_s, + eport, + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + pcp_msg_info->protocol, + pcp_msg_info->desc, + timestamp); + if (r < 0) + return PCP_ERR_NO_RESOURCES; + pcp_msg_info->ext_port = eport; + return PCP_SUCCESS; +} + +static void CreatePCPPeer(pcp_info_t *pcp_msg_info) +{ + char peerip_s[INET6_ADDRSTRLEN]; + int r = -1; + + if (!inet_n46top(pcp_msg_info->peer_ip, peerip_s, sizeof(peerip_s))) { + syslog(LOG_ERR, "inet_n46top(peer_ip): %m"); + return; + } + + if (pcp_msg_info->is_fw) { +#if 0 + /* Someday, something like this is available.. and we're ready! */ +#ifdef ENABLE_UPNPPINHOLE + pcp_msg_info->ext_port = pcp_msg_info->int_port; + r = upnp_add_outbound_pinhole(peerip_s, + pcp_msg_info->peer_port, + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + pcp_msg_info->protocol, + pcp_msg_info->desc, + pcp_msg_info->lifetime, NULL); +#endif /* ENABLE_UPNPPINHOLE */ +#else + r = PCP_ERR_UNSUPP_OPCODE; +#endif /* 0 */ + } else { + r = CreatePCPPeer_NAT(pcp_msg_info); + } + /* TODO: add upnp function for PI */ + pcp_msg_info->result_code = r; + syslog(r == PCP_SUCCESS ? LOG_INFO : LOG_ERR, + "PCP PEER: %s peer mapping %s %s:%hu(%hu)->%s:%hu '%s'", + r == PCP_SUCCESS ? "added" : "failed to add", + (pcp_msg_info->protocol==IPPROTO_TCP)?"TCP":"UDP", + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + pcp_msg_info->ext_port, + peerip_s, + pcp_msg_info->peer_port, + pcp_msg_info->desc); +} + +static void DeletePCPPeer(pcp_info_t *pcp_msg_info) +{ + uint16_t iport = pcp_msg_info->int_port; /* private port */ + uint16_t rport = pcp_msg_info->peer_port; /* private port */ + uint8_t proto = pcp_msg_info->protocol; + char rhost[INET6_ADDRSTRLEN]; + int r = -1; + + /* remove requested mappings for this client */ + int index = 0; + unsigned short eport2, iport2, rport2; + char iaddr2[INET6_ADDRSTRLEN], rhost2[INET6_ADDRSTRLEN]; + int proto2; + char desc[64]; + unsigned int timestamp; +#if 0 + int uid; +#endif /* 0 */ + + if (pcp_msg_info->is_fw) { + pcp_msg_info->result_code = PCP_ERR_UNSUPP_OPCODE; + return; + } + + inet_n46top((struct in6_addr*)pcp_msg_info->peer_ip, rhost, sizeof(rhost)); + + for (index = 0 ; + (!pcp_msg_info->is_fw && + get_peer_rule_by_index(index, 0, + &eport2, iaddr2, sizeof(iaddr2), + &iport2, &proto2, + desc, sizeof(desc), + rhost2, sizeof(rhost2), &rport2, + ×tamp, 0, 0) >= 0) +#if 0 + /* Some day if outbound pinholes are supported.. */ + || + (pcp_msg_info->is_fw && + (uid=upnp_get_pinhole_uid_by_index(index))>=0 && + upnp_get_pinhole_info((unsigned short)uid, + rhost2, sizeof(rhost2), &rport2, + iaddr2, sizeof(iaddr2), &iport2, + &proto2, desc, sizeof(desc), + ×tamp, NULL) >= 0) +#endif /* 0 */ + ; + index++) + if((0 == strcmp(iaddr2, pcp_msg_info->mapped_str)) + && (0 == strcmp(rhost2, rhost)) + && (proto2==proto) + && 0 == strcmp(desc, pcp_msg_info->desc) + && (iport2==iport) && (rport2==rport)) { + if (!pcp_msg_info->is_fw) + r = _upnp_delete_redir(eport2, proto2); +#if 0 + else + r = upnp_delete_outboundpinhole(uid); +#endif /* 0 */ + if(r<0) { + syslog(LOG_ERR, "PCP PEER: failed to remove peer mapping"); + } else { + syslog(LOG_INFO, "PCP PEER: %s port %hu peer mapping removed", + proto2==IPPROTO_TCP?"TCP":"UDP", eport2); + } + return; + } + if (r==-1) { + syslog(LOG_ERR, "PCP PEER: Failed to find PCP mapping internal port %hu, protocol %s", + iport, (pcp_msg_info->protocol == IPPROTO_TCP)?"TCP":"UDP"); + pcp_msg_info->result_code = PCP_ERR_NO_RESOURCES; + } +} +#endif /* PCP_PEER */ + +static int CreatePCPMap_NAT(pcp_info_t *pcp_msg_info) +{ + int r = 0; + char iaddr_old[INET6_ADDRSTRLEN]; + uint16_t iport_old, eport_first = 0; + int any_eport_allowed = 0; + unsigned int timestamp = upnp_time() + pcp_msg_info->lifetime; + + if (pcp_msg_info->ext_port == 0) { + pcp_msg_info->ext_port = pcp_msg_info->int_port; + } + + /* TODO: Support non-TCP/UDP */ + if (pcp_msg_info->ext_port == 0) { + return PCP_ERR_MALFORMED_REQUEST; + } + + do { + if (eport_first == 0) { /* first time in loop */ + eport_first = pcp_msg_info->ext_port; + } else if (pcp_msg_info->ext_port == eport_first) { /* no eport available */ + /* all eports rejected by permissions? */ + if (any_eport_allowed == 0) + return PCP_ERR_NOT_AUTHORIZED; + /* at least one eport allowed (but none available) */ + return PCP_ERR_NO_RESOURCES; + } + if ((IN6_IS_ADDR_V4MAPPED(pcp_msg_info->mapped_ip) && + (!check_upnp_rule_against_permissions(upnppermlist, + num_upnpperm, pcp_msg_info->ext_port, + ((struct in_addr*)pcp_msg_info->mapped_ip->s6_addr)[3], + pcp_msg_info->int_port)))) { + if (pcp_msg_info->pfailure_present) { + return PCP_ERR_CANNOT_PROVIDE_EXTERNAL; + } + pcp_msg_info->ext_port++; + if (pcp_msg_info->ext_port == 0) { /* skip port zero */ + pcp_msg_info->ext_port++; + } + continue; + } + any_eport_allowed = 1; +#ifdef CHECK_PORTINUSE + if (port_in_use(ext_if_name, pcp_msg_info->ext_port, pcp_msg_info->protocol, + pcp_msg_info->mapped_str, pcp_msg_info->int_port) > 0) { + syslog(LOG_INFO, "port %hu protocol %s already in use", + pcp_msg_info->ext_port, + (pcp_msg_info->protocol==IPPROTO_TCP)?"tcp":"udp"); + pcp_msg_info->ext_port++; + if (pcp_msg_info->ext_port == 0) { /* skip port zero */ + pcp_msg_info->ext_port++; + } + continue; + } +#endif + r = get_redirect_rule(ext_if_name, + pcp_msg_info->ext_port, + pcp_msg_info->protocol, + iaddr_old, sizeof(iaddr_old), + &iport_old, 0, 0, 0, 0, + NULL/*×tamp*/, 0, 0); + + if(r==0) { + if((strcmp(pcp_msg_info->mapped_str, iaddr_old)!=0) + || (pcp_msg_info->int_port != iport_old)) { + /* redirection already existing */ + if (pcp_msg_info->pfailure_present) { + return PCP_ERR_CANNOT_PROVIDE_EXTERNAL; + } + } else { + syslog(LOG_INFO, "port %hu %s already redirected to %s:%hu, replacing", + pcp_msg_info->ext_port, (pcp_msg_info->protocol==IPPROTO_TCP)?"tcp":"udp", + iaddr_old, iport_old); + /* remove and then add again */ + if (_upnp_delete_redir(pcp_msg_info->ext_port, + pcp_msg_info->protocol)==0) { + break; + } else if (pcp_msg_info->pfailure_present) { + return PCP_ERR_CANNOT_PROVIDE_EXTERNAL; + } + } + pcp_msg_info->ext_port++; + if (pcp_msg_info->ext_port == 0) { /* skip port zero */ + pcp_msg_info->ext_port++; + } + } + } while (r==0); + + r = upnp_redirect_internal(NULL, + pcp_msg_info->ext_port, + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + pcp_msg_info->protocol, + pcp_msg_info->desc, + timestamp); + if (r < 0) + return PCP_ERR_NO_RESOURCES; + return PCP_SUCCESS; +} + +static int CreatePCPMap_FW(pcp_info_t *pcp_msg_info) +{ +#ifdef ENABLE_UPNPPINHOLE + int uid; + int r; + /* first check if pinhole already exists */ + uid = upnp_find_inboundpinhole(NULL, 0, + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + pcp_msg_info->protocol, + NULL, 0, /* desc */ + NULL /* lifetime */); + if(uid >= 0) { + /* pinhole already exists, updating */ + syslog(LOG_INFO, "updating pinhole to %s:%hu %s", + pcp_msg_info->mapped_str, pcp_msg_info->int_port, + (pcp_msg_info->protocol == IPPROTO_TCP)?"TCP":"UDP"); + r = upnp_update_inboundpinhole((unsigned short)uid, pcp_msg_info->lifetime); + return r >= 0 ? PCP_SUCCESS : PCP_ERR_NO_RESOURCES; + } else { + r = upnp_add_inboundpinhole(NULL, 0, + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + pcp_msg_info->protocol, + pcp_msg_info->desc, + pcp_msg_info->lifetime, + &uid); + if (r < 0) + return PCP_ERR_NO_RESOURCES; + pcp_msg_info->ext_port = pcp_msg_info->int_port; + return PCP_SUCCESS; + } +#else + UNUSED(pcp_msg_info); + return PCP_ERR_NO_RESOURCES; +#endif /* ENABLE_UPNPPINHOLE */ +} + + +/* internal external PCP remote peer actual remote peer + * -------- ------- --------------- ------------------ + * IPv4 firewall IPv4 IPv4 IPv4 IPv4 + * IPv6 firewall IPv6 IPv6 IPv6 IPv6 + * NAT44 IPv4 IPv4 IPv4 IPv4 + * NAT46 IPv4 IPv6 IPv4 IPv6 + * NAT64 IPv6 IPv4 IPv6 IPv4 + * NPTv6 IPv6 IPv6 IPv6 IPv6 + * + * Address Families with MAP and PEER + * + * The 'internal' address is implicitly the same as the source IP + * address of the PCP request, except when the THIRD_PARTY option is + * used. + * + * The 'external' address is the Suggested External Address field of the + * MAP or PEER request, and its address family is usually the same as + * the 'internal' address family, except when technologies like NAT64 + * are used. + * + * The 'remote peer' address is the remote peer IP address of the PEER + * request or the FILTER option of the MAP request, and is always the + * same address family as the 'internal' address, even when NAT64 is + * used. In NAT64, the IPv6 PCP client is not necessarily aware of the + * NAT64 or aware of the actual IPv4 address of the remote peer, so it + * expresses the IPv6 address from its perspective. */ + +/* TODO: Support more than basic NAT44 / IPv6 firewall cases. */ +static void CreatePCPMap(pcp_info_t *pcp_msg_info) +{ + int r; + + if (pcp_msg_info->is_fw) + r = CreatePCPMap_FW(pcp_msg_info); + else + r = CreatePCPMap_NAT(pcp_msg_info); + pcp_msg_info->result_code = r; + syslog(r == PCP_SUCCESS ? LOG_INFO : LOG_ERR, + "PCP MAP: %s mapping %s %hu->%s:%hu '%s'", + r == PCP_SUCCESS ? "added" : "failed to add", + (pcp_msg_info->protocol==IPPROTO_TCP)?"TCP":"UDP", + pcp_msg_info->ext_port, + pcp_msg_info->mapped_str, + pcp_msg_info->int_port, + pcp_msg_info->desc); +} + +static void DeletePCPMap(pcp_info_t *pcp_msg_info) +{ + uint16_t iport = pcp_msg_info->int_port; /* private port */ + uint8_t proto = pcp_msg_info->protocol; + int r=-1; + /* remove the mapping */ + /* remove all the mappings for this client */ + int index; + unsigned short eport2, iport2; + char iaddr2[INET6_ADDRSTRLEN]; + int proto2; + char desc[64]; + unsigned int timestamp; +#ifdef ENABLE_UPNPPINHOLE + int uid = -1; +#endif /* ENABLE_UPNPPINHOLE */ + + /* iterate through all rules and delete the requested ones */ + for (index = 0 ; + (!pcp_msg_info->is_fw && + get_redirect_rule_by_index(index, 0, + &eport2, iaddr2, sizeof(iaddr2), + &iport2, &proto2, + desc, sizeof(desc), + 0, 0, ×tamp, 0, 0) >= 0) +#ifdef ENABLE_UPNPPINHOLE + || + (pcp_msg_info->is_fw && + (uid=upnp_get_pinhole_uid_by_index(index))>=0 && + upnp_get_pinhole_info((unsigned short)uid, + NULL, 0, NULL, + iaddr2, sizeof(iaddr2), &iport2, + &proto2, desc, sizeof(desc), + ×tamp, NULL) >= 0) +#endif /* ENABLE_UPNPPINHOLE */ + ; + index++) + if(0 == strcmp(iaddr2, pcp_msg_info->mapped_str) + && (proto2==proto) + && ((iport2==iport) || (iport==0))) { + if(0 != strcmp(desc, pcp_msg_info->desc)) { + /* nonce does not match */ + pcp_msg_info->result_code = PCP_ERR_NOT_AUTHORIZED; + syslog(LOG_ERR, "Unauthorized to remove PCP mapping internal port %hu, protocol %s", + iport, (pcp_msg_info->protocol == IPPROTO_TCP)?"TCP":"UDP"); + return; + } else if (!pcp_msg_info->is_fw) { + r = _upnp_delete_redir(eport2, proto2); + } else { +#ifdef ENABLE_UPNPPINHOLE + r = upnp_delete_inboundpinhole(uid); +#endif /* ENABLE_UPNPPINHOLE */ + } + break; + } + if (r >= 0) { + syslog(LOG_INFO, "PCP: %s port %hu mapping removed", + proto2==IPPROTO_TCP?"TCP":"UDP", eport2); + } else { + syslog(LOG_ERR, "Failed to remove PCP mapping internal port %hu, protocol %s", + iport, (pcp_msg_info->protocol == IPPROTO_TCP)?"TCP":"UDP"); + pcp_msg_info->result_code = PCP_ERR_NO_RESOURCES; + } +} + +static int ValidatePCPMsg(pcp_info_t *pcp_msg_info) +{ + if (pcp_msg_info->result_code) { + return 0; + } + + /* RFC 6887, section 8.2: MUST return address mismatch if NAT + * in middle. */ + if (memcmp(pcp_msg_info->int_ip, + &pcp_msg_info->sender_ip, + sizeof(pcp_msg_info->sender_ip)) != 0) { + pcp_msg_info->result_code = PCP_ERR_ADDRESS_MISMATCH; + return 0; + } + + if (pcp_msg_info->thirdp_ip) { + if (!GETFLAG(PCP_ALLOWTHIRDPARTYMASK)) { + pcp_msg_info->result_code = PCP_ERR_UNSUPP_OPTION; + return 0; + } + + /* RFC687, section 13.1 - if sender ip == THIRD_PARTY, + * it's an error. */ + if (memcmp(pcp_msg_info->thirdp_ip, + &pcp_msg_info->sender_ip, + sizeof(pcp_msg_info->sender_ip)) == 0) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return 0; + } + } + + /* Produce mapped_str for future use. */ + if (!inet_n46top(pcp_msg_info->mapped_ip, pcp_msg_info->mapped_str, + sizeof(pcp_msg_info->mapped_str))) { + syslog(LOG_ERR, "inet_ntop(pcpserver): %m"); + return 0; + } + + /* protocol zero means 'all protocols' : internal port MUST be zero */ + if (pcp_msg_info->protocol == 0 && pcp_msg_info->int_port != 0) { + syslog(LOG_ERR, "PCP %s: Protocol was ZERO, but internal port " + "has non-ZERO value.", getPCPOpCodeStr(pcp_msg_info->opcode)); + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return 0; + } + + if (pcp_msg_info->pfailure_present) { + if ( (IN6_IS_ADDR_UNSPECIFIED(pcp_msg_info->ext_ip) || + ((IN6_IS_ADDR_V4MAPPED(pcp_msg_info->ext_ip)) && + (((uint32_t*)pcp_msg_info->ext_ip->s6_addr)[3] == 0))) && + (pcp_msg_info->ext_port == 0) + ) + { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return 0; + } + } + + if (CheckExternalAddress(pcp_msg_info)) { + return 0; + } + + /* Fill in the desc that describes uniquely what flow we're + * dealing with (same code used in both create + delete of + * MAP/PEER) */ + switch (pcp_msg_info->opcode) { + case PCP_OPCODE_MAP: + case PCP_OPCODE_PEER: + snprintf(pcp_msg_info->desc, sizeof(pcp_msg_info->desc), + "PCP %s %08x%08x%08x", + getPCPOpCodeStr(pcp_msg_info->opcode), + pcp_msg_info->nonce[0], + pcp_msg_info->nonce[1], pcp_msg_info->nonce[2]); + break; + } + return 1; +} + +/* + * return value indicates whether the request is valid or not. + * Based on the return value simple response can be formed. + */ +static int processPCPRequest(void * req, int req_size, pcp_info_t *pcp_msg_info) +{ + int remainingSize; + + /* start with PCP_SUCCESS as result code, + * if everything is OK value will be unchanged */ + pcp_msg_info->result_code = PCP_SUCCESS; + + remainingSize = req_size; + + /* discard request that exceeds maximal length, + or that is shorter than PCP_MIN_LEN (=24) + or that is not the multiple of 4 */ + if (req_size < 3) + return 0; /* ignore msg */ + + if (req_size < PCP_MIN_LEN) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return 1; /* send response */ + } + + if ( (req_size > PCP_MAX_LEN) || ( (req_size & 3) != 0)) { + syslog(LOG_ERR, "PCP: Size of PCP packet(%d) is larger than %d bytes or " + "the size is not multiple of 4.\n", req_size, PCP_MAX_LEN); + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return 1; /* send response */ + } + + /* first parse request header */ + if (parseCommonRequestHeader(req, pcp_msg_info) ) { + return 1; + } + + remainingSize -= PCP_COMMON_REQUEST_SIZE; + req += PCP_COMMON_REQUEST_SIZE; + + if (pcp_msg_info->version == 1) { + /* legacy PCP version 1 support */ + switch (pcp_msg_info->opcode) { + case PCP_OPCODE_MAP: + + remainingSize -= PCP_MAP_V1_SIZE; + if (remainingSize < 0) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return pcp_msg_info->result_code; + } + +#ifdef DEBUG + printMAPOpcodeVersion1(req); +#endif /* DEBUG */ + parsePCPMAP_version1(req, pcp_msg_info); + + req += PCP_MAP_V1_SIZE; + + parsePCPOptions(req, remainingSize, pcp_msg_info); + if (ValidatePCPMsg(pcp_msg_info)) { + if (pcp_msg_info->lifetime == 0) { + DeletePCPMap(pcp_msg_info); + } else { + CreatePCPMap(pcp_msg_info); + } + } else { + syslog(LOG_ERR, "PCP: Invalid PCP v1 MAP message."); + return pcp_msg_info->result_code; + } + break; + +#ifdef PCP_PEER + case PCP_OPCODE_PEER: + + remainingSize -= PCP_PEER_V1_SIZE; + if (remainingSize < 0) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return pcp_msg_info->result_code; + } + +#ifdef DEBUG + printPEEROpcodeVersion1(req); +#endif /* DEBUG */ + parsePCPPEER_version1(req, pcp_msg_info); + + req += PCP_PEER_V1_SIZE; + + parsePCPOptions(req, remainingSize, pcp_msg_info); + + if (ValidatePCPMsg(pcp_msg_info)) { + if (pcp_msg_info->lifetime == 0) { + DeletePCPPeer(pcp_msg_info); + } else { + CreatePCPPeer(pcp_msg_info); + } + } else { + syslog(LOG_ERR, "PCP: Invalid PCP v1 PEER message."); + return pcp_msg_info->result_code; + } + + + break; +#endif /* PCP_PEER */ + default: + pcp_msg_info->result_code = PCP_ERR_UNSUPP_OPCODE; + break; + } + + } else if (pcp_msg_info->version == 2) { + /* RFC 6887 PCP support + * http://tools.ietf.org/html/rfc6887 */ + switch (pcp_msg_info->opcode) { + case PCP_OPCODE_ANNOUNCE: + /* should check PCP Client's IP Address in request */ + /* see http://tools.ietf.org/html/rfc6887#section-14.1 */ + break; + case PCP_OPCODE_MAP: + + remainingSize -= PCP_MAP_V2_SIZE; + if (remainingSize < 0) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return pcp_msg_info->result_code; + } + +#ifdef DEBUG + printMAPOpcodeVersion2(req); +#endif /* DEBUG */ + parsePCPMAP_version2(req, pcp_msg_info); + req += PCP_MAP_V2_SIZE; + + parsePCPOptions(req, remainingSize, pcp_msg_info); + + if (ValidatePCPMsg(pcp_msg_info)) { + if (pcp_msg_info->lifetime == 0) { + DeletePCPMap(pcp_msg_info); + } else { + CreatePCPMap(pcp_msg_info); + } + } else { + syslog(LOG_ERR, "PCP: Invalid PCP v2 MAP message."); + return pcp_msg_info->result_code; + } + + + break; + +#ifdef PCP_PEER + case PCP_OPCODE_PEER: + + remainingSize -= PCP_PEER_V2_SIZE; + if (remainingSize < 0) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return pcp_msg_info->result_code; + } + +#ifdef DEBUG + printPEEROpcodeVersion2(req); +#endif /* DEBUG */ + parsePCPPEER_version2(req, pcp_msg_info); + req += PCP_PEER_V2_SIZE; + + if (pcp_msg_info->result_code != 0) { + return pcp_msg_info->result_code; + } + + parsePCPOptions(req, remainingSize, pcp_msg_info); + + if (ValidatePCPMsg(pcp_msg_info)) { + if (pcp_msg_info->lifetime == 0) { + DeletePCPPeer(pcp_msg_info); + } else { + CreatePCPPeer(pcp_msg_info); + } + } else { + syslog(LOG_ERR, "PCP: Invalid PCP v2 PEER message."); + } + + break; +#endif /* PCP_PEER */ + +#ifdef PCP_SADSCP + case PCP_OPCODE_SADSCP: + remainingSize -= PCP_SADSCP_REQ_SIZE; + if (remainingSize < 0) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_REQUEST; + return pcp_msg_info->result_code; + } + + remainingSize -= ((uint8_t *)req)[13]; /* app_name_length */ + if (remainingSize < 0) { + pcp_msg_info->result_code = PCP_ERR_MALFORMED_OPTION; + return pcp_msg_info->result_code; + } + +#ifdef DEBUG + printSADSCPOpcode(req); +#endif + parseSADSCP(req, pcp_msg_info); + req += PCP_SADSCP_REQ_SIZE; + if (pcp_msg_info->result_code != 0) { + return pcp_msg_info->result_code; + } + req += pcp_msg_info->app_name_len; + + get_dscp_value(pcp_msg_info); + + + break; +#endif + default: + pcp_msg_info->result_code = PCP_ERR_UNSUPP_OPCODE; + break; + } + } else { + pcp_msg_info->result_code = PCP_ERR_UNSUPP_VERSION; + return pcp_msg_info->result_code; + } + return 1; +} + + +static void createPCPResponse(unsigned char *response, pcp_info_t *pcp_msg_info) +{ + response[2] = 0; /* reserved */ + memset(response + 12, 0, 12); /* reserved */ + if (pcp_msg_info->result_code == PCP_ERR_UNSUPP_VERSION ) { + /* highest supported version */ + response[0] = this_server_info.server_version; + } else { + response[0] = pcp_msg_info->version; + } + + response[1] |= 0x80; /* r_opcode */ + response[3] = pcp_msg_info->result_code; + if(epoch_origin == 0) { + epoch_origin = startup_time; + } + WRITENU32(response + 8, upnp_time() - epoch_origin); /* epochtime */ + switch (pcp_msg_info->result_code) { + /*long lifetime errors*/ + case PCP_ERR_UNSUPP_VERSION: + case PCP_ERR_NOT_AUTHORIZED: + case PCP_ERR_MALFORMED_REQUEST: + case PCP_ERR_UNSUPP_OPCODE: + case PCP_ERR_UNSUPP_OPTION: + case PCP_ERR_MALFORMED_OPTION: + case PCP_ERR_UNSUPP_PROTOCOL: + case PCP_ERR_ADDRESS_MISMATCH: + case PCP_ERR_CANNOT_PROVIDE_EXTERNAL: + case PCP_ERR_EXCESSIVE_REMOTE_PEERS: + WRITENU32(response + 4, 0); /* lifetime */ + break; + + case PCP_ERR_NETWORK_FAILURE: + case PCP_ERR_NO_RESOURCES: + case PCP_ERR_USER_EX_QUOTA: + WRITENU32(response + 4, 30); /* lifetime */ + break; + + case PCP_SUCCESS: + default: + WRITENU32(response + 4, pcp_msg_info->lifetime); /* lifetime */ + break; + } + + if (response[1] == 0x81) { /* MAP response */ + if (response[0] == 1) { /* version */ + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 4, pcp_msg_info->int_port); + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 6, pcp_msg_info->ext_port); + copyIPv6IfDifferent(response + PCP_COMMON_RESPONSE_SIZE + 8, + pcp_msg_info->ext_ip); + } + else if (response[0] == 2) { + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 16, pcp_msg_info->int_port); + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 18, pcp_msg_info->ext_port); + copyIPv6IfDifferent(response + PCP_COMMON_RESPONSE_SIZE + 20, + pcp_msg_info->ext_ip); + } + } +#ifdef PCP_PEER + else if (response[1] == 0x82) { /* PEER response */ + if (response[0] == 1) { + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 4, pcp_msg_info->int_port); + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 6, pcp_msg_info->ext_port); + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 24, pcp_msg_info->peer_port); + copyIPv6IfDifferent(response + PCP_COMMON_RESPONSE_SIZE + 8, + pcp_msg_info->ext_ip); + } + else if (response[0] == 2) { + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 16, pcp_msg_info->int_port); + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 18, pcp_msg_info->ext_port); + WRITENU16(response + PCP_COMMON_RESPONSE_SIZE + 36, pcp_msg_info->peer_port); + copyIPv6IfDifferent(response + PCP_COMMON_RESPONSE_SIZE + 20, + pcp_msg_info->ext_ip); + } + } +#endif /* PCP_PEER */ + +#ifdef PCP_SADSCP + else if (response[1] == 0x83) { /*SADSCP response*/ + response[PCP_COMMON_RESPONSE_SIZE + 12] + = ((pcp_msg_info->matched_name<<7) & ~(1<<6)) | + (pcp_msg_info->sadscp_dscp & PCP_SADSCP_MASK); + memset(response + PCP_COMMON_RESPONSE_SIZE + 13, 0, 3); + } +#endif /* PCP_SADSCP */ +} + +int ProcessIncomingPCPPacket(int s, unsigned char *buff, int len, + const struct sockaddr *senderaddr, + const struct sockaddr_in6 *receiveraddr) +{ + pcp_info_t pcp_msg_info; + struct lan_addr_s * lan_addr; + char addr_str[64]; + + memset(&pcp_msg_info, 0, sizeof(pcp_info_t)); + + if(senderaddr->sa_family == AF_INET) { + const struct sockaddr_in * senderaddr_v4 = + (const struct sockaddr_in *)senderaddr; + pcp_msg_info.sender_ip.s6_addr[11] = 0xff; + pcp_msg_info.sender_ip.s6_addr[10] = 0xff; + memcpy(pcp_msg_info.sender_ip.s6_addr+12, + &senderaddr_v4->sin_addr, 4); + } else if(senderaddr->sa_family == AF_INET6) { + const struct sockaddr_in6 * senderaddr_v6 = + (const struct sockaddr_in6 *)senderaddr; + pcp_msg_info.sender_ip = senderaddr_v6->sin6_addr; + } else { + syslog(LOG_WARNING, "unknown PCP packet sender address family %d", + senderaddr->sa_family); + return 0; + } + + if(sockaddr_to_string(senderaddr, addr_str, sizeof(addr_str))) + syslog(LOG_DEBUG, "PCP request received from %s %dbytes", + addr_str, len); + + if(buff[1] & 128) { + /* discarding PCP responses silently */ + return 0; + } + + /* If we're in allow third party-mode, we probably don't care + * about locality either. Let's hope firewall is ok. */ + if (!GETFLAG(PCP_ALLOWTHIRDPARTYMASK)) { + lan_addr = get_lan_for_peer(senderaddr); + if(lan_addr == NULL) { + syslog(LOG_WARNING, "PCP packet sender %s not from a LAN, ignoring", + addr_str); + return 0; + } + } + + if (processPCPRequest(buff, len, &pcp_msg_info) ) { + + createPCPResponse(buff, &pcp_msg_info); + + if(len < PCP_MIN_LEN) + len = PCP_MIN_LEN; + else + len = (len + 3) & ~3; /* round up resp. length to multiple of 4 */ + len = sendto_or_schedule2(s, buff, len, 0, senderaddr, + (senderaddr->sa_family == AF_INET) ? + sizeof(struct sockaddr_in) : + sizeof(struct sockaddr_in6), + receiveraddr); + if( len < 0 ) { + syslog(LOG_ERR, "sendto(pcpserver): %m"); + } + } + + return 0; +} + +#ifdef ENABLE_IPV6 +int OpenAndConfPCPv6Socket(void) +{ + int s; + int i = 1; + struct sockaddr_in6 addr; + s = socket(PF_INET6, SOCK_DGRAM, 0/*IPPROTO_UDP*/); + if(s < 0) { + syslog(LOG_ERR, "%s: socket(): %m", "OpenAndConfPCPv6Socket"); + return -1; + } + if(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i)) < 0) { + syslog(LOG_WARNING, "%s: setsockopt(SO_REUSEADDR): %m", + "OpenAndConfPCPv6Socket"); + } +#ifdef IPV6_V6ONLY + /* force IPV6 only for IPV6 socket. + * see http://www.ietf.org/rfc/rfc3493.txt section 5.3 */ + if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &i, sizeof(i)) < 0) { + syslog(LOG_WARNING, "%s: setsockopt(IPV6_V6ONLY): %m", + "OpenAndConfPCPv6Socket"); + } +#endif +#ifdef IPV6_RECVPKTINFO + /* see RFC3542 */ + if(setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, &i, sizeof(i)) < 0) { + syslog(LOG_WARNING, "%s: setsockopt(IPV6_RECVPKTINFO): %m", + "OpenAndConfPCPv6Socket"); + } +#endif + if(!set_non_blocking(s)) { + syslog(LOG_WARNING, "%s: set_non_blocking(): %m", + "OpenAndConfPCPv6Socket"); + } + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_port = htons(NATPMP_PORT); + addr.sin6_addr = ipv6_bind_addr; + if(bind(s, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + syslog(LOG_ERR, "%s: bind(): %m", "OpenAndConfPCPv6Socket"); + close(s); + return -1; + } + return s; +} +#endif /*ENABLE_IPV6*/ + +#ifdef ENABLE_IPV6 +static void PCPSendUnsolicitedAnnounce(int * sockets, int n_sockets, int socket6) +#else /* IPv4 only */ +static void PCPSendUnsolicitedAnnounce(int * sockets, int n_sockets) +#endif +{ + int i; + unsigned char buff[PCP_MIN_LEN]; + pcp_info_t info; + ssize_t len; + struct sockaddr_in addr; +#ifdef ENABLE_IPV6 + struct sockaddr_in6 addr6; +#endif /* ENABLE_IPV6 */ + /* this is an Unsolicited ANNOUNCE response */ + + info.version = this_server_info.server_version; + info.opcode = PCP_OPCODE_ANNOUNCE; + info.result_code = PCP_SUCCESS; + info.lifetime = 0; + createPCPResponse(buff, &info); + /* Multicast PCP restart announcements are sent to + * 224.0.0.1:5350 and/or [ff02::1]:5350 */ + memset(&addr, 0, sizeof(struct sockaddr_in)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr("224.0.0.1"); + addr.sin_port = htons(5350); + for(i = 0; i < n_sockets; i++) { + len = sendto_or_schedule(sockets[i], buff, PCP_MIN_LEN, 0, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)); + if( len < 0 ) { + syslog(LOG_ERR, "PCPSendUnsolicitedAnnounce() sendto(): %m"); + } + } +#ifdef ENABLE_IPV6 + memset(&addr6, 0, sizeof(struct sockaddr_in6)); + addr6.sin6_family = AF_INET6; + inet_pton(AF_INET6, "FF02::1", &(addr6.sin6_addr)); + addr6.sin6_port = htons(5350); + len = sendto_or_schedule(socket6, buff, PCP_MIN_LEN, 0, (struct sockaddr *)&addr6, sizeof(struct sockaddr_in6)); + if( len < 0 ) { + syslog(LOG_ERR, "PCPSendUnsolicitedAnnounce() IPv6 sendto(): %m"); + } +#endif /* ENABLE_IPV6 */ +} + +#ifdef ENABLE_IPV6 +void PCPPublicAddressChanged(int * sockets, int n_sockets, int socket6) +#else /* IPv4 only */ +void PCPPublicAddressChanged(int * sockets, int n_sockets) +#endif +{ + /* according to RFC 6887 8.5 : + * if the external IP address(es) of the NAT (controlled by + * the PCP server) changes, the Epoch time MUST be reset. */ + epoch_origin = upnp_time(); +#ifdef ENABLE_IPV6 + PCPSendUnsolicitedAnnounce(sockets, n_sockets, socket6); +#else /* IPv4 Only */ + PCPSendUnsolicitedAnnounce(sockets, n_sockets); +#endif +} +#endif /*ENABLE_PCP*/ diff --git a/src/contrib/miniupnp/miniupnpd/pcpserver.h b/src/contrib/miniupnp/miniupnpd/pcpserver.h new file mode 100644 index 0000000..3357e4f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pcpserver.h @@ -0,0 +1,65 @@ +/* $Id: pcpserver.h,v 1.3 2014/03/24 10:49:46 nanard Exp $ */ +/* MiniUPnP project + * Website : http://miniupnp.free.fr/ + * Author : Peter Tatrai + +Copyright (c) 2013 by Cisco Systems, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PCPSERVER_H_INCLUDED +#define PCPSERVER_H_INCLUDED + +#define PCP_MIN_LEN 24 +#define PCP_MAX_LEN 1100 + +struct sockaddr; + +/* + * receiveraddr is only used for IPV6 + * + * returns 0 upon success 1 otherwise + */ +int ProcessIncomingPCPPacket(int s, unsigned char *msg_buff, int len, + const struct sockaddr *senderaddr, + const struct sockaddr_in6 *receiveraddr); + +/* + * returns the socket + */ +int OpenAndConfPCPv6Socket(void); + + +/* + * To be called when Public IP address changed (IPv4) + */ +#ifdef ENABLE_IPV6 +void PCPPublicAddressChanged(int * sockets, int n_sockets, int socket6); +#else /* IPV4 Only */ +void PCPPublicAddressChanged(int * sockets, int n_sockets); +#endif + +#endif /* PCPSERVER_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpd/pf/Makefile b/src/contrib/miniupnp/miniupnpd/pf/Makefile new file mode 100644 index 0000000..f94c074 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pf/Makefile @@ -0,0 +1,25 @@ +# $Id: Makefile,v 1.4 2012/04/18 20:45:33 nanard Exp $ +# made for GNU Make (and BSD make) +CFLAGS = -Wall -g -DTEST +CFLAGS += -Wextra +EXECUTABLES = testobsdrdr testpfpinhole + +all: $(EXECUTABLES) + +clean: + rm -f *.o $(EXECUTABLES) + +testobsdrdr: testobsdrdr.o obsdrdr.o + $(CC) $(CFLAGS) -o $@ $> + +testpfpinhole: testpfpinhole.o obsdrdr.o pfpinhole.o + $(CC) $(CFLAGS) -o $@ $> + +obsdrdr.o: obsdrdr.c obsdrdr.h + +pfpinhole.o: pfpinhole.c pfpinhole.h + +testobsdrdr.o: testobsdrdr.c obsdrdr.h + +testpfpinhole.o: testpfpinhole.c pfpinhole.h + diff --git a/src/contrib/miniupnp/miniupnpd/pf/obsdrdr.c b/src/contrib/miniupnp/miniupnpd/pf/obsdrdr.c new file mode 100644 index 0000000..0b00ad9 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pf/obsdrdr.c @@ -0,0 +1,1160 @@ +/* $Id: obsdrdr.c,v 1.88 2018/04/12 09:27:53 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +/* + * pf rules created (with ext_if = xl1) + * - OpenBSD up to version 4.6 : + * rdr pass on xl1 inet proto udp from any to any port = 54321 \ + * keep state label "test label" -> 192.168.0.42 port 12345 + * or a rdr rule + a pass rule : + * rdr quick on xl1 inet proto udp from any to any port = 54321 \ + * keep state label "test label" -> 192.168.0.42 port 12345 + * pass in quick on xl1 inet proto udp from any to 192.168.0.42 port = 12345 \ + * flags S/SA keep state label "test label" + * + * - OpenBSD starting from version 4.7 + * match in on xl1 inet proto udp from any to any port 54321 \ + * label "test label" rdr-to 192.168.0.42 port 12345 + * or + * pass in quick on xl1 inet proto udp from any to any port 54321 \ + * label "test label" rdr-to 192.168.0.42 port 12345 + * + * + * + * Macros/#defines : + * - PF_ENABLE_FILTER_RULES + * If set, two rules are created : rdr + pass. Else a rdr/pass rule + * is created. + * - USE_IFNAME_IN_RULES + * If set the interface name is set in the rule. + * - PFRULE_INOUT_COUNTS + * Must be set with OpenBSD version 3.8 and up. + * - PFRULE_HAS_RTABLEID + * Must be set with OpenBSD version 4.0 and up. + * - PF_NEWSSTYLE + * Must be set with OpenBSD version 4.7 and up. + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef __DragonFly__ +#include +#else +#ifdef __APPLE__ +#define PRIVATE 1 +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include + +#include "../macros.h" +#include "../config.h" +#include "obsdrdr.h" +#include "../upnpglobalvars.h" + +#ifndef USE_PF +#error "USE_PF macro is undefined, check consistency between config.h and Makefile" +#else + +/* list to keep timestamps for port mappings having a lease duration */ +struct timestamp_entry { + struct timestamp_entry * next; + unsigned int timestamp; + unsigned short eport; + short protocol; +}; + +static struct timestamp_entry * timestamp_list = NULL; + +static unsigned int +get_timestamp(unsigned short eport, int proto) +{ + struct timestamp_entry * e; + e = timestamp_list; + while(e) { + if(e->eport == eport && e->protocol == (short)proto) + return e->timestamp; + e = e->next; + } + return 0; +} + +static void +remove_timestamp_entry(unsigned short eport, int proto) +{ + struct timestamp_entry * e; + struct timestamp_entry * * p; + p = ×tamp_list; + e = *p; + while(e) { + if(e->eport == eport && e->protocol == (short)proto) { + /* remove the entry */ + *p = e->next; + free(e); + return; + } + p = &(e->next); + e = *p; + } +} + +static void +add_timestamp_entry(unsigned short eport, int proto, unsigned timestamp) +{ + struct timestamp_entry * tmp; + tmp = malloc(sizeof(struct timestamp_entry)); + if(tmp) + { + tmp->next = timestamp_list; + tmp->timestamp = timestamp; + tmp->eport = eport; + tmp->protocol = (short)proto; + timestamp_list = tmp; + } + else + { + syslog(LOG_ERR, "add_timestamp_entry() malloc(%lu) error", + sizeof(struct timestamp_entry)); + } +} + +/* /dev/pf when opened */ +int dev = -1; + +/* shutdown_redirect() : + * close the /dev/pf device */ +void +shutdown_redirect(void) +{ + if(close(dev)<0) + syslog(LOG_ERR, "close(\"/dev/pf\"): %m"); + dev = -1; +} + +/* open the device */ +int +init_redirect(void) +{ + struct pf_status status; + if(dev>=0) + shutdown_redirect(); + dev = open("/dev/pf", O_RDWR); + if(dev<0) { + syslog(LOG_ERR, "open(\"/dev/pf\"): %m"); + return -1; + } + if(ioctl(dev, DIOCGETSTATUS, &status)<0) { + syslog(LOG_ERR, "DIOCGETSTATUS: %m"); + return -1; + } + if(!status.running) { + syslog(LOG_ERR, "pf is disabled"); + return -1; + } + return 0; +} + +#if TEST +/* for debug */ +int +clear_redirect_rules(void) +{ + struct pfioc_trans io; + struct pfioc_trans_e ioe; + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + memset(&ioe, 0, sizeof(ioe)); + io.size = 1; + io.esize = sizeof(ioe); + io.array = &ioe; +#ifndef PF_NEWSTYLE + ioe.rs_num = PF_RULESET_RDR; +#else + ioe.type = PF_TRANS_RULESET; +#endif + strlcpy(ioe.anchor, anchor_name, MAXPATHLEN); + if(ioctl(dev, DIOCXBEGIN, &io) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCXBEGIN, ...): %m"); + goto error; + } + if(ioctl(dev, DIOCXCOMMIT, &io) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCXCOMMIT, ...): %m"); + goto error; + } + return 0; +error: + return -1; +} + +int +clear_filter_rules(void) +{ +#ifndef PF_ENABLE_FILTER_RULES + return 0; +#else + struct pfioc_trans io; + struct pfioc_trans_e ioe; + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + memset(&ioe, 0, sizeof(ioe)); + io.size = 1; + io.esize = sizeof(ioe); + io.array = &ioe; +#ifndef PF_NEWSTYLE + ioe.rs_num = PF_RULESET_FILTER; +#else + /* ? */ + ioe.type = PF_TRANS_RULESET; +#endif + strlcpy(ioe.anchor, anchor_name, MAXPATHLEN); + if(ioctl(dev, DIOCXBEGIN, &io) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCXBEGIN, ...): %m"); + goto error; + } + if(ioctl(dev, DIOCXCOMMIT, &io) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCXCOMMIT, ...): %m"); + goto error; + } + return 0; +error: + return -1; +#endif +} +#endif + +/* add_redirect_rule2() : + * create a rdr rule */ +int +add_redirect_rule2(const char * ifname, + const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp) +{ + int r; + struct pfioc_rule pcr; +#ifndef PF_NEWSTYLE + struct pfioc_pooladdr pp; + struct pf_pooladdr *a; +#endif + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + r = 0; + memset(&pcr, 0, sizeof(pcr)); + strlcpy(pcr.anchor, anchor_name, MAXPATHLEN); + +#ifndef PF_NEWSTYLE + memset(&pp, 0, sizeof(pp)); + strlcpy(pp.anchor, anchor_name, MAXPATHLEN); + if(ioctl(dev, DIOCBEGINADDRS, &pp) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCBEGINADDRS, ...): %m"); + r = -1; + } + else + { + pcr.pool_ticket = pp.ticket; +#else + if(1) + { + pcr.rule.direction = PF_IN; + /*pcr.rule.src.addr.type = PF_ADDR_NONE;*/ + pcr.rule.src.addr.type = PF_ADDR_ADDRMASK; + pcr.rule.dst.addr.type = PF_ADDR_ADDRMASK; + pcr.rule.nat.addr.type = PF_ADDR_NONE; + pcr.rule.rdr.addr.type = PF_ADDR_ADDRMASK; +#endif + +#ifdef __APPLE__ + pcr.rule.dst.xport.range.op = PF_OP_EQ; + pcr.rule.dst.xport.range.port[0] = htons(eport); + pcr.rule.dst.xport.range.port[1] = htons(eport); +#else + pcr.rule.dst.port_op = PF_OP_EQ; + pcr.rule.dst.port[0] = htons(eport); + pcr.rule.dst.port[1] = htons(eport); +#endif +#ifndef PF_NEWSTYLE + pcr.rule.action = PF_RDR; +#ifndef PF_ENABLE_FILTER_RULES + pcr.rule.natpass = 1; +#else + pcr.rule.natpass = 0; +#endif +#else +#ifndef PF_ENABLE_FILTER_RULES + pcr.rule.action = PF_PASS; +#else + pcr.rule.action = PF_MATCH; +#endif +#endif + pcr.rule.af = AF_INET; +#ifdef USE_IFNAME_IN_RULES + if(ifname) + strlcpy(pcr.rule.ifname, ifname, IFNAMSIZ); +#endif + pcr.rule.proto = proto; + pcr.rule.log = (GETFLAG(LOGPACKETSMASK))?1:0; /*logpackets;*/ +#ifdef PFRULE_HAS_RTABLEID + pcr.rule.rtableid = -1; /* first appeared in OpenBSD 4.0 */ +#endif +#ifdef PFRULE_HAS_ONRDOMAIN + pcr.rule.onrdomain = -1; /* first appeared in OpenBSD 5.0 */ +#endif + pcr.rule.quick = 1; + pcr.rule.keep_state = PF_STATE_NORMAL; + if(tag) + strlcpy(pcr.rule.tagname, tag, PF_TAG_NAME_SIZE); + strlcpy(pcr.rule.label, desc, PF_RULE_LABEL_SIZE); + if(rhost && rhost[0] != '\0' && rhost[0] != '*') + { + inet_pton(AF_INET, rhost, &pcr.rule.src.addr.v.a.addr.v4.s_addr); + pcr.rule.src.addr.v.a.mask.v4.s_addr = htonl(INADDR_NONE); + } + if(use_ext_ip_addr && use_ext_ip_addr[0] != '\0') + { + inet_pton(AF_INET, use_ext_ip_addr, &pcr.rule.dst.addr.v.a.addr.v4.s_addr); + pcr.rule.dst.addr.v.a.mask.v4.s_addr = htonl(INADDR_NONE); + } +#ifndef PF_NEWSTYLE + pcr.rule.rpool.proxy_port[0] = iport; + pcr.rule.rpool.proxy_port[1] = iport; + TAILQ_INIT(&pcr.rule.rpool.list); + a = calloc(1, sizeof(struct pf_pooladdr)); + inet_pton(AF_INET, iaddr, &a->addr.v.a.addr.v4.s_addr); + a->addr.v.a.mask.v4.s_addr = htonl(INADDR_NONE); + TAILQ_INSERT_TAIL(&pcr.rule.rpool.list, a, entries); + + memcpy(&pp.addr, a, sizeof(struct pf_pooladdr)); + if(ioctl(dev, DIOCADDADDR, &pp) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCADDADDR, ...): %m"); + r = -1; + } + else + { +#else + pcr.rule.rdr.proxy_port[0] = iport; + pcr.rule.rdr.proxy_port[1] = iport; + inet_pton(AF_INET, iaddr, &pcr.rule.rdr.addr.v.a.addr.v4.s_addr); + pcr.rule.rdr.addr.v.a.mask.v4.s_addr = htonl(INADDR_NONE); + if(1) + { +#endif + pcr.action = PF_CHANGE_GET_TICKET; + if(ioctl(dev, DIOCCHANGERULE, &pcr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_GET_TICKET: %m"); + r = -1; + } + else + { + pcr.action = PF_CHANGE_ADD_TAIL; + if(ioctl(dev, DIOCCHANGERULE, &pcr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_ADD_TAIL: %m"); + r = -1; + } + } + } +#ifndef PF_NEWSTYLE + free(a); +#endif + } + if(r == 0 && timestamp > 0) + add_timestamp_entry(eport, proto, timestamp); + return r; +} + +/* thanks to Seth Mos for this function */ +int +add_filter_rule2(const char * ifname, + const char * rhost, const char * iaddr, + unsigned short eport, unsigned short iport, + int proto, const char * desc) +{ +#ifndef PF_ENABLE_FILTER_RULES + UNUSED(ifname); + UNUSED(rhost); UNUSED(iaddr); + UNUSED(eport); UNUSED(iport); + UNUSED(proto); UNUSED(desc); + return 0; +#else + int r; + struct pfioc_rule pcr; +#ifndef PF_NEWSTYLE + struct pfioc_pooladdr pp; +#endif +#ifndef USE_IFNAME_IN_RULES + UNUSED(ifname); +#endif + UNUSED(eport); + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + r = 0; + memset(&pcr, 0, sizeof(pcr)); + strlcpy(pcr.anchor, anchor_name, MAXPATHLEN); + +#ifndef PF_NEWSTYLE + memset(&pp, 0, sizeof(pp)); + strlcpy(pp.anchor, anchor_name, MAXPATHLEN); + if(ioctl(dev, DIOCBEGINADDRS, &pp) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCBEGINADDRS, ...): %m"); + r = -1; + } + else + { + pcr.pool_ticket = pp.ticket; +#else + if(1) + { +#endif + pcr.rule.dst.port_op = PF_OP_EQ; + pcr.rule.dst.port[0] = htons(iport); + pcr.rule.direction = PF_IN; + pcr.rule.action = PF_PASS; + pcr.rule.af = AF_INET; +#ifdef USE_IFNAME_IN_RULES + if(ifname) + strlcpy(pcr.rule.ifname, ifname, IFNAMSIZ); +#endif + pcr.rule.proto = proto; + pcr.rule.quick = (GETFLAG(PFNOQUICKRULESMASK))?0:1; + pcr.rule.log = (GETFLAG(LOGPACKETSMASK))?1:0; /*logpackets;*/ +/* see the discussion on the forum : + * http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=638 */ + pcr.rule.flags = TH_SYN; + pcr.rule.flagset = (TH_SYN|TH_ACK); +#ifdef PFRULE_HAS_RTABLEID + pcr.rule.rtableid = -1; /* first appeared in OpenBSD 4.0 */ +#endif +#ifdef PFRULE_HAS_ONRDOMAIN + pcr.rule.onrdomain = -1; /* first appeared in OpenBSD 5.0 */ +#endif + pcr.rule.keep_state = 1; + strlcpy(pcr.rule.label, desc, PF_RULE_LABEL_SIZE); + if(queue) + strlcpy(pcr.rule.qname, queue, PF_QNAME_SIZE); + if(tag) + strlcpy(pcr.rule.tagname, tag, PF_TAG_NAME_SIZE); + + if(rhost && rhost[0] != '\0' && rhost[0] != '*') + { + inet_pton(AF_INET, rhost, &pcr.rule.src.addr.v.a.addr.v4.s_addr); + pcr.rule.src.addr.v.a.mask.v4.s_addr = htonl(INADDR_NONE); + } + /* we want any - iaddr port = # keep state label */ + inet_pton(AF_INET, iaddr, &pcr.rule.dst.addr.v.a.addr.v4.s_addr); + pcr.rule.dst.addr.v.a.mask.v4.s_addr = htonl(INADDR_NONE); +#ifndef PF_NEWSTYLE + pcr.rule.rpool.proxy_port[0] = iport; + pcr.rule.rpool.proxy_port[1] = iport; + TAILQ_INIT(&pcr.rule.rpool.list); +#endif + if(1) + { + pcr.action = PF_CHANGE_GET_TICKET; + if(ioctl(dev, DIOCCHANGERULE, &pcr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_GET_TICKET: %m"); + r = -1; + } + else + { + pcr.action = PF_CHANGE_ADD_TAIL; + if(ioctl(dev, DIOCCHANGERULE, &pcr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_ADD_TAIL: %m"); + r = -1; + } + } + } + } + return r; +#endif +} + +/* get_redirect_rule() + * return value : 0 success (found) + * -1 = error or rule not found */ +int +get_redirect_rule(const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes) +{ + int i, n; + struct pfioc_rule pr; +#ifndef PF_NEWSTYLE + struct pfioc_pooladdr pp; +#endif + UNUSED(ifname); + + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + memset(&pr, 0, sizeof(pr)); + strlcpy(pr.anchor, anchor_name, MAXPATHLEN); +#ifndef PF_NEWSTYLE + pr.rule.action = PF_RDR; +#endif + if(ioctl(dev, DIOCGETRULES, &pr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCGETRULES, ...): %m"); + goto error; + } + n = pr.nr; + for(i=0; i 0) + { + if (pr.rule.src.addr.v.a.addr.v4.s_addr == 0) + { + rhost[0] = '\0'; /* empty string */ + } + else + { + inet_ntop(AF_INET, &pr.rule.src.addr.v.a.addr.v4.s_addr, + rhost, rhostlen); + } + } + if(timestamp) + *timestamp = get_timestamp(eport, proto); + return 0; + } + } +error: + return -1; +} + +#define priv_delete_redirect_rule(ifname, eport, proto, iport, \ + iaddr, rhost, rhostlen) \ + priv_delete_redirect_rule_check_desc(ifname, eport, proto, iport, \ + iaddr, rhost, rhostlen, 0, NULL) +/* if check_desc is true, only delete the rule if the description differs. + * returns : -1 : error / rule not found + * 0 : rule deleted + * 1 : rule untouched + */ +static int +priv_delete_redirect_rule_check_desc(const char * ifname, unsigned short eport, + int proto, unsigned short * iport, + in_addr_t * iaddr, char * rhost, int rhostlen, + int check_desc, const char * desc) +{ + int i, n; + struct pfioc_rule pr; + UNUSED(ifname); + + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + memset(&pr, 0, sizeof(pr)); + strlcpy(pr.anchor, anchor_name, MAXPATHLEN); +#ifndef PF_NEWSTYLE + pr.rule.action = PF_RDR; +#endif + if(ioctl(dev, DIOCGETRULES, &pr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCGETRULES, ...): %m"); + goto error; + } + n = pr.nr; + for(i=0; i 0) + { + if (pr.rule.src.addr.v.a.addr.v4.s_addr == 0) + rhost[0] = '\0'; /* empty string */ + else + inet_ntop(AF_INET, &pr.rule.src.addr.v.a.addr.v4.s_addr, + rhost, rhostlen); + } + if(check_desc) { + if((desc == NULL && pr.rule.label[0] == '\0') || + (desc && 0 == strcmp(desc, pr.rule.label))) { + return 1; + } + } + pr.action = PF_CHANGE_GET_TICKET; + if(ioctl(dev, DIOCCHANGERULE, &pr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_GET_TICKET: %m"); + goto error; + } + pr.action = PF_CHANGE_REMOVE; + pr.nr = i; + if(ioctl(dev, DIOCCHANGERULE, &pr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_REMOVE: %m"); + goto error; + } + remove_timestamp_entry(eport, proto); + return 0; + } + } +error: + return -1; +} + +int +delete_redirect_rule(const char * ifname, unsigned short eport, + int proto) +{ + return priv_delete_redirect_rule(ifname, eport, proto, NULL, NULL, NULL, 0); +} + +static int +priv_delete_filter_rule(const char * ifname, unsigned short iport, + int proto, in_addr_t iaddr) +{ +#ifndef PF_ENABLE_FILTER_RULES + UNUSED(ifname); UNUSED(iport); UNUSED(proto); UNUSED(iaddr); + return 0; +#else + int i, n; + struct pfioc_rule pr; + UNUSED(ifname); + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + memset(&pr, 0, sizeof(pr)); + strlcpy(pr.anchor, anchor_name, MAXPATHLEN); + pr.rule.action = PF_PASS; + if(ioctl(dev, DIOCGETRULES, &pr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCGETRULES, ...): %m"); + goto error; + } + n = pr.nr; + for(i=0; i= n) + goto error; + pr.nr = index; + if(ioctl(dev, DIOCGETRULE, &pr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCGETRULE): %m"); + goto error; + } + *proto = pr.rule.proto; +#ifdef __APPLE__ + *eport = ntohs(pr.rule.dst.xport.range.port[0]); +#else + *eport = ntohs(pr.rule.dst.port[0]); +#endif +#ifndef PF_NEWSTYLE + *iport = pr.rule.rpool.proxy_port[0]; +#else + *iport = pr.rule.rdr.proxy_port[0]; +#endif + if(ifname) + strlcpy(ifname, pr.rule.ifname, IFNAMSIZ); + if(desc) + strlcpy(desc, pr.rule.label, desclen); +#ifdef PFRULE_INOUT_COUNTS + if(packets) + *packets = pr.rule.packets[0] + pr.rule.packets[1]; + if(bytes) + *bytes = pr.rule.bytes[0] + pr.rule.bytes[1]; +#else + if(packets) + *packets = pr.rule.packets; + if(bytes) + *bytes = pr.rule.bytes; +#endif +#ifndef PF_NEWSTYLE + memset(&pp, 0, sizeof(pp)); + strlcpy(pp.anchor, anchor_name, MAXPATHLEN); + pp.r_action = PF_RDR; + pp.r_num = index; + pp.ticket = pr.ticket; + if(ioctl(dev, DIOCGETADDRS, &pp) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCGETADDRS, ...): %m"); + goto error; + } + if(pp.nr != 1) + { + syslog(LOG_NOTICE, "No address associated with pf rule"); + goto error; + } + pp.nr = 0; /* first */ + if(ioctl(dev, DIOCGETADDR, &pp) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCGETADDR, ...): %m"); + goto error; + } + inet_ntop(AF_INET, &pp.addr.addr.v.a.addr.v4.s_addr, + iaddr, iaddrlen); +#else + inet_ntop(AF_INET, &pr.rule.rdr.addr.v.a.addr.v4.s_addr, + iaddr, iaddrlen); +#endif + if(rhost && rhostlen > 0) + { + if (pr.rule.src.addr.v.a.addr.v4.s_addr == 0) + { + rhost[0] = '\0'; /* empty string */ + } + else + { + inet_ntop(AF_INET, &pr.rule.src.addr.v.a.addr.v4.s_addr, + rhost, rhostlen); + } + } + if(timestamp) + *timestamp = get_timestamp(*eport, *proto); + return 0; +error: + return -1; +} + +/* return an (malloc'ed) array of "external" port for which there is + * a port mapping. number is the size of the array */ +unsigned short * +get_portmappings_in_range(unsigned short startport, unsigned short endport, + int proto, unsigned int * number) +{ + unsigned short * array; + unsigned int capacity; + int i, n; + unsigned short eport; + struct pfioc_rule pr; + + *number = 0; + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return NULL; + } + capacity = 128; + array = calloc(capacity, sizeof(unsigned short)); + if(!array) + { + syslog(LOG_ERR, "get_portmappings_in_range() : calloc error"); + return NULL; + } + memset(&pr, 0, sizeof(pr)); + strlcpy(pr.anchor, anchor_name, MAXPATHLEN); +#ifndef PF_NEWSTYLE + pr.rule.action = PF_RDR; +#endif + if(ioctl(dev, DIOCGETRULES, &pr) < 0) + { + syslog(LOG_ERR, "ioctl(dev, DIOCGETRULES, ...): %m"); + free(array); + return NULL; + } + n = pr.nr; + for(i=0; i= capacity) + { + /* need to increase the capacity of the array */ + unsigned short * tmp; + capacity += 128; + tmp = realloc(array, sizeof(unsigned short)*capacity); + if(!tmp) + { + syslog(LOG_ERR, "get_portmappings_in_range() : realloc(%lu) error", sizeof(unsigned short)*capacity); + *number = 0; + free(array); + return NULL; + } + array = tmp; + } + array[*number] = eport; + (*number)++; + } + } + return array; +} + +/* update the port mapping internal port, decription and timestamp */ +int +update_portmapping(const char * ifname, unsigned short eport, int proto, + unsigned short iport, const char * desc, + unsigned int timestamp) +{ + unsigned short old_iport; + in_addr_t iaddr; + char iaddr_str[16]; + char rhost[32]; + + if(priv_delete_redirect_rule(ifname, eport, proto, &old_iport, &iaddr, rhost, sizeof(rhost)) < 0) + return -1; + if (priv_delete_filter_rule(ifname, old_iport, proto, iaddr) < 0) + return -1; + + inet_ntop(AF_INET, &iaddr, iaddr_str, sizeof(iaddr_str)); + + if(add_redirect_rule2(ifname, rhost, eport, iaddr_str, iport, proto, + desc, timestamp) < 0) + return -1; + if(add_filter_rule2(ifname, rhost, iaddr_str, eport, iport, proto, desc) < 0) + return -1; + + return 0; +} + +/* update the port mapping decription and timestamp */ +int +update_portmapping_desc_timestamp(const char * ifname, + unsigned short eport, int proto, + const char * desc, unsigned int timestamp) +{ + unsigned short iport; + in_addr_t iaddr; + char iaddr_str[16]; + char rhost[32]; + int r; + + r = priv_delete_redirect_rule_check_desc(ifname, eport, proto, &iport, &iaddr, rhost, sizeof(rhost), 1, desc); + if(r < 0) + return -1; + if(r == 1) { + /* only change timestamp */ + remove_timestamp_entry(eport, proto); + add_timestamp_entry(eport, proto, timestamp); + return 0; + } + if (priv_delete_filter_rule(ifname, iport, proto, iaddr) < 0) + return -1; + + inet_ntop(AF_INET, &iaddr, iaddr_str, sizeof(iaddr_str)); + + if(add_redirect_rule2(ifname, rhost, eport, iaddr_str, iport, proto, + desc, timestamp) < 0) + return -1; + if(add_filter_rule2(ifname, rhost, iaddr_str, eport, iport, proto, desc) < 0) + return -1; + + return 0; +} + + +/* this function is only for testing */ +#if TEST +void +list_rules(void) +{ + char buf[32]; + char buf2[32]; + int i, n; + struct pfioc_rule pr; +#ifndef PF_NEWSTYLE + struct pfioc_pooladdr pp; +#endif + + if(dev<0) + { + perror("pf dev not open"); + return ; + } + memset(&pr, 0, sizeof(pr)); + strlcpy(pr.anchor, anchor_name, MAXPATHLEN); + pr.rule.action = PF_RDR; + if(ioctl(dev, DIOCGETRULES, &pr) < 0) + perror("DIOCGETRULES"); + printf("ticket = %d, nr = %d\n", pr.ticket, pr.nr); + n = pr.nr; + for(i=0; i %s %d:%d proto %d keep_state=%d action=%d\n", + pr.rule.ifname, + inet_ntop(AF_INET, &pr.rule.src.addr.v.a.addr.v4.s_addr, buf, 32), + (int)ntohs(pr.rule.dst.port[0]), + (int)ntohs(pr.rule.dst.port[1]), + inet_ntop(AF_INET, &pr.rule.dst.addr.v.a.addr.v4.s_addr, buf2, 32), +#ifndef PF_NEWSTYLE + (int)pr.rule.rpool.proxy_port[0], + (int)pr.rule.rpool.proxy_port[1], +#else + (int)pr.rule.rdr.proxy_port[0], + (int)pr.rule.rdr.proxy_port[1], +#endif + (int)pr.rule.proto, + (int)pr.rule.keep_state, + (int)pr.rule.action); + printf(" description: \"%s\"\n", pr.rule.label); +#ifndef PF_NEWSTYLE + memset(&pp, 0, sizeof(pp)); + strlcpy(pp.anchor, anchor_name, MAXPATHLEN); + pp.r_action = PF_RDR; + pp.r_num = i; + pp.ticket = pr.ticket; + if(ioctl(dev, DIOCGETADDRS, &pp) < 0) + perror("DIOCGETADDRS"); + printf(" nb pool addr = %d ticket=%d\n", pp.nr, pp.ticket); + /*if(ioctl(dev, DIOCGETRULE, &pr) < 0) + perror("DIOCGETRULE"); */ + pp.nr = 0; /* first */ + if(ioctl(dev, DIOCGETADDR, &pp) < 0) + perror("DIOCGETADDR"); + /* addr.v.a.addr.v4.s_addr */ + printf(" %s\n", inet_ntop(AF_INET, &pp.addr.addr.v.a.addr.v4.s_addr, buf, 32)); +#else + printf(" rule_flag=%08x action=%d direction=%d log=%d logif=%d " + "quick=%d ifnot=%d af=%d type=%d code=%d rdr.port_op=%d rdr.opts=%d\n", + pr.rule.rule_flag, pr.rule.action, pr.rule.direction, + pr.rule.log, pr.rule.logif, pr.rule.quick, pr.rule.ifnot, + pr.rule.af, pr.rule.type, pr.rule.code, + pr.rule.rdr.port_op, pr.rule.rdr.opts); + printf(" %s\n", inet_ntop(AF_INET, &pr.rule.rdr.addr.v.a.addr.v4.s_addr, buf, 32)); +#endif + } +} +#endif /* TEST */ + +#endif /* USE_PF */ diff --git a/src/contrib/miniupnp/miniupnpd/pf/obsdrdr.h b/src/contrib/miniupnp/miniupnpd/pf/obsdrdr.h new file mode 100644 index 0000000..3defa8e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pf/obsdrdr.h @@ -0,0 +1,70 @@ +/* $Id: obsdrdr.h,v 1.23 2014/03/06 12:24:33 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef OBSDRDR_H_INCLUDED +#define OBSDRDR_H_INCLUDED + +#include "../commonrdr.h" + +/* add_redirect_rule2() uses DIOCCHANGERULE ioctl + * proto can take the values IPPROTO_UDP or IPPROTO_TCP + */ +int +add_redirect_rule2(const char * ifname, + const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, int proto, + const char * desc, unsigned int timestamp); + +/* add_filter_rule2() uses DIOCCHANGERULE ioctl + * proto can take the values IPPROTO_UDP or IPPROTO_TCP + */ +int +add_filter_rule2(const char * ifname, + const char * rhost, const char * iaddr, + unsigned short eport, unsigned short iport, + int proto, const char * desc); + + +/* get_redirect_rule() gets internal IP and port from + * interface, external port and protocl + */ +#if 0 +int +get_redirect_rule(const char * ifname, unsigned short eport, int proto, + char * iaddr, int iaddrlen, unsigned short * iport, + char * desc, int desclen, + u_int64_t * packets, u_int64_t * bytes); + +int +get_redirect_rule_by_index(int index, + char * ifname, unsigned short * eport, + char * iaddr, int iaddrlen, unsigned short * iport, + int * proto, char * desc, int desclen, + u_int64_t * packets, u_int64_t * bytes); +#endif + +/* delete_redirect_rule() + */ +int +delete_redirect_rule(const char * ifname, unsigned short eport, int proto); + +/* delete_redirect_and_filter_rules() + */ +int +delete_redirect_and_filter_rules(const char * ifname, unsigned short eport, + int proto); + +#ifdef TEST +int +clear_redirect_rules(void); +int +clear_filter_rules(void); +#endif + +#endif + + diff --git a/src/contrib/miniupnp/miniupnpd/pf/pfpinhole.c b/src/contrib/miniupnp/miniupnpd/pf/pfpinhole.c new file mode 100644 index 0000000..e0f0959 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pf/pfpinhole.c @@ -0,0 +1,445 @@ +/* $Id: pfpinhole.c,v 1.27 2018/03/13 23:05:21 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2012-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef __DragonFly__ +#include +#else +#ifdef __APPLE__ +#define PRIVATE 1 +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include + +#include "../config.h" +#include "pfpinhole.h" +#include "../upnpglobalvars.h" +#include "../macros.h" + +/* the pass rules created by add_pinhole() are as follow : + * + * pass in quick on ep0 inet6 proto udp + * from any to dead:beef::42:42 port = 8080 + * flags S/SA keep state + * label "pinhole-2 ts-4321000" + * + * with the label "pinhole-$uid ts-$timestamp: $description" + */ + +#ifdef ENABLE_UPNPPINHOLE + +/* /dev/pf when opened */ +extern int dev; + +static int next_uid = 1; + +#define PINEHOLE_LABEL_FORMAT "pinhole-%d ts-%u: %s" +#define PINEHOLE_LABEL_FORMAT_SKIPDESC "pinhole-%d ts-%u: %*s" + +int add_pinhole(const char * ifname, + const char * rem_host, unsigned short rem_port, + const char * int_client, unsigned short int_port, + int proto, const char * desc, unsigned int timestamp) +{ + int uid; + struct pfioc_rule pcr; +#ifndef PF_NEWSTYLE + struct pfioc_pooladdr pp; +#endif + + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + memset(&pcr, 0, sizeof(pcr)); + strlcpy(pcr.anchor, anchor_name, MAXPATHLEN); + +#ifndef PF_NEWSTYLE + memset(&pp, 0, sizeof(pp)); + strlcpy(pp.anchor, anchor_name, MAXPATHLEN); + if(ioctl(dev, DIOCBEGINADDRS, &pp) < 0) { + syslog(LOG_ERR, "ioctl(dev, DIOCBEGINADDRS, ...): %m"); + return -1; + } else { + pcr.pool_ticket = pp.ticket; +#else + { +#endif + pcr.rule.direction = PF_IN; + pcr.rule.action = PF_PASS; + pcr.rule.af = AF_INET6; +#ifdef PF_NEWSTYLE + pcr.rule.nat.addr.type = PF_ADDR_NONE; + pcr.rule.rdr.addr.type = PF_ADDR_NONE; +#endif +#ifdef USE_IFNAME_IN_RULES + if(ifname) + strlcpy(pcr.rule.ifname, ifname, IFNAMSIZ); +#endif + pcr.rule.proto = proto; + + pcr.rule.quick = 1;/*(GETFLAG(PFNOQUICKRULESMASK))?0:1;*/ + pcr.rule.log = (GETFLAG(LOGPACKETSMASK))?1:0; /*logpackets;*/ +/* see the discussion on the forum : + * http://miniupnp.tuxfamily.org/forum/viewtopic.php?p=638 */ + pcr.rule.flags = TH_SYN; + pcr.rule.flagset = (TH_SYN|TH_ACK); +#ifdef PFRULE_HAS_RTABLEID + pcr.rule.rtableid = -1; /* first appeared in OpenBSD 4.0 */ +#endif +#ifdef PFRULE_HAS_ONRDOMAIN + pcr.rule.onrdomain = -1; /* first appeared in OpenBSD 5.0 */ +#endif + pcr.rule.keep_state = 1; + uid = next_uid; + snprintf(pcr.rule.label, PF_RULE_LABEL_SIZE, + PINEHOLE_LABEL_FORMAT, uid, timestamp, desc); + if(queue) + strlcpy(pcr.rule.qname, queue, PF_QNAME_SIZE); + if(tag) + strlcpy(pcr.rule.tagname, tag, PF_TAG_NAME_SIZE); + + if(rem_port) { + pcr.rule.src.port_op = PF_OP_EQ; + pcr.rule.src.port[0] = htons(rem_port); + } + if(rem_host && rem_host[0] != '\0' && rem_host[0] != '*') { + pcr.rule.src.addr.type = PF_ADDR_ADDRMASK; + if(inet_pton(AF_INET6, rem_host, &pcr.rule.src.addr.v.a.addr.v6) != 1) { + syslog(LOG_ERR, "inet_pton(%s) failed", rem_host); + } + memset(&pcr.rule.src.addr.v.a.mask.addr8, 255, 16); + } + + pcr.rule.dst.port_op = PF_OP_EQ; + pcr.rule.dst.port[0] = htons(int_port); + pcr.rule.dst.addr.type = PF_ADDR_ADDRMASK; + if(inet_pton(AF_INET6, int_client, &pcr.rule.dst.addr.v.a.addr.v6) != 1) { + syslog(LOG_ERR, "inet_pton(%s) failed", int_client); + } + memset(&pcr.rule.dst.addr.v.a.mask.addr8, 255, 16); + + if(ifname) + strlcpy(pcr.rule.ifname, ifname, IFNAMSIZ); + + pcr.action = PF_CHANGE_GET_TICKET; + if(ioctl(dev, DIOCCHANGERULE, &pcr) < 0) { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_GET_TICKET: %m"); + return -1; + } else { + pcr.action = PF_CHANGE_ADD_TAIL; + if(ioctl(dev, DIOCCHANGERULE, &pcr) < 0) { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_ADD_TAIL: %m"); + return -1; + } + } + } + + if(++next_uid >= 65535) { + next_uid = 1; + } + return uid; +} + +int find_pinhole(const char * ifname, + const char * rem_host, unsigned short rem_port, + const char * int_client, unsigned short int_port, + int proto, + char *desc, int desc_len, unsigned int * timestamp) +{ + int uid; + unsigned int ts; + int i, n; + struct pfioc_rule pr; + struct in6_addr saddr; + struct in6_addr daddr; + UNUSED(ifname); + + if(dev<0) { + syslog(LOG_ERR, "pf device is not open"); + return -1; + } + if(rem_host && (rem_host[0] != '\0')) { + inet_pton(AF_INET6, rem_host, &saddr); + } else { + memset(&saddr, 0, sizeof(struct in6_addr)); + } + inet_pton(AF_INET6, int_client, &daddr); + memset(&pr, 0, sizeof(pr)); + strlcpy(pr.anchor, anchor_name, MAXPATHLEN); +#ifndef PF_NEWSTYLE + pr.rule.action = PF_PASS; +#endif + if(ioctl(dev, DIOCGETRULES, &pr) < 0) { + syslog(LOG_ERR, "ioctl(dev, DIOCGETRULES, ...): %m"); + return -1; + } + n = pr.nr; + for(i=0; i= 0; i--) { + pr.nr = i; + if(ioctl(dev, DIOCGETRULE, &pr) < 0) { + syslog(LOG_ERR, "ioctl(dev, DIOCGETRULE): %m"); + return -1; + } + if(sscanf(pr.rule.label, PINEHOLE_LABEL_FORMAT_SKIPDESC, &uid, &ts) != 2) { + syslog(LOG_DEBUG, "rule with label '%s' is not a IGD pinhole", pr.rule.label); + continue; + } + if(ts <= (unsigned int)current_time) { + syslog(LOG_INFO, "removing expired pinhole '%s'", pr.rule.label); + pr.action = PF_CHANGE_GET_TICKET; + if(ioctl(dev, DIOCCHANGERULE, &pr) < 0) { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_GET_TICKET: %m"); + return -1; + } + pr.action = PF_CHANGE_REMOVE; + pr.nr = i; + if(ioctl(dev, DIOCCHANGERULE, &pr) < 0) { + syslog(LOG_ERR, "ioctl(dev, DIOCCHANGERULE, ...) PF_CHANGE_REMOVE: %m"); + return -1; + } + n++; +#ifndef PF_NEWSTYLE + pr.rule.action = PF_PASS; +#endif + if(ioctl(dev, DIOCGETRULES, &pr) < 0) { + syslog(LOG_ERR, "ioctl(dev, DIOCGETRULES, ...): %m"); + return -1; + } + } else { + if(uid > max_uid) + max_uid = uid; + else if(uid < min_uid) + min_uid = uid; + if(ts < min_ts) + min_ts = ts; + } + } + if(next_timestamp && (min_ts != UINT_MAX)) + *next_timestamp = min_ts; + if(max_uid > 0) { + if(((min_uid - 32000) <= next_uid) && (next_uid <= max_uid)) { + next_uid = max_uid + 1; + } + if(next_uid >= 65535) { + next_uid = 1; + } + } + return n; /* number of rules removed */ +} + +#endif /* ENABLE_UPNPPINHOLE */ diff --git a/src/contrib/miniupnp/miniupnpd/pf/pfpinhole.h b/src/contrib/miniupnp/miniupnpd/pf/pfpinhole.h new file mode 100644 index 0000000..5d695df --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pf/pfpinhole.h @@ -0,0 +1,41 @@ +/* $Id: pfpinhole.h,v 1.9 2012/05/01 22:37:53 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2012-2016 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef PFPINHOLE_H_INCLUDED +#define PFPINHOLE_H_INCLUDED + +#ifdef ENABLE_UPNPPINHOLE + +int find_pinhole(const char * ifname, + const char * rem_host, unsigned short rem_port, + const char * int_client, unsigned short int_port, + int proto, + char *desc, int desc_len, unsigned int * timestamp); + +int add_pinhole(const char * ifname, + const char * rem_host, unsigned short rem_port, + const char * int_client, unsigned short int_port, + int proto, const char * desc, unsigned int timestamp); + +int delete_pinhole(unsigned short uid); + +int +get_pinhole_info(unsigned short uid, + char * rem_host, int rem_hostlen, unsigned short * rem_port, + char * int_client, int int_clientlen, unsigned short * int_port, + int * proto, char * desc, int desclen, + unsigned int * timestamp, + u_int64_t * packets, u_int64_t * bytes); + +int update_pinhole(unsigned short uid, unsigned int timestamp); + +int clean_pinhole_list(unsigned int * next_timestamp); + +#endif /* ENABLE_UPNPPINHOLE */ + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/pf/testobsdrdr.c b/src/contrib/miniupnp/miniupnpd/pf/testobsdrdr.c new file mode 100644 index 0000000..cd0bbba --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pf/testobsdrdr.c @@ -0,0 +1,147 @@ +/* $Id: testobsdrdr.c,v 1.30 2018/04/12 09:27:54 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include + +#include "obsdrdr.h" + +/*int logpackets = 1;*/ +int runtime_flags = 0; +const char * tag = 0; +const char * anchor_name = "miniupnpd"; +const char * queue = NULL; +const char * use_ext_ip_addr = "42.42.42.42"; + +void +list_rules(void); + +void +list_eports_tcp(void) +{ + unsigned short * port_list; + unsigned int number = 0; + unsigned int i; + port_list = get_portmappings_in_range(0, 65535, IPPROTO_TCP, &number); + printf("%u ports redirected (TCP) :", number); + for(i = 0; i < number; i++) + { + printf(" %hu", port_list[i]); + } + printf("\n"); + free(port_list); +} + +void +test_index(void) +{ + char ifname[16/*IFNAMSIZ*/]; + char iaddr[32]; + char desc[64]; + char rhost[32]; + unsigned short iport = 0; + unsigned short eport = 0; + int proto = 0; + unsigned int timestamp; + ifname[0] = '\0'; + iaddr[0] = '\0'; + rhost[0] = '\0'; + if(get_redirect_rule_by_index(0, ifname, &eport, iaddr, sizeof(iaddr), + &iport, &proto, desc, sizeof(desc), + rhost, sizeof(rhost), + ×tamp, 0, 0) < 0) + { + printf("get.._by_index : no rule\n"); + } + else + { + printf("%s %u -> %s:%u proto %d\n", ifname, (unsigned int)eport, + iaddr, (unsigned int)iport, proto); + printf("description: \"%s\"\n", desc); + } +} + +int +main(int argc, char * * argv) +{ + char buf[32]; + char desc[64]; + char rhost[64]; + /*char rhost[32];*/ + unsigned short iport; + unsigned int timestamp; + u_int64_t packets = 0; + u_int64_t bytes = 0; + int clear = 0; + + if(argc > 1) { + if(0 == strcmp(argv[1], "--clear") || 0 == strcmp(argv[1], "-c")) + clear = 1; + } + + openlog("testobsdrdr", LOG_PERROR, LOG_USER); + if(init_redirect() < 0) + { + fprintf(stderr, "init_redirect() failed\n"); + return 1; + } +#if 0 + add_redirect_rule("ep0", 12123, "192.168.1.23", 1234); + add_redirect_rule2("ep0", 12155, "192.168.1.155", 1255, IPPROTO_TCP); +#endif + if(add_redirect_rule2("ep0", "8.8.8.8", 12123, "192.168.1.125", 1234, + IPPROTO_UDP, "test description", 0) < 0) + printf("add_redirect_rule2() #3 failed\n"); + use_ext_ip_addr = NULL; + if(add_redirect_rule2("em0", NULL, 12123, "127.1.2.3", 1234, + IPPROTO_TCP, "test description tcp", 0) < 0) + printf("add_redirect_rule2() #4 failed\n"); + if(add_filter_rule2("em0", NULL, "127.1.2.3", 12123, 1234, IPPROTO_TCP, + "test description tcp") < 0) + printf("add_filter_rule2() #1 failed\n"); + + list_rules(); + list_eports_tcp(); + + + if(get_redirect_rule("xl1", 4662, IPPROTO_TCP, + buf, sizeof(buf), &iport, desc, sizeof(desc), + rhost, sizeof(rhost), + ×tamp, + &packets, &bytes) < 0) + printf("get_redirect_rule() failed\n"); + else + { + printf("\n%s:%d '%s' packets=%" PRIu64 " bytes=%" PRIu64 "\n", buf, (int)iport, desc, + packets, bytes); + } + + if(delete_redirect_rule("ep0", 12123, IPPROTO_UDP) < 0) + printf("delete_redirect_rule() failed\n"); + + if(delete_redirect_rule("ep0", 12123, IPPROTO_UDP) < 0) + printf("delete_redirect_rule() failed\n"); + + if(delete_redirect_and_filter_rules("em0", 12123, IPPROTO_TCP) < 0) + printf("delete_redirect_and_filter_rules() failed\n"); + + test_index(); + + if(clear) { + clear_redirect_rules(); + clear_filter_rules(); + } + /*list_rules();*/ + + return 0; +} + + diff --git a/src/contrib/miniupnp/miniupnpd/pf/testpfpinhole.c b/src/contrib/miniupnp/miniupnpd/pf/testpfpinhole.c new file mode 100644 index 0000000..de3d4e0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/pf/testpfpinhole.c @@ -0,0 +1,105 @@ +/* $Id: testpfpinhole.c,v 1.12 2014/05/15 21:23:43 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2012-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include + +#include "../config.h" +#include "obsdrdr.h" +#include "pfpinhole.h" + +int runtime_flags = 0; +const char * tag = NULL; + +const char * anchor_name = "miniupnpd"; +const char * queue = NULL; + +#ifdef ENABLE_IPV6 +static int print_pinhole(int uid) +{ + int r; + char rem_host[64]; + unsigned short rem_port; + char int_client[64]; + unsigned short int_port; + int proto; + unsigned int timestamp; + u_int64_t packets, bytes; + char desc[64]; + + r = get_pinhole_info((unsigned short)uid, + rem_host, sizeof(rem_host), &rem_port, + int_client, sizeof(int_client), &int_port, + &proto, + desc, sizeof(desc), + ×tamp, + &packets, &bytes); + if(r < 0) { + fprintf(stderr, "get_pinhole(%d) returned %d\n", uid, r); + } else { + printf("pinhole %d : [%s]:%hu => [%s]:%hu proto=%d ts=%u\n", + uid, rem_host, rem_port, int_client, int_port, + proto, timestamp); + printf(" desc='%s'\n", desc); + printf(" packets=%" PRIu64 " bytes=%" PRIu64 "\n", packets, bytes); + } + return r; +} +#endif + +int main(int argc, char * *argv) +{ +#ifndef ENABLE_IPV6 + fprintf(stderr,"nothing to test, ENABLE_IPV6 is not defined in config.h\n"); + return 1; +#else + int uid; + int uid2; + int ret; + unsigned int timestamp; + (void)argc; (void)argv; + + openlog("testpfpinhole", LOG_PERROR, LOG_USER); + if(init_redirect() < 0) { + fprintf(stderr, "init_redirect() failed\n"); + return 1; + } + + uid = add_pinhole("ep0", "2001::1:2:3", 12345, "123::ff", 54321, IPPROTO_UDP, "description test 1", 424242); + if(uid < 0) { + fprintf(stderr, "add_pinhole() failed\n"); + } + printf("add_pinhole() returned %d\n", uid); + uid = add_pinhole("ep0", NULL, 0, "dead:beef::42:42", 8080, IPPROTO_UDP, "description test 2", 4321000); + if(uid < 0) { + fprintf(stderr, "add_pinhole() failed\n"); + } + printf("add_pinhole() returned %d\n", uid); + + uid2 = find_pinhole("ep0", NULL, 0, "dead:beef::42:42", 8080, IPPROTO_UDP, NULL, 0, ×tamp); + if(uid2 < 0) { + fprintf(stderr, "find_pinhole() failed\n"); + } else { + printf("find_pinhole() uid=%d timestamp=%u\n", uid2, timestamp); + } + + print_pinhole(1); + print_pinhole(2); + clean_pinhole_list(NULL); + + ret = delete_pinhole(1); + printf("delete_pinhole() returned %d\n", ret); + ret = delete_pinhole(2); + printf("delete_pinhole() returned %d\n", ret); +#endif + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/portinuse.c b/src/contrib/miniupnp/miniupnpd/portinuse.c new file mode 100644 index 0000000..bfff889 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/portinuse.c @@ -0,0 +1,409 @@ +/* $Id: portinuse.c,v 1.3 2014/04/01 12:52:50 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * (c) 2007-2017 Thomas Bernard + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#if defined(__DragonFly__) || defined(__FreeBSD__) +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__OpenBSD__) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#if defined(__DragonFly__) || defined(__FreeBSD__) +#include +#include +/* sys/socketvar.h must be included above the following headers */ +#include +#include +#endif + +#include "macros.h" +#include "config.h" +#include "upnpglobalvars.h" +#include "getifaddr.h" +#include "portinuse.h" + +#if defined(USE_NETFILTER) +#include "netfilter/iptcrdr.h" +#endif + +#ifdef CHECK_PORTINUSE + +#if defined(USE_NETFILTER) +/* Hardcoded for now. Ideally would come from .conf file */ +# ifdef TOMATO + const char *chains_to_check[] = { "WANPREROUTING" , 0 }; +# else + const char *chains_to_check[] = { "PREROUTING" , 0 }; +# endif +#endif + +int +port_in_use(const char *if_name, + unsigned eport, int proto, + const char *iaddr, unsigned iport) +{ + int found = 0; + char ip_addr_str[INET_ADDRSTRLEN]; + struct in_addr ip_addr; +#ifdef __linux__ + /* linux code */ + char line[256]; + FILE *f; + const char * tcpfile = "/proc/net/tcp"; + const char * udpfile = "/proc/net/udp"; +#endif + + if(getifaddr(if_name, ip_addr_str, INET_ADDRSTRLEN, &ip_addr, NULL) < 0) { + ip_addr.s_addr = 0; + ip_addr_str[0] = '\0'; + } + + syslog(LOG_DEBUG, "Check protocol %s for port %u on ext_if %s %s, %08X", + (proto==IPPROTO_TCP)?"tcp":"udp", eport, if_name, + ip_addr_str, (unsigned)ip_addr.s_addr); + + /* Phase 1 : check for local sockets (would be listed by netstat) */ +#if defined(__linux__) + f = fopen((proto==IPPROTO_TCP)?tcpfile:udpfile, "r"); + if (!f) { + syslog(LOG_ERR, "cannot open %s", (proto==IPPROTO_TCP)?tcpfile:udpfile); + return -1; + } + + while (fgets(line, 255, f)) { + char eaddr[68]; + unsigned tmp_port; + if (sscanf(line, "%*d: %64[0-9A-Fa-f]:%x %*x:%*x %*x %*x:%*x " + "%*x:%*x %*x %*d %*d %*llu", + eaddr, &tmp_port) == 2 + ) { + /* TODO add IPV6 support if enabled + * Presently assumes IPV4 */ +#ifdef DEBUG + syslog(LOG_DEBUG, "port_in_use check port %u and address %s", tmp_port, eaddr); +#endif + if (tmp_port == eport) { + char tmp_addr[4]; + struct in_addr *tmp_ip_addr = (struct in_addr *)tmp_addr; + if (sscanf(eaddr,"%2hhx%2hhx%2hhx%2hhx", + &tmp_addr[3],&tmp_addr[2],&tmp_addr[1],&tmp_addr[0]) == 4) + { + if (tmp_ip_addr->s_addr == 0 || tmp_ip_addr->s_addr == ip_addr.s_addr) + { + found++; + break; /* don't care how many, just that we found at least one */ + } + } + } + } + } + fclose(f); + +#elif defined(__OpenBSD__) +static struct nlist list[] = { +#if 0 + {"_tcpstat", 0, 0, 0, 0}, + {"_udpstat", 0, 0, 0, 0}, + {"_tcbinfo", 0, 0, 0, 0}, + {"_udbinfo", 0, 0, 0, 0}, +#endif + {"_tcbtable", 0, 0, 0, 0}, + {"_udbtable", 0, 0, 0, 0}, + {NULL,0, 0, 0, 0} +}; + char errstr[_POSIX2_LINE_MAX]; + kvm_t *kd; + ssize_t n; + struct inpcbtable table; + struct inpcb *next; + struct inpcb inpcb; + kd = kvm_openfiles(NULL, NULL, NULL, O_RDONLY, errstr); + if(!kd) { + syslog(LOG_ERR, "%s: kvm_openfiles(): %s", + "portinuse()", errstr); + return -1; + } + if(kvm_nlist(kd, list) < 0) { + syslog(LOG_ERR, "%s: kvm_nlist(): %s", + "portinuse()", kvm_geterr(kd)); + kvm_close(kd); + return -1; + } + n = kvm_read(kd, list[(proto==IPPROTO_TCP)?0:1].n_value, &table, sizeof(table)); + if(n < 0) { + syslog(LOG_ERR, "%s: kvm_read(): %s", + "portinuse()", kvm_geterr(kd)); + kvm_close(kd); + return -1; + } + next = CIRCLEQ_FIRST(&table.inpt_queue); /*TAILQ_FIRST(&table.inpt_queue);*/ + while(next != NULL) { + if(((u_long)next & 3) != 0) break; + n = kvm_read(kd, (u_long)next, &inpcb, sizeof(inpcb)); + if(n < 0) { + syslog(LOG_ERR, "kvm_read(): %s", kvm_geterr(kd)); + break; + } + next = CIRCLEQ_NEXT(&inpcb, inp_queue); /*TAILQ_NEXT(&inpcb, inp_queue);*/ + /* skip IPv6 sockets */ + if((inpcb.inp_flags & INP_IPV6) != 0) + continue; +#ifdef DEBUG + syslog(LOG_DEBUG, "%08lx:%hu %08lx:%hu", + (u_long)inpcb.inp_laddr.s_addr, ntohs(inpcb.inp_lport), + (u_long)inpcb.inp_faddr.s_addr, ntohs(inpcb.inp_fport)); +#endif + if(eport == (unsigned)ntohs(inpcb.inp_lport)) { + if(inpcb.inp_laddr.s_addr == INADDR_ANY || inpcb.inp_laddr.s_addr == ip_addr.s_addr) { + found++; + break; /* don't care how many, just that we found at least one */ + } + } + } + kvm_close(kd); + +#elif defined(__DragonFly__) + const char *varname; + struct xinpcb *xip; + struct xtcpcb *xtp; + struct inpcb *inp; + void *buf = NULL; + void *so_begin, *so_end; + + size_t len; + + switch (proto) { + case IPPROTO_TCP: + varname = "net.inet.tcp.pcblist"; + break; + case IPPROTO_UDP: + varname = "net.inet.udp.pcblist"; + break; + default: + syslog(LOG_ERR, "port_in_use() unknown proto=%d", proto); + return -1; + } + + if (sysctlbyname(varname, NULL, &len, NULL, 0) < 0) { + syslog(LOG_ERR, "sysctlbyname(%s, NULL, ...): %m", varname); + return -1; + } + buf = malloc(len); + if (buf == NULL) { + syslog(LOG_ERR, "malloc(%u) failed", (unsigned)len); + return -1; + } + if (sysctlbyname(varname, buf, &len, NULL, 0) < 0) { + syslog(LOG_ERR, "sysctlbyname(%s, buf, ...): %m", varname); + free(buf); + return -1; + } + + so_begin = buf; + so_end = (uint8_t *)buf + len; + for (so_begin = buf, so_end = (uint8_t *)so_begin + len; + (uint8_t *)so_begin + sizeof(size_t) < (uint8_t *)so_end && + (uint8_t *)so_begin + *(size_t *)so_begin <= (uint8_t *)so_end; + so_begin = (uint8_t *)so_begin + *(size_t *)so_begin) { + switch (proto) { + case IPPROTO_TCP: + xtp = (struct xtcpcb *)so_begin; + if (xtp->xt_len != sizeof *xtp) { + syslog(LOG_WARNING, "struct xtcpcb size mismatch; %ld vs %ld", + (long)xtp->xt_len, sizeof *xtp); + free(buf); + return -1; + } + inp = &xtp->xt_inp; + break; + case IPPROTO_UDP: + xip = (struct xinpcb *)so_begin; + if (xip->xi_len != sizeof *xip) { + syslog(LOG_WARNING, "struct xinpcb size mismatch : %ld vs %ld", + (long)xip->xi_len, sizeof *xip); + free(buf); + return -1; + } + inp = &xip->xi_inp; + break; + default: + abort(); + } + /* no support for IPv6 */ + if (INP_ISIPV6(inp) != 0) + continue; + syslog(LOG_DEBUG, "%08lx:%hu %08lx:%hu <=> %hu %08lx:%hu", + (u_long)inp->inp_laddr.s_addr, ntohs(inp->inp_lport), + (u_long)inp->inp_faddr.s_addr, ntohs(inp->inp_fport), + eport, (u_long)ip_addr.s_addr, iport + ); + if (eport == (unsigned)ntohs(inp->inp_lport)) { + if (inp->inp_laddr.s_addr == INADDR_ANY || inp->inp_laddr.s_addr == ip_addr.s_addr) { + found++; + break; /* don't care how many, just that we found at least one */ + } + } + } + if (buf) { + free(buf); + buf = NULL; + } +#elif defined(__FreeBSD__) + const char *varname; + struct xinpgen *xig, *exig; + struct xinpcb *xip; + struct xtcpcb *xtp; + struct inpcb *inp; + void *buf = NULL; + size_t len; + + switch (proto) { + case IPPROTO_TCP: + varname = "net.inet.tcp.pcblist"; + break; + case IPPROTO_UDP: + varname = "net.inet.udp.pcblist"; + break; + default: + syslog(LOG_ERR, "port_in_use() unknown proto=%d", proto); + return -1; + } + + if (sysctlbyname(varname, NULL, &len, NULL, 0) < 0) { + syslog(LOG_ERR, "sysctlbyname(%s, NULL, ...): %m", varname); + return -1; + } + buf = malloc(len); + if (buf == NULL) { + syslog(LOG_ERR, "malloc(%u) failed", (unsigned)len); + return -1; + } + if (sysctlbyname(varname, buf, &len, NULL, 0) < 0) { + syslog(LOG_ERR, "sysctlbyname(%s, buf, ...): %m", varname); + free(buf); + return -1; + } + + xig = (struct xinpgen *)buf; + exig = (struct xinpgen *)(void *)((char *)buf + len - sizeof *exig); + if (xig->xig_len != sizeof *xig) { + syslog(LOG_WARNING, "struct xinpgen size mismatch; %ld vs %ld", + (long)xig->xig_len, sizeof *xig); + free(buf); + return -1; + } + if (exig->xig_len != sizeof *exig) { + syslog(LOG_WARNING, "struct xinpgen size mismatch; %ld vs %ld", + (long)exig->xig_len, sizeof *exig); + free(buf); + return -1; + } + + while (1) { + xig = (struct xinpgen *)(void *)((char *)xig + xig->xig_len); + if (xig >= exig) + break; + switch (proto) { + case IPPROTO_TCP: + xtp = (struct xtcpcb *)xig; + if (xtp->xt_len != sizeof *xtp) { + syslog(LOG_WARNING, "struct xtcpcb size mismatch; %ld vs %ld", + (long)xtp->xt_len, sizeof *xtp); + free(buf); + return -1; + } + inp = &xtp->xt_inp; + break; + case IPPROTO_UDP: + xip = (struct xinpcb *)xig; + if (xip->xi_len != sizeof *xip) { + syslog(LOG_WARNING, "struct xinpcb size mismatch : %ld vs %ld", + (long)xip->xi_len, sizeof *xip); + free(buf); + return -1; + } + inp = &xip->xi_inp; + break; + default: + abort(); + } + /* no support for IPv6 */ + if ((inp->inp_vflag & INP_IPV6) != 0) + continue; + syslog(LOG_DEBUG, "%08lx:%hu %08lx:%hu <=> %hu %08lx:%hu", + (u_long)inp->inp_laddr.s_addr, ntohs(inp->inp_lport), + (u_long)inp->inp_faddr.s_addr, ntohs(inp->inp_fport), + eport, (u_long)ip_addr.s_addr, iport + ); + if (eport == (unsigned)ntohs(inp->inp_lport)) { + if (inp->inp_laddr.s_addr == INADDR_ANY || inp->inp_laddr.s_addr == ip_addr.s_addr) { + found++; + break; /* don't care how many, just that we found at least one */ + } + } + } + if (buf) { + free(buf); + buf = NULL; + } +/* #elif __NetBSD__ */ +#else +/* TODO : NetBSD / Darwin (OS X) / Solaris code */ +#error "No port_in_use() implementation available for this OS" +#endif + + /* Phase 2 : check existing mappings + * TODO : implement for pf/ipfw/etc. */ +#if defined(USE_NETFILTER) + if (!found) { + char iaddr_old[16]; + unsigned short iport_old; + int i; + for (i = 0; chains_to_check[i]; i++) { + if (get_nat_redirect_rule(chains_to_check[i], if_name, eport, proto, + iaddr_old, sizeof(iaddr_old), &iport_old, + 0, 0, 0, 0, 0, 0, 0) == 0) + { + syslog(LOG_DEBUG, "port_in_use check port %d on nat chain %s redirected to %s port %d", eport, + chains_to_check[i], iaddr_old, iport_old); + if (!(strcmp(iaddr, iaddr_old)==0 && iport==iport_old)) { + /* only "in use" if redirected to somewhere else */ + found++; + break; /* don't care how many, just that we found at least one */ + } + } + } + } +#else /* USE_NETFILTER */ + UNUSED(iport); UNUSED(iaddr); +#endif /* USE_NETFILTER */ + return found; +} +#endif /* CHECK_PORTINUSE */ diff --git a/src/contrib/miniupnp/miniupnpd/portinuse.h b/src/contrib/miniupnp/miniupnpd/portinuse.h new file mode 100644 index 0000000..01842f6 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/portinuse.h @@ -0,0 +1,23 @@ +/* $Id: $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef __PORTINUSE_H__ +#define __PORTINUSE_H__ + +#ifdef CHECK_PORTINUSE +/* portinuse() + * determine wether a port is already in use + * on a given interface. + * returns: 0 not in use, > 0 in use + * -1 in case of error */ +int +port_in_use(const char *if_name, + unsigned port, int proto, + const char *iaddr, unsigned iport); +#endif /* CHECK_PORTINUSE */ + +#endif diff --git a/src/contrib/miniupnp/miniupnpd/solaris/getifstats.c b/src/contrib/miniupnp/miniupnpd/solaris/getifstats.c new file mode 100644 index 0000000..19170c7 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/solaris/getifstats.c @@ -0,0 +1,97 @@ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * author: Ryan Wagoner and Thomas Bernard + * (c) 2007 Darren Reed + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include + +#include "../getifstats.h" + +int +getifstats(const char * ifname, struct ifdata * data) +{ + char buffer[64], *s; + kstat_named_t *kn; + kstat_ctl_t *kc; + int instance; + kstat_t *ksp; + + if (data == NULL) + goto error; + + if (ifname == NULL || *ifname == '\0') + goto error; + + s = (char *)ifname + strlen(ifname); + s--; + while ((s > ifname) && isdigit(*s)) + s--; + + s++; + instance = atoi(s); + strlcpy(buffer, ifname, MIN(s - ifname + 1, 64)); + + kc = kstat_open(); + if (kc != NULL) { + ksp = kstat_lookup(kc, buffer, instance, (char *)ifname); + if (ksp && (kstat_read(kc, ksp, NULL) != -1)) { + /* found the right interface */ + if (sizeof(long) == 8) { + kn = kstat_data_lookup(ksp, "rbytes64"); + if (kn != NULL) { + data->ibytes = kn->value.i64; + } + kn = kstat_data_lookup(ksp, "ipackets64"); + if (kn != NULL) { + data->ipackets = kn->value.i64; + } + kn = kstat_data_lookup(ksp, "obytes64"); + if (kn != NULL) { + data->obytes = kn->value.i64; + } + kn = kstat_data_lookup(ksp, "opackets64"); + if (kn != NULL) { + data->opackets = kn->value.i64; + } + } else { + kn = kstat_data_lookup(ksp, "rbytes"); + if (kn != NULL) { + data->ibytes = kn->value.i32; + } + kn = kstat_data_lookup(ksp, "ipackets"); + if (kn != NULL) { + data->ipackets = kn->value.i32; + } + kn = kstat_data_lookup(ksp, "obytes"); + if (kn != NULL) { + data->obytes = kn->value.i32; + } + kn = kstat_data_lookup(ksp, "opackets"); + if (kn != NULL) { + data->ipackets = kn->value.i32; + } + } + kn = kstat_data_lookup(ksp, "ifspeed"); + if (kn != NULL) { + data->baudrate = kn->value.i32; + } + kstat_close(kc); + return 0; /* ok */ + } + syslog(LOG_ERR, "kstat_lookup/read() failed: %m"); + kstat_close(kc); + return -1; + } else { + syslog(LOG_ERR, "kstat_open() failed: %m"); + } +error: + return -1; /* not found or error */ +} + diff --git a/src/contrib/miniupnp/miniupnpd/testasyncsendto.c b/src/contrib/miniupnp/miniupnpd/testasyncsendto.c new file mode 100644 index 0000000..6d441ea --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testasyncsendto.c @@ -0,0 +1,129 @@ +/* $Id: testasyncsendto.c,v 1.5 2018/03/13 10:25:52 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "miniupnpdtypes.h" +#include "upnputils.h" +#include "asyncsendto.h" + +struct lan_addr_list lan_addrs; +int runtime_flags = 0; +time_t startup_time = 0; + +#define DEST_IP "239.255.255.250" +#define DEST_PORT 1900 +/* +ssize_t +sendto_schedule(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen, + unsigned int delay) +*/ + +int test(void) +{ + int s; + ssize_t n; + int i; + struct sockaddr_in addr; + struct sockaddr_in dest_addr; + struct timeval next_send; + if( (s = socket(PF_INET, SOCK_DGRAM, 0)) < 0) { + syslog(LOG_ERR, "socket(): %m"); + return 1; + } + set_non_blocking(s); + memset(&addr, 0, sizeof(struct sockaddr_in)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + if(bind(s, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + syslog(LOG_ERR, "bind(): %m"); + close(s); + return 1; + } + memset(&dest_addr, 0, sizeof(struct sockaddr_in)); + dest_addr.sin_family = AF_INET; + dest_addr.sin_addr.s_addr = inet_addr(DEST_IP); + dest_addr.sin_port = htons(DEST_PORT); + n = sendto_or_schedule(s, "1234", 4, 0, + (struct sockaddr *)&dest_addr, sizeof(dest_addr)); + syslog(LOG_DEBUG, "sendto_or_schedule : %d", (int)n); + n = sendto_schedule(s, "1234", 4, 0, + (struct sockaddr *)&dest_addr, sizeof(dest_addr), + 4400); + syslog(LOG_DEBUG, "sendto_schedule : %d", (int)n); + n = sendto_schedule(s, "1234", 4, 0, + (struct sockaddr *)&dest_addr, sizeof(dest_addr), + 3000); + syslog(LOG_DEBUG, "sendto_schedule : %d", (int)n); + while ((i = get_next_scheduled_send(&next_send)) > 0) { + fd_set writefds; + int max_fd; + struct timeval timeout; + struct timeval now; + syslog(LOG_DEBUG, "get_next_scheduled_send : %d next_send=%ld.%06ld", + i, (long)next_send.tv_sec, (long)next_send.tv_usec); + FD_ZERO(&writefds); + max_fd = 0; + gettimeofday(&now, NULL); + i = get_sendto_fds(&writefds, &max_fd, &now); + if(now.tv_sec > next_send.tv_sec || + (now.tv_sec == next_send.tv_sec && now.tv_usec >= next_send.tv_usec)) { + if(i > 0) { + /* don't wait */ + timeout.tv_sec = 0; + } else { + /* wait 10sec :) */ + timeout.tv_sec = 10; + } + timeout.tv_usec = 0; + } else { + /* ... */ + timeout.tv_sec = (next_send.tv_sec - now.tv_sec); + timeout.tv_usec = (next_send.tv_usec - now.tv_usec); + if(timeout.tv_usec < 0) { + timeout.tv_usec += 1000000; + timeout.tv_sec--; + } + } + syslog(LOG_DEBUG, "get_sendto_fds() returned %d", i); + syslog(LOG_DEBUG, "select(%d, NULL, xx, NULL, %ld.%06ld)", + max_fd, (long)timeout.tv_sec, (long)timeout.tv_usec); + i = select(max_fd, NULL, &writefds, NULL, &timeout); + if(i < 0) { + syslog(LOG_ERR, "select: %m"); + if(errno != EINTR) + break; + } else if(try_sendto(&writefds) < 0) { + syslog(LOG_ERR, "try_sendto: %m"); + break; + } + } + close(s); + return 0; +} + +int main(int argc, char * * argv) +{ + int r; + (void)argc; + (void)argv; + openlog("testasyncsendto", LOG_CONS|LOG_PERROR, LOG_USER); + r = test(); + closelog(); + return r; +} + diff --git a/src/contrib/miniupnp/miniupnpd/testgetifaddr.c b/src/contrib/miniupnp/miniupnpd/testgetifaddr.c new file mode 100644 index 0000000..8045b89 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testgetifaddr.c @@ -0,0 +1,54 @@ +/* $Id: testgetifaddr.c,v 1.7 2013/04/27 15:38:57 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#include +#include +#include +#include +#include +#include +#include "config.h" +#include "getifaddr.h" + +#if defined(__sun) +/* solaris 10 does not define LOG_PERROR */ +#define LOG_PERROR 0 +#endif + +int main(int argc, char * * argv) { + char str_addr[64]; + struct in_addr addr; + struct in_addr mask; +#ifdef ENABLE_IPV6 + int r; + char str_addr6[64]; +#endif + if(argc < 2) { + fprintf(stderr, "Usage:\t%s interface_name\n", argv[0]); + return 1; + } + + openlog("testgetifaddr", LOG_CONS|LOG_PERROR, LOG_USER); + if(getifaddr(argv[1], str_addr, sizeof(str_addr), &addr, &mask) < 0) { + fprintf(stderr, "Cannot get address for interface %s.\n", argv[1]); + return 1; + } + printf("Interface %s has IP address %s.\n", argv[1], str_addr); + printf("addr=%s ", inet_ntoa(addr)); + printf("mask=%s\n", inet_ntoa(mask)); +#ifdef ENABLE_IPV6 + r = find_ipv6_addr(argv[1], str_addr6, sizeof(str_addr6)); + if(r < 0) { + fprintf(stderr, "find_ipv6_addr() failed\n"); + return 1; + } else if(r == 0) { + printf("Interface %s has no IPv6 address.\n", argv[1]); + } else { + printf("Interface %s has IPv6 address %s.\n", argv[1], str_addr6); + } +#endif + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/testgetifaddr.sh b/src/contrib/miniupnp/miniupnpd/testgetifaddr.sh new file mode 100644 index 0000000..0ab012c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testgetifaddr.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# $Id: testgetifaddr.sh,v 1.2 2015/09/22 14:48:09 nanard Exp $ + +OS=`uname -s` +case $OS in + FreeBSD | OpenBSD | Darwin | SunOS) + NS="`which netstat`" || exit 1 + IFCONFIG="`which ifconfig`" || exit 1 + EXTIF="`$NS -r -f inet | grep 'default' | awk '{ print $NF }' `" || exit 1 + EXTIP="`$IFCONFIG $EXTIF | awk '/inet / { print $2 }' `" + ;; + *) + IP="`which ip`" || exit 1 + EXTIF="`LC_ALL=C $IP -4 route | grep 'default' | sed -e 's/.*dev[[:space:]]*//' -e 's/[[:space:]].*//'`" || exit 1 + EXTIP="`LC_ALL=C $IP -4 addr show $EXTIF | awk '/inet/ { print $2 }' | cut -d "/" -f 1`" + ;; +esac + +#echo "Interface : $EXTIF IP address : $EXTIP" +RES=`./testgetifaddr $EXTIF | head -n1 | sed 's/Interface .* has IP address \(.*\)\./\1/'` || exit 1 + + +if [ "$RES" = "$EXTIP" ] ; then + echo "testgetifaddr test OK" + exit 0 +else + echo "testgetifaddr test FAILED : $EXTIP != $RES" + exit 1 +fi diff --git a/src/contrib/miniupnp/miniupnpd/testgetifstats.c b/src/contrib/miniupnp/miniupnpd/testgetifstats.c new file mode 100644 index 0000000..9c29b4e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testgetifstats.c @@ -0,0 +1,41 @@ +/* $Id: testgetifstats.c,v 1.5 2012/03/05 20:36:17 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include + +#include "getifstats.h" + +#if defined(__sun) +/* solaris 10 does not define LOG_PERROR */ +#define LOG_PERROR 0 +#endif + +int +main(int argc, char **argv) +{ + int r; + struct ifdata data; + if(argc<2) + { + fprintf(stderr, "usage : %s \n", argv[0]); + return 1; + } + openlog("testgetifstats", LOG_CONS|LOG_PERROR, LOG_USER); + memset(&data, 0, sizeof(data)); + r = getifstats(argv[1], &data); + printf("getifstats() returned %d\n", r); + printf("stats for interface %s :\n", argv[1]); + printf("bitrate = %lu\n", data.baudrate); + printf(" input packets : %9lu\t input bytes : %9lu\n", + data.ipackets, data.ibytes); + printf("output packets : %9lu\toutput bytes : %9lu\n", + data.opackets, data.obytes); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/testgetroute.c b/src/contrib/miniupnp/miniupnpd/testgetroute.c new file mode 100644 index 0000000..61491be --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testgetroute.c @@ -0,0 +1,101 @@ +/* $Id: testgetroute.c,v 1.7 2018/03/13 10:25:52 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include + +#include "getroute.h" +#include "upnputils.h" +#include "upnpglobalvars.h" + +#ifndef LOG_PERROR +/* solaris does not define LOG_PERROR */ +#define LOG_PERROR 0 +#endif + +struct lan_addr_list lan_addrs; +int runtime_flags = 0; +time_t startup_time = 0; + +int +main(int argc, char ** argv) +{ + struct sockaddr_in dst4; + struct sockaddr_in6 dst6; + struct sockaddr * dst; + void * src; + size_t src_len; + int r; + int index = -1; + + memset(&dst4, 0, sizeof(dst4)); + memset(&dst6, 0, sizeof(dst6)); + dst = NULL; + if(argc < 2) { + fprintf(stderr, "usage: %s \n", argv[0]); + fprintf(stderr, "both v4 and v6 IP addresses are supported.\n"); + return 1; + } + openlog("testgetroute", LOG_CONS|LOG_PERROR, LOG_USER); + r = inet_pton (AF_INET, argv[1], &dst4.sin_addr); + if(r < 0) { + syslog(LOG_ERR, "inet_pton(AF_INET, %s) : %m", argv[1]); + closelog(); + return 2; + } + if (r == 0) { + r = inet_pton (AF_INET6, argv[1], &dst6.sin6_addr); + if(r < 0) { + syslog(LOG_ERR, "inet_pton(AF_INET6, %s) : %m", argv[1]); + closelog(); + return 2; + } else if(r > 0) { + dst6.sin6_family = AF_INET6; + dst = (struct sockaddr *)&dst6; + src = &dst6.sin6_addr; + src_len = sizeof(dst6.sin6_addr); + } else { + /* r == 0 */ + syslog(LOG_ERR, "%s is not a correct IPv4 or IPv6 address", argv[1]); + closelog(); + return 1; + } + } else { + dst4.sin_family = AF_INET; + dst = (struct sockaddr *)&dst4; + src = &dst4.sin_addr; + src_len = sizeof(dst4.sin_addr); + } + + if (dst) { + syslog(LOG_DEBUG, "calling get_src_for_route_to(%p, NULL, NULL, %p)", + dst, &index); + r = get_src_for_route_to (dst, NULL, NULL, &index); + syslog(LOG_DEBUG, "get_src_for_route_to() returned %d", r); + if(r >= 0) { + syslog(LOG_DEBUG, "index=%d", index); + } + syslog(LOG_DEBUG, "calling get_src_for_route_to(%p, %p, %p(%u), %p)", + dst, src, &src_len, (unsigned)src_len, &index); + r = get_src_for_route_to (dst, src, &src_len, &index); + syslog(LOG_DEBUG, "get_src_for_route_to() returned %d", r); + if(r >= 0) { + char src_str[128]; + sockaddr_to_string(dst, src_str, sizeof(src_str)); + syslog(LOG_DEBUG, "src=%s", src_str); + syslog(LOG_DEBUG, "index=%d", index); + } + } + closelog(); + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/testminissdp.c b/src/contrib/miniupnp/miniupnpd/testminissdp.c new file mode 100644 index 0000000..fffadb1 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testminissdp.c @@ -0,0 +1,82 @@ +/* $Id: testminissdp.c,v 1.3 2018/01/15 16:46:48 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2017-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include "config.h" +#include "minissdp.h" +#include "upnpglobalvars.h" + +void test(const char * buffer, size_t n) +{ + int s = 0; + struct sockaddr_in dummy_sender; + + memset(&dummy_sender, 0, sizeof(dummy_sender)); + dummy_sender.sin_family = AF_INET; + + ProcessSSDPData(s, buffer, n, + (struct sockaddr *)&dummy_sender, 0, +#ifdef ENABLE_HTTPS + 80, 443 +#else + 80 +#endif + ); + +} + +int main(int argc, char * * argv) +{ + FILE * f = NULL; + char buffer[1500]; + size_t n = 0; + struct lan_addr_s * lan_addr; + + if((argc > 1) && ((strcmp(argv[1], "--help") == 0) || (strcmp(argv[1], "-h") == 0))) { + fprintf(stderr, "Usage:\t%s [file]\nIf no file is specified, the program is reading the standard input.\n", argv[0]); + return 1; + } + openlog("testminissdp", LOG_CONS|LOG_PERROR, LOG_USER); + + /* populate lan_addrs */ + LIST_INIT(&lan_addrs); + lan_addr = (struct lan_addr_s *) malloc(sizeof(struct lan_addr_s)); + memset(lan_addr, 0, sizeof(struct lan_addr_s)); + LIST_INSERT_HEAD(&lan_addrs, lan_addr, list); + + if(argc > 1) { + f = fopen(argv[1], "rb"); + if(f == NULL) { + syslog(LOG_ERR, "error opening file %s", argv[1]); + return 1; + } + } + n = fread(buffer, 1, sizeof(buffer), f ? f : stdin); + if(n <= 0) { + syslog(LOG_ERR, "error reading"); + return 1; + } + + test(buffer, n); + + if(f) fclose(f); + /* free memory */ + while(lan_addrs.lh_first != NULL) + { + lan_addr = lan_addrs.lh_first; + LIST_REMOVE(lan_addrs.lh_first, list); + free(lan_addr); + } + + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/testportinuse.c b/src/contrib/miniupnp/miniupnpd/testportinuse.c new file mode 100644 index 0000000..48504bc --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testportinuse.c @@ -0,0 +1,54 @@ +/* $Id: testportinuse.c,v 1.3 2014/03/28 12:13:17 nanard Exp $ */ +/* MiniUPnP project + * (c) 2014 Thomas Bernard + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ +#include +#include +#include +#include +#include + +#include "macros.h" +#include "config.h" +#include "portinuse.h" + +#ifdef USE_NETFILTER +const char * miniupnpd_nat_chain = "MINIUPNPD"; +const char * miniupnpd_nat_postrouting_chain = "MINIUPNPD-POSTROUTING"; +const char * miniupnpd_forward_chain = "MINIUPNPD"; +#endif /* USE_NETFILTER */ + +int main(int argc, char * * argv) +{ +#ifndef CHECK_PORTINUSE + UNUSED(argc); UNUSED(argv); + printf("CHECK_PORTINUSE is not defined.\n"); +#else /* CHECK_PORTINUSE */ + int r; + const char * if_name; + unsigned eport; + int proto; + const char * iaddr; + unsigned iport; + + if(argc <= 5) { + fprintf(stderr, "usage: %s if_name eport (tcp|udp) iaddr iport\n", + argv[0]); + return 1; + } + openlog("testportinuse", LOG_CONS|LOG_PERROR, LOG_USER); + if_name = argv[1]; + eport = (unsigned)atoi(argv[2]); + proto = (0==strcmp(argv[3], "tcp"))?IPPROTO_TCP:IPPROTO_UDP; + iaddr = argv[4]; + iport = (unsigned)atoi(argv[5]); + + r = port_in_use(if_name, eport, proto, iaddr, iport); + printf("port_in_use(%s, %u, %d, %s, %u) returned %d\n", + if_name, eport, proto, iaddr, iport, r); + closelog(); +#endif /* CHECK_PORTINUSE */ + return 0; +} diff --git a/src/contrib/miniupnp/miniupnpd/testssdppktgen.c b/src/contrib/miniupnp/miniupnpd/testssdppktgen.c new file mode 100644 index 0000000..5de7998 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testssdppktgen.c @@ -0,0 +1,100 @@ +/* $Id: $ */ +#include +#include +#include + +#include "config.h" +#include "miniupnpdpath.h" +#include "upnphttp.h" +#include "macros.h" + +#define SSDP_PORT 1900 + +char uuidvalue_igd[] = "uuid:12345678-0000-0000-0000-000000abcd01"; +unsigned upnp_bootid; +unsigned upnp_configid; + +static int +MakeSSDPPacket(const char * dest_str, + const char * host, unsigned short http_port, +#ifdef ENABLE_HTTPS + unsigned short https_port, +#endif + const char * nt, const char * suffix, + const char * usn1, const char * usn2, const char * usn3, + unsigned int lifetime) +{ + char bufr[SSDP_PACKET_MAX_LEN]; + int l; + + l = snprintf(bufr, sizeof(bufr), + "NOTIFY * HTTP/1.1\r\n" + "HOST: %s:%d\r\n" + "CACHE-CONTROL: max-age=%u\r\n" + "LOCATION: http://%s:%u" ROOTDESC_PATH "\r\n" +#ifdef ENABLE_HTTPS + "SECURELOCATION.UPNP.ORG: https://%s:%u" ROOTDESC_PATH "\r\n" +#endif + "SERVER: " MINIUPNPD_SERVER_STRING "\r\n" + "NT: %s%s\r\n" + "USN: %s%s%s%s\r\n" + "NTS: ssdp:alive\r\n" + "OPT: \"http://schemas.upnp.org/upnp/1/0/\"; ns=01\r\n" /* UDA v1.1 */ + "01-NLS: %u\r\n" /* same as BOOTID field. UDA v1.1 */ + "BOOTID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "CONFIGID.UPNP.ORG: %u\r\n" /* UDA v1.1 */ + "\r\n", + dest_str, SSDP_PORT, /* HOST: */ + lifetime, /* CACHE-CONTROL: */ + host, (unsigned int)http_port, /* LOCATION: */ +#ifdef ENABLE_HTTPS + host, (unsigned int)https_port, /* SECURE-LOCATION: */ +#endif + nt, suffix, /* NT: */ + usn1, usn2, usn3, suffix, /* USN: */ + upnp_bootid, /* 01-NLS: */ + upnp_bootid, /* BOOTID.UPNP.ORG: */ + upnp_configid ); /* CONFIGID.UPNP.ORG: */ + if(l<0) { + syslog(LOG_ERR, "%s: snprintf error", "MakeSSDPPacket()"); + return -1; + } else if((unsigned int)l >= sizeof(bufr)) { + syslog(LOG_WARNING, "%s: truncated output (%u>=%u)", + "MakeSSDPPacket()", (unsigned)l, (unsigned)sizeof(bufr)); + l = sizeof(bufr) - 1; + return -1; + } + return 0; +} + + +int main(int argc, char * * argv) +{ + int r; + UNUSED(argc); UNUSED(argv); + + openlog("testssdppktgen", LOG_CONS|LOG_PERROR, LOG_USER); + upnp_bootid = (unsigned)time(NULL); + upnp_configid = 1234567890; + r = MakeSSDPPacket("123.456.789.123", "222.222.222.222", 12345, +#ifdef ENABLE_HTTPS + 54321, +#endif /* ENABLE_HTTPS */ + "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:", "1", + uuidvalue_igd, "", "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:", + 1234567890); + if(r < 0) return 1; +#ifdef ENABLE_IPV6 + r = MakeSSDPPacket("[1234:5678:abcd:ef00:1234:5678:abcd:ef00]", + "[1000:2000:3000:4000:5000:6000:7000:8000]", 12345, +#ifdef ENABLE_HTTPS + 54321, +#endif /* ENABLE_HTTPS */ + "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:", "1", + uuidvalue_igd, "", "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:", + 1234567890); + if(r < 0) return 1; +#endif /* ENABLE_IPV6 */ + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/testupnpdescgen.c b/src/contrib/miniupnp/miniupnpd/testupnpdescgen.c new file mode 100644 index 0000000..5b88c2a --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testupnpdescgen.c @@ -0,0 +1,267 @@ +/* $Id: testupnpdescgen.c,v 1.34 2017/05/27 07:47:57 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +/* for mkdir */ +#include +#include +#include + +#include "macros.h" +#include "config.h" +#include "upnpglobalvars.h" +#include "upnpdescgen.h" +#include "upnpdescstrings.h" +#include "getifaddr.h" + +char uuidvalue_igd[] = "uuid:12345678-0000-0000-0000-000000abcd01"; +char uuidvalue_wan[] = "uuid:12345678-0000-0000-0000-000000abcd02"; +char uuidvalue_wcd[] = "uuid:12345678-0000-0000-0000-000000abcd03"; +char serialnumber[] = "12345678"; +char modelnumber[] = "1"; +char presentationurl[] = "http://192.168.0.1:8080/"; +/*char presentationurl[] = "";*/ +#ifdef ENABLE_MANUFACTURER_INFO_CONFIGURATION +char friendly_name[] = OS_NAME " router"; +char manufacturer_name[] = ROOTDEV_MANUFACTURER; +char manufacturer_url[] = ROOTDEV_MANUFACTURERURL; +char model_name[] = ROOTDEV_MODELNAME; +char model_description[] = ROOTDEV_MODELDESCRIPTION; +char model_url[] = ROOTDEV_MODELURL; +#endif /* ENABLE_MANUFACTURER_INFO_CONFIGURATION */ +#ifdef RANDOMIZE_URLS +char random_url[] = "RANDOM"; +#endif /* RANDOMIZE_URLS */ +unsigned int upnp_configid = 666; + +const char * use_ext_ip_addr = NULL; +const char * ext_if_name = "eth0"; + +int runtime_flags = 0; + +int getifaddr(const char * ifname, char * buf, int len, struct in_addr * addr, struct in_addr * mask) +{ + UNUSED(ifname); + UNUSED(addr); + UNUSED(mask); + strncpy(buf, "1.2.3.4", len); + return 0; +} + +int upnp_get_portmapping_number_of_entries(void) +{ + return 42; +} + +int get_wan_connection_status(const char * ifname) +{ + UNUSED(ifname); + return 2; +} + +/* To be improved */ +int +xml_pretty_print(const char * s, int len, FILE * f) +{ + int n = 0, i; + int elt_close = 0; + int c, indent = 0; + + if(!s) + return n; + while(len > 0) + { + c = *(s++); len--; + switch(c) + { + case '<': + if(len>0 && *s == '/') + elt_close++; + else if(len>0 && *s == '?') + elt_close = 1; + else + elt_close = 0; + if(elt_close!=1) + { + if(elt_close > 1) + indent--; + fputc('\n', f); n++; + for(i=indent; i>0; i--) + fputc(' ', f); + n += indent; + } + fputc(c, f); n++; + break; + case '>': + fputc(c, f); n++; + if(elt_close==1) + { + /*fputc('\n', f); n++; */ + /* elt_close = 0; */ + if(indent > 0) + indent--; + } + else if(elt_close == 0) + indent++; + break; + case '\n': + /* remove existing LF */ + break; + default: + fputc(c, f); n++; + } + } + return n; +} + +/* stupid test */ +const char * str1 = "Prefix123String"; +const char * str2 = "123String"; + +void stupid_test(void) +{ + printf("str1:'%s' str2:'%s'\n", str1, str2); + printf("str1:%p str2:%p str2-str1:%ld\n", str1, str2, (long)(str2-str1)); +} + +/* main */ + +int +main(int argc, char * * argv) +{ + char * rootDesc; + int rootDescLen; + char * s; + int l; + FILE * f; + + for(l = 1; l < argc; l++) { + if(0 == strcmp(argv[l], "--help") || 0 == strcmp(argv[l], "-h")) { + printf("Usage:\t%s [options]\n", argv[0]); + printf("options:\n"); +#ifdef IGD_V2 + printf("\t--forceigdv1 Force versions of devices to be 1\n"); +#else + printf("\tNone\n"); +#endif + return 0; +#ifdef IGD_V2 + } else if(0 == strcmp(argv[l], "--forceigdv1")) { + SETFLAG(FORCEIGDDESCV1MASK); +#endif + } else { + fprintf(stderr, "unknown option %s\n", argv[l]); + } + } + + if(mkdir("testdescs", 0777) < 0) { + if(errno != EEXIST) { + perror("mkdir"); + } + } + printf("Root Description :\n"); + rootDesc = genRootDesc(&rootDescLen); + xml_pretty_print(rootDesc, rootDescLen, stdout); + f = fopen("testdescs/rootdesc.xml", "w"); + if(f) { + xml_pretty_print(rootDesc, rootDescLen, f); + fclose(f); + } + free(rootDesc); + printf("\n-------------\n"); + printf("WANIPConnection Description :\n"); + s = genWANIPCn(&l); + xml_pretty_print(s, l, stdout); + f = fopen("testdescs/wanipc_scpd.xml", "w"); + if(f) { + xml_pretty_print(s, l, f); + fclose(f); + } + free(s); + printf("\n-------------\n"); + printf("WANConfig Description :\n"); + s = genWANCfg(&l); + xml_pretty_print(s, l, stdout); + f = fopen("testdescs/wanconfig_scpd.xml", "w"); + if(f) { + xml_pretty_print(s, l, f); + fclose(f); + } + free(s); + printf("\n-------------\n"); +#ifdef ENABLE_L3F_SERVICE + printf("Layer3Forwarding service :\n"); + s = genL3F(&l); + xml_pretty_print(s, l, stdout); + f = fopen("testdescs/l3f_scpd.xml", "w"); + if(f) { + xml_pretty_print(s, l, f); + fclose(f); + } + free(s); + printf("\n-------------\n"); +#endif +#ifdef ENABLE_6FC_SERVICE + printf("WANIPv6FirewallControl service :\n"); + s = gen6FC(&l); + xml_pretty_print(s, l, stdout); + f = fopen("testdescs/wanipv6fc_scpd.xml", "w"); + if(f) { + xml_pretty_print(s, l, f); + fclose(f); + } + free(s); + printf("\n-------------\n"); +#endif +#ifdef ENABLE_DP_SERVICE + printf("DeviceProtection service :\n"); + s = genDP(&l); + xml_pretty_print(s, l, stdout); + f = fopen("testdescs/dp_scpd.xml", "w"); + if(f) { + xml_pretty_print(s, l, f); + fclose(f); + } + free(s); + printf("\n-------------\n"); +#endif +#ifdef ENABLE_EVENTS + s = getVarsWANIPCn(&l); + xml_pretty_print(s, l, stdout); + free(s); + printf("\n-------------\n"); + s = getVarsWANCfg(&l); + xml_pretty_print(s, l, stdout); + free(s); + printf("\n-------------\n"); +#ifdef ENABLE_L3F_SERVICE + s = getVarsL3F(&l); + xml_pretty_print(s, l, stdout); + free(s); + printf("\n-------------\n"); +#ifdef ENABLE_6FC_SERVICE + s = getVars6FC(&l); + xml_pretty_print(s, l, stdout); + free(s); + printf("\n-------------\n"); +#endif +#ifdef ENABLE_DP_SERVICE + s = getVarsDP(&l); + xml_pretty_print(s, l, stdout); + free(s); + printf("\n-------------\n"); +#endif +#endif +#endif +/* + stupid_test(); +*/ + return 0; +} + diff --git a/src/contrib/miniupnp/miniupnpd/testupnppermissions.c b/src/contrib/miniupnp/miniupnpd/testupnppermissions.c new file mode 100644 index 0000000..16174c6 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/testupnppermissions.c @@ -0,0 +1,62 @@ +/* $Id: testupnppermissions.c,v 1.3 2009/09/14 15:24:46 nanard Exp $ */ +/* (c) 2007-2015 Thomas Bernard + * MiniUPnP Project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + */ +#include +#include +#include +#include +#include +#include +#include +#include "upnppermissions.h" + +void +print_upnpperm(const struct upnpperm * p) +{ + switch(p->type) + { + case UPNPPERM_ALLOW: + printf("allow "); + break; + case UPNPPERM_DENY: + printf("deny "); + break; + default: + printf("error ! "); + } + printf("%hu-%hu ", p->eport_min, p->eport_max); + printf("%s/", inet_ntoa(p->address)); + printf("%s ", inet_ntoa(p->mask)); + printf("%hu-%hu", p->iport_min, p->iport_max); + putchar('\n'); +} + +int main(int argc, char * * argv) +{ + int i, r, ret; + struct upnpperm p; + if(argc < 2) { + fprintf(stderr, "Usage: %s \"permission line\" [...]\n", argv[0]); + fprintf(stderr, "Example: %s \"allow 1234 10.10.10.10/32 1234\"\n", argv[0]); + return 1; + } + openlog("testupnppermissions", LOG_PERROR, LOG_USER); + ret = 0; + for(i=1; i +#include +#include + +#include "config.h" +#ifdef ENABLE_EVENTS +#include "getifaddr.h" +#include "upnpredirect.h" +#endif +#include "upnpdescgen.h" +#include "miniupnpdpath.h" +#include "upnpglobalvars.h" +#include "upnpdescstrings.h" +#include "upnpurns.h" +#include "getconnstatus.h" + + +/* Event magical values codes */ +#define SETUPREADY_MAGICALVALUE (248) +#define CONNECTIONSTATUS_MAGICALVALUE (249) +#define FIREWALLENABLED_MAGICALVALUE (250) +#define INBOUNDPINHOLEALLOWED_MAGICALVALUE (251) +#define SYSTEMUPDATEID_MAGICALVALUE (252) +#define PORTMAPPINGNUMBEROFENTRIES_MAGICALVALUE (253) +#define EXTERNALIPADDRESS_MAGICALVALUE (254) +#define DEFAULTCONNECTIONSERVICE_MAGICALVALUE (255) + + +static const char * const upnptypes[] = +{ + "string", + "boolean", + "ui2", + "ui4", + "bin.base64" +}; + +static const char * const upnpdefaultvalues[] = +{ + 0, + "IP_Routed"/*"Unconfigured"*/, /* 1 default value for ConnectionType */ + "3600", /* 2 default value for PortMappingLeaseDuration */ + "Unconfigured", /* 3 default value for ConnectionStatus */ + "0", /* 4 default value for RSIPAvailable */ + "1", /* 5 default value for NATEnabled */ + "ERROR_NONE", /* 6 default value for LastConnectionError */ +}; + +static const char * const upnpallowedvalues[] = +{ + 0, /* 0 */ + "DSL", /* 1 */ + "POTS", + "Cable", + "Ethernet", + 0, + "Up", /* 6 */ + "Down", + "Initializing", + "Unavailable", + 0, + "TCP", /* 11 */ + "UDP", + 0, + "Unconfigured", /* 14 */ + "IP_Routed", + "IP_Bridged", + 0, + "Unconfigured", /* 18 */ + "Connecting", + "Connected", + "PendingDisconnect", + "Disconnecting", + "Disconnected", + 0, + "ERROR_NONE", /* 25 */ +/* Optionals values : + * ERROR_COMMAND_ABORTED + * ERROR_NOT_ENABLED_FOR_INTERNET + * ERROR_USER_DISCONNECT + * ERROR_ISP_DISCONNECT + * ERROR_IDLE_DISCONNECT + * ERROR_FORCED_DISCONNECT + * ERROR_NO_CARRIER + * ERROR_IP_CONFIGURATION + * ERROR_UNKNOWN */ + 0, + "", /* 27 */ + 0 +}; + +static const int upnpallowedranges[] = { + 0, + /* 1 PortMappingLeaseDuration */ + 0, + 604800, + /* 3 InternalPort */ + 1, + 65535, + /* 5 LeaseTime */ + 1, + 86400, + /* 7 OutboundPinholeTimeout */ + 100, + 200, +}; + +static const char * magicargname[] = { + 0, + "StartPort", /* 1 */ + "EndPort", /* 2 */ + "RemoteHost", /* 3 */ + "RemotePort", /* 4 */ + "InternalClient", /* 5 */ + "InternalPort", /* 6 */ + "IsWorking", /* 7 */ +#ifdef ENABLE_DP_SERVICE + "ProtocolType", /* 8 */ + "InMessage", /* 9 */ + "OutMessage", /* 10 */ + "ProtocolList", /* 11 */ + "RoleList", /* 12 */ +#endif /* ENABLE_DP_SERVICE */ +}; + +static const char xmlver[] = + "\r\n"; +static const char root_service[] = + "scpd xmlns=\"urn:schemas-upnp-org:service-1-0\""; +static const char root_device[] = + "root xmlns=\"urn:schemas-upnp-org:device-1-0\""; + +/* root Description of the UPnP Device + * fixed to match UPnP_IGD_InternetGatewayDevice 1.0.pdf + * Needs to be checked with UPnP-gw-InternetGatewayDevice-v2-Device.pdf + * presentationURL is only "recommended" but the router doesn't appears + * in "Network connections" in Windows XP if it is not present. */ +static const struct XMLElt rootDesc[] = +{ +/* 0 */ + {root_device, INITHELPER(1,2)}, + {"specVersion", INITHELPER(3,2)}, +#if defined(ENABLE_L3F_SERVICE) || defined(HAS_DUMMY_SERVICE) || defined(ENABLE_DP_SERVICE) + {"device", INITHELPER(5,13)}, +#else + {"device", INITHELPER(5,12)}, +#endif + {"/major", UPNP_VERSION_MAJOR_STR}, + {"/minor", UPNP_VERSION_MINOR_STR}, +/* 5 */ + {"/deviceType", DEVICE_TYPE_IGD}, + /* urn:schemas-upnp-org:device:InternetGatewayDevice:1 or 2 */ +#ifdef ENABLE_MANUFACTURER_INFO_CONFIGURATION + {"/friendlyName", friendly_name/*ROOTDEV_FRIENDLYNAME*/}, /* required */ + {"/manufacturer", manufacturer_name/*ROOTDEV_MANUFACTURER*/}, /* required */ +/* 8 */ + {"/manufacturerURL", manufacturer_url/*ROOTDEV_MANUFACTURERURL*/}, /* optional */ + {"/modelDescription", model_description/*ROOTDEV_MODELDESCRIPTION*/}, /* recommended */ + {"/modelName", model_name/*ROOTDEV_MODELNAME*/}, /* required */ + {"/modelNumber", modelnumber}, + {"/modelURL", model_url/*ROOTDEV_MODELURL*/}, +#else + {"/friendlyName", ROOTDEV_FRIENDLYNAME}, /* required */ + {"/manufacturer", ROOTDEV_MANUFACTURER}, /* required */ +/* 8 */ + {"/manufacturerURL", ROOTDEV_MANUFACTURERURL}, /* optional */ + {"/modelDescription", ROOTDEV_MODELDESCRIPTION}, /* recommended */ + {"/modelName", ROOTDEV_MODELNAME}, /* required */ + {"/modelNumber", modelnumber}, + {"/modelURL", ROOTDEV_MODELURL}, +#endif + {"/serialNumber", serialnumber}, + {"/UDN", uuidvalue_igd}, /* required */ + /* see if /UPC is needed. */ +#ifdef ENABLE_6FC_SERVICE +#define SERVICES_OFFSET 63 +#else +#define SERVICES_OFFSET 58 +#endif +#if defined(ENABLE_L3F_SERVICE) || defined(HAS_DUMMY_SERVICE) || defined(ENABLE_DP_SERVICE) + /* here we dening Services for the root device : + * L3F and DUMMY and DeviceProtection */ +#ifdef ENABLE_L3F_SERVICE +#define NSERVICES1 1 +#else +#define NSERVICES1 0 +#endif +#ifdef HAS_DUMMY_SERVICE +#define NSERVICES2 1 +#else +#define NSERVICES2 0 +#endif +#ifdef ENABLE_DP_SERVICE +#define NSERVICES3 1 +#else +#define NSERVICES3 0 +#endif +#define NSERVICES (NSERVICES1+NSERVICES2+NSERVICES3) + {"serviceList", INITHELPER(SERVICES_OFFSET,NSERVICES)}, + {"deviceList", INITHELPER(18,1)}, + {"/presentationURL", presentationurl}, /* recommended */ +#else + {"deviceList", INITHELPER(18,1)}, + {"/presentationURL", presentationurl}, /* recommended */ + {0,0}, +#endif +/* 18 */ + {"device", INITHELPER(19,13)}, +/* 19 */ + {"/deviceType", DEVICE_TYPE_WAN}, /* required */ + /* urn:schemas-upnp-org:device:WANDevice:1 or 2 */ + {"/friendlyName", WANDEV_FRIENDLYNAME}, + {"/manufacturer", WANDEV_MANUFACTURER}, + {"/manufacturerURL", WANDEV_MANUFACTURERURL}, + {"/modelDescription" , WANDEV_MODELDESCRIPTION}, + {"/modelName", WANDEV_MODELNAME}, + {"/modelNumber", WANDEV_MODELNUMBER}, + {"/modelURL", WANDEV_MODELURL}, + {"/serialNumber", serialnumber}, + {"/UDN", uuidvalue_wan}, + {"/UPC", WANDEV_UPC}, /* UPC (=12 digit barcode) is optional */ +/* 30 */ + {"serviceList", INITHELPER(32,1)}, + {"deviceList", INITHELPER(38,1)}, +/* 32 */ + {"service", INITHELPER(33,5)}, +/* 33 */ + {"/serviceType", + "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1"}, + /*{"/serviceId", "urn:upnp-org:serviceId:WANCommonInterfaceConfig"}, */ + {"/serviceId", "urn:upnp-org:serviceId:WANCommonIFC1"}, /* required */ + {"/SCPDURL", WANCFG_PATH}, + {"/controlURL", WANCFG_CONTROLURL}, + {"/eventSubURL", WANCFG_EVENTURL}, +/* 38 */ + {"device", INITHELPER(39,12)}, +/* 39 */ + {"/deviceType", DEVICE_TYPE_WANC}, + /* urn:schemas-upnp-org:device:WANConnectionDevice:1 or 2 */ + {"/friendlyName", WANCDEV_FRIENDLYNAME}, + {"/manufacturer", WANCDEV_MANUFACTURER}, + {"/manufacturerURL", WANCDEV_MANUFACTURERURL}, + {"/modelDescription", WANCDEV_MODELDESCRIPTION}, + {"/modelName", WANCDEV_MODELNAME}, + {"/modelNumber", WANCDEV_MODELNUMBER}, + {"/modelURL", WANCDEV_MODELURL}, + {"/serialNumber", serialnumber}, + {"/UDN", uuidvalue_wcd}, + {"/UPC", WANCDEV_UPC}, /* UPC (=12 digit Barcode) is optional */ +#ifdef ENABLE_6FC_SERVICE + {"serviceList", INITHELPER(51,2)}, +#else + {"serviceList", INITHELPER(51,1)}, +#endif +/* 51 */ + {"service", INITHELPER(53,5)}, + {"service", INITHELPER(58,5)}, +/* 53 */ + {"/serviceType", SERVICE_TYPE_WANIPC}, + /* urn:schemas-upnp-org:service:WANIPConnection:2 for v2 */ + {"/serviceId", SERVICE_ID_WANIPC}, + /* urn:upnp-org:serviceId:WANIPConn1 or 2 */ + {"/SCPDURL", WANIPC_PATH}, + {"/controlURL", WANIPC_CONTROLURL}, + {"/eventSubURL", WANIPC_EVENTURL}, +#ifdef ENABLE_6FC_SERVICE +/* 58 */ + {"/serviceType", "urn:schemas-upnp-org:service:WANIPv6FirewallControl:1"}, + {"/serviceId", "urn:upnp-org:serviceId:WANIPv6Firewall1"}, + {"/SCPDURL", WANIP6FC_PATH}, + {"/controlURL", WANIP6FC_CONTROLURL}, + {"/eventSubURL", WANIP6FC_EVENTURL}, +#endif +/* 58 / 63 = SERVICES_OFFSET*/ +#if defined(HAS_DUMMY_SERVICE) || defined(ENABLE_L3F_SERVICE) || defined(ENABLE_DP_SERVICE) + {"service", INITHELPER(SERVICES_OFFSET+2,5)}, + {"service", INITHELPER(SERVICES_OFFSET+7,5)}, +#endif +#ifdef HAS_DUMMY_SERVICE +/* 60 / 65 = SERVICES_OFFSET+2 */ + {"/serviceType", "urn:schemas-dummy-com:service:Dummy:1"}, + {"/serviceId", "urn:dummy-com:serviceId:dummy1"}, + {"/SCPDURL", DUMMY_PATH}, + {"/controlURL", "/dummy"}, + {"/eventSubURL", "/dummy"}, +#endif +#ifdef ENABLE_L3F_SERVICE +/* 60 / 65 = SERVICES_OFFSET+2 */ + {"/serviceType", "urn:schemas-upnp-org:service:Layer3Forwarding:1"}, + {"/serviceId", "urn:upnp-org:serviceId:L3Forwarding1"}, + {"/SCPDURL", L3F_PATH}, + {"/controlURL", L3F_CONTROLURL}, /* The Layer3Forwarding service is only */ + {"/eventSubURL", L3F_EVENTURL}, /* recommended, not mandatory */ +#endif +#ifdef ENABLE_DP_SERVICE +/* InternetGatewayDevice v2 : + * it is RECOMMEDED that DeviceProtection service is implemented and applied. + * If DeviceProtection is not implemented and applied, it is RECOMMENDED + * that control points are able to access only actions and parameters defined + * as Public role. */ +/* 65 / 70 = SERVICES_OFFSET+7 */ + {"/serviceType", "urn:schemas-upnp-org:service:DeviceProtection:1"}, + {"/serviceId", "urn:upnp-org:serviceId:DeviceProtection1"}, + {"/SCPDURL", DP_PATH}, + {"/controlURL", DP_CONTROLURL}, + {"/eventSubURL", DP_EVENTURL}, +#endif + {0, 0} +}; + +/* WANIPCn.xml */ +/* see UPnP_IGD_WANIPConnection 1.0.pdf +static struct XMLElt scpdWANIPCn[] = +{ + {root_service, {INITHELPER(1,2)}}, + {0, {0}} +}; +*/ +static const struct argument AddPortMappingArgs[] = +{ + {1, 11}, /* RemoteHost */ + {1, 12}, /* ExternalPort */ + {1, 14}, /* PortMappingProtocol */ + {1, 13}, /* InternalPort */ + {1, 15}, /* InternalClient */ + {1, 9}, /* PortMappingEnabled */ + {1, 16}, /* PortMappingDescription */ + {1, 10}, /* PortMappingLeaseDuration */ + {0, 0} +}; + +#ifdef IGD_V2 +static const struct argument AddAnyPortMappingArgs[] = +{ + {1, 11}, /* RemoteHost */ + {1, 12}, /* ExternalPort */ + {1, 14}, /* PortMappingProtocol */ + {1, 13}, /* InternalPort */ + {1, 15}, /* InternalClient */ + {1, 9}, /* PortMappingEnabled */ + {1, 16}, /* PortMappingDescription */ + {1, 10}, /* PortMappingLeaseDuration */ + {2, 12}, /* NewReservedPort / ExternalPort */ + {0, 0} +}; + +static const struct argument DeletePortMappingRangeArgs[] = +{ + {1|(1<<2), 12}, /* NewStartPort / ExternalPort */ + {1|(2<<2), 12}, /* NewEndPort / ExternalPort */ + {1, 14}, /* NewProtocol / PortMappingProtocol */ + {1, 18}, /* NewManage / A_ARG_TYPE_Manage */ + {0, 0} +}; + +static const struct argument GetListOfPortMappingsArgs[] = +{ + {1|(1<<2), 12}, /* NewStartPort / ExternalPort */ + {1|(2<<2), 12}, /* NewEndPort / ExternalPort */ + {1, 14}, /* NewProtocol / PortMappingProtocol */ + {1, 18}, /* NewManage / A_ARG_TYPE_Manage */ + {1, 8}, /* NewNumberOfPorts / PortMappingNumberOfEntries */ + {2, 19}, /* NewPortListing / A_ARG_TYPE_PortListing */ + {0, 0} +}; +#endif + +static const struct argument GetExternalIPAddressArgs[] = +{ + {2, 7}, + {0, 0} +}; + +static const struct argument DeletePortMappingArgs[] = +{ + {1, 11}, /* RemoteHost */ + {1, 12}, /* ExternalPort */ + {1, 14}, /* PortMappingProtocol */ + {0, 0} +}; + +static const struct argument SetConnectionTypeArgs[] = +{ + {1, 0}, + {0, 0} +}; + +static const struct argument GetConnectionTypeInfoArgs[] = +{ + {2, 0}, + {2, 1}, + {0, 0} +}; + +static const struct argument GetStatusInfoArgs[] = +{ + {2, 2}, + {2, 4}, + {2, 3}, + {0, 0} +}; + +static const struct argument GetNATRSIPStatusArgs[] = +{ + {2, 5}, + {2, 6}, + {0, 0} +}; + +static const struct argument GetGenericPortMappingEntryArgs[] = +{ + {1, 8}, + {2, 11}, + {2, 12}, + {2, 14}, + {2, 13}, + {2, 15}, + {2, 9}, + {2, 16}, + {2, 10}, + {0, 0} +}; + +static const struct argument GetSpecificPortMappingEntryArgs[] = +{ + {1, 11}, + {1, 12}, + {1, 14}, + {2, 13}, + {2, 15}, + {2, 9}, + {2, 16}, + {2, 10}, + {0, 0} +}; + +static const struct action WANIPCnActions[] = +{ + {"SetConnectionType", SetConnectionTypeArgs}, /* R */ + {"GetConnectionTypeInfo", GetConnectionTypeInfoArgs}, /* R */ + {"RequestConnection", 0}, /* R */ + /*{"RequestTermination", 0},*/ /* O */ + {"ForceTermination", 0}, /* R */ + /*{"SetAutoDisconnectTime", 0},*/ /* O */ + /*{"SetIdleDisconnectTime", 0},*/ /* O */ + /*{"SetWarnDisconnectDelay", 0}, */ /* O */ + {"GetStatusInfo", GetStatusInfoArgs}, /* R */ + /*GetAutoDisconnectTime*/ /* O */ + /*GetIdleDisconnectTime*/ /* O */ + /*GetWarnDisconnectDelay*/ /* O */ + {"GetNATRSIPStatus", GetNATRSIPStatusArgs}, /* R */ + {"GetGenericPortMappingEntry", GetGenericPortMappingEntryArgs}, /* R */ + {"GetSpecificPortMappingEntry", GetSpecificPortMappingEntryArgs}, /* R */ + {"AddPortMapping", AddPortMappingArgs}, /* R */ + {"DeletePortMapping", DeletePortMappingArgs}, /* R */ + {"GetExternalIPAddress", GetExternalIPAddressArgs}, /* R */ +#ifdef IGD_V2 + {"DeletePortMappingRange", DeletePortMappingRangeArgs}, /* R, IGD v2 */ + {"GetListOfPortMappings", GetListOfPortMappingsArgs}, /* R, IGD v2 */ + {"AddAnyPortMapping", AddAnyPortMappingArgs}, /* R, IGD v2 */ +#endif +#if 0 + {"AddPortMapping", AddPortMappingArgs}, /* R */ + {"GetExternalIPAddress", GetExternalIPAddressArgs}, /* R */ + {"DeletePortMapping", DeletePortMappingArgs}, /* R */ + {"SetConnectionType", SetConnectionTypeArgs}, /* R */ + {"GetConnectionTypeInfo", GetConnectionTypeInfoArgs}, /* R */ + {"RequestConnection", 0}, /* R */ + {"ForceTermination", 0}, /* R */ + {"GetStatusInfo", GetStatusInfoArgs}, /* R */ + {"GetNATRSIPStatus", GetNATRSIPStatusArgs}, /* R */ + {"GetGenericPortMappingEntry", GetGenericPortMappingEntryArgs}, /* R */ + {"GetSpecificPortMappingEntry", GetSpecificPortMappingEntryArgs}, /* R */ +/* added in v2 UPnP-gw-WANIPConnection-v2-Service.pdf */ +#ifdef IGD_V2 + {"AddAnyPortMapping", AddAnyPortMappingArgs}, + {"DeletePortMappingRange", DeletePortMappingRangeArgs}, + {"GetListOfPortMappings", GetListOfPortMappingsArgs}, +#endif +#endif + {0, 0} +}; +/* R=Required, O=Optional */ + +/* ignore "warning: missing initializer" */ +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" + +/* struct stateVar : + * {name, itype(&event), idefault, iallowedlist, ieventvalue} */ +static const struct stateVar WANIPCnVars[] = +{ +/* 0 */ +#if 0 + {"ConnectionType", 0, 0/*1*/}, /* required */ + {"PossibleConnectionTypes", 0|0x80, 0, 14, 15}, +#elif 0 + {"ConnectionType", 0, 1, 14, 15}, /* required */ + {"PossibleConnectionTypes", 0|0x80, 0, 0, 15}, +#else + {"ConnectionType", 0, 1, 0, 15}, /* required */ + {"PossibleConnectionTypes", 0|0x80, 0, 14, 15}, +#endif + /* Required + * Allowed values : Unconfigured / IP_Routed / IP_Bridged */ + {"ConnectionStatus", 0|0x80, 3, 18, + CONNECTIONSTATUS_MAGICALVALUE }, /* required */ + /* Allowed Values : Unconfigured / Connecting(opt) / Connected + * PendingDisconnect(opt) / Disconnecting (opt) + * Disconnected */ + {"Uptime", 3, 0}, /* Required */ + {"LastConnectionError", 0, 6, 25}, /* required : */ + /* Allowed Values : ERROR_NONE(req) / ERROR_COMMAND_ABORTED(opt) + * ERROR_NOT_ENABLED_FOR_INTERNET(opt) + * ERROR_USER_DISCONNECT(opt) + * ERROR_ISP_DISCONNECT(opt) + * ERROR_IDLE_DISCONNECT(opt) + * ERROR_FORCED_DISCONNECT(opt) + * ERROR_NO_CARRIER(opt) + * ERROR_IP_CONFIGURATION(opt) + * ERROR_UNKNOWN(opt) */ + {"RSIPAvailable", 1, 4}, /* required */ + {"NATEnabled", 1, 5}, /* required */ + {"ExternalIPAddress", 0|0x80, 0, 0, + EXTERNALIPADDRESS_MAGICALVALUE}, /* required. Default : empty string */ + {"PortMappingNumberOfEntries", 2|0x80, 0, 0, + PORTMAPPINGNUMBEROFENTRIES_MAGICALVALUE}, /* required >= 0 */ + {"PortMappingEnabled", 1, 0}, /* Required */ +/* 10 */ + {"PortMappingLeaseDuration", 3, 2, 1}, /* required */ + {"RemoteHost", 0, 0}, /* required. Default : empty string */ + {"ExternalPort", 2, 0}, /* required */ + {"InternalPort", 2, 0, 3}, /* required */ + {"PortMappingProtocol", 0, 0, 11}, /* required allowedValues: TCP/UDP */ + {"InternalClient", 0, 0}, /* required */ + {"PortMappingDescription", 0, 0}, /* required default: empty string */ +/* added in v2 UPnP-gw-WANIPConnection-v2-Service.pdf */ +#ifdef IGD_V2 + {"SystemUpdateID", 3|0x80, 0, 0, SYSTEMUPDATEID_MAGICALVALUE}, + {"A_ARG_TYPE_Manage", 1, 0}, + {"A_ARG_TYPE_PortListing", 0, 0}, +#endif + {0, 0} +}; + +static const struct serviceDesc scpdWANIPCn = +{ WANIPCnActions, WANIPCnVars }; + +/* WANCfg.xml */ +/* See UPnP_IGD_WANCommonInterfaceConfig 1.0.pdf */ + +static const struct argument GetCommonLinkPropertiesArgs[] = +{ + {2, 0}, + {2, 1}, + {2, 2}, + {2, 3}, + {0, 0} +}; + +static const struct argument GetTotalBytesSentArgs[] = +{ + {2, 4}, + {0, 0} +}; + +static const struct argument GetTotalBytesReceivedArgs[] = +{ + {2, 5}, + {0, 0} +}; + +static const struct argument GetTotalPacketsSentArgs[] = +{ + {2, 6}, + {0, 0} +}; + +static const struct argument GetTotalPacketsReceivedArgs[] = +{ + {2, 7}, + {0, 0} +}; + +static const struct action WANCfgActions[] = +{ + {"GetCommonLinkProperties", GetCommonLinkPropertiesArgs}, /* Required */ + {"GetTotalBytesSent", GetTotalBytesSentArgs}, /* optional */ + {"GetTotalBytesReceived", GetTotalBytesReceivedArgs}, /* optional */ + {"GetTotalPacketsSent", GetTotalPacketsSentArgs}, /* optional */ + {"GetTotalPacketsReceived", GetTotalPacketsReceivedArgs}, /* optional */ + {0, 0} +}; + +/* See UPnP_IGD_WANCommonInterfaceConfig 1.0.pdf */ +static const struct stateVar WANCfgVars[] = +{ + {"WANAccessType", 0, 0, 1}, + /* Allowed Values : DSL / POTS / Cable / Ethernet + * Default value : empty string */ + {"Layer1UpstreamMaxBitRate", 3, 0}, + {"Layer1DownstreamMaxBitRate", 3, 0}, + {"PhysicalLinkStatus", 0|0x80, 0, 6, 6}, + /* allowed values : + * Up / Down / Initializing (optional) / Unavailable (optionnal) + * no Default value + * Evented */ + {"TotalBytesSent", 3, 0}, /* Optional */ + {"TotalBytesReceived", 3, 0}, /* Optional */ + {"TotalPacketsSent", 3, 0}, /* Optional */ + {"TotalPacketsReceived", 3, 0},/* Optional */ + /*{"MaximumActiveConnections", 2, 0}, // allowed Range value // OPTIONAL */ + /*{"WANAccessProvider", 0, 0},*/ /* Optional */ + {0, 0} +}; + +static const struct serviceDesc scpdWANCfg = +{ WANCfgActions, WANCfgVars }; + +#ifdef ENABLE_L3F_SERVICE +/* Read UPnP_IGD_Layer3Forwarding_1.0.pdf */ +static const struct argument SetDefaultConnectionServiceArgs[] = +{ + {1, 0}, /* in */ + {0, 0} +}; + +static const struct argument GetDefaultConnectionServiceArgs[] = +{ + {2, 0}, /* out */ + {0, 0} +}; + +static const struct action L3FActions[] = +{ + {"SetDefaultConnectionService", SetDefaultConnectionServiceArgs}, /* Req */ + {"GetDefaultConnectionService", GetDefaultConnectionServiceArgs}, /* Req */ + {0, 0} +}; + +static const struct stateVar L3FVars[] = +{ + {"DefaultConnectionService", 0|0x80, 0, 0, + DEFAULTCONNECTIONSERVICE_MAGICALVALUE}, /* Required */ + {0, 0} +}; + +static const struct serviceDesc scpdL3F = +{ L3FActions, L3FVars }; +#endif + +#ifdef ENABLE_6FC_SERVICE +/* see UPnP-gw-WANIPv6FirewallControl-v1-Service.pdf */ +static const struct argument GetFirewallStatusArgs[] = +{ + {2|0x80, 0}, /* OUT : FirewallEnabled */ + {2|0x80, 6}, /* OUT : InboundPinholeAllowed */ + {0, 0} +}; + +static const struct argument GetOutboundPinholeTimeoutArgs[] = +{ + {1|0x80|(3<<2), 1}, /* RemoteHost IN A_ARG_TYPE_IPv6Address */ + {1|0x80|(4<<2), 2}, /* RemotePort IN A_ARG_TYPE_Port */ + {1|0x80|(5<<2), 1}, /* InternalClient IN A_ARG_TYPE_IPv6Address */ + {1|0x80|(6<<2), 2}, /* InternalPort IN A_ARG_TYPE_Port */ + {1|0x80, 3}, /* Protocol IN A_ARG_TYPE_Protocol */ + {2|0x80, 7}, /* OutboundPinholeTimeout OUT A_ARG_TYPE_OutboundPinholeTimeout */ + {0, 0} +}; + +static const struct argument AddPinholeArgs[] = +{ + {1|0x80|(3<<2), 1}, /* RemoteHost IN A_ARG_TYPE_IPv6Address */ + {1|0x80|(4<<2), 2}, /* RemotePort IN A_ARG_TYPE_Port */ + {1|0x80|(5<<2), 1}, /* InternalClient IN A_ARG_TYPE_IPv6Address */ + {1|0x80|(6<<2), 2}, /* InternalPort IN A_ARG_TYPE_Port */ + {1|0x80, 3}, /* Protocol IN A_ARG_TYPE_Protocol */ + {1|0x80, 5}, /* LeaseTime IN A_ARG_TYPE_LeaseTime */ + {2|0x80, 4}, /* UniqueID OUT A_ARG_TYPE_UniqueID */ + {0, 0} +}; + +static const struct argument UpdatePinholeArgs[] = +{ + {1|0x80, 4}, /* UniqueID IN A_ARG_TYPE_UniqueID */ + {1, 5}, /* LeaseTime IN A_ARG_TYPE_LeaseTime */ + {0, 0} +}; + +static const struct argument DeletePinholeArgs[] = +{ + {1|0x80, 4}, /* UniqueID IN A_ARG_TYPE_UniqueID */ + {0, 0} +}; + +static const struct argument GetPinholePacketsArgs[] = +{ + {1|0x80, 4}, /* UniqueID IN A_ARG_TYPE_UniqueID */ + {2|0x80, 9}, /* PinholePackets OUT A_ARG_TYPE_PinholePackets */ + {0, 0} +}; + +static const struct argument CheckPinholeWorkingArgs[] = +{ + {1|0x80, 4}, /* UniqueID IN A_ARG_TYPE_UniqueID */ + {2|0x80|(7<<2), 8}, /* IsWorking OUT A_ARG_TYPE_Boolean */ + {0, 0} +}; + +static const struct action IPv6FCActions[] = +{ + {"GetFirewallStatus", GetFirewallStatusArgs}, /* Req */ + {"GetOutboundPinholeTimeout", GetOutboundPinholeTimeoutArgs}, /* Opt */ + {"AddPinhole", AddPinholeArgs}, /* Req */ + {"UpdatePinhole", UpdatePinholeArgs}, /* Req */ + {"DeletePinhole", DeletePinholeArgs}, /* Req */ + {"GetPinholePackets", GetPinholePacketsArgs}, /* Req */ + {"CheckPinholeWorking", CheckPinholeWorkingArgs}, /* Opt */ + {0, 0} +}; + +static const struct stateVar IPv6FCVars[] = +{ + {"FirewallEnabled", 1|0x80, 0, 0, + FIREWALLENABLED_MAGICALVALUE}, /* Required */ + {"A_ARG_TYPE_IPv6Address", 0, 0, 0, 0}, /* Required */ + {"A_ARG_TYPE_Port", 2, 0, 0, 0}, /* Required */ + {"A_ARG_TYPE_Protocol", 2, 0, 0, 0}, /* Required */ +/* 4 */ + {"A_ARG_TYPE_UniqueID", 2, 0, 0, 0}, /* Required */ + {"A_ARG_TYPE_LeaseTime", 3, 0, 5, 0}, /* Required */ + {"InboundPinholeAllowed", 1|0x80, 0, 0, + INBOUNDPINHOLEALLOWED_MAGICALVALUE}, /* Required */ + {"A_ARG_TYPE_OutboundPinholeTimeout", 3, 0, 7, 0}, /* Optional */ +/* 8 */ + {"A_ARG_TYPE_Boolean", 1, 0, 0, 0}, /* Optional */ + {"A_ARG_TYPE_PinholePackets", 3, 0, 0, 0}, /* Required */ + {0, 0} +}; + +static const struct serviceDesc scpd6FC = +{ IPv6FCActions, IPv6FCVars }; +#endif + +#ifdef ENABLE_DP_SERVICE +/* UPnP-gw-DeviceProtection-v1-Service.pdf */ + +static const struct argument SendSetupMessageArgs[] = +{ + {1|0x80|(8<<2), 6}, /* ProtocolType : in ProtocolType / A_ARG_TYPE_String */ + {1|0x80|(9<<2), 5}, /* InMessage : in InMessage / A_ARG_TYPE_Base64 */ + {2|0x80|(10<<2), 5}, /* OutMessage : out OutMessage / A_ARG_TYPE_Base64 */ + {0, 0} +}; + +static const struct argument GetSupportedProtocolsArgs[] = +{ + {2|0x80|(11<<2), 1}, /* ProtocolList : out ProtocolList / SupportedProtocols */ + {0, 0} +}; + +static const struct argument GetAssignedRolesArgs[] = +{ + {2|0x80|(12<<2), 6}, /* RoleList : out RoleList / A_ARG_TYPE_String */ + {0, 0} +}; + +static const struct action DPActions[] = +{ + {"SendSetupMessage", SendSetupMessageArgs}, + {"GetSupportedProtocols", GetSupportedProtocolsArgs}, + {"GetAssignedRoles", GetAssignedRolesArgs}, + {0, 0} +}; + +static const struct stateVar DPVars[] = +{ + {"SetupReady", 1|0x80, 0, 0, SETUPREADY_MAGICALVALUE}, + {"SupportedProtocols", 0}, + {"A_ARG_TYPE_ACL", 0}, + {"A_ARG_TYPE_IdentityList", 0}, + {"A_ARG_TYPE_Identity", 0}, + {"A_ARG_TYPE_Base64", 4}, + {"A_ARG_TYPE_String", 0}, + {0, 0} +}; + +static const struct serviceDesc scpdDP = +{ DPActions, DPVars }; +#endif + +/* strcat_str() + * concatenate the string and use realloc to increase the + * memory buffer if needed. */ +static char * +strcat_str(char * str, int * len, int * tmplen, const char * s2) +{ + int s2len; + int newlen; + char * p; + + s2len = (int)strlen(s2); + if(*tmplen <= (*len + s2len)) + { + if(s2len < 256) + newlen = *tmplen + 256; + else + newlen = *tmplen + s2len + 1; + p = (char *)realloc(str, newlen); + if(p == NULL) /* handle a failure of realloc() */ + return str; + str = p; + *tmplen = newlen; + } + /*strcpy(str + *len, s2); */ + memcpy(str + *len, s2, s2len + 1); + *len += s2len; + return str; +} + +/* strcat_char() : + * concatenate a character and use realloc to increase the + * size of the memory buffer if needed */ +static char * +strcat_char(char * str, int * len, int * tmplen, char c) +{ + char * p; + + if(*tmplen <= (*len + 1)) + { + *tmplen += 256; + p = (char *)realloc(str, *tmplen); + if(p == NULL) /* handle a failure of realloc() */ + { + *tmplen -= 256; + return str; + } + str = p; + } + str[*len] = c; + (*len)++; + return str; +} + +/* strcat_int() + * concatenate the string representation of the integer. + * call strcat_char() */ +static char * +strcat_int(char * str, int * len, int * tmplen, int i) +{ + char buf[16]; + int j; + + if(i < 0) { + str = strcat_char(str, len, tmplen, '-'); + i = -i; + } else if(i == 0) { + /* special case for 0 */ + str = strcat_char(str, len, tmplen, '0'); + return str; + } + j = 0; + while(i && j < (int)sizeof(buf)) { + buf[j++] = '0' + (i % 10); + i = i / 10; + } + while(j > 0) { + str = strcat_char(str, len, tmplen, buf[--j]); + } + return str; +} + +/* iterative subroutine using a small stack + * This way, the progam stack usage is kept low */ +static char * +genXML(char * str, int * len, int * tmplen, + const struct XMLElt * p) +{ +#define GENXML_STACK_SIZE 16 + unsigned short i, j; + unsigned long k; + int top; + const char * eltname, *s; + char c; + struct { + unsigned short i; + unsigned short j; + const char * eltname; + } pile[GENXML_STACK_SIZE]; /* stack */ + top = -1; + i = 0; /* current node */ + j = 1; /* i + number of nodes*/ + for(;;) + { + eltname = p[i].eltname; + if(!eltname) + return str; + if(eltname[0] == '/') + { + /* leaf node */ + if(p[i].data && p[i].data[0]) + { + /*printf("<%s>%s<%s>\n", eltname+1, p[i].data, eltname); */ + str = strcat_char(str, len, tmplen, '<'); + str = strcat_str(str, len, tmplen, eltname+1); + str = strcat_char(str, len, tmplen, '>'); +#ifdef RANDOMIZE_URLS + if(p[i].data[0] == '/') + { + /* prepend all URL paths with a "random" value */ + str = strcat_char(str, len, tmplen, '/'); + str = strcat_str(str, len, tmplen, random_url); + } +#endif /* RANDOMIZE_URLS */ + str = strcat_str(str, len, tmplen, p[i].data); +#ifdef IGD_V2 + /* checking a single 'u' saves us 4 strcmp() calls most of the time */ + if (GETFLAG(FORCEIGDDESCV1MASK) && (p[i].data[0] == 'u')) + { + if ((strcmp(p[i].data, DEVICE_TYPE_IGD) == 0) || + (strcmp(p[i].data, DEVICE_TYPE_WAN) == 0) || + (strcmp(p[i].data, DEVICE_TYPE_WANC) == 0) || + (strcmp(p[i].data, SERVICE_TYPE_WANIPC) == 0) ) + { + str[*len - 1] = '1'; /* Change the version number to 1 */ + } + } +#endif + str = strcat_char(str, len, tmplen, '<'); + str = strcat_str(str, len, tmplen, eltname); + str = strcat_char(str, len, tmplen, '>'); + } + for(;;) + { + if(top < 0) + return str; + i = ++(pile[top].i); + j = pile[top].j; + /*printf(" pile[%d]\t%d %d\n", top, i, j); */ + if(i==j) + { + /*printf("\n", pile[top].eltname); */ + str = strcat_char(str, len, tmplen, '<'); + str = strcat_char(str, len, tmplen, '/'); + s = pile[top].eltname; + for(c = *s; c > ' '; c = *(++s)) + str = strcat_char(str, len, tmplen, c); + str = strcat_char(str, len, tmplen, '>'); + top--; + } + else + break; + } + } + else + { + /* node with child(ren) */ + /*printf("<%s>\n", eltname); */ + str = strcat_char(str, len, tmplen, '<'); + str = strcat_str(str, len, tmplen, eltname); + if(memcmp(eltname, "root ", 5) == 0) { + char configid_str[16]; + /* add configId attribute, required by UDA 1.1 */ + snprintf(configid_str, sizeof(configid_str), "\"%u\"", upnp_configid); + str = strcat_str(str, len, tmplen, " configId="); + str = strcat_str(str, len, tmplen, configid_str); + } + str = strcat_char(str, len, tmplen, '>'); + k = (unsigned long)p[i].data; + i = k & 0xffff; + j = i + (k >> 16); + if(top < (GENXML_STACK_SIZE - 1)) { + top++; + /*printf(" +pile[%d]\t%d %d\n", top, i, j); */ + pile[top].i = i; + pile[top].j = j; + pile[top].eltname = eltname; +#ifdef DEBUG + } else { + fprintf(stderr, "*** GenXML(): stack OVERFLOW ***\n"); +#endif /* DEBUG */ + } + } + } +} + +/* genRootDesc() : + * - Generate the root description of the UPnP device. + * - the len argument is used to return the length of + * the returned string. + * - tmp_uuid argument is used to build the uuid string */ +char * +genRootDesc(int * len) +{ + char * str; + int tmplen; + tmplen = 2048; + str = (char *)malloc(tmplen); + if(str == NULL) + return NULL; + * len = strlen(xmlver); + /*strcpy(str, xmlver); */ + memcpy(str, xmlver, *len + 1); + str = genXML(str, len, &tmplen, rootDesc); + str[*len] = '\0'; + return str; +} + +/* genServiceDesc() : + * Generate service description with allowed methods and + * related variables. */ +static char * +genServiceDesc(int * len, const struct serviceDesc * s) +{ + int i, j; + const struct action * acts; + const struct stateVar * vars; + const struct argument * args; + const char * p; + char * str; + int tmplen; + tmplen = 2048; + str = (char *)malloc(tmplen); + if(str == NULL) + return NULL; + /*strcpy(str, xmlver); */ + *len = strlen(xmlver); + memcpy(str, xmlver, *len + 1); + + acts = s->actionList; + vars = s->serviceStateTable; + + str = strcat_char(str, len, &tmplen, '<'); + str = strcat_str(str, len, &tmplen, root_service); + str = strcat_char(str, len, &tmplen, '>'); + + str = strcat_str(str, len, &tmplen, + "" UPNP_VERSION_MAJOR_STR "" + "" UPNP_VERSION_MINOR_STR ""); + + i = 0; + str = strcat_str(str, len, &tmplen, ""); + while(acts[i].name) + { + str = strcat_str(str, len, &tmplen, ""); + str = strcat_str(str, len, &tmplen, acts[i].name); + str = strcat_str(str, len, &tmplen, ""); + /* argument List */ + args = acts[i].args; + if(args) + { + str = strcat_str(str, len, &tmplen, ""); + j = 0; + while(args[j].dir) + { + str = strcat_str(str, len, &tmplen, ""); + if((args[j].dir & 0x80) == 0) { + str = strcat_str(str, len, &tmplen, "New"); + } + p = vars[args[j].relatedVar].name; + if(args[j].dir & 0x7c) { + /* use magic values ... */ + str = strcat_str(str, len, &tmplen, magicargname[(args[j].dir & 0x7c) >> 2]); + } else if(0 == memcmp(p, "PortMapping", 11) + && 0 != memcmp(p + 11, "Description", 11)) { + if(0 == memcmp(p + 11, "NumberOfEntries", 15)) { + /* PortMappingNumberOfEntries */ +#ifdef IGD_V2 + if(0 == memcmp(acts[i].name, "GetListOfPortMappings", 22)) { + str = strcat_str(str, len, &tmplen, "NumberOfPorts"); + } else { + str = strcat_str(str, len, &tmplen, "PortMappingIndex"); + } +#else + str = strcat_str(str, len, &tmplen, "PortMappingIndex"); +#endif + } else { + /* PortMappingEnabled + * PortMappingLeaseDuration + * PortMappingProtocol */ + str = strcat_str(str, len, &tmplen, p + 11); + } +#ifdef IGD_V2 + } else if(0 == memcmp(p, "A_ARG_TYPE_", 11)) { + str = strcat_str(str, len, &tmplen, p + 11); + } else if(0 == memcmp(p, "ExternalPort", 13) + && args[j].dir == 2 + && 0 == memcmp(acts[i].name, "AddAnyPortMapping", 18)) { + str = strcat_str(str, len, &tmplen, "ReservedPort"); +#endif + } else { + str = strcat_str(str, len, &tmplen, p); + } + str = strcat_str(str, len, &tmplen, ""); + str = strcat_str(str, len, &tmplen, (args[j].dir&0x03)==1?"in":"out"); + str = strcat_str(str, len, &tmplen, + ""); + str = strcat_str(str, len, &tmplen, p); + str = strcat_str(str, len, &tmplen, + ""); + j++; + } + str = strcat_str(str, len, &tmplen,""); + } + str = strcat_str(str, len, &tmplen, ""); + /*str = strcat_char(str, len, &tmplen, '\n'); // TEMP ! */ + i++; + } + str = strcat_str(str, len, &tmplen, ""); + i = 0; + while(vars[i].name) + { + str = strcat_str(str, len, &tmplen, + ""); + str = strcat_str(str, len, &tmplen, vars[i].name); + str = strcat_str(str, len, &tmplen, ""); + str = strcat_str(str, len, &tmplen, upnptypes[vars[i].itype & 0x0f]); + str = strcat_str(str, len, &tmplen, ""); + /*if(vars[i].defaultValue) */ + if(vars[i].idefault) + { + str = strcat_str(str, len, &tmplen, ""); + /*str = strcat_str(str, len, &tmplen, vars[i].defaultValue); */ + str = strcat_str(str, len, &tmplen, upnpdefaultvalues[vars[i].idefault]); + str = strcat_str(str, len, &tmplen, ""); + } + if(vars[i].iallowedlist) + { + if((vars[i].itype & 0x0f) == 0) + { + /* string */ + str = strcat_str(str, len, &tmplen, ""); + for(j=vars[i].iallowedlist; upnpallowedvalues[j]; j++) + { + str = strcat_str(str, len, &tmplen, ""); + str = strcat_str(str, len, &tmplen, upnpallowedvalues[j]); + str = strcat_str(str, len, &tmplen, ""); + } + str = strcat_str(str, len, &tmplen, ""); + } else { + /* ui2 and ui4 */ + str = strcat_str(str, len, &tmplen, ""); + str = strcat_int(str, len, &tmplen, upnpallowedranges[vars[i].iallowedlist]); + str = strcat_str(str, len, &tmplen, ""); + str = strcat_int(str, len, &tmplen, upnpallowedranges[vars[i].iallowedlist+1]); + str = strcat_str(str, len, &tmplen, ""); + } + } + str = strcat_str(str, len, &tmplen, ""); + /*str = strcat_char(str, len, &tmplen, '\n'); // TEMP ! */ + i++; + } + str = strcat_str(str, len, &tmplen, ""); + str[*len] = '\0'; + return str; +} + +/* genWANIPCn() : + * Generate the WANIPConnection xml description */ +char * +genWANIPCn(int * len) +{ + return genServiceDesc(len, &scpdWANIPCn); +} + +/* genWANCfg() : + * Generate the WANInterfaceConfig xml description. */ +char * +genWANCfg(int * len) +{ + return genServiceDesc(len, &scpdWANCfg); +} + +#ifdef ENABLE_L3F_SERVICE +char * +genL3F(int * len) +{ + return genServiceDesc(len, &scpdL3F); +} +#endif + +#ifdef ENABLE_6FC_SERVICE +char * +gen6FC(int * len) +{ + return genServiceDesc(len, &scpd6FC); +} +#endif + +#ifdef ENABLE_DP_SERVICE +char * +genDP(int * len) +{ + return genServiceDesc(len, &scpdDP); +} +#endif + +#ifdef ENABLE_EVENTS +static char * +genEventVars(int * len, const struct serviceDesc * s) +{ + char tmp[16]; + const struct stateVar * v; + char * str; + int tmplen; + tmplen = 512; + str = (char *)malloc(tmplen); + if(str == NULL) + return NULL; + *len = 0; + v = s->serviceStateTable; + str = strcat_str(str, len, &tmplen, ""); + while(v->name) { + if(v->itype & 0x80) { + str = strcat_str(str, len, &tmplen, "<"); + str = strcat_str(str, len, &tmplen, v->name); + str = strcat_str(str, len, &tmplen, ">"); + /*printf("<%s>", v->name);*/ + switch(v->ieventvalue) { + case 0: + break; +#ifdef ENABLE_DP_SERVICE + case SETUPREADY_MAGICALVALUE: + /* always ready for setup */ + snprintf(tmp, sizeof(tmp), "%d", 1); + str = strcat_str(str, len, &tmplen, tmp); + break; +#endif + case CONNECTIONSTATUS_MAGICALVALUE: + /* or get_wan_connection_status_str(ext_if_name) */ + str = strcat_str(str, len, &tmplen, + upnpallowedvalues[18 + get_wan_connection_status(ext_if_name)]); + break; +#ifdef ENABLE_6FC_SERVICE + case FIREWALLENABLED_MAGICALVALUE: + /* see 2.4.2 of UPnP-gw-WANIPv6FirewallControl-v1-Service.pdf */ + snprintf(tmp, sizeof(tmp), "%d", + GETFLAG(IPV6FCFWDISABLEDMASK) ? 0 : 1); + str = strcat_str(str, len, &tmplen, tmp); + break; + case INBOUNDPINHOLEALLOWED_MAGICALVALUE: + /* see 2.4.3 of UPnP-gw-WANIPv6FirewallControl-v1-Service.pdf */ + snprintf(tmp, sizeof(tmp), "%d", + GETFLAG(IPV6FCINBOUNDDISALLOWEDMASK) ? 0 : 1); + str = strcat_str(str, len, &tmplen, tmp); + break; +#endif +#ifdef IGD_V2 + case SYSTEMUPDATEID_MAGICALVALUE: + /* Please read section 2.3.23 SystemUpdateID + * of UPnP-gw-WANIPConnection-v2-Service.pdf */ + snprintf(tmp, sizeof(tmp), "%d", + 1/* system update id */); + str = strcat_str(str, len, &tmplen, tmp); + break; +#endif + case PORTMAPPINGNUMBEROFENTRIES_MAGICALVALUE: + /* Port mapping number of entries magical value */ + snprintf(tmp, sizeof(tmp), "%d", + upnp_get_portmapping_number_of_entries()); + str = strcat_str(str, len, &tmplen, tmp); + break; + case EXTERNALIPADDRESS_MAGICALVALUE: + /* External ip address magical value */ + if(use_ext_ip_addr) + str = strcat_str(str, len, &tmplen, use_ext_ip_addr); + else { + char ext_ip_addr[INET_ADDRSTRLEN]; + if(getifaddr(ext_if_name, ext_ip_addr, INET_ADDRSTRLEN, NULL, NULL) < 0) { + str = strcat_str(str, len, &tmplen, "0.0.0.0"); + } else { + str = strcat_str(str, len, &tmplen, ext_ip_addr); + } + } + break; + case DEFAULTCONNECTIONSERVICE_MAGICALVALUE: + /* DefaultConnectionService magical value */ + str = strcat_str(str, len, &tmplen, uuidvalue_wcd); +#ifdef IGD_V2 + str = strcat_str(str, len, &tmplen, ":WANConnectionDevice:2,urn:upnp-org:serviceId:WANIPConn1"); +#else + str = strcat_str(str, len, &tmplen, ":WANConnectionDevice:1,urn:upnp-org:serviceId:WANIPConn1"); +#endif + break; + default: + str = strcat_str(str, len, &tmplen, upnpallowedvalues[v->ieventvalue]); + } + str = strcat_str(str, len, &tmplen, "name); + str = strcat_str(str, len, &tmplen, ">"); + /*printf("\n", v->name);*/ + } + v++; + } + str = strcat_str(str, len, &tmplen, ""); +#if 0 + printf("\n"); + printf("\n"); + printf("%d\n", tmplen); +#endif + str[*len] = '\0'; + return str; +} + +char * +getVarsWANIPCn(int * l) +{ + return genEventVars(l, + &scpdWANIPCn); +} + +char * +getVarsWANCfg(int * l) +{ + return genEventVars(l, + &scpdWANCfg); +} + +#ifdef ENABLE_L3F_SERVICE +char * +getVarsL3F(int * l) +{ + return genEventVars(l, + &scpdL3F); +} +#endif + +#ifdef ENABLE_6FC_SERVICE +char * +getVars6FC(int * l) +{ + return genEventVars(l, + &scpd6FC); +} +#endif + +#ifdef ENABLE_DP_SERVICE +char * +getVarsDP(int * l) +{ + return genEventVars(l, + &scpdDP); +} +#endif + +#endif /* ENABLE_EVENTS */ diff --git a/src/contrib/miniupnp/miniupnpd/upnpdescgen.h b/src/contrib/miniupnp/miniupnpd/upnpdescgen.h new file mode 100644 index 0000000..deaed69 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpdescgen.h @@ -0,0 +1,102 @@ +/* $Id: upnpdescgen.h,v 1.22 2011/05/18 22:22:24 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2011 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPDESCGEN_H_INCLUDED +#define UPNPDESCGEN_H_INCLUDED + +#include "config.h" + +/* for the root description + * The child list reference is stored in "data" member using the + * INITHELPER macro with index/nchild always in the + * same order, whatever the endianness */ +struct XMLElt { + const char * eltname; /* begin with '/' if no child */ + const char * data; /* Value */ +}; + +/* for service description */ +struct serviceDesc { + const struct action * actionList; + const struct stateVar * serviceStateTable; +}; + +struct action { + const char * name; + const struct argument * args; +}; + +struct argument { /* the name of the arg is obtained from the variable */ + unsigned char dir; /* MSB : don't append "New" Flag, + * 5 Medium bits : magic argument name index + * 2 LSB : 1 = in, 2 = out */ + unsigned char relatedVar; /* index of the related variable */ +}; + +struct stateVar { + const char * name; + unsigned char itype; /* MSB: sendEvent flag, 7 LSB: index in upnptypes */ + unsigned char idefault; /* default value */ + unsigned char iallowedlist; /* index in allowed values list + * or in allowed range list */ + unsigned char ieventvalue; /* fixed value returned or magical values */ +}; + +/* little endian + * The code has now be tested on big endian architecture */ +#define INITHELPER(i, n) ((char *)(((n)<<16)|(i))) + +/* char * genRootDesc(int *); + * returns: NULL on error, string allocated on the heap */ +char * +genRootDesc(int * len); + +/* for the two following functions */ +char * +genWANIPCn(int * len); + +char * +genWANCfg(int * len); + +#ifdef ENABLE_L3F_SERVICE +char * +genL3F(int * len); +#endif + +#ifdef ENABLE_6FC_SERVICE +char * +gen6FC(int * len); +#endif + +#ifdef ENABLE_DP_SERVICE +char * +genDP(int * len); +#endif + +#ifdef ENABLE_EVENTS +char * +getVarsWANIPCn(int * len); + +char * +getVarsWANCfg(int * len); + +#ifdef ENABLE_L3F_SERVICE +char * +getVarsL3F(int * len); +#endif +#ifdef ENABLE_6FC_SERVICE +char * +getVars6FC(int * len); +#endif +#ifdef ENABLE_DP_SERVICE +char * +getVarsDP(int * len); +#endif +#endif /* ENABLE_EVENTS */ + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnpdescstrings.h b/src/contrib/miniupnp/miniupnpd/upnpdescstrings.h new file mode 100644 index 0000000..4fd800e --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpdescstrings.h @@ -0,0 +1,41 @@ +/* $Id: upnpdescstrings.h,v 1.8 2012/09/27 16:00:10 nanard Exp $ */ +/* miniupnp project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the coditions detailed in + * the LICENCE file provided within the distribution */ +#ifndef UPNPDESCSTRINGS_H_INCLUDED +#define UPNPDESCSTRINGS_H_INCLUDED + +#include "config.h" + +/* strings used in the root device xml description */ +#define ROOTDEV_FRIENDLYNAME OS_NAME " router" +#define ROOTDEV_MANUFACTURER OS_NAME +#define ROOTDEV_MANUFACTURERURL OS_URL +#define ROOTDEV_MODELNAME OS_NAME " router" +#define ROOTDEV_MODELDESCRIPTION OS_NAME " router" +#define ROOTDEV_MODELURL OS_URL + +#define WANDEV_FRIENDLYNAME "WANDevice" +#define WANDEV_MANUFACTURER "MiniUPnP" +#define WANDEV_MANUFACTURERURL "http://miniupnp.free.fr/" +#define WANDEV_MODELNAME "WAN Device" +#define WANDEV_MODELDESCRIPTION "WAN Device" +#define WANDEV_MODELNUMBER MINIUPNPD_DATE +#define WANDEV_MODELURL "http://miniupnp.free.fr/" +#define WANDEV_UPC "000000000000" +/* UPC is 12 digit (barcode) */ + +#define WANCDEV_FRIENDLYNAME "WANConnectionDevice" +#define WANCDEV_MANUFACTURER WANDEV_MANUFACTURER +#define WANCDEV_MANUFACTURERURL WANDEV_MANUFACTURERURL +#define WANCDEV_MODELNAME "MiniUPnPd" +#define WANCDEV_MODELDESCRIPTION "MiniUPnP daemon" +#define WANCDEV_MODELNUMBER MINIUPNPD_DATE +#define WANCDEV_MODELURL "http://miniupnp.free.fr/" +#define WANCDEV_UPC "000000000000" +/* UPC is 12 digit (barcode) */ + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnpevents.c b/src/contrib/miniupnp/miniupnpd/upnpevents.c new file mode 100644 index 0000000..d96bccb --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpevents.c @@ -0,0 +1,670 @@ +/* $Id: upnpevents.c,v 1.39 2018/03/12 22:41:54 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2008-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "config.h" +#if defined(LIB_UUID) +/* as found on linux */ +#include +#elif defined(BSD_UUID) +#include +#endif /* LIB_UUID / BSD_UUID */ +#include "upnpevents.h" +#include "miniupnpdpath.h" +#include "upnpglobalvars.h" +#include "upnpdescgen.h" +#include "upnputils.h" + +#ifdef ENABLE_EVENTS +/*enum subscriber_service_enum { + EWanCFG = 1, + EWanIPC, + EL3F +};*/ + +/* stuctures definitions */ +struct subscriber { + LIST_ENTRY(subscriber) entries; + struct upnp_event_notify * notify; + time_t timeout; + uint32_t seq; + enum subscriber_service_enum service; + char uuid[42]; + char callback[]; +}; + +struct upnp_event_notify { + LIST_ENTRY(upnp_event_notify) entries; + int s; /* socket */ + enum { ECreated=1, + EConnecting, + ESending, + EWaitingForResponse, + EFinished, + EError } state; + struct subscriber * sub; + char * buffer; + int buffersize; + int tosend; + int sent; + const char * path; +#ifdef ENABLE_IPV6 + int ipv6; + char addrstr[48]; +#else + char addrstr[16]; +#endif + char portstr[8]; +}; + +/* prototypes */ +static void +upnp_event_create_notify(struct subscriber * sub); + +/* Subscriber list */ +LIST_HEAD(listhead, subscriber) subscriberlist = { NULL }; + +/* notify list */ +LIST_HEAD(listheadnotif, upnp_event_notify) notifylist = { NULL }; + +/* create a new subscriber */ +static struct subscriber * +newSubscriber(const char * eventurl, const char * callback, int callbacklen) +{ + struct subscriber * tmp; + if(!eventurl || !callback || !callbacklen) + return NULL; + tmp = calloc(1, sizeof(struct subscriber)+callbacklen+1); + if(!tmp) + return NULL; + if(strcmp(eventurl, WANCFG_EVENTURL)==0) + tmp->service = EWanCFG; + else if(strcmp(eventurl, WANIPC_EVENTURL)==0) + tmp->service = EWanIPC; +#ifdef ENABLE_L3F_SERVICE + else if(strcmp(eventurl, L3F_EVENTURL)==0) + tmp->service = EL3F; +#endif +#ifdef ENABLE_6FC_SERVICE + else if(strcmp(eventurl, WANIP6FC_EVENTURL)==0) + tmp->service = E6FC; +#endif +#ifdef ENABLE_DP_SERVICE + else if(strcmp(eventurl, DP_EVENTURL)==0) + tmp->service = EDP; +#endif + else { + free(tmp); + return NULL; + } + memcpy(tmp->callback, callback, callbacklen); + tmp->callback[callbacklen] = '\0'; +#if defined(LIB_UUID) + { + uuid_t uuid; + uuid_generate(uuid); + memcpy(tmp->uuid, "uuid:", 5); + uuid_unparse(uuid, tmp->uuid + 5); + } +#elif defined(BSD_UUID) + { + uuid_t uuid; + uint32_t status; + uuid_create(&uuid, &status); + if(status != uuid_s_ok) { + syslog(LOG_ERR, "uuid_create() failed (%u)", status); + } else { + char * uuid_str; + uuid_to_string(&uuid, &uuid_str, &status); + if(status != uuid_s_ok) { + syslog(LOG_ERR, "uuid_to_string() failed (%u)", status); + } else { + if(strlen(uuid_str) != 36) { + syslog(LOG_ERR, "uuid_to_string() returned %s", uuid_str); + status = (uint32_t)-1; + } else { + memcpy(tmp->uuid, "uuid:", 5); + memcpy(tmp->uuid + 5, uuid_str, 36); + tmp->uuid[sizeof(tmp->uuid)-1] = '\0'; + } + free(uuid_str); + } + } + if(status != uuid_s_ok) { + /* make a dummy uuid */ + strncpy(tmp->uuid, uuidvalue_igd, sizeof(tmp->uuid)); + tmp->uuid[sizeof(tmp->uuid)-1] = '\0'; + snprintf(tmp->uuid+sizeof(tmp->uuid)-5, 5, "%04lx", random() & 0xffff); + } + } +#else + /* make a dummy uuid */ + strncpy(tmp->uuid, uuidvalue_igd, sizeof(tmp->uuid)); + tmp->uuid[sizeof(tmp->uuid)-1] = '\0'; + snprintf(tmp->uuid+sizeof(tmp->uuid)-5, 5, "%04lx", random() & 0xffff); +#endif + return tmp; +} + +/* creates a new subscriber and adds it to the subscriber list + * also initiate 1st notify + * TODO : add a check on the number of subscriber in order to + * prevent memory overflow... */ +const char * +upnpevents_addSubscriber(const char * eventurl, + const char * callback, int callbacklen, + int timeout) +{ + struct subscriber * tmp; + /*static char uuid[42];*/ + /* "uuid:00000000-0000-0000-0000-000000000000"; 5+36+1=42bytes */ + syslog(LOG_DEBUG, "addSubscriber(%s, %.*s, %d)", + eventurl, callbacklen, callback, timeout); + /*strncpy(uuid, uuidvalue, sizeof(uuid)); + uuid[sizeof(uuid)-1] = '\0';*/ + tmp = newSubscriber(eventurl, callback, callbacklen); + if(!tmp) + return NULL; + if(timeout) + tmp->timeout = upnp_time() + timeout; + LIST_INSERT_HEAD(&subscriberlist, tmp, entries); + upnp_event_create_notify(tmp); + return tmp->uuid; +} + +/* renew a subscription (update the timeout) */ +const char * +upnpevents_renewSubscription(const char * sid, int sidlen, int timeout) +{ + struct subscriber * sub; + for(sub = subscriberlist.lh_first; sub != NULL; sub = sub->entries.le_next) { + if((sidlen == 41) && (memcmp(sid, sub->uuid, 41) == 0)) { +#ifdef UPNP_STRICT + /* check if the subscription already timeouted */ + if(sub->timeout && upnp_time() > sub->timeout) + continue; +#endif + sub->timeout = (timeout ? upnp_time() + timeout : 0); + return sub->uuid; + } + } + return NULL; +} + +int +upnpevents_removeSubscriber(const char * sid, int sidlen) +{ + struct subscriber * sub; + if(!sid) + return -1; + for(sub = subscriberlist.lh_first; sub != NULL; sub = sub->entries.le_next) { + if((sidlen == 41) && (memcmp(sid, sub->uuid, 41) == 0)) { + if(sub->notify) { + sub->notify->sub = NULL; + } + LIST_REMOVE(sub, entries); + free(sub); + return 0; + } + } + return -1; +} + +/* notifies all subscriber of a number of port mapping change + * or external ip address change */ +void +upnp_event_var_change_notify(enum subscriber_service_enum service) +{ + struct subscriber * sub; + for(sub = subscriberlist.lh_first; sub != NULL; sub = sub->entries.le_next) { + if(sub->service == service && sub->notify == NULL) + upnp_event_create_notify(sub); + } +} + +/* create and add the notify object to the list */ +static void +upnp_event_create_notify(struct subscriber * sub) +{ + struct upnp_event_notify * obj; + /*struct timeval sock_timeout;*/ + + obj = calloc(1, sizeof(struct upnp_event_notify)); + if(!obj) { + syslog(LOG_ERR, "%s: calloc(): %m", "upnp_event_create_notify"); + return; + } + obj->sub = sub; + obj->state = ECreated; +#ifdef ENABLE_IPV6 + obj->s = socket((obj->sub->callback[7] == '[') ? PF_INET6 : PF_INET, + SOCK_STREAM, 0); +#else + obj->s = socket(PF_INET, SOCK_STREAM, 0); +#endif + if(obj->s<0) { + syslog(LOG_ERR, "%s: socket(): %m", "upnp_event_create_notify"); + goto error; + } +#if 0 /* does not work for non blocking connect() */ + /* set timeout to 3 seconds */ + sock_timeout.tv_sec = 3; + sock_timeout.tv_usec = 0; + if(setsockopt(obj->s, SOL_SOCKET, SO_RCVTIMEO, &sock_timeout, sizeof(struct timeval)) < 0) { + syslog(LOG_WARNING, "%s: setsockopt(SO_RCVTIMEO): %m", + "upnp_event_create_notify"); + } + sock_timeout.tv_sec = 3; + sock_timeout.tv_usec = 0; + if(setsockopt(obj->s, SOL_SOCKET, SO_SNDTIMEO, &sock_timeout, sizeof(struct timeval)) < 0) { + syslog(LOG_WARNING, "%s: setsockopt(SO_SNDTIMEO): %m", + "upnp_event_create_notify"); + } +#endif + /* set socket non blocking */ + if(!set_non_blocking(obj->s)) { + syslog(LOG_ERR, "%s: set_non_blocking(): %m", + "upnp_event_create_notify"); + goto error; + } + if(sub) + sub->notify = obj; + LIST_INSERT_HEAD(¬ifylist, obj, entries); + return; +error: + if(obj->s >= 0) + close(obj->s); + free(obj); +} + +static void +upnp_event_notify_connect(struct upnp_event_notify * obj) +{ + unsigned int i; + const char * p; + unsigned short port; +#ifdef ENABLE_IPV6 + struct sockaddr_storage addr; + socklen_t addrlen; +#else + struct sockaddr_in addr; + socklen_t addrlen; +#endif + + if(!obj) + return; + memset(&addr, 0, sizeof(addr)); + i = 0; + if(obj->sub == NULL) { + obj->state = EError; + return; + } + p = obj->sub->callback; + p += 7; /* http:// */ +#ifdef ENABLE_IPV6 + if(*p == '[') { /* ip v6 */ + obj->addrstr[i++] = '['; + p++; + obj->ipv6 = 1; + while(*p != ']' && i < (sizeof(obj->addrstr)-1)) + obj->addrstr[i++] = *(p++); + if(*p == ']') + p++; + if(i < (sizeof(obj->addrstr)-1)) + obj->addrstr[i++] = ']'; + } else { +#endif + while(*p != '/' && *p != ':' && i < (sizeof(obj->addrstr)-1)) + obj->addrstr[i++] = *(p++); +#ifdef ENABLE_IPV6 + } +#endif + obj->addrstr[i] = '\0'; + if(*p == ':') { + obj->portstr[0] = *p; + i = 1; + p++; + port = (unsigned short)atoi(p); + while(*p != '/') { + if(i<7) obj->portstr[i++] = *p; + p++; + } + obj->portstr[i] = 0; + } else { + port = 80; + obj->portstr[0] = '\0'; + } + obj->path = p; +#ifdef ENABLE_IPV6 + if(obj->ipv6) { + char addrstr_tmp[48]; + struct sockaddr_in6 * sa = (struct sockaddr_in6 *)&addr; + sa->sin6_family = AF_INET6; + i = (int)strlen(obj->addrstr); + if(i > 2) { + i -= 2; + memcpy(addrstr_tmp, obj->addrstr + 1, i); + addrstr_tmp[i] = '\0'; + inet_pton(AF_INET6, addrstr_tmp, &(sa->sin6_addr)); + } + sa->sin6_port = htons(port); + addrlen = sizeof(struct sockaddr_in6); + } else { + struct sockaddr_in * sa = (struct sockaddr_in *)&addr; + sa->sin_family = AF_INET; + inet_pton(AF_INET, obj->addrstr, &(sa->sin_addr)); + sa->sin_port = htons(port); + addrlen = sizeof(struct sockaddr_in); + } +#else + addr.sin_family = AF_INET; + inet_aton(obj->addrstr, &addr.sin_addr); + addr.sin_port = htons(port); + addrlen = sizeof(struct sockaddr_in); +#endif + syslog(LOG_DEBUG, "%s: '%s' %hu '%s'", "upnp_event_notify_connect", + obj->addrstr, port, obj->path); + obj->state = EConnecting; + if(connect(obj->s, (struct sockaddr *)&addr, addrlen) < 0) { + if(errno != EINPROGRESS && errno != EWOULDBLOCK) { + syslog(LOG_ERR, "%s: connect(%d, %s, %u): %m", + "upnp_event_notify_connect", obj->s, + obj->addrstr, addrlen); + obj->state = EError; + } + } +} + +static void upnp_event_prepare(struct upnp_event_notify * obj) +{ + static const char notifymsg[] = + "NOTIFY %s HTTP/1.1\r\n" + "Host: %s%s\r\n" +#if (UPNP_VERSION_MAJOR == 1) && (UPNP_VERSION_MINOR == 0) + "Content-Type: text/xml\r\n" /* UDA v1.0 */ +#else + "Content-Type: text/xml; charset=\"utf-8\"\r\n" /* UDA v1.1 or later */ +#endif + "Content-Length: %d\r\n" + "NT: upnp:event\r\n" + "NTS: upnp:propchange\r\n" + "SID: %s\r\n" + "SEQ: %u\r\n" + "Connection: close\r\n" + "Cache-Control: no-cache\r\n" + "\r\n" + "%.*s\r\n"; + char * xml; + int l; + if(obj->sub == NULL) { + obj->state = EError; + return; + } + switch(obj->sub->service) { + case EWanCFG: + xml = getVarsWANCfg(&l); + break; + case EWanIPC: + xml = getVarsWANIPCn(&l); + break; +#ifdef ENABLE_L3F_SERVICE + case EL3F: + xml = getVarsL3F(&l); + break; +#endif +#ifdef ENABLE_6FC_SERVICE + case E6FC: + xml = getVars6FC(&l); + break; +#endif +#ifdef ENABLE_DP_SERVICE + case EDP: + xml = getVarsDP(&l); + break; +#endif + default: + xml = NULL; + l = 0; + } + obj->buffersize = 1024; + obj->buffer = malloc(obj->buffersize); + if(!obj->buffer) { + syslog(LOG_ERR, "%s: malloc returned NULL", "upnp_event_prepare"); + if(xml) { + free(xml); + } + obj->state = EError; + return; + } + obj->tosend = snprintf(obj->buffer, obj->buffersize, notifymsg, + obj->path, obj->addrstr, obj->portstr, l+2, + obj->sub->uuid, obj->sub->seq, + l, xml); + if(xml) { + free(xml); + xml = NULL; + } + obj->state = ESending; +} + +static void upnp_event_send(struct upnp_event_notify * obj) +{ + int i; + + syslog(LOG_DEBUG, "%s: sending event notify message to %s%s", + "upnp_event_send", obj->addrstr, obj->portstr); + syslog(LOG_DEBUG, "%s: msg: %s", + "upnp_event_send", obj->buffer + obj->sent); + i = send(obj->s, obj->buffer + obj->sent, obj->tosend - obj->sent, 0); + if(i<0) { + if(errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) { + syslog(LOG_NOTICE, "%s: send(%s%s): %m", "upnp_event_send", + obj->addrstr, obj->portstr); + obj->state = EError; + return; + } else { + /* EAGAIN or EWOULDBLOCK or EINTR : no data sent */ + i = 0; + } + } + if(i != (obj->tosend - obj->sent)) + syslog(LOG_NOTICE, "%s: %d bytes send out of %d", + "upnp_event_send", i, obj->tosend - obj->sent); + obj->sent += i; + if(obj->sent == obj->tosend) + obj->state = EWaitingForResponse; +} + +static void upnp_event_recv(struct upnp_event_notify * obj) +{ + int n; + n = recv(obj->s, obj->buffer, obj->buffersize, 0); + if(n<0) { + if(errno != EAGAIN && + errno != EWOULDBLOCK && + errno != EINTR) { + syslog(LOG_ERR, "%s: recv(): %m", "upnp_event_recv"); + obj->state = EError; + } + return; + } + syslog(LOG_DEBUG, "%s: (%dbytes) %.*s", "upnp_event_recv", + n, n, obj->buffer); + /* TODO : do something with the data recevied ? + * right now, n (number of bytes received) is ignored + * We may need to recv() more bytes. */ + obj->state = EFinished; + if(obj->sub) + obj->sub->seq++; +} + +static void +upnp_event_process_notify(struct upnp_event_notify * obj) +{ + int err; + socklen_t len; + switch(obj->state) { + case EConnecting: + /* now connected or failed to connect */ + len = sizeof(err); + if(getsockopt(obj->s, SOL_SOCKET, SO_ERROR, &err, &len) < 0) { + syslog(LOG_ERR, "%s: getsockopt: %m", "upnp_event_process_notify"); + obj->state = EError; + break; + } + if(err != 0) { + errno = err; + syslog(LOG_WARNING, "%s: connect(%s%s): %m", + "upnp_event_process_notify", + obj->addrstr, obj->portstr); + obj->state = EError; + break; + } + upnp_event_prepare(obj); + if(obj->state == ESending) + upnp_event_send(obj); + break; + case ESending: + upnp_event_send(obj); + break; + case EWaitingForResponse: + upnp_event_recv(obj); + break; + case EFinished: + close(obj->s); + obj->s = -1; + break; + default: + syslog(LOG_ERR, "%s: unknown state", "upnp_event_process_notify"); + } +} + +void upnpevents_selectfds(fd_set *readset, fd_set *writeset, int * max_fd) +{ + struct upnp_event_notify * obj; + for(obj = notifylist.lh_first; obj != NULL; obj = obj->entries.le_next) { + syslog(LOG_DEBUG, "upnpevents_selectfds: %p %d %d", + obj, obj->state, obj->s); + if(obj->s >= 0) { + switch(obj->state) { + case ECreated: + upnp_event_notify_connect(obj); + if(obj->state != EConnecting) + break; + case EConnecting: + case ESending: + FD_SET(obj->s, writeset); + if(obj->s > *max_fd) + *max_fd = obj->s; + break; + case EWaitingForResponse: + FD_SET(obj->s, readset); + if(obj->s > *max_fd) + *max_fd = obj->s; + break; + default: + ; + } + } + } +} + +void upnpevents_processfds(fd_set *readset, fd_set *writeset) +{ + struct upnp_event_notify * obj; + struct upnp_event_notify * next; + struct subscriber * sub; + struct subscriber * subnext; + time_t curtime; + for(obj = notifylist.lh_first; obj != NULL; obj = obj->entries.le_next) { + syslog(LOG_DEBUG, "%s: %p %d %d %d %d", + "upnpevents_processfds", obj, obj->state, obj->s, + FD_ISSET(obj->s, readset), FD_ISSET(obj->s, writeset)); + if(obj->s >= 0) { + if(FD_ISSET(obj->s, readset) || FD_ISSET(obj->s, writeset)) + upnp_event_process_notify(obj); + } + } + obj = notifylist.lh_first; + while(obj != NULL) { + next = obj->entries.le_next; + if(obj->state == EError || obj->state == EFinished) { + if(obj->s >= 0) { + close(obj->s); + } + if(obj->sub) + obj->sub->notify = NULL; + /* remove also the subscriber from the list if there was an error */ + if(obj->state == EError && obj->sub) { + syslog(LOG_ERR, "%s: %p, remove subscriber %s after an ERROR cb: %s", + "upnpevents_processfds", obj, obj->sub->uuid, obj->sub->callback); + LIST_REMOVE(obj->sub, entries); + free(obj->sub); + } + if(obj->buffer) { + free(obj->buffer); + } + LIST_REMOVE(obj, entries); + free(obj); + } + obj = next; + } + /* remove timeouted subscribers */ + curtime = upnp_time(); + for(sub = subscriberlist.lh_first; sub != NULL; ) { + subnext = sub->entries.le_next; + if(sub->timeout && curtime > sub->timeout && sub->notify == NULL) { + syslog(LOG_INFO, "subscriber timeouted : %u > %u SID=%s", + (unsigned)curtime, (unsigned)sub->timeout, sub->uuid); + LIST_REMOVE(sub, entries); + free(sub); + } + sub = subnext; + } +} + +#ifdef USE_MINIUPNPDCTL +void write_events_details(int s) { + int n; + char buff[80]; + struct upnp_event_notify * obj; + struct subscriber * sub; + write(s, "Events details :\n", 17); + for(obj = notifylist.lh_first; obj != NULL; obj = obj->entries.le_next) { + n = snprintf(buff, sizeof(buff), " %p sub=%p state=%d s=%d\n", + obj, obj->sub, obj->state, obj->s); + write(s, buff, n); + } + write(s, "Subscribers :\n", 14); + for(sub = subscriberlist.lh_first; sub != NULL; sub = sub->entries.le_next) { + n = snprintf(buff, sizeof(buff), " %p timeout=%d seq=%u service=%d\n", + sub, (int)sub->timeout, sub->seq, sub->service); + write(s, buff, n); + n = snprintf(buff, sizeof(buff), " notify=%p %s\n", + sub->notify, sub->uuid); + write(s, buff, n); + n = snprintf(buff, sizeof(buff), " %s\n", + sub->callback); + write(s, buff, n); + } +} +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnpevents.h b/src/contrib/miniupnp/miniupnpd/upnpevents.h new file mode 100644 index 0000000..ed9104f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpevents.h @@ -0,0 +1,52 @@ +/* $Id: upnpevents.h,v 1.12 2017/11/02 15:48:29 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2008-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPEVENTS_H_INCLUDED +#define UPNPEVENTS_H_INCLUDED + +/* for fd_set */ +#include +#include "config.h" + +#ifdef ENABLE_EVENTS +enum subscriber_service_enum { + EWanCFG = 1, + EWanIPC, +#ifdef ENABLE_L3F_SERVICE + EL3F, +#endif +#ifdef ENABLE_6FC_SERVICE + E6FC, +#endif +#ifdef ENABLE_DP_SERVICE + EDP, +#endif +}; + +void +upnp_event_var_change_notify(enum subscriber_service_enum service); + +const char * +upnpevents_addSubscriber(const char * eventurl, + const char * callback, int callbacklen, + int timeout); + +int +upnpevents_removeSubscriber(const char * sid, int sidlen); + +const char * +upnpevents_renewSubscription(const char * sid, int sidlen, int timeout); + +void upnpevents_selectfds(fd_set *readset, fd_set *writeset, int * max_fd); +void upnpevents_processfds(fd_set *readset, fd_set *writeset); + +#ifdef USE_MINIUPNPDCTL +void write_events_details(int s); +#endif + +#endif +#endif diff --git a/src/contrib/miniupnp/miniupnpd/upnpglobalvars.c b/src/contrib/miniupnp/miniupnpd/upnpglobalvars.c new file mode 100644 index 0000000..290059c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpglobalvars.c @@ -0,0 +1,170 @@ +/* $Id: upnpglobalvars.c,v 1.41 2017/05/27 07:47:57 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include + +#include "config.h" +#include "upnpglobalvars.h" +#include "upnpdescstrings.h" + +/* network interface for internet */ +const char * ext_if_name = 0; + +/* file to store leases */ +#ifdef ENABLE_LEASEFILE +const char* lease_file = 0; +#endif + +/* forced ip address to use for this interface + * when NULL, getifaddr() is used */ +const char * use_ext_ip_addr = 0; + +unsigned long downstream_bitrate = 0; +unsigned long upstream_bitrate = 0; + +/* startup time */ +time_t startup_time = 0; + +#if defined(ENABLE_NATPMP) || defined(ENABLE_PCP) +/* origin for "epoch time" sent into NATPMP and PCP responses */ +time_t epoch_origin = 0; +#endif /* defined(ENABLE_NATPMP) || defined(ENABLE_PCP) */ + +#ifdef ENABLE_PCP +/* for PCP */ +unsigned long int min_lifetime = 120; +unsigned long int max_lifetime = 86400; +#endif + +int runtime_flags = 0; + +const char * pidfilename = "/var/run/miniupnpd.pid"; + +char uuidvalue_igd[] = "uuid:00000000-0000-0000-0000-000000000000"; +char uuidvalue_wan[] = "uuid:00000000-0000-0000-0000-000000000000"; +char uuidvalue_wcd[] = "uuid:00000000-0000-0000-0000-000000000000"; +char serialnumber[SERIALNUMBER_MAX_LEN] = "00000000"; + +char modelnumber[MODELNUMBER_MAX_LEN] = "1"; + +/* presentation url : + * http://nnn.nnn.nnn.nnn:ppppp/ => max 30 bytes including terminating 0 */ +char presentationurl[PRESENTATIONURL_MAX_LEN]; + +#ifdef ENABLE_MANUFACTURER_INFO_CONFIGURATION +/* friendly name for root devices in XML description */ +char friendly_name[FRIENDLY_NAME_MAX_LEN] = OS_NAME " router"; + +/* manufacturer name for root devices in XML description */ +char manufacturer_name[MANUFACTURER_NAME_MAX_LEN] = ROOTDEV_MANUFACTURER; + +/* manufacturer url for root devices in XML description */ +char manufacturer_url[MANUFACTURER_URL_MAX_LEN] = ROOTDEV_MANUFACTURERURL; + +/* model name for root devices in XML description */ +char model_name[MODEL_NAME_MAX_LEN] = ROOTDEV_MODELNAME; + +/* model description for root devices in XML description */ +char model_description[MODEL_DESCRIPTION_MAX_LEN] = ROOTDEV_MODELDESCRIPTION; + +/* model url for root devices in XML description */ +char model_url[MODEL_URL_MAX_LEN] = ROOTDEV_MODELURL; +#endif + +/* UPnP permission rules : */ +struct upnpperm * upnppermlist = 0; +unsigned int num_upnpperm = 0; + +#ifdef PCP_SADSCP +struct dscp_values* dscp_values_list = 0; +unsigned int num_dscp_values = 0; +#endif /*PCP_SADSCP*/ + +/* For automatic removal of expired rules (with LeaseDuration) */ +unsigned int nextruletoclean_timestamp = 0; + +#ifdef USE_PF +/* "rdr-anchor miniupnpd" or/and "anchor miniupnpd" in pf.conf */ +const char * anchor_name = "miniupnpd"; +const char * queue = 0; +const char * tag = 0; +#endif + +#ifdef USE_NETFILTER +/* chain names to use in the nat and filter tables. */ + +/* iptables -t nat -N MINIUPNPD + * iptables -t nat -A PREROUTING -i -j MINIUPNPD */ +const char * miniupnpd_nat_chain = "MINIUPNPD"; + +/* iptables -t nat -N MINIUPNPD-POSTROUTING + * iptables -t nat -A POSTROUTING -o -j MINIUPNPD-POSTROUTING */ +const char * miniupnpd_nat_postrouting_chain = "MINIUPNPD-POSTROUTING"; + +/* iptables -t filter -N MINIUPNPD + * iptables -t filter -A FORWARD -i ! -o -j MINIUPNPD */ +const char * miniupnpd_forward_chain = "MINIUPNPD"; + +#ifdef ENABLE_UPNPPINHOLE +/* ip6tables -t filter -N MINIUPNPD + * ip6tables -t filter -A FORWARD -i ! -o -j MINIUPNPD */ +const char * miniupnpd_v6_filter_chain = "MINIUPNPD"; +#endif /* ENABLE_UPNPPINHOLE */ + +#endif /* USE_NETFILTER */ + +#ifdef ENABLE_NFQUEUE +int nfqueue = -1; +int n_nfqix = 0; +unsigned nfqix[MAX_LAN_ADDR]; +#endif /* ENABLE_NFQUEUE */ + +struct lan_addr_list lan_addrs; + +#ifdef ENABLE_IPV6 +/* ipv6 address used for HTTP */ +char ipv6_addr_for_http_with_brackets[64]; + +/* address used to bind local services */ +struct in6_addr ipv6_bind_addr; +#endif + +/* Path of the Unix socket used to communicate with MiniSSDPd */ +const char * minissdpdsocketpath = "/var/run/minissdpd.sock"; + +/* BOOTID.UPNP.ORG and CONFIGID.UPNP.ORG */ +/* See UPnP Device Architecture v1.1 section 1.2 Advertisement : + * The field value of the BOOTID.UPNP.ORG header field MUST be increased + * each time a device (re)joins the network and sends an initial announce + * (a "reboot" in UPnP terms), or adds a UPnP-enabled interface. + * Unless the device explicitly announces a change in the BOOTID.UPNP.ORG + * field value using an SSDP message, as long as the device remains + * continuously available in the network, the same BOOTID.UPNP.ORG field + * value MUST be used in all repeat announcements, search responses, + * update messages and eventually bye-bye messages. */ +unsigned int upnp_bootid = 1; /* BOOTID.UPNP.ORG */ +/* The field value of the CONFIGID.UPNP.ORG header field identifies the + * current set of device and service descriptions; control points can + * parse this header field to detect whether they need to send new + * description query messages. */ +/* UPnP 1.1 devices MAY freely assign configid numbers from 0 to + * 16777215 (2^24-1). Higher numbers are reserved for future use, and + * can be assigned by the Technical Committee. The configuration of a + * root device consists of the following information: the DDD of the + * root device and all its embedded devices, and the SCPDs of all the + * contained services. If any part of the configuration changes, the + * CONFIGID.UPNP.ORG field value MUST be changed. + * DDD = Device Description Document + * SCPD = Service Control Protocol Description */ +unsigned int upnp_configid = 1337; /* CONFIGID.UPNP.ORG */ + +#ifdef RANDOMIZE_URLS +char random_url[RANDOM_URL_MAX_LEN] = "random"; +#endif /* RANDOMIZE_URLS */ + diff --git a/src/contrib/miniupnp/miniupnpd/upnpglobalvars.h b/src/contrib/miniupnp/miniupnpd/upnpglobalvars.h new file mode 100644 index 0000000..80f7a02 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpglobalvars.h @@ -0,0 +1,168 @@ +/* $Id: upnpglobalvars.h,v 1.45 2018/01/16 00:50:49 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPGLOBALVARS_H_INCLUDED +#define UPNPGLOBALVARS_H_INCLUDED + +#include +#include "upnppermissions.h" +#include "miniupnpdtypes.h" +#include "config.h" + +/* name of the network interface used to access internet */ +extern const char * ext_if_name; + +/* file to store all leases */ +#ifdef ENABLE_LEASEFILE +extern const char * lease_file; +#endif + +/* forced ip address to use for this interface + * when NULL, getifaddr() is used */ +extern const char * use_ext_ip_addr; + +/* parameters to return to upnp client when asked */ +extern unsigned long downstream_bitrate; +extern unsigned long upstream_bitrate; + +/* statup time */ +extern time_t startup_time; +#if defined(ENABLE_NATPMP) || defined(ENABLE_PCP) +/* origin for "epoch time" sent into NATPMP and PCP responses */ +extern time_t epoch_origin; +#endif /* defined(ENABLE_NATPMP) || defined(ENABLE_PCP) */ + +extern unsigned long int min_lifetime; +extern unsigned long int max_lifetime; + +/* runtime boolean flags */ +extern int runtime_flags; +#define LOGPACKETSMASK 0x0001 +#define SYSUPTIMEMASK 0x0002 +#ifdef ENABLE_NATPMP +#define ENABLENATPMPMASK 0x0004 +#endif +#define CHECKCLIENTIPMASK 0x0008 +#define SECUREMODEMASK 0x0010 + +#define ENABLEUPNPMASK 0x0020 + +#ifdef PF_ENABLE_FILTER_RULES +#define PFNOQUICKRULESMASK 0x0040 +#endif +#ifdef ENABLE_IPV6 +#define IPV6DISABLEDMASK 0x0080 +#endif +#ifdef ENABLE_6FC_SERVICE +#define IPV6FCFWDISABLEDMASK 0x0100 +#define IPV6FCINBOUNDDISALLOWEDMASK 0x0200 +#endif +#ifdef ENABLE_PCP +#define PCP_ALLOWTHIRDPARTYMASK 0x0400 +#endif +#ifdef IGD_V2 +#define FORCEIGDDESCV1MASK 0x0800 +#endif + +#define SETFLAG(mask) runtime_flags |= mask +#define GETFLAG(mask) (runtime_flags & mask) +#define CLEARFLAG(mask) runtime_flags &= ~mask + +extern const char * pidfilename; + +extern char uuidvalue_igd[]; /* uuid of root device (IGD) */ +extern char uuidvalue_wan[]; /* uuid of WAN Device */ +extern char uuidvalue_wcd[]; /* uuid of WAN Connection Device */ + +#define SERIALNUMBER_MAX_LEN (10) +extern char serialnumber[]; + +#define MODELNUMBER_MAX_LEN (48) +extern char modelnumber[]; + +#define PRESENTATIONURL_MAX_LEN (64) +extern char presentationurl[]; + +#ifdef ENABLE_MANUFACTURER_INFO_CONFIGURATION +#define FRIENDLY_NAME_MAX_LEN (64) +extern char friendly_name[]; + +#define MANUFACTURER_NAME_MAX_LEN (64) +extern char manufacturer_name[]; + +#define MANUFACTURER_URL_MAX_LEN (64) +extern char manufacturer_url[]; + +#define MODEL_NAME_MAX_LEN (64) +extern char model_name[]; + +#define MODEL_DESCRIPTION_MAX_LEN (64) +extern char model_description[]; + +#define MODEL_URL_MAX_LEN (64) +extern char model_url[]; +#endif + +/* UPnP permission rules : */ +extern struct upnpperm * upnppermlist; +extern unsigned int num_upnpperm; + +#ifdef PCP_SADSCP +extern struct dscp_values* dscp_values_list; +extern unsigned int num_dscp_values; +#endif + +/* For automatic removal of expired rules (with LeaseDuration) */ +extern unsigned int nextruletoclean_timestamp; + +#ifdef USE_PF +extern const char * anchor_name; +/* queue and tag for PF rules */ +extern const char * queue; +extern const char * tag; +#endif + +#ifdef USE_NETFILTER +extern const char * miniupnpd_nat_chain; +extern const char * miniupnpd_nat_postrouting_chain; +extern const char * miniupnpd_forward_chain; +#ifdef ENABLE_UPNPPINHOLE +extern const char * miniupnpd_v6_filter_chain; +#endif +#endif + +#ifdef ENABLE_NFQUEUE +extern int nfqueue; +extern int n_nfqix; +extern unsigned nfqix[]; +#endif + +/* lan addresses to listen to SSDP traffic */ +extern struct lan_addr_list lan_addrs; + +#ifdef ENABLE_IPV6 +/* ipv6 address used for HTTP */ +extern char ipv6_addr_for_http_with_brackets[64]; + +/* address used to bind local services */ +extern struct in6_addr ipv6_bind_addr; + +#endif /* ENABLE_IPV6 */ + +extern const char * minissdpdsocketpath; + +/* BOOTID.UPNP.ORG and CONFIGID.UPNP.ORG */ +extern unsigned int upnp_bootid; +extern unsigned int upnp_configid; + +#ifdef RANDOMIZE_URLS +#define RANDOM_URL_MAX_LEN (16) +extern char random_url[]; +#endif /* RANDOMIZE_URLS */ + +#endif /* UPNPGLOBALVARS_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpd/upnphttp.c b/src/contrib/miniupnp/miniupnpd/upnphttp.c new file mode 100644 index 0000000..b12d180 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnphttp.c @@ -0,0 +1,1307 @@ +/* $Id: upnphttp.c,v 1.107 2018/01/16 00:50:49 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp + * Website : http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * Author : Thomas Bernard + * Copyright (c) 2005-2018 Thomas Bernard + * This software is subject to the conditions detailed in the + * LICENCE file included in this distribution. + * */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "config.h" +#ifdef ENABLE_HTTP_DATE +#include +#endif +#include "upnphttp.h" +#include "upnpdescgen.h" +#include "miniupnpdpath.h" +#include "upnpsoap.h" +#include "upnpevents.h" +#include "upnputils.h" +#ifdef RANDOMIZE_URLS +#include "upnpglobalvars.h" +#endif /* RANDOMIZE_URLS */ + +#ifdef ENABLE_HTTPS +#include +#include +#include +static SSL_CTX *ssl_ctx = NULL; + +#ifndef HTTPS_CERTFILE +#define HTTPS_CERTFILE "/etc/ssl/certs/ssl-cert-snakeoil.pem" +#endif +#ifndef HTTPS_KEYFILE +#define HTTPS_KEYFILE "/etc/ssl/private/ssl-cert-snakeoil.key" +#endif + +static void +syslogsslerr(void) +{ + unsigned long err; + char buffer[256]; + while((err = ERR_get_error()) != 0) { + syslog(LOG_ERR, "%s", ERR_error_string(err, buffer)); + } +} + +static int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) +{ + syslog(LOG_DEBUG, "verify_callback(%d, %p)", preverify_ok, ctx); + return preverify_ok; +} + +int init_ssl(void) +{ + const SSL_METHOD *method; + SSL_library_init(); + SSL_load_error_strings(); + method = TLSv1_server_method(); + if(method == NULL) { + syslog(LOG_ERR, "TLSv1_server_method() failed"); + syslogsslerr(); + return -1; + } + ssl_ctx = SSL_CTX_new(method); + if(ssl_ctx == NULL) { + syslog(LOG_ERR, "SSL_CTX_new() failed"); + syslogsslerr(); + return -1; + } + /* set the local certificate */ + if(!SSL_CTX_use_certificate_file(ssl_ctx, HTTPS_CERTFILE, SSL_FILETYPE_PEM)) { + syslog(LOG_ERR, "SSL_CTX_use_certificate_file(%s) failed", HTTPS_CERTFILE); + syslogsslerr(); + return -1; + } + /* set the private key */ + if(!SSL_CTX_use_PrivateKey_file(ssl_ctx, HTTPS_KEYFILE, SSL_FILETYPE_PEM)) { + syslog(LOG_ERR, "SSL_CTX_use_PrivateKey_file(%s) failed", HTTPS_KEYFILE); + syslogsslerr(); + return -1; + } + /* verify private key */ + if(!SSL_CTX_check_private_key(ssl_ctx)) { + syslog(LOG_ERR, "SSL_CTX_check_private_key() failed"); + syslogsslerr(); + return -1; + } + /*SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER|SSL_VERIFY_CLIENT_ONCE, verify_callback);*/ + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_NONE, verify_callback); + /*SSL_CTX_set_verify_depth(depth);*/ + syslog(LOG_INFO, "using %s", SSLeay_version(SSLEAY_VERSION)); + return 0; +} + +void free_ssl(void) +{ + /* free context */ + if(ssl_ctx != NULL) { + SSL_CTX_free(ssl_ctx); + ssl_ctx = NULL; + } + ERR_remove_state(0); + ENGINE_cleanup(); + CONF_modules_unload(1); + ERR_free_strings(); + EVP_cleanup(); + CRYPTO_cleanup_all_ex_data(); +} +#endif /* ENABLE_HTTPS */ + +struct upnphttp * +New_upnphttp(int s) +{ + struct upnphttp * ret; + if(s<0) + return NULL; + ret = (struct upnphttp *)malloc(sizeof(struct upnphttp)); + if(ret == NULL) + return NULL; + memset(ret, 0, sizeof(struct upnphttp)); + ret->socket = s; + if(!set_non_blocking(s)) + syslog(LOG_WARNING, "New_upnphttp::set_non_blocking(): %m"); + return ret; +} + +#ifdef ENABLE_HTTPS +void +InitSSL_upnphttp(struct upnphttp * h) +{ + int r; + h->ssl = SSL_new(ssl_ctx); + if(h->ssl == NULL) { + syslog(LOG_ERR, "SSL_new() failed"); + syslogsslerr(); + abort(); + } + if(!SSL_set_fd(h->ssl, h->socket)) { + syslog(LOG_ERR, "SSL_set_fd() failed"); + syslogsslerr(); + abort(); + } + r = SSL_accept(h->ssl); /* start the handshaking */ + if(r < 0) { + int err; + err = SSL_get_error(h->ssl, r); + syslog(LOG_DEBUG, "SSL_accept() returned %d, SSL_get_error() %d", r, err); + if(err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) { + syslog(LOG_ERR, "SSL_accept() failed"); + syslogsslerr(); + abort(); + } + } +} +#endif /* ENABLE_HTTPS */ + +void +CloseSocket_upnphttp(struct upnphttp * h) +{ + /* SSL_shutdown() ? */ + if(close(h->socket) < 0) + { + syslog(LOG_ERR, "CloseSocket_upnphttp: close(%d): %m", h->socket); + } + h->socket = -1; + h->state = EToDelete; +} + +void +Delete_upnphttp(struct upnphttp * h) +{ + if(h) + { +#ifdef ENABLE_HTTPS + if(h->ssl) + SSL_free(h->ssl); +#endif + if(h->socket >= 0) + CloseSocket_upnphttp(h); + if(h->req_buf) + free(h->req_buf); + if(h->res_buf) + free(h->res_buf); + free(h); + } +} + +/* parse HttpHeaders of the REQUEST + * This function is called after the \r\n\r\n character + * sequence has been found in h->req_buf */ +static void +ParseHttpHeaders(struct upnphttp * h) +{ + char * line; + char * colon; + char * p; + int n; + if((h->req_buf == NULL) || (h->req_contentoff <= 0)) + return; + line = h->req_buf; + while(line < (h->req_buf + h->req_contentoff)) + { + colon = line; + while(*colon != ':') + { + if(*colon == '\r' || *colon == '\n') + { + colon = NULL; /* no ':' character found on the line */ + break; + } + colon++; + } + if(colon) + { + if(strncasecmp(line, "Content-Length:", 15)==0) + { + p = colon; + while((*p < '0' || *p > '9') && (*p != '\r') && (*p != '\n')) + p++; + h->req_contentlen = atoi(p); + if(h->req_contentlen < 0) { + syslog(LOG_WARNING, "ParseHttpHeaders() invalid Content-Length %d", h->req_contentlen); + h->req_contentlen = 0; + } + /*printf("*** Content-Lenght = %d ***\n", h->req_contentlen); + printf(" readbufflen=%d contentoff = %d\n", + h->req_buflen, h->req_contentoff);*/ + } + else if(strncasecmp(line, "Host:", 5)==0) + { + p = colon; + n = 0; + while(*p == ':' || *p == ' ' || *p == '\t') + p++; + while(p[n]>' ') + n++; + h->req_HostOff = p - h->req_buf; + h->req_HostLen = n; + } + else if(strncasecmp(line, "SOAPAction:", 11)==0) + { + p = colon; + n = 0; + while(*p == ':' || *p == ' ' || *p == '\t') + p++; + while(p[n]>=' ') + n++; + if((p[0] == '"' && p[n-1] == '"') + || (p[0] == '\'' && p[n-1] == '\'')) + { + p++; n -= 2; + } + h->req_soapActionOff = p - h->req_buf; + h->req_soapActionLen = n; + } + else if(strncasecmp(line, "accept-language:", 16) == 0) + { + p = colon; + n = 0; + while(*p == ':' || *p == ' ' || *p == '\t') + p++; + while(p[n]>=' ') + n++; + syslog(LOG_DEBUG, "accept-language HTTP header : '%.*s'", n, p); + /* keep only the 1st accepted language */ + n = 0; + while(p[n]>' ' && p[n] != ',') + n++; + if(n >= (int)sizeof(h->accept_language)) + n = (int)sizeof(h->accept_language) - 1; + memcpy(h->accept_language, p, n); + h->accept_language[n] = '\0'; + } + else if(strncasecmp(line, "expect:", 7) == 0) + { + p = colon; + n = 0; + while(*p == ':' || *p == ' ' || *p == '\t') + p++; + while(p[n]>=' ') + n++; + if(strncasecmp(p, "100-continue", 12) == 0) { + h->respflags |= FLAG_CONTINUE; + syslog(LOG_DEBUG, "\"Expect: 100-Continue\" header detected"); + } + } +#ifdef ENABLE_EVENTS + else if(strncasecmp(line, "Callback:", 9)==0) + { + /* The Callback can contain several urls : + * If there is more than one URL, when the service sends + * events, it will try these URLs in order until one + * succeeds. One or more URLs each enclosed by angle + * brackets ("<" and ">") */ + p = colon; + while(*p != '<' && *p != '\r' ) + p++; + n = 0; + while(p[n] != '\r') + n++; + while(n > 0 && p[n] != '>') + n--; + /* found last > character */ + h->req_CallbackOff = p - h->req_buf; + h->req_CallbackLen = MAX(0, n + 1); + } + else if(strncasecmp(line, "SID:", 4)==0) + { + p = colon + 1; + while((*p == ' ') || (*p == '\t')) + p++; + n = 0; + while(!isspace(p[n])) + n++; + h->req_SIDOff = p - h->req_buf; + h->req_SIDLen = n; + } + /* Timeout: Seconds-nnnn */ +/* TIMEOUT +Recommended. Requested duration until subscription expires, +either number of seconds or infinite. Recommendation +by a UPnP Forum working committee. Defined by UPnP vendor. + Consists of the keyword "Second-" followed (without an +intervening space) by either an integer or the keyword "infinite". */ + else if(strncasecmp(line, "Timeout:", 8)==0) + { + p = colon + 1; + while((*p == ' ') || (*p == '\t')) + p++; + if(strncasecmp(p, "Second-", 7)==0) { + h->req_Timeout = atoi(p+7); + } + } +#ifdef UPNP_STRICT + else if(strncasecmp(line, "nt:", 3)==0) + { + p = colon + 1; + while((*p == ' ') || (*p == '\t')) + p++; + n = 0; + while(!isspace(p[n])) + n++; + h->req_NTOff = p - h->req_buf; + h->req_NTLen = n; + } +#endif /* UPNP_STRICT */ +#endif /* ENABLE_EVENTS */ + } + /* the loop below won't run off the end of the buffer + * because the buffer is guaranteed to contain the \r\n\r\n + * character sequence */ + while(!(line[0] == '\r' && line[1] == '\n')) + line++; + line += 2; + } +} + +/* very minimalistic 404 error message */ +static void +Send404(struct upnphttp * h) +{ + static const char body404[] = + "404 Not Found" + "

Not Found

The requested URL was not found" + " on this server.\r\n"; + + h->respflags = FLAG_HTML; + BuildResp2_upnphttp(h, 404, "Not Found", + body404, sizeof(body404) - 1); + SendRespAndClose_upnphttp(h); +} + +static void +Send405(struct upnphttp * h) +{ + static const char body405[] = + "405 Method Not Allowed" + "

Method Not Allowed

The HTTP Method " + "is not allowed on this resource.\r\n"; + + h->respflags |= FLAG_HTML; + BuildResp2_upnphttp(h, 405, "Method Not Allowed", + body405, sizeof(body405) - 1); + SendRespAndClose_upnphttp(h); +} + +/* very minimalistic 501 error message */ +static void +Send501(struct upnphttp * h) +{ + static const char body501[] = + "501 Not Implemented" + "

Not Implemented

The HTTP Method " + "is not implemented by this server.\r\n"; + + h->respflags = FLAG_HTML; + BuildResp2_upnphttp(h, 501, "Not Implemented", + body501, sizeof(body501) - 1); + SendRespAndClose_upnphttp(h); +} + +/* findendheaders() find the \r\n\r\n character sequence and + * return a pointer to it. + * It returns NULL if not found */ +static const char * +findendheaders(const char * s, int len) +{ + while(len-->3) + { + if(s[0]=='\r' && s[1]=='\n' && s[2]=='\r' && s[3]=='\n') + return s; + s++; + } + return NULL; +} + +#ifdef HAS_DUMMY_SERVICE +static void +sendDummyDesc(struct upnphttp * h) +{ + static const char xml_desc[] = "\r\n" + "" + " " + " 1" + " 0" + " " + " " + " " + "\r\n"; + BuildResp_upnphttp(h, xml_desc, sizeof(xml_desc)-1); + SendRespAndClose_upnphttp(h); +} +#endif + +/* Sends the description generated by the parameter */ +static void +sendXMLdesc(struct upnphttp * h, char * (f)(int *)) +{ + char * desc; + int len; + desc = f(&len); + if(!desc) + { + static const char error500[] = "Error 500" + "Internal Server Error\r\n"; + syslog(LOG_ERR, "Failed to generate XML description"); + h->respflags = FLAG_HTML; + BuildResp2_upnphttp(h, 500, "Internal Server Error", + error500, sizeof(error500)-1); + } + else + { + BuildResp_upnphttp(h, desc, len); + } + SendRespAndClose_upnphttp(h); + free(desc); +} + +/* ProcessHTTPPOST_upnphttp() + * executes the SOAP query if it is possible */ +static void +ProcessHTTPPOST_upnphttp(struct upnphttp * h) +{ + if((h->req_buflen - h->req_contentoff) >= h->req_contentlen) + { + /* the request body is received */ + if(h->req_soapActionOff > 0) + { + /* we can process the request */ + syslog(LOG_INFO, "SOAPAction: %.*s", + h->req_soapActionLen, h->req_buf + h->req_soapActionOff); + ExecuteSoapAction(h, + h->req_buf + h->req_soapActionOff, + h->req_soapActionLen); + } + else + { + static const char err400str[] = + "Bad request"; + syslog(LOG_INFO, "No SOAPAction in HTTP headers"); + h->respflags = FLAG_HTML; + BuildResp2_upnphttp(h, 400, "Bad Request", + err400str, sizeof(err400str) - 1); + SendRespAndClose_upnphttp(h); + } + } + else if(h->respflags & FLAG_CONTINUE) + { + /* Sending the 100 Continue response */ + if(!h->res_buf) { + h->res_buf = malloc(256); + h->res_buf_alloclen = 256; + } + h->res_buflen = snprintf(h->res_buf, h->res_buf_alloclen, + "%s 100 Continue\r\n\r\n", h->HttpVer); + h->res_sent = 0; + h->state = ESendingContinue; + if(SendResp_upnphttp(h)) + h->state = EWaitingForHttpContent; + } + else + { + /* waiting for remaining data */ + h->state = EWaitingForHttpContent; + } +} + +#ifdef ENABLE_EVENTS +/** + * checkCallbackURL() + * check that url is on originating IP + * extract first correct URL + * returns 0 if the callback header value is not valid + * 1 if it is valid. + */ +static int +checkCallbackURL(struct upnphttp * h) +{ + char addrstr[48]; + int ipv6; + const char * p; + unsigned int i; + +start_again: + if(h->req_CallbackOff <= 0 || h->req_CallbackLen < 10) + return 0; + if(memcmp(h->req_buf + h->req_CallbackOff, "req_buf + h->req_CallbackOff + 1; + goto invalid; + } + /* extract host from url to addrstr[] */ + i = 0; + p = h->req_buf + h->req_CallbackOff + 8; + if(*p == '[') { + p++; + ipv6 = 1; + while(*p != ']' && *p != '>' && i < (sizeof(addrstr)-1) + && p < (h->req_buf + h->req_CallbackOff + h->req_CallbackLen)) + addrstr[i++] = *(p++); + } else { + ipv6 = 0; + while(*p != '/' && *p != ':' && *p != '>' && i < (sizeof(addrstr)-1) + && p < (h->req_buf + h->req_CallbackOff + h->req_CallbackLen)) + addrstr[i++] = *(p++); + } + addrstr[i] = '\0'; + /* check addrstr */ + if(ipv6) { +#ifdef ENABLE_IPV6 + struct in6_addr addr; + if(inet_pton(AF_INET6, addrstr, &addr) <= 0) + goto invalid; + if(!h->ipv6 + || (0!=memcmp(&addr, &(h->clientaddr_v6), sizeof(struct in6_addr)))) + goto invalid; +#else + goto invalid; +#endif + } else { + struct in_addr addr; + if(inet_pton(AF_INET, addrstr, &addr) <= 0) + goto invalid; +#ifdef ENABLE_IPV6 + if(h->ipv6) { + if(!IN6_IS_ADDR_V4MAPPED(&(h->clientaddr_v6))) + goto invalid; + if(0!=memcmp(&addr, ((const char *)&(h->clientaddr_v6) + 12), 4)) + goto invalid; + } else { + if(0!=memcmp(&addr, &(h->clientaddr), sizeof(struct in_addr))) + goto invalid; + } +#else + if(0!=memcmp(&addr, &(h->clientaddr), sizeof(struct in_addr))) + goto invalid; +#endif + } + /* select only the good callback url */ + while(p < h->req_buf + h->req_CallbackOff + h->req_CallbackLen && *p != '>') + p++; + h->req_CallbackOff++; /* skip initial '<' */ + h->req_CallbackLen = (int)(p - h->req_buf - h->req_CallbackOff); + return 1; +invalid: + while(p < h->req_buf + h->req_CallbackOff + h->req_CallbackLen && *p != '>') + p++; + if(*p != '>') return 0; + while(p < h->req_buf + h->req_CallbackOff + h->req_CallbackLen && *p != '<') + p++; + if(*p != '<') return 0; + h->req_CallbackLen -= (int)(p - h->req_buf - h->req_CallbackOff); + h->req_CallbackOff = (int)(p - h->req_buf); + goto start_again; +} + +static void +ProcessHTTPSubscribe_upnphttp(struct upnphttp * h, const char * path) +{ + const char * sid; + syslog(LOG_DEBUG, "ProcessHTTPSubscribe %s", path); + syslog(LOG_DEBUG, "Callback '%.*s' Timeout=%d", + h->req_CallbackLen, h->req_buf + h->req_CallbackOff, + h->req_Timeout); + syslog(LOG_DEBUG, "SID '%.*s'", h->req_SIDLen, h->req_buf + h->req_SIDOff); +#if defined(UPNP_STRICT) && (UPNP_VERSION_MAJOR > 1) || (UPNP_VERSION_MINOR > 0) + /*if(h->req_Timeout < 1800) {*/ + if(h->req_Timeout == 0) { + /* Second-infinite is forbidden with UDA v1.1 and later : + * (UDA 1.1 : 4.1.1 Subscription) + * UPnP 1.1 control points MUST NOT subscribe using keyword infinite, + * UPnP 1.1 devices MUST NOT set actual subscription durations to + * "infinite". The presence of infinite in a request MUST be silently + * ignored by a UPnP 1.1 device (the presence of infinite is handled + * by the device as if the TIMEOUT header field in a request was not + * present) . The keyword infinite MUST NOT be returned by a UPnP 1.1 + * device. */ + h->req_Timeout = 1800; /* default to 30 minutes */ + } +#endif /* UPNP_STRICT */ + if((h->req_CallbackOff <= 0) && (h->req_SIDOff <= 0)) { + /* Missing or invalid CALLBACK : 412 Precondition Failed. + * If CALLBACK header is missing or does not contain a valid HTTP URL, + * the publisher must respond with HTTP error 412 Precondition Failed*/ + BuildResp2_upnphttp(h, 412, "Precondition Failed", 0, 0); + SendRespAndClose_upnphttp(h); + } else { + /* - add to the subscriber list + * - respond HTTP/x.x 200 OK + * - Send the initial event message */ +/* Server:, SID:; Timeout: Second-(xx|infinite) */ + /* Check that the callback URL is on the same IP as + * the request, and not on the internet, nor on ourself (DOS attack ?) */ + if(h->req_CallbackOff > 0) { +#ifdef UPNP_STRICT + /* SID: and Callback: are incompatible */ + if(h->req_SIDOff > 0) { + syslog(LOG_WARNING, "Both Callback: and SID: in SUBSCRIBE"); + BuildResp2_upnphttp(h, 400, "Incompatible header fields", 0, 0); + /* "NT: upnp:event" header is mandatory */ + } else if(h->req_NTOff <= 0 || h->req_NTLen != 10 || + 0 != memcmp("upnp:event", h->req_buf + h->req_NTOff, 10)) { + syslog(LOG_WARNING, "Invalid NT in SUBSCRIBE %.*s", + h->req_NTLen, h->req_buf + h->req_NTOff); + BuildResp2_upnphttp(h, 412, "Precondition Failed", 0, 0); + } else +#endif + if(checkCallbackURL(h)) { + sid = upnpevents_addSubscriber(path, h->req_buf + h->req_CallbackOff, + h->req_CallbackLen, h->req_Timeout); + h->respflags = FLAG_TIMEOUT; + if(sid) { + syslog(LOG_DEBUG, "generated sid=%s", sid); + h->respflags |= FLAG_SID; + h->res_SID = sid; + } + BuildResp_upnphttp(h, 0, 0); + } else { + syslog(LOG_WARNING, "Invalid Callback in SUBSCRIBE %.*s", + h->req_CallbackLen, h->req_buf + h->req_CallbackOff); + BuildResp2_upnphttp(h, 412, "Precondition Failed", 0, 0); + } + } else { + /* subscription renew */ + /* Invalid SID +412 Precondition Failed. If a SID does not correspond to a known, +un-expired subscription, the publisher must respond +with HTTP error 412 Precondition Failed. */ +#ifdef UPNP_STRICT + /* SID: and NT: headers are incompatibles */ + if(h->req_NTOff > 0) { + syslog(LOG_WARNING, "Both NT: and SID: in SUBSCRIBE"); + BuildResp2_upnphttp(h, 400, "Incompatible header fields", 0, 0); + } else { +#endif /* UPNP_STRICT */ + sid = upnpevents_renewSubscription(h->req_buf + h->req_SIDOff, + h->req_SIDLen, h->req_Timeout); + if(!sid) { + BuildResp2_upnphttp(h, 412, "Precondition Failed", 0, 0); + } else { + h->respflags = FLAG_TIMEOUT | FLAG_SID; + h->res_SID = sid; + BuildResp_upnphttp(h, 0, 0); + } +#ifdef UPNP_STRICT + } +#endif /* UPNP_STRICT */ + } + SendRespAndClose_upnphttp(h); + } +} + +static void +ProcessHTTPUnSubscribe_upnphttp(struct upnphttp * h, const char * path) +{ + syslog(LOG_DEBUG, "ProcessHTTPUnSubscribe %s", path); + syslog(LOG_DEBUG, "SID '%.*s'", h->req_SIDLen, h->req_buf + h->req_SIDOff); + /* Remove from the list */ +#ifdef UPNP_STRICT + if(h->req_SIDOff <= 0 || h->req_SIDLen == 0) { + /* SID: header missing or empty */ + BuildResp2_upnphttp(h, 412, "Precondition Failed", 0, 0); + } else if(h->req_CallbackOff > 0 || h->req_NTOff > 0) { + /* no NT: or Callback: header must be present */ + BuildResp2_upnphttp(h, 400, "Incompatible header fields", 0, 0); + } else +#endif + if(upnpevents_removeSubscriber(h->req_buf + h->req_SIDOff, h->req_SIDLen) < 0) { + BuildResp2_upnphttp(h, 412, "Precondition Failed", 0, 0); + } else { + BuildResp_upnphttp(h, 0, 0); + } + SendRespAndClose_upnphttp(h); +} +#endif + +/* Parse and process Http Query + * called once all the HTTP headers have been received, + * so it is guaranteed that h->req_buf contains the \r\n\r\n + * character sequence */ +static void +ProcessHttpQuery_upnphttp(struct upnphttp * h) +{ + static const struct { + const char * path; + char * (* f)(int *); + } path_desc[] = { + { ROOTDESC_PATH, genRootDesc}, + { WANIPC_PATH, genWANIPCn}, + { WANCFG_PATH, genWANCfg}, +#ifdef HAS_DUMMY_SERVICE + { DUMMY_PATH, NULL}, +#endif +#ifdef ENABLE_L3F_SERVICE + { L3F_PATH, genL3F}, +#endif +#ifdef ENABLE_6FC_SERVICE + { WANIP6FC_PATH, gen6FC}, +#endif +#ifdef ENABLE_DP_SERVICE + { DP_PATH, genDP}, +#endif + { NULL, NULL} + }; + char HttpCommand[16]; + char HttpUrl[128]; + char * HttpVer; + char * p; + int i; + + p = h->req_buf; + if(!p) + return; + /* note : checking (*p != '\r') is enough to avoid running off the + * end of the buffer, because h->req_buf is guaranteed to contain + * the \r\n\r\n character sequence */ + for(i = 0; i<15 && *p != ' ' && *p != '\r'; i++) + HttpCommand[i] = *(p++); + HttpCommand[i] = '\0'; + while(*p==' ') + p++; + for(i = 0; i<127 && *p != ' ' && *p != '\r'; i++) + HttpUrl[i] = *(p++); + HttpUrl[i] = '\0'; + while(*p==' ') + p++; + HttpVer = h->HttpVer; + for(i = 0; i<15 && *p != '\r'; i++) + HttpVer[i] = *(p++); + HttpVer[i] = '\0'; + syslog(LOG_INFO, "HTTP REQUEST from %s : %s %s (%s)", + h->clientaddr_str, HttpCommand, HttpUrl, HttpVer); + ParseHttpHeaders(h); + if(h->req_HostOff > 0 && h->req_HostLen > 0) { + syslog(LOG_DEBUG, "Host: %.*s", h->req_HostLen, h->req_buf + h->req_HostOff); + p = h->req_buf + h->req_HostOff; + if(*p == '[') { + /* IPv6 */ + p++; + while(p < h->req_buf + h->req_HostOff + h->req_HostLen) { + if(*p == ']') break; + /* TODO check *p in [0-9a-f:.] */ + p++; + } + if(*p != ']') { + syslog(LOG_NOTICE, "DNS rebinding attack suspected (Host: %.*s)", h->req_HostLen, h->req_buf + h->req_HostOff); + Send404(h);/* 403 */ + return; + } + p++; + /* TODO : Check port */ + } else { + for(i = 0; i < h->req_HostLen; i++) { + if(*p != ':' && *p != '.' && (*p > '9' || *p < '0')) { + syslog(LOG_NOTICE, "DNS rebinding attack suspected (Host: %.*s)", h->req_HostLen, h->req_buf + h->req_HostOff); + Send404(h);/* 403 */ + return; + } + p++; + } + } + } +#ifdef RANDOMIZE_URLS + /* first check if the URL begins with the randomized string */ + if(HttpUrl[0] != '/' || memcmp(HttpUrl+1, random_url, strlen(random_url)) != 0) + { + Send404(h); + return; + } + /* remove "random" from the start of the URL */ + p = HttpUrl + strlen(random_url) + 1; + memmove(HttpUrl, p, strlen(p) + 1); +#endif /* RANDOMIZE_URLS */ + if(strcmp("POST", HttpCommand) == 0) + { + h->req_command = EPost; + ProcessHTTPPOST_upnphttp(h); + } + else if(strcmp("GET", HttpCommand) == 0) + { + h->req_command = EGet; + for(i=0; path_desc[i].path; i++) { + if(strcasecmp(path_desc[i].path, HttpUrl) == 0) { + if(path_desc[i].f) + sendXMLdesc(h, path_desc[i].f); + else +#ifdef HAS_DUMMY_SERVICE + sendDummyDesc(h); +#else + continue; +#endif + return; + } + } + if(0 == memcmp(HttpUrl, "/ctl/", 5)) { + /* 405 Method Not Allowed + * Allow: POST */ + h->respflags = FLAG_ALLOW_POST; + Send405(h); + return; + } +#ifdef ENABLE_EVENTS + if(0 == memcmp(HttpUrl, "/evt/", 5)) { + /* 405 Method Not Allowed + * Allow: SUBSCRIBE, UNSUBSCRIBE */ + h->respflags = FLAG_ALLOW_SUB_UNSUB; + Send405(h); + return; + } +#endif + syslog(LOG_NOTICE, "%s not found, responding ERROR 404", HttpUrl); + Send404(h); + } +#ifdef ENABLE_EVENTS + else if(strcmp("SUBSCRIBE", HttpCommand) == 0) + { + h->req_command = ESubscribe; + ProcessHTTPSubscribe_upnphttp(h, HttpUrl); + } + else if(strcmp("UNSUBSCRIBE", HttpCommand) == 0) + { + h->req_command = EUnSubscribe; + ProcessHTTPUnSubscribe_upnphttp(h, HttpUrl); + } +#else + else if(strcmp("SUBSCRIBE", HttpCommand) == 0) + { + syslog(LOG_NOTICE, "SUBSCRIBE not implemented. ENABLE_EVENTS compile option disabled"); + Send501(h); + } +#endif + else + { + syslog(LOG_NOTICE, "Unsupported HTTP Command %s", HttpCommand); + Send501(h); + } +} + + +void +Process_upnphttp(struct upnphttp * h) +{ + char * h_tmp; + char buf[2048]; + int n; + + if(!h) + return; + switch(h->state) + { + case EWaitingForHttpRequest: +#ifdef ENABLE_HTTPS + if(h->ssl) { + n = SSL_read(h->ssl, buf, sizeof(buf)); + } else { + n = recv(h->socket, buf, sizeof(buf), 0); + } +#else + n = recv(h->socket, buf, sizeof(buf), 0); +#endif + if(n<0) + { +#ifdef ENABLE_HTTPS + if(h->ssl) { + int err; + err = SSL_get_error(h->ssl, n); + if(err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) + { + syslog(LOG_ERR, "SSL_read() failed"); + syslogsslerr(); + h->state = EToDelete; + } + } else { +#endif + if(errno != EAGAIN && + errno != EWOULDBLOCK && + errno != EINTR) + { + syslog(LOG_ERR, "recv (state0): %m"); + h->state = EToDelete; + } + /* if errno is EAGAIN, EWOULDBLOCK or EINTR, try again later */ +#ifdef ENABLE_HTTPS + } +#endif + } + else if(n==0) + { + syslog(LOG_WARNING, "HTTP Connection from %s closed unexpectedly", inet_ntoa(h->clientaddr)); + h->state = EToDelete; + } + else + { + const char * endheaders; + /* if 1st arg of realloc() is null, + * realloc behaves the same as malloc() */ + h_tmp = (char *)realloc(h->req_buf, n + h->req_buflen + 1); + if (h_tmp == NULL) + { + syslog(LOG_WARNING, "Unable to allocate new memory for h->req_buf)"); + h->state = EToDelete; + } + else + { + h->req_buf = h_tmp; + memcpy(h->req_buf + h->req_buflen, buf, n); + h->req_buflen += n; + h->req_buf[h->req_buflen] = '\0'; + } + /* search for the string "\r\n\r\n" */ + endheaders = findendheaders(h->req_buf, h->req_buflen); + if(endheaders) + { + /* at this point, the request buffer (h->req_buf) + * is guaranteed to contain the \r\n\r\n character sequence */ + h->req_contentoff = endheaders - h->req_buf + 4; + ProcessHttpQuery_upnphttp(h); + } + } + break; + case EWaitingForHttpContent: +#ifdef ENABLE_HTTPS + if(h->ssl) { + n = SSL_read(h->ssl, buf, sizeof(buf)); + } else { + n = recv(h->socket, buf, sizeof(buf), 0); + } +#else + n = recv(h->socket, buf, sizeof(buf), 0); +#endif + if(n<0) + { +#ifdef ENABLE_HTTPS + if(h->ssl) { + int err; + err = SSL_get_error(h->ssl, n); + if(err != SSL_ERROR_WANT_READ && err != SSL_ERROR_WANT_WRITE) + { + syslog(LOG_ERR, "SSL_read() failed"); + syslogsslerr(); + h->state = EToDelete; + } + } else { +#endif + if(errno != EAGAIN && + errno != EWOULDBLOCK && + errno != EINTR) + { + syslog(LOG_ERR, "recv (state1): %m"); + h->state = EToDelete; + } + /* if errno is EAGAIN, EWOULDBLOCK or EINTR, try again later */ +#ifdef ENABLE_HTTPS + } +#endif + } + else if(n==0) + { + syslog(LOG_WARNING, "HTTP Connection from %s closed unexpectedly", inet_ntoa(h->clientaddr)); + h->state = EToDelete; + } + else + { + void * tmp = realloc(h->req_buf, n + h->req_buflen); + if(!tmp) + { + syslog(LOG_ERR, "memory allocation error %m"); + h->state = EToDelete; + } + else + { + h->req_buf = tmp; + memcpy(h->req_buf + h->req_buflen, buf, n); + h->req_buflen += n; + if((h->req_buflen - h->req_contentoff) >= h->req_contentlen) + { + ProcessHTTPPOST_upnphttp(h); + } + } + } + break; + case ESendingContinue: + if(SendResp_upnphttp(h)) + h->state = EWaitingForHttpContent; + break; + case ESendingAndClosing: + SendRespAndClose_upnphttp(h); + break; + default: + syslog(LOG_WARNING, "Unexpected state: %d", h->state); + } +} + +static const char httpresphead[] = + "%s %d %s\r\n" + "Content-Type: %s\r\n" + "Connection: close\r\n" + "Content-Length: %d\r\n" + "Server: " MINIUPNPD_SERVER_STRING "\r\n" + "Ext:\r\n" + ; /*"\r\n";*/ +/* + "\n" + "" + "" + + "" + ""; +*/ +/* with response code and response message + * also allocate enough memory */ + +int +BuildHeader_upnphttp(struct upnphttp * h, int respcode, + const char * respmsg, + int bodylen) +{ + int templen; + if(!h->res_buf || + h->res_buf_alloclen < ((int)sizeof(httpresphead) + 256 + bodylen)) { + if(h->res_buf) + free(h->res_buf); + templen = sizeof(httpresphead) + 256 + bodylen; + h->res_buf = (char *)malloc(templen); + if(!h->res_buf) { + syslog(LOG_ERR, "malloc error in BuildHeader_upnphttp()"); + return -1; + } + h->res_buf_alloclen = templen; + } + h->res_sent = 0; + h->res_buflen = snprintf(h->res_buf, h->res_buf_alloclen, + httpresphead, h->HttpVer, + respcode, respmsg, + (h->respflags&FLAG_HTML)?"text/html":"text/xml; charset=\"utf-8\"", + bodylen); + /* Content-Type MUST be 'text/xml; charset="utf-8"' according to UDA v1.1 */ + /* Content-Type MUST be 'text/xml' according to UDA v1.0 */ + /* Additional headers */ +#ifdef ENABLE_HTTP_DATE + { + char http_date[64]; + time_t t; + struct tm tm; + time(&t); + gmtime_r(&t, &tm); + /* %a and %b depend on locale */ + strftime(http_date, sizeof(http_date), + "%a, %d %b %Y %H:%M:%S GMT", &tm); + h->res_buflen += snprintf(h->res_buf + h->res_buflen, + h->res_buf_alloclen - h->res_buflen, + "Date: %s\r\n", http_date); + } +#endif +#ifdef ENABLE_EVENTS + if(h->respflags & FLAG_TIMEOUT) { + h->res_buflen += snprintf(h->res_buf + h->res_buflen, + h->res_buf_alloclen - h->res_buflen, + "Timeout: Second-"); + if(h->req_Timeout) { + h->res_buflen += snprintf(h->res_buf + h->res_buflen, + h->res_buf_alloclen - h->res_buflen, + "%d\r\n", h->req_Timeout); + } else { + h->res_buflen += snprintf(h->res_buf + h->res_buflen, + h->res_buf_alloclen - h->res_buflen, + "infinite\r\n"); + } + } + if(h->respflags & FLAG_SID) { + h->res_buflen += snprintf(h->res_buf + h->res_buflen, + h->res_buf_alloclen - h->res_buflen, + "SID: %s\r\n", h->res_SID); + } +#endif + if(h->respflags & FLAG_ALLOW_POST) { + h->res_buflen += snprintf(h->res_buf + h->res_buflen, + h->res_buf_alloclen - h->res_buflen, + "Allow: %s\r\n", "POST"); + } else if(h->respflags & FLAG_ALLOW_SUB_UNSUB) { + h->res_buflen += snprintf(h->res_buf + h->res_buflen, + h->res_buf_alloclen - h->res_buflen, + "Allow: %s\r\n", "SUBSCRIBE, UNSUBSCRIBE"); + } + if(h->accept_language[0] != '\0') { + /* defaulting to "en" */ + h->res_buflen += snprintf(h->res_buf + h->res_buflen, + h->res_buf_alloclen - h->res_buflen, + "Content-Language: %s\r\n", + h->accept_language[0] == '*' ? "en" : h->accept_language); + } + h->res_buf[h->res_buflen++] = '\r'; + h->res_buf[h->res_buflen++] = '\n'; + if(h->res_buf_alloclen < (h->res_buflen + bodylen)) + { + char * tmp; + tmp = (char *)realloc(h->res_buf, (h->res_buflen + bodylen)); + if(tmp) + { + h->res_buf = tmp; + h->res_buf_alloclen = h->res_buflen + bodylen; + } + else + { + syslog(LOG_ERR, "realloc error in BuildHeader_upnphttp()"); + return -1; + } + } + return 0; +} + +void +BuildResp2_upnphttp(struct upnphttp * h, int respcode, + const char * respmsg, + const char * body, int bodylen) +{ + int r = BuildHeader_upnphttp(h, respcode, respmsg, bodylen); + if(body && (r >= 0)) { + memcpy(h->res_buf + h->res_buflen, body, bodylen); + h->res_buflen += bodylen; + } +} + +/* responding 200 OK ! */ +void +BuildResp_upnphttp(struct upnphttp * h, + const char * body, int bodylen) +{ + BuildResp2_upnphttp(h, 200, "OK", body, bodylen); +} + +int +SendResp_upnphttp(struct upnphttp * h) +{ + ssize_t n; + + while (h->res_sent < h->res_buflen) + { +#ifdef ENABLE_HTTPS + if(h->ssl) { + n = SSL_write(h->ssl, h->res_buf + h->res_sent, + h->res_buflen - h->res_sent); + } else { + n = send(h->socket, h->res_buf + h->res_sent, + h->res_buflen - h->res_sent, 0); + } +#else + n = send(h->socket, h->res_buf + h->res_sent, + h->res_buflen - h->res_sent, 0); +#endif + if(n<0) + { +#ifdef ENABLE_HTTPS + if(h->ssl) { + int err; + err = SSL_get_error(h->ssl, n); + if(err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) { + /* try again later */ + return 0; + } + syslog(LOG_ERR, "SSL_write() failed"); + syslogsslerr(); + break; + } else { +#endif + if(errno == EINTR) + continue; /* try again immediately */ + if(errno == EAGAIN || errno == EWOULDBLOCK) + { + /* try again later */ + return 0; + } + syslog(LOG_ERR, "send(res_buf): %m"); + break; /* avoid infinite loop */ +#ifdef ENABLE_HTTPS + } +#endif + } + else if(n == 0) + { + syslog(LOG_ERR, "send(res_buf): %d bytes sent (out of %d)", + h->res_sent, h->res_buflen); + break; + } + else + { + h->res_sent += n; + } + } + return 1; /* finished */ +} + +void +SendRespAndClose_upnphttp(struct upnphttp * h) +{ + ssize_t n; + + while (h->res_sent < h->res_buflen) + { +#ifdef ENABLE_HTTPS + if(h->ssl) { + n = SSL_write(h->ssl, h->res_buf + h->res_sent, + h->res_buflen - h->res_sent); + } else { + n = send(h->socket, h->res_buf + h->res_sent, + h->res_buflen - h->res_sent, 0); + } +#else + n = send(h->socket, h->res_buf + h->res_sent, + h->res_buflen - h->res_sent, 0); +#endif + if(n<0) + { +#ifdef ENABLE_HTTPS + if(h->ssl) { + int err; + err = SSL_get_error(h->ssl, n); + if(err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) { + /* try again later */ + h->state = ESendingAndClosing; + return; + } + syslog(LOG_ERR, "SSL_write() failed"); + syslogsslerr(); + break; /* avoid infinite loop */ + } else { +#endif + if(errno == EINTR) + continue; /* try again immediately */ + if(errno == EAGAIN || errno == EWOULDBLOCK) + { + /* try again later */ + h->state = ESendingAndClosing; + return; + } + syslog(LOG_ERR, "send(res_buf): %m"); + break; /* avoid infinite loop */ +#ifdef ENABLE_HTTPS + } +#endif + } + else if(n == 0) + { + syslog(LOG_ERR, "send(res_buf): %d bytes sent (out of %d)", + h->res_sent, h->res_buflen); + break; + } + else + { + h->res_sent += n; + } + } + CloseSocket_upnphttp(h); +} + diff --git a/src/contrib/miniupnp/miniupnpd/upnphttp.h b/src/contrib/miniupnp/miniupnpd/upnphttp.h new file mode 100644 index 0000000..bf48b47 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnphttp.h @@ -0,0 +1,165 @@ +/* $Id: upnphttp.h,v 1.42 2015/12/16 10:21:49 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2015 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPHTTP_H_INCLUDED +#define UPNPHTTP_H_INCLUDED + +#include +#include + +#include "config.h" + +#ifdef ENABLE_HTTPS +#include +#endif /* ENABLE_HTTPS */ + +#define UPNP_VERSION_STRING "UPnP/" UPNP_VERSION_MAJOR_STR "." UPNP_VERSION_MINOR_STR + +/* server: HTTP header returned in all HTTP responses : */ +#define MINIUPNPD_SERVER_STRING OS_VERSION " " UPNP_VERSION_STRING " MiniUPnPd/" MINIUPNPD_VERSION + +/* + states : + 0 - waiting for data to read + 1 - waiting for HTTP Post Content. + ... + >= 100 - to be deleted +*/ +enum httpStates { + EWaitingForHttpRequest = 0, + EWaitingForHttpContent, + ESendingContinue, + ESendingAndClosing, + EToDelete = 100 +}; + +enum httpCommands { + EUnknown = 0, + EGet, + EPost, + ESubscribe, + EUnSubscribe +}; + +struct upnphttp { + int socket; + struct in_addr clientaddr; /* client address */ +#ifdef ENABLE_IPV6 + int ipv6; + struct in6_addr clientaddr_v6; +#endif /* ENABLE_IPV6 */ +#ifdef ENABLE_HTTPS + SSL * ssl; +#endif /* ENABLE_HTTPS */ + char clientaddr_str[64]; /* used for syslog() output */ + enum httpStates state; + char HttpVer[16]; + /* request */ + char * req_buf; + char accept_language[8]; + int req_buflen; + int req_contentlen; + int req_contentoff; /* header length */ + enum httpCommands req_command; + int req_soapActionOff; + int req_soapActionLen; + int req_HostOff; /* Host: header */ + int req_HostLen; +#ifdef ENABLE_EVENTS + int req_CallbackOff; /* For SUBSCRIBE */ + int req_CallbackLen; + int req_Timeout; + int req_SIDOff; /* For UNSUBSCRIBE */ + int req_SIDLen; + const char * res_SID; +#ifdef UPNP_STRICT + int req_NTOff; + int req_NTLen; +#endif +#endif + int respflags; /* see FLAG_* constants below */ + /* response */ + char * res_buf; + int res_buflen; + int res_sent; + int res_buf_alloclen; + LIST_ENTRY(upnphttp) entries; +}; + +/* Include the "Timeout:" header in response */ +#define FLAG_TIMEOUT 0x01 +/* Include the "SID:" header in response */ +#define FLAG_SID 0x02 + +/* If set, the POST request included a "Expect: 100-continue" header */ +#define FLAG_CONTINUE 0x40 + +/* If set, the Content-Type is set to text/xml, otherwise it is text/xml */ +#define FLAG_HTML 0x80 + +/* If set, the corresponding Allow: header is set */ +#define FLAG_ALLOW_POST 0x100 +#define FLAG_ALLOW_SUB_UNSUB 0x200 + +#ifdef ENABLE_HTTPS +int init_ssl(void); +void free_ssl(void); +#endif /* ENABLE_HTTPS */ + +/* New_upnphttp() */ +struct upnphttp * +New_upnphttp(int); + +#ifdef ENABLE_HTTPS +void +InitSSL_upnphttp(struct upnphttp *); +#endif /* ENABLE_HTTPS */ + +/* CloseSocket_upnphttp() */ +void +CloseSocket_upnphttp(struct upnphttp *); + +/* Delete_upnphttp() */ +void +Delete_upnphttp(struct upnphttp *); + +/* Process_upnphttp() */ +void +Process_upnphttp(struct upnphttp *); + +/* BuildHeader_upnphttp() + * build the header for the HTTP Response + * also allocate the buffer for body data + * return -1 on error */ +int +BuildHeader_upnphttp(struct upnphttp * h, int respcode, + const char * respmsg, + int bodylen); + +/* BuildResp_upnphttp() + * fill the res_buf buffer with the complete + * HTTP 200 OK response from the body passed as argument */ +void +BuildResp_upnphttp(struct upnphttp *, const char *, int); + +/* BuildResp2_upnphttp() + * same but with given response code/message */ +void +BuildResp2_upnphttp(struct upnphttp * h, int respcode, + const char * respmsg, + const char * body, int bodylen); + +int +SendResp_upnphttp(struct upnphttp *); + +/* SendRespAndClose_upnphttp() */ +void +SendRespAndClose_upnphttp(struct upnphttp *); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnppermissions.c b/src/contrib/miniupnp/miniupnpd/upnppermissions.c new file mode 100644 index 0000000..ad02d8c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnppermissions.c @@ -0,0 +1,264 @@ +/* $Id: upnppermissions.c,v 1.18 2014/03/07 10:43:29 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2014 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "config.h" +#include "upnppermissions.h" + +/* read_permission_line() + * parse the a permission line which format is : + * (deny|allow) [0-9]+(-[0-9]+) ip/mask [0-9]+(-[0-9]+) + * ip/mask is either 192.168.1.1/24 or 192.168.1.1/255.255.255.0 + */ +int +read_permission_line(struct upnpperm * perm, + char * p) +{ + char * q; + int n_bits; + int i; + + /* first token: (allow|deny) */ + while(isspace(*p)) + p++; + if(0 == memcmp(p, "allow", 5)) + { + perm->type = UPNPPERM_ALLOW; + p += 5; + } + else if(0 == memcmp(p, "deny", 4)) + { + perm->type = UPNPPERM_DENY; + p += 4; + } + else + { + return -1; + } + while(isspace(*p)) + p++; + + /* second token: eport or eport_min-eport_max */ + if(!isdigit(*p)) + return -1; + for(q = p; isdigit(*q); q++); + if(*q=='-') + { + *q = '\0'; + i = atoi(p); + if(i > 65535) + return -1; + perm->eport_min = (u_short)i; + q++; + p = q; + while(isdigit(*q)) + q++; + *q = '\0'; + i = atoi(p); + if(i > 65535) + return -1; + perm->eport_max = (u_short)i; + if(perm->eport_min > perm->eport_max) + return -1; + } + else if(isspace(*q)) + { + *q = '\0'; + i = atoi(p); + if(i > 65535) + return -1; + perm->eport_min = perm->eport_max = (u_short)i; + } + else + { + return -1; + } + p = q + 1; + while(isspace(*p)) + p++; + + /* third token: ip/mask */ + if(!isdigit(*p)) + return -1; + for(q = p; isdigit(*q) || (*q == '.'); q++); + if(*q=='/') + { + *q = '\0'; + if(!inet_aton(p, &perm->address)) + return -1; + q++; + p = q; + while(isdigit(*q)) + q++; + if(*q == '.') + { + while(*q == '.' || isdigit(*q)) + q++; + if(!isspace(*q)) + return -1; + *q = '\0'; + if(!inet_aton(p, &perm->mask)) + return -1; + } + else if(!isspace(*q)) + return -1; + else + { + *q = '\0'; + n_bits = atoi(p); + if(n_bits > 32) + return -1; + perm->mask.s_addr = htonl(n_bits ? (0xffffffffu << (32 - n_bits)) : 0); + } + } + else if(isspace(*q)) + { + *q = '\0'; + if(!inet_aton(p, &perm->address)) + return -1; + perm->mask.s_addr = 0xffffffffu; + } + else + { + return -1; + } + p = q + 1; + + /* fourth token: iport or iport_min-iport_max */ + while(isspace(*p)) + p++; + if(!isdigit(*p)) + return -1; + for(q = p; isdigit(*q); q++); + if(*q=='-') + { + *q = '\0'; + i = atoi(p); + if(i > 65535) + return -1; + perm->iport_min = (u_short)i; + q++; + p = q; + while(isdigit(*q)) + q++; + *q = '\0'; + i = atoi(p); + if(i > 65535) + return -1; + perm->iport_max = (u_short)i; + if(perm->iport_min > perm->iport_max) + return -1; + } + else if(isspace(*q) || *q == '\0') + { + *q = '\0'; + i = atoi(p); + if(i > 65535) + return -1; + perm->iport_min = perm->iport_max = (u_short)i; + } + else + { + return -1; + } +#ifdef DEBUG + printf("perm rule added : %s %hu-%hu %08x/%08x %hu-%hu\n", + (perm->type==UPNPPERM_ALLOW)?"allow":"deny", + perm->eport_min, perm->eport_max, ntohl(perm->address.s_addr), + ntohl(perm->mask.s_addr), perm->iport_min, perm->iport_max); +#endif + return 0; +} + +#ifdef USE_MINIUPNPDCTL +void +write_permlist(int fd, const struct upnpperm * permary, + int nperms) +{ + int l; + const struct upnpperm * perm; + int i; + char buf[128]; + write(fd, "Permissions :\n", 14); + for(i = 0; itype==UPNPPERM_ALLOW)?"allow":"deny", + perm->eport_min, perm->eport_max, ntohl(perm->address.s_addr), + ntohl(perm->mask.s_addr), perm->iport_min, perm->iport_max); + if(l<0) + return; + write(fd, buf, l); + } +} +#endif + +/* match_permission() + * returns: 1 if eport, address, iport matches the permission rule + * 0 if no match */ +static int +match_permission(const struct upnpperm * perm, + u_short eport, struct in_addr address, u_short iport) +{ + if( (eport < perm->eport_min) || (perm->eport_max < eport)) + return 0; + if( (iport < perm->iport_min) || (perm->iport_max < iport)) + return 0; + if( (address.s_addr & perm->mask.s_addr) + != (perm->address.s_addr & perm->mask.s_addr) ) + return 0; + return 1; +} + +#if 0 +/* match_permission_internal() + * returns: 1 if address, iport matches the permission rule + * 0 if no match */ +static int +match_permission_internal(const struct upnpperm * perm, + struct in_addr address, u_short iport) +{ + if( (iport < perm->iport_min) || (perm->iport_max < iport)) + return 0; + if( (address.s_addr & perm->mask.s_addr) + != (perm->address.s_addr & perm->mask.s_addr) ) + return 0; + return 1; +} +#endif + +int +check_upnp_rule_against_permissions(const struct upnpperm * permary, + int n_perms, + u_short eport, struct in_addr address, + u_short iport) +{ + int i; + for(i=0; i +#include +#include +#include "config.h" + +/* UPnP permission rule samples: + * allow 1024-65535 192.168.3.0/24 1024-65535 + * deny 0-65535 192.168.1.125/32 0-65535 */ +struct upnpperm { + enum {UPNPPERM_ALLOW=1, UPNPPERM_DENY=2 } type; + /* is it an allow or deny permission rule ? */ + u_short eport_min, eport_max; /* external port range */ + struct in_addr address, mask; /* ip/mask */ + u_short iport_min, iport_max; /* internal port range */ +}; + +/* read_permission_line() + * returns: 0 line read okay + * -1 error reading line + * + * line sample : + * allow 1024-65535 192.168.3.0/24 1024-65535 + * allow 22 192.168.4.33/32 22 + * deny 0-65535 0.0.0.0/0 0-65535 */ +int +read_permission_line(struct upnpperm * perm, + char * p); + +/* check_upnp_rule_against_permissions() + * returns: 0 if the upnp rule should be rejected, + * 1 if it could be accepted */ +int +check_upnp_rule_against_permissions(const struct upnpperm * permary, + int n_perms, + u_short eport, struct in_addr address, + u_short iport); + +#ifdef USE_MINIUPNPDCTL +void +write_permlist(int fd, const struct upnpperm * permary, + int nperms); +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnppinhole.c b/src/contrib/miniupnp/miniupnpd/upnppinhole.c new file mode 100644 index 0000000..00a74b0 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnppinhole.c @@ -0,0 +1,545 @@ +/* $Id: upnppinhole.c,v 1.13 2018/03/13 10:49:13 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "macros.h" +#include "config.h" +#include "upnpredirect.h" +#include "upnpglobalvars.h" +#include "upnpevents.h" +#include "upnputils.h" +#include "upnppinhole.h" +#ifdef __APPLE__ +/* XXX - Apple version of PF API seems to differ from what + * pf/pfpinhole.c expects so don't use that at least.. */ +#ifdef USE_PF +#undef USE_PF +#endif /* USE_PF */ +#endif /* __APPLE__ */ +#if defined(USE_NETFILTER) +#include "netfilter/iptpinhole.h" +#endif +#if defined(USE_PF) +#include "pf/pfpinhole.h" +#endif +#if defined(USE_IPF) +#endif +#if defined(USE_IPFW) +#endif + +#ifdef ENABLE_UPNPPINHOLE + +#if 0 +int +upnp_check_outbound_pinhole(int proto, int * timeout) +{ + int s, tmptimeout, tmptime_out; + switch(proto) + { + case IPPROTO_UDP: + s = retrieve_timeout("udp_timeout", timeout); + return s; + break; + case IPPROTO_UDPLITE: + s = retrieve_timeout("udp_timeout_stream", timeout); + return s; + break; + case IPPROTO_TCP: + s = retrieve_timeout("tcp_timeout_established", timeout); + return s; + break; + case 65535: + s = retrieve_timeout("udp_timeout", timeout); + s = retrieve_timeout("udp_timeout_stream", &tmptimeout); + s = retrieve_timeout("tcp_timeout_established", &tmptime_out); + if(tmptimeout= 0) { + syslog(LOG_INFO, "Pinhole for inbound traffic from [%s]:%hu to [%s]:%hu with proto %d found uid=%d. Updating it.", raddr, rport, iaddr, iport, proto, *uid); + r = upnp_update_inboundpinhole(*uid, timestamp); + return (r >= 0) ? 1 : r; + } +#if defined(USE_PF) || defined(USE_NETFILTER) + *uid = add_pinhole (ext_if_name, raddr, rport, + iaddr, iport, proto, desc, timestamp); + return *uid >= 0 ? 1 : -1; +#else + return -42; /* not implemented */ +#endif +} + +#if 0 +int +upnp_add_inboundpinhole_internal(const char * raddr, unsigned short rport, + const char * iaddr, unsigned short iport, + const char * proto, int * uid) +{ + int c = 9999; + char cmd[256], cmd_raw[256], cuid[42]; +#if 0 + static const char cmdval_full_udptcp[] = "ip6tables -I %s %d -p %s -i %s -s %s --sport %hu -d %s --dport %hu -j ACCEPT"; + static const char cmdval_udptcp[] = "ip6tables -I %s %d -p %s -i %s --sport %hu -d %s --dport %hu -j ACCEPT"; + static const char cmdval_full_udplite[] = "ip6tables -I %s %d -p %s -i %s -s %s -d %s -j ACCEPT"; + static const char cmdval_udplite[] = "ip6tables -I %s %d -p %s -i %s -d %s -j ACCEPT"; + // raw table command + static const char cmdval_full_udptcp_raw[] = "ip6tables -t raw -I PREROUTING %d -p %s -i %s -s %s --sport %hu -d %s --dport %hu -j TRACE"; + static const char cmdval_udptcp_raw[] = "ip6tables -t raw -I PREROUTING %d -p %s -i %s --sport %hu -d %s --dport %hu -j TRACE"; + static const char cmdval_full_udplite_raw[] = "ip6tables -t raw -I PREROUTING %d -p %s -i %s -s %s -d %s -j TRACE"; + static const char cmdval_udplite_raw[] = "ip6tables -t raw -I PREROUTING %d -p %s -i %s -d %s -j TRACE"; +#endif + /*printf("%s\n", raddr);*/ + if(raddr!=NULL) + { +#ifdef IPPROTO_UDPLITE + if(atoi(proto) == IPPROTO_UDPLITE) + { + /* snprintf(cmd, sizeof(cmd), cmdval_full_udplite, miniupnpd_forward_chain, line_number, proto, ext_if_name, raddr, iaddr); + snprintf(cmd_raw, sizeof(cmd_raw), cmdval_full_udplite_raw, line_number, proto, ext_if_name, raddr, iaddr);*/ + } + else +#endif + { + /* snprintf(cmd, sizeof(cmd), cmdval_full_udptcp, miniupnpd_forward_chain, line_number, proto, ext_if_name, raddr, rport, iaddr, iport); + snprintf(cmd_raw, sizeof(cmd_raw), cmdval_full_udptcp_raw, line_number, proto, ext_if_name, raddr, rport, iaddr, iport);*/ + } + } + else + { +#ifdef IPPROTO_UDPLITE + if(atoi(proto) == IPPROTO_UDPLITE) + { + /*snprintf(cmd, sizeof(cmd), cmdval_udplite, miniupnpd_forward_chain, line_number, proto, ext_if_name, iaddr); + snprintf(cmd_raw, sizeof(cmd_raw), cmdval_udplite_raw, line_number, proto, ext_if_name, iaddr);*/ + } + else +#endif + { + /*snprintf(cmd, sizeof(cmd), cmdval_udptcp, miniupnpd_forward_chain, line_number, proto, ext_if_name, rport, iaddr, iport); + snprintf(cmd_raw, sizeof(cmd_raw), cmdval_udptcp_raw, line_number, proto, ext_if_name, rport, iaddr, iport); +*/ + } + } +#ifdef DEBUG + syslog(LOG_INFO, "Adding following ip6tables rule:"); + syslog(LOG_INFO, " -> %s", cmd); + syslog(LOG_INFO, " -> %s", cmd_raw); +#endif + /* TODO Add a better checking error.*/ + if(system(cmd) < 0 || system(cmd_raw) < 0) + { + return 0; + } + srand(time(NULL)); + snprintf(cuid, sizeof(cuid), "%.4d", rand()%c); + *uid = atoi(cuid); + printf("\t_add_ uid: %s\n", cuid); + return 1; +} +#endif + +/* upnp_get_pinhole_info() + * return values : + * 0 OK + * -1 Internal error + * -2 NOT FOUND (no such entry) + * .. + * -42 Not implemented + */ +int +upnp_get_pinhole_info(unsigned short uid, + char * raddr, int raddrlen, + unsigned short * rport, + char * iaddr, int iaddrlen, + unsigned short * iport, + int * proto, char * desc, int desclen, + unsigned int * leasetime, + unsigned int * packets) +{ + /* Call Firewall specific code to get IPv6 pinhole infos */ +#if defined(USE_PF) || defined(USE_NETFILTER) + int r; + unsigned int timestamp; + u_int64_t packets_tmp; + /*u_int64_t bytes_tmp;*/ + + r = get_pinhole_info(uid, raddr, raddrlen, rport, + iaddr, iaddrlen, iport, + proto, desc, desclen, + leasetime ? ×tamp : NULL, + packets ? &packets_tmp : NULL, + NULL/*&bytes_tmp*/); + if(r >= 0) { + if(leasetime) { + time_t current_time; + current_time = upnp_time(); + if(timestamp > (unsigned int)current_time) + *leasetime = timestamp - current_time; + else + *leasetime = 0; + } + if(packets) + *packets = (unsigned int)packets_tmp; + } + return r; +#else + UNUSED(uid); + UNUSED(raddr); UNUSED(raddrlen); UNUSED(rport); + UNUSED(iaddr); UNUSED(iaddrlen); UNUSED(iport); + UNUSED(proto); UNUSED(desc); UNUSED(desclen); + UNUSED(leasetime); UNUSED(packets); + return -42; /* not implemented */ +#endif +} + +int +upnp_get_pinhole_uid_by_index(int index) +{ +#if defined (USE_NETFILTER) + return get_pinhole_uid_by_index(index); +#else + UNUSED(index); + return -42; +#endif /* defined (USE_NETFILTER) */ +} + +int +upnp_update_inboundpinhole(unsigned short uid, unsigned int leasetime) +{ +#if defined(USE_PF) || defined(USE_NETFILTER) + unsigned int timestamp; + + timestamp = upnp_time() + leasetime; + return update_pinhole(uid, timestamp); +#else + UNUSED(uid); UNUSED(leasetime); + + return -42; /* not implemented */ +#endif +} + +int +upnp_delete_inboundpinhole(unsigned short uid) +{ +#if defined(USE_PF) || defined(USE_NETFILTER) + return delete_pinhole(uid); +#else + UNUSED(uid); + + return -1; +#endif +} + +#if 0 +/* + * Result: + * 1: Found Result + * -4: No result + * -5: Result in another table + * -6: Result in another chain + * -7: Result in a chain not a rule +*/ +int +upnp_check_pinhole_working(const char * uid, + char * eaddr, + char * iaddr, + unsigned short * eport, + unsigned short * iport, + char * protocol, + int * rulenum_used) +{ + /* TODO : to be implemented */ +#if 0 + FILE * fd; + time_t expire = upnp_time(); + char buf[1024], filename[] = "/var/log/kern.log", expire_time[32]=""; + int res = -4, str_len; + + str_len = strftime(expire_time, sizeof(expire_time), "%b %d %H:%M:%S", localtime(&expire)); + + fd = fopen(filename, "r"); + if (fd==NULL) + { + syslog(LOG_ERR, "Get_rule: could not open file: %s", filename); + return -1; + } + + syslog(LOG_INFO, "Get_rule: Starting getting info in file %s for %s\n", filename, uid); + buf[sizeof(buf)-1] = 0; + while(fgets(buf, sizeof(buf)-1, fd) != NULL && res != 1) + { + //printf("line: %s\n", buf); + char * r, * t, * c, * p; + // looking for something like filter:FORWARD:rule: or filter:MINIUPNPD:rule: + r = strstr(buf, ":rule:"); + p = strstr(buf, ":policy:"); + t = strstr(buf, "TRACE:"); // table pointeur + t += 7; + c = t + 7; // chain pointeur + if(r) + { + printf("\t** Found %.*s\n", 24 ,t); + char * src, * dst, * sport, * dport, * proto, * line; + char time[15]="", src_addr[40], dst_addr[40], proto_tmp[8]; + int proto_int; + strncpy(time, buf, sizeof(time)); + /*if(compare_time(time, expire_time)<0) + { + printf("\t\tNot corresponding time\n"); + continue; + }*/ + + line = r + 6; + printf("\trule line = %d\n", atoi(line)); + + src = strstr(buf, "SRC="); + src += 4; + snprintf(src_addr, sizeof(src_addr), "%.*s", 39, src); +#if 0 + del_char(src_addr); + add_char(src_addr); +#endif + + dst = strstr(buf, "DST="); + dst += 4; + snprintf(dst_addr, sizeof(dst_addr), "%.*s", 39, dst); +#if 0 + del_char(dst_addr); + add_char(dst_addr); +#endif + + proto = strstr(buf, "PROTO="); + proto += 6; + proto_int = atoi(protocol); + if(proto_int == IPPROTO_UDP) + strcpy(proto_tmp, "UDP"); + else if(proto_int == IPPROTO_TCP) + strcpy(proto_tmp, "TCP"); +#ifdef IPPROTO_UDPLITE + else if(proto_int == IPPROTO_UDPLITE) + strcpy(proto_tmp, "UDPLITE"); +#endif + else + strcpy(proto_tmp, "UnsupportedProto"); + + // printf("\tCompare eaddr: %s // protocol: %s\n\t to addr: %s // protocol: %.*s\n", eaddr, proto_tmp, src_addr, strlen(proto_tmp), proto); + // printf("\tCompare iaddr: %s // protocol: %s\n\t to addr: %s // protocol: %.*s\n", iaddr, proto_tmp, dst_addr, strlen(proto_tmp), proto); + // TODO Check time + // Check that the paquet found in trace correspond to the one we are looking for + if( /*(strcmp(eaddr, src_addr) == 0) &&*/ (strcmp(iaddr, dst_addr) == 0) && (strncmp(proto_tmp, proto, strlen(proto_tmp))==0)) + { + sport = strstr(buf, "SPT="); + sport += 4; + dport = strstr(buf, "DPT="); + dport += 4; + printf("\tCompare eport: %hu\n\t to port: %d\n", *eport, atoi(sport)); + printf("\tCompare iport: %hu\n\t to port: %d\n", *iport, atoi(dport)); + if(/*eport != atoi(sport) &&*/ *iport != atoi(dport)) + { + printf("\t\tPort not corresponding\n"); + continue; + } + printf("\ttable found: %.*s\n", 6, t); + printf("\tchain found: %.*s\n", 9, c); + // Check that the table correspond to the filter table + if(strncmp(t, "filter", 6)==0) + { + // Check that the table correspond to the MINIUPNP table + if(strncmp(c, "MINIUPNPD", 9)==0) + { + *rulenum_used = atoi(line); + res = 1; + } + else + { + res = -6; + continue; + } + } + else + { + res = -5; + continue; + } + } + else + { + printf("Packet information not corresponding\n"); + continue; + } + } + if(!r && p) + { + printf("\t** Policy case\n"); + char * src, * dst, * sport, * dport, * proto, * line; + char time[15], src_addr[40], dst_addr[40], proto_tmp[8]; + int proto_int; + strncpy(time, buf, sizeof(time)); + /*if(compare_time(time, expire_time)<0) + { + printf("\t\tNot corresponding time\n"); + continue; + }*/ + + line = p + 8; + printf("\trule line = %d\n", atoi(line)); + + src = strstr(buf, "SRC="); + src += 4; + snprintf(src_addr, sizeof(src_addr), "%.*s", 39, src); +#if 0 + del_char(src_addr); + add_char(src_addr); +#endif + + dst = strstr(buf, "DST="); + dst += 4; + snprintf(dst_addr, sizeof(dst_addr), "%.*s", 39, dst); +#if 0 + del_char(dst_addr); + add_char(dst_addr); +#endif + + proto = strstr(buf, "PROTO="); + proto += 6; + proto_int = atoi(protocol); + if(proto_int == IPPROTO_UDP) + strcpy(proto_tmp, "UDP"); + else if(proto_int == IPPROTO_TCP) + strcpy(proto_tmp, "TCP"); +#ifdef IPPROTO_UDPLITE + else if(proto_int == IPPROTO_UDPLITE) + strcpy(proto_tmp, "UDPLITE"); +#endif + else + strcpy(proto_tmp, "UnsupportedProto"); + + // printf("\tCompare eaddr: %s // protocol: %s\n\t to addr: %s // protocol: %.*s\n", eaddr, proto_tmp, src_addr, strlen(proto_tmp), proto); + // printf("\tCompare iaddr: %s // protocol: %s\n\t to addr: %s // protocol: %.*s\n", iaddr, proto_tmp, dst_addr, strlen(proto_tmp), proto); + // Check that the paquet found in trace correspond to the one we are looking for + if( (strcmp(eaddr, src_addr) == 0) && (strcmp(iaddr, dst_addr) == 0) && (strncmp(proto_tmp, proto, 5)==0)) + { + sport = strstr(buf, "SPT="); + sport += 4; + dport = strstr(buf, "DPT="); + dport += 4; + printf("\tCompare eport: %hu\n\t to port: %d\n", *eport, atoi(sport)); + printf("\tCompare iport: %hu\n\t to port: %d\n", *iport, atoi(dport)); + if(*eport != atoi(sport) && *iport != atoi(dport)) + { + printf("\t\tPort not corresponding\n"); + continue; + } + else + { + printf("Find a corresponding policy trace in the chain: %.*s\n", 10, c); + res = -7; + continue; + } + } + else + continue; + } + } + fclose(fd); + return res; +#else + return -42; /* to be implemented */ +#endif +} +#endif + +int +upnp_clean_expired_pinholes(unsigned int * next_timestamp) +{ +#if defined(USE_PF) || defined(USE_NETFILTER) + return clean_pinhole_list(next_timestamp); +#else + UNUSED(next_timestamp); + + return 0; /* nothing to do */ +#endif +} + +#endif /* ENABLE_UPNPPINHOLE */ diff --git a/src/contrib/miniupnp/miniupnpd/upnppinhole.h b/src/contrib/miniupnp/miniupnpd/upnppinhole.h new file mode 100644 index 0000000..c979401 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnppinhole.h @@ -0,0 +1,89 @@ +/* $Id: upnppinhole.h,v 1.2 2012/09/18 08:29:49 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2016 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPPINHOLE_H_INCLUDED +#define UPNPPINHOLE_H_INCLUDED + +#include "config.h" + +#ifdef ENABLE_UPNPPINHOLE + +/* functions to be used by WANIPv6_FirewallControl implementation + * and PCP (IPv6) */ + +#if 0 +/* retrieve outbound pinhole timeout */ +int +upnp_check_outbound_pinhole(int proto, int * timeout); +#endif + +/* find an inbound pinhole base on remove host:port / local host:port + * return the (positive) uid or a negative value if not found */ +int +upnp_find_inboundpinhole(const char * raddr, unsigned short rport, + const char * iaddr, unsigned short iport, + int proto, + char * desc, int desc_len, unsigned int * leasetime); + + +/* add an inbound pinehole + * return value : + * 1 = success + * -1 = Pinhole space exhausted + * .. = error */ +int +upnp_add_inboundpinhole(const char * raddr, unsigned short rport, + const char * iaddr, unsigned short iport, + int proto, char * desc, + unsigned int leasetime, int * uid); + +/* get from uid + * return values : + * -1 not found + * */ +int +upnp_get_pinhole_info(unsigned short uid, + char * raddr, int raddrlen, + unsigned short * rport, + char * iaddr, int iaddrlen, + unsigned short * iport, + int * proto, char * desc, int desclen, + unsigned int * leasetime, + unsigned int * packets); + +/* + * return values: + * -1 = not found + * 0 .. 65535 = uid of the rule for the index + */ +int +upnp_get_pinhole_uid_by_index(int index); + +/* update the lease time */ +int +upnp_update_inboundpinhole(unsigned short uid, unsigned int leasetime); + +/* remove the inbound pinhole */ +int +upnp_delete_inboundpinhole(unsigned short uid); + +/* ... */ +#if 0 +int +upnp_check_pinhole_working(const char * uid, char * eaddr, char * iaddr, unsigned short * eport, unsigned short * iport, char * protocol, int * rulenum_used); +#endif + +/* return the number of expired pinhole removed + * write timestamp to next pinhole to exprire to next_timestamp + * next_timestamp is left untouched if there is no pinhole lest */ +int +upnp_clean_expired_pinholes(unsigned int * next_timestamp); + +#endif /* ENABLE_UPNPPINHOLE */ + +#endif /* !UPNPPINHOLE_H_INCLUDED */ diff --git a/src/contrib/miniupnp/miniupnpd/upnpredirect.c b/src/contrib/miniupnp/miniupnpd/upnpredirect.c new file mode 100644 index 0000000..2306ca5 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpredirect.c @@ -0,0 +1,782 @@ +/* $Id: upnpredirect.c,v 1.93 2018/03/12 22:41:53 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "macros.h" +#include "config.h" +#include "upnpredirect.h" +#include "upnpglobalvars.h" +#include "upnpevents.h" +#include "portinuse.h" +#include "upnputils.h" +#if defined(USE_NETFILTER) +#include "netfilter/iptcrdr.h" +#endif +#if defined(USE_PF) +#include "pf/obsdrdr.h" +#endif +#if defined(USE_IPF) +#include "ipf/ipfrdr.h" +#endif +#if defined(USE_IPFW) +#include "ipfw/ipfwrdr.h" +#endif +#ifdef USE_MINIUPNPDCTL +#include +#include +#endif +#ifdef ENABLE_LEASEFILE +#include +#endif + +/* from */ +#ifndef PRIu64 +#define PRIu64 "llu" +#endif + +/* proto_atoi() + * convert the string "UDP" or "TCP" to IPPROTO_UDP and IPPROTO_UDP */ +static int +proto_atoi(const char * protocol) +{ + int proto = IPPROTO_TCP; + if(strcasecmp(protocol, "UDP") == 0) + proto = IPPROTO_UDP; +#ifdef IPPROTO_UDPLITE + else if(strcasecmp(protocol, "UDPLITE") == 0) + proto = IPPROTO_UDPLITE; +#endif /* IPPROTO_UDPLITE */ + return proto; +} + +/* proto_itoa() + * convert IPPROTO_UDP, IPPROTO_UDP, etc. to "UDP", "TCP" */ +static const char * +proto_itoa(int proto) +{ + const char * protocol; + switch(proto) { + case IPPROTO_UDP: + protocol = "UDP"; + break; + case IPPROTO_TCP: + protocol = "TCP"; + break; +#ifdef IPPROTO_UDPLITE + case IPPROTO_UDPLITE: + protocol = "UDPLITE"; + break; +#endif /* IPPROTO_UDPLITE */ + default: + protocol = "*UNKNOWN*"; + } + return protocol; +} + +#ifdef ENABLE_LEASEFILE +static int +lease_file_add(unsigned short eport, + const char * iaddr, + unsigned short iport, + int proto, + const char * desc, + unsigned int timestamp) +{ + FILE * fd; + + if (lease_file == NULL) return 0; + + fd = fopen( lease_file, "a"); + if (fd==NULL) { + syslog(LOG_ERR, "could not open lease file: %s", lease_file); + return -1; + } + + /* convert our time to unix time + * if LEASEFILE_USE_REMAINING_TIME is defined, only the remaining time is stored */ + if (timestamp != 0) { + timestamp -= upnp_time(); +#ifndef LEASEFILE_USE_REMAINING_TIME + timestamp += time(NULL); +#endif + } + + fprintf(fd, "%s:%hu:%s:%hu:%u:%s\n", + proto_itoa(proto), eport, iaddr, iport, + timestamp, desc); + fclose(fd); + + return 0; +} + +static int +lease_file_remove(unsigned short eport, int proto) +{ + FILE* fd, *fdt; + int tmp; + char buf[512]; + char str[32]; + char tmpfilename[128]; + int str_size, buf_size; + + + if (lease_file == NULL) return 0; + + if (strlen( lease_file) + 7 > sizeof(tmpfilename)) { + syslog(LOG_ERR, "Lease filename is too long"); + return -1; + } + + snprintf( tmpfilename, sizeof(tmpfilename), "%sXXXXXX", lease_file); + + fd = fopen( lease_file, "r"); + if (fd==NULL) { + return 0; + } + + snprintf( str, sizeof(str), "%s:%u", proto_itoa(proto), eport); + str_size = strlen(str); + + tmp = mkstemp(tmpfilename); + if (tmp==-1) { + fclose(fd); + syslog(LOG_ERR, "could not open temporary lease file"); + return -1; + } + fchmod(tmp, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + fdt = fdopen(tmp, "a"); + + buf[sizeof(buf)-1] = 0; + while( fgets(buf, sizeof(buf)-1, fd) != NULL) { + buf_size = strlen(buf); + + if (buf_size < str_size || strncmp(str, buf, str_size)!=0) { + fwrite(buf, buf_size, 1, fdt); + } + } + fclose(fdt); + fclose(fd); + + if (rename(tmpfilename, lease_file) < 0) { + syslog(LOG_ERR, "could not rename temporary lease file to %s", lease_file); + remove(tmpfilename); + } + + return 0; + +} + +/* reload_from_lease_file() + * read lease_file and add the rules contained + */ +int reload_from_lease_file() +{ + FILE * fd; + char * p; + unsigned short eport, iport; + char * proto; + char * iaddr; + char * desc; + char * rhost; + unsigned int leaseduration; + unsigned int timestamp; + time_t current_time; +#ifndef LEASEFILE_USE_REMAINING_TIME + time_t current_unix_time; +#endif + char line[128]; + int r; + + if(!lease_file) return -1; + fd = fopen( lease_file, "r"); + if (fd==NULL) { + syslog(LOG_ERR, "could not open lease file: %s", lease_file); + return -1; + } + if(unlink(lease_file) < 0) { + syslog(LOG_WARNING, "could not unlink file %s : %m", lease_file); + } + + current_time = upnp_time(); +#ifndef LEASEFILE_USE_REMAINING_TIME + current_unix_time = time(NULL); +#endif + while(fgets(line, sizeof(line), fd)) { + syslog(LOG_DEBUG, "parsing lease file line '%s'", line); + proto = line; + p = strchr(line, ':'); + if(!p) { + syslog(LOG_ERR, "unrecognized data in lease file"); + continue; + } + *(p++) = '\0'; + iaddr = strchr(p, ':'); + if(!iaddr) { + syslog(LOG_ERR, "unrecognized data in lease file"); + continue; + } + *(iaddr++) = '\0'; + eport = (unsigned short)atoi(p); + p = strchr(iaddr, ':'); + if(!p) { + syslog(LOG_ERR, "unrecognized data in lease file"); + continue; + } + *(p++) = '\0'; + iport = (unsigned short)atoi(p); + p = strchr(p, ':'); + if(!p) { + syslog(LOG_ERR, "unrecognized data in lease file"); + continue; + } + *(p++) = '\0'; + desc = strchr(p, ':'); + if(!desc) { + syslog(LOG_ERR, "unrecognized data in lease file"); + continue; + } + *(desc++) = '\0'; + /*timestamp = (unsigned int)atoi(p);*/ + timestamp = (unsigned int)strtoul(p, NULL, 10); + /* trim description */ + while(isspace(*desc)) + desc++; + p = desc; + while(*(p+1)) + p++; + while(isspace(*p) && (p > desc)) + *(p--) = '\0'; + + if(timestamp > 0) { +#ifdef LEASEFILE_USE_REMAINING_TIME + leaseduration = timestamp; + timestamp += current_time; /* convert to our time */ +#else + if(timestamp <= (unsigned int)current_unix_time) { + syslog(LOG_NOTICE, "already expired lease in lease file"); + continue; + } else { + leaseduration = timestamp - current_unix_time; + timestamp = leaseduration + current_time; /* convert to our time */ + } +#endif + } else { + leaseduration = 0; /* default value */ + } + rhost = NULL; + r = upnp_redirect(rhost, eport, iaddr, iport, proto, desc, leaseduration); + if(r == -1) { + syslog(LOG_ERR, "Failed to redirect %hu -> %s:%hu protocol %s", + eport, iaddr, iport, proto); + } else if(r == -2) { + /* Add the redirection again to the lease file */ + lease_file_add(eport, iaddr, iport, proto_atoi(proto), + desc, timestamp); + } + } + fclose(fd); + + return 0; +} + +#ifdef LEASEFILE_USE_REMAINING_TIME +void lease_file_rewrite(void) +{ + int index; + unsigned short eport, iport; + int proto; + char iaddr[32]; + char desc[64]; + char rhost[40]; + unsigned int timestamp; + + if (lease_file == NULL) return; + remove(lease_file); + for(index = 0; ; index++) { + if(get_redirect_rule_by_index(index, 0/*ifname*/, &eport, iaddr, sizeof(iaddr), + &iport, &proto, desc, sizeof(desc), + rhost, sizeof(rhost), ×tamp, + 0, 0) < 0) + break; + if(lease_file_add(eport, iaddr, iport, proto, desc, timestamp) < 0) + break; + } +} +#endif +#endif + +/* upnp_redirect() + * calls OS/fw dependent implementation of the redirection. + * protocol should be the string "TCP" or "UDP" + * returns: 0 on success + * -1 failed to redirect + * -2 already redirected + * -3 permission check failed + * -4 already redirected (other mechanism) + */ +int +upnp_redirect(const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, + const char * protocol, const char * desc, + unsigned int leaseduration) +{ + int proto, r; + char iaddr_old[32]; + char rhost_old[32]; + unsigned short iport_old; + struct in_addr address; + unsigned int timestamp; + + proto = proto_atoi(protocol); + if(inet_aton(iaddr, &address) <= 0) { + syslog(LOG_ERR, "inet_aton(%s) FAILED", iaddr); + return -1; + } + + if(!check_upnp_rule_against_permissions(upnppermlist, num_upnpperm, + eport, address, iport)) { + syslog(LOG_INFO, "redirection permission check failed for " + "%hu->%s:%hu %s", eport, iaddr, iport, protocol); + return -3; + } + /* IGDv1 (WANIPConnection:1 Service Template Version 1.01 / Nov 12, 2001) + * - 2.2.20.PortMappingDescription : + * Overwriting Previous / Existing Port Mappings: + * If the RemoteHost, ExternalPort, PortMappingProtocol and InternalClient + * are exactly the same as an existing mapping, the existing mapping values + * for InternalPort, PortMappingDescription, PortMappingEnabled and + * PortMappingLeaseDuration are overwritten. + * Rejecting a New Port Mapping: + * In cases where the RemoteHost, ExternalPort and PortMappingProtocol + * are the same as an existing mapping, but the InternalClient is + * different, the action is rejected with an appropriate error. + * Add or Reject New Port Mapping behavior based on vendor implementation: + * In cases where the ExternalPort, PortMappingProtocol and InternalClient + * are the same, but RemoteHost is different, the vendor can choose to + * support both mappings simultaneously, or reject the second mapping + * with an appropriate error. + * + * - 2.4.16.AddPortMapping + * This action creates a new port mapping or overwrites an existing + * mapping with the same internal client. If the ExternalPort and + * PortMappingProtocol pair is already mapped to another internal client, + * an error is returned. + * + * IGDv2 (WANIPConnection:2 Service Standardized DCP (SDCP) Sep 10, 2010) + * Protocol ExternalPort RemoteHost InternalClient Result + * = = ≠ ≠ Failure + * = = ≠ = Failure or success + * (vendor specific) + * = = = ≠ Failure + * = = = = Success (overwrite) + */ + rhost_old[0] = '\0'; + r = get_redirect_rule(ext_if_name, eport, proto, + iaddr_old, sizeof(iaddr_old), &iport_old, 0, 0, + rhost_old, sizeof(rhost_old), + ×tamp, 0, 0); + if(r == 0) { + if(strcmp(iaddr, iaddr_old)==0 && + ((rhost == NULL && rhost_old[0]=='\0') || + (rhost && (strcmp(rhost, "*") == 0) && rhost_old[0]=='\0') || + (rhost && (strcmp(rhost, rhost_old) == 0)))) { + syslog(LOG_INFO, "updating existing port mapping %hu %s (rhost '%s') => %s:%hu", + eport, protocol, rhost_old, iaddr_old, iport_old); + timestamp = (leaseduration > 0) ? upnp_time() + leaseduration : 0; + if(iport != iport_old) { + r = update_portmapping(ext_if_name, eport, proto, iport, desc, timestamp); + } else { + r = update_portmapping_desc_timestamp(ext_if_name, eport, proto, desc, timestamp); + } +#ifdef ENABLE_LEASEFILE + if(r == 0) { + lease_file_remove(eport, proto); + lease_file_add(eport, iaddr, iport, proto, desc, timestamp); + } +#endif /* ENABLE_LEASEFILE */ + return r; + } else { + syslog(LOG_INFO, "port %hu %s (rhost '%s') already redirected to %s:%hu", + eport, protocol, rhost_old, iaddr_old, iport_old); + return -2; + } +#ifdef CHECK_PORTINUSE + } else if (port_in_use(ext_if_name, eport, proto, iaddr, iport) > 0) { + syslog(LOG_INFO, "port %hu protocol %s already in use", + eport, protocol); + return -4; +#endif /* CHECK_PORTINUSE */ + } else { + timestamp = (leaseduration > 0) ? upnp_time() + leaseduration : 0; + syslog(LOG_INFO, "redirecting port %hu to %s:%hu protocol %s for: %s", + eport, iaddr, iport, protocol, desc); + return upnp_redirect_internal(rhost, eport, iaddr, iport, proto, + desc, timestamp); + } +} + +int +upnp_redirect_internal(const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, + int proto, const char * desc, + unsigned int timestamp) +{ + /*syslog(LOG_INFO, "redirecting port %hu to %s:%hu protocol %s for: %s", + eport, iaddr, iport, protocol, desc); */ + if(add_redirect_rule2(ext_if_name, rhost, eport, iaddr, iport, proto, + desc, timestamp) < 0) { + return -1; + } + +#ifdef ENABLE_LEASEFILE + lease_file_add( eport, iaddr, iport, proto, desc, timestamp); +#endif +/* syslog(LOG_INFO, "creating pass rule to %s:%hu protocol %s for: %s", + iaddr, iport, protocol, desc);*/ + if(add_filter_rule2(ext_if_name, rhost, iaddr, eport, iport, proto, desc) < 0) { + /* clean up the redirect rule */ +#if !defined(__linux__) + delete_redirect_rule(ext_if_name, eport, proto); +#endif + return -1; + } + if(timestamp > 0) { + if(!nextruletoclean_timestamp || (timestamp < nextruletoclean_timestamp)) + nextruletoclean_timestamp = timestamp; + } +#ifdef ENABLE_EVENTS + /* the number of port mappings changed, we must + * inform the subscribers */ + upnp_event_var_change_notify(EWanIPC); +#endif + return 0; +} + + + +/* Firewall independent code which call the FW dependent code. */ +int +upnp_get_redirection_infos(unsigned short eport, const char * protocol, + unsigned short * iport, + char * iaddr, int iaddrlen, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * leaseduration) +{ + int r; + unsigned int timestamp; + time_t current_time; + + if(desc && (desclen > 0)) + desc[0] = '\0'; + if(rhost && (rhostlen > 0)) + rhost[0] = '\0'; + r = get_redirect_rule(ext_if_name, eport, proto_atoi(protocol), + iaddr, iaddrlen, iport, desc, desclen, + rhost, rhostlen, ×tamp, + 0, 0); + if(r == 0 && + timestamp > 0 && + timestamp > (unsigned int)(current_time = upnp_time())) { + *leaseduration = timestamp - current_time; + } else { + *leaseduration = 0; + } + return r; +} + +int +upnp_get_redirection_infos_by_index(int index, + unsigned short * eport, char * protocol, + unsigned short * iport, + char * iaddr, int iaddrlen, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * leaseduration) +{ + /*char ifname[IFNAMSIZ];*/ + int proto = 0; + unsigned int timestamp; + time_t current_time; + + if(desc && (desclen > 0)) + desc[0] = '\0'; + if(rhost && (rhostlen > 0)) + rhost[0] = '\0'; + if(get_redirect_rule_by_index(index, 0/*ifname*/, eport, iaddr, iaddrlen, + iport, &proto, desc, desclen, + rhost, rhostlen, ×tamp, + 0, 0) < 0) + return -1; + else + { + current_time = upnp_time(); + *leaseduration = (timestamp > (unsigned int)current_time) + ? (timestamp - current_time) + : 0; + if(proto == IPPROTO_TCP) + memcpy(protocol, "TCP", 4); +#ifdef IPPROTO_UDPLITE + else if(proto == IPPROTO_UDPLITE) + memcpy(protocol, "UDPLITE", 8); +#endif /* IPPROTO_UDPLITE */ + else + memcpy(protocol, "UDP", 4); + return 0; + } +} + +/* called from natpmp.c too */ +int +_upnp_delete_redir(unsigned short eport, int proto) +{ + int r; +#if defined(__linux__) + r = delete_redirect_and_filter_rules(eport, proto); +#elif defined(USE_PF) + r = delete_redirect_and_filter_rules(ext_if_name, eport, proto); +#else + r = delete_redirect_rule(ext_if_name, eport, proto); + delete_filter_rule(ext_if_name, eport, proto); +#endif +#ifdef ENABLE_LEASEFILE + lease_file_remove( eport, proto); +#endif + +#ifdef ENABLE_EVENTS + upnp_event_var_change_notify(EWanIPC); +#endif + return r; +} + +int +upnp_delete_redirection(unsigned short eport, const char * protocol) +{ + syslog(LOG_INFO, "removing redirect rule port %hu %s", eport, protocol); + return _upnp_delete_redir(eport, proto_atoi(protocol)); +} + +/* upnp_get_portmapping_number_of_entries() + * TODO: improve this code. */ +int +upnp_get_portmapping_number_of_entries() +{ + int n = 0, r = 0; + unsigned short eport, iport; + char protocol[4], iaddr[32], desc[64], rhost[32]; + unsigned int leaseduration; + do { + protocol[0] = '\0'; iaddr[0] = '\0'; desc[0] = '\0'; + r = upnp_get_redirection_infos_by_index(n, &eport, protocol, &iport, + iaddr, sizeof(iaddr), + desc, sizeof(desc), + rhost, sizeof(rhost), + &leaseduration); + n++; + } while(r==0); + return (n-1); +} + +/* functions used to remove unused rules + * As a side effect, delete expired rules (based on LeaseDuration) */ +struct rule_state * +get_upnp_rules_state_list(int max_rules_number_target) +{ + /*char ifname[IFNAMSIZ];*/ + int proto; + unsigned short iport; + unsigned int timestamp; + struct rule_state * tmp; + struct rule_state * list = 0; + struct rule_state * * p; + int i = 0; + time_t current_time; + + /*ifname[0] = '\0';*/ + tmp = malloc(sizeof(struct rule_state)); + if(!tmp) + return 0; + current_time = upnp_time(); + nextruletoclean_timestamp = 0; + while(get_redirect_rule_by_index(i, /*ifname*/0, &tmp->eport, 0, 0, + &iport, &proto, 0, 0, 0,0, ×tamp, + &tmp->packets, &tmp->bytes) >= 0) + { + tmp->to_remove = 0; + if(timestamp > 0) { + /* need to remove this port mapping ? */ + if(timestamp <= (unsigned int)current_time) + tmp->to_remove = 1; + else if((nextruletoclean_timestamp <= (unsigned int)current_time) + || (timestamp < nextruletoclean_timestamp)) + nextruletoclean_timestamp = timestamp; + } + tmp->proto = (short)proto; + /* add tmp to list */ + tmp->next = list; + list = tmp; + /* prepare next iteration */ + i++; + tmp = malloc(sizeof(struct rule_state)); + if(!tmp) + break; + } +#ifdef PCP_PEER + i=0; + while(get_peer_rule_by_index(i, /*ifname*/0, &tmp->eport, 0, 0, + &iport, &proto, 0, 0, 0,0,0, ×tamp, + &tmp->packets, &tmp->bytes) >= 0) + { + tmp->to_remove = 0; + if(timestamp > 0) { + /* need to remove this port mapping ? */ + if(timestamp <= (unsigned int)current_time) + tmp->to_remove = 1; + else if((nextruletoclean_timestamp <= (unsigned int)current_time) + || (timestamp < nextruletoclean_timestamp)) + nextruletoclean_timestamp = timestamp; + } + tmp->proto = (short)proto; + /* add tmp to list */ + tmp->next = list; + list = tmp; + /* prepare next iteration */ + i++; + tmp = malloc(sizeof(struct rule_state)); + if(!tmp) + break; + } +#endif + free(tmp); + /* remove the redirections that need to be removed */ + for(p = &list, tmp = list; tmp; tmp = *p) + { + if(tmp->to_remove) + { + syslog(LOG_NOTICE, "remove port mapping %hu %s because it has expired", + tmp->eport, proto_itoa(tmp->proto)); + _upnp_delete_redir(tmp->eport, tmp->proto); + *p = tmp->next; + free(tmp); + i--; + } else { + p = &(tmp->next); + } + } + /* return empty list if not enough redirections */ + if(i<=max_rules_number_target) + while(list) + { + tmp = list; + list = tmp->next; + free(tmp); + } + /* return list */ + return list; +} + +void +remove_unused_rules(struct rule_state * list) +{ + char ifname[IFNAMSIZ]; + unsigned short iport; + struct rule_state * tmp; + u_int64_t packets; + u_int64_t bytes; + unsigned int timestamp; + int n = 0; + + while(list) + { + /* remove the rule if no traffic has used it */ + if(get_redirect_rule(ifname, list->eport, list->proto, + 0, 0, &iport, 0, 0, 0, 0, ×tamp, + &packets, &bytes) >= 0) + { + if(packets == list->packets && bytes == list->bytes) + { + syslog(LOG_DEBUG, "removing unused mapping %hu %s : still " + "%" PRIu64 "packets %" PRIu64 "bytes", + list->eport, proto_itoa(list->proto), + packets, bytes); + _upnp_delete_redir(list->eport, list->proto); + n++; + } + } + tmp = list; + list = tmp->next; + free(tmp); + } + if(n>0) + syslog(LOG_NOTICE, "removed %d unused rules", n); +} + +/* upnp_get_portmappings_in_range() + * return a list of all "external" ports for which a port + * mapping exists */ +unsigned short * +upnp_get_portmappings_in_range(unsigned short startport, + unsigned short endport, + const char * protocol, + unsigned int * number) +{ + int proto; + proto = proto_atoi(protocol); + if(!number) + return NULL; + return get_portmappings_in_range(startport, endport, proto, number); +} + +/* stuff for miniupnpdctl */ +#ifdef USE_MINIUPNPDCTL +void +write_ruleset_details(int s) +{ + int proto = 0; + unsigned short eport, iport; + char desc[64]; + char iaddr[32]; + char rhost[32]; + unsigned int timestamp; + u_int64_t packets; + u_int64_t bytes; + int i = 0; + char buffer[256]; + int n; + + write(s, "Ruleset :\n", 10); + while(get_redirect_rule_by_index(i, 0/*ifname*/, &eport, iaddr, sizeof(iaddr), + &iport, &proto, desc, sizeof(desc), + rhost, sizeof(rhost), + ×tamp, + &packets, &bytes) >= 0) + { + n = snprintf(buffer, sizeof(buffer), + "%2d %s %s:%hu->%s:%hu " + "'%s' %u %" PRIu64 " %" PRIu64 "\n", + /*"'%s' %llu %llu\n",*/ + i, proto_itoa(proto), rhost, + eport, iaddr, iport, desc, timestamp, packets, bytes); + write(s, buffer, n); + i++; + } +} +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnpredirect.h b/src/contrib/miniupnp/miniupnpd/upnpredirect.h new file mode 100644 index 0000000..bb105de --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpredirect.h @@ -0,0 +1,123 @@ +/* $Id: upnpredirect.h,v 1.36 2018/01/16 00:50:49 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPREDIRECT_H_INCLUDED +#define UPNPREDIRECT_H_INCLUDED + +/* for u_int64_t */ +#include + +#include "config.h" + +#ifdef ENABLE_LEASEFILE +int reload_from_lease_file(void); +#ifdef LEASEFILE_USE_REMAINING_TIME +void lease_file_rewrite(void); +#endif +#endif + +/* upnp_redirect() + * calls OS/fw dependent implementation of the redirection. + * protocol should be the string "TCP" or "UDP" + * returns: 0 on success + * -1 failed to redirect + * -2 already redirected + * -3 permission check failed + */ +int +upnp_redirect(const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, + const char * protocol, const char * desc, + unsigned int leaseduration); + +/* upnp_redirect_internal() + * same as upnp_redirect() without any check */ +int +upnp_redirect_internal(const char * rhost, unsigned short eport, + const char * iaddr, unsigned short iport, + int proto, const char * desc, + unsigned int timestamp); + +/* upnp_get_redirection_infos() + * returns : 0 on success + * -1 failed to get the port mapping entry or no entry exists */ +int +upnp_get_redirection_infos(unsigned short eport, const char * protocol, + unsigned short * iport, char * iaddr, int iaddrlen, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * leaseduration); + +/* upnp_get_redirection_infos_by_index() + * returns : 0 on success + * -1 failed to get the port mapping or index out of range */ +int +upnp_get_redirection_infos_by_index(int index, + unsigned short * eport, char * protocol, + unsigned short * iport, + char * iaddr, int iaddrlen, + char * desc, int desclen, + char * rhost, int rhostlen, + unsigned int * leaseduration); + +/* upnp_delete_redirection() + * returns: 0 on success + * -1 on failure*/ +int +upnp_delete_redirection(unsigned short eport, const char * protocol); + +/* _upnp_delete_redir() + * same as above */ +int +_upnp_delete_redir(unsigned short eport, int proto); + +/* Periodic cleanup functions + */ +struct rule_state +{ + u_int64_t packets; + u_int64_t bytes; + struct rule_state * next; + unsigned short eport; + unsigned char proto; + unsigned char to_remove; +}; + +/* return a linked list of all rules + * or an empty list if there are not enough + * As a "side effect", delete rules which are expired */ +struct rule_state * +get_upnp_rules_state_list(int max_rules_number_target); + +/* return the number of port mapping entries */ +int +upnp_get_portmapping_number_of_entries(void); + +/* remove_unused_rules() : + * also free the list */ +void +remove_unused_rules(struct rule_state * list); + +/* upnp_get_portmappings_in_range() + * return a list of all "external" ports for which a port + * mapping exists */ +unsigned short * +upnp_get_portmappings_in_range(unsigned short startport, + unsigned short endport, + const char * protocol, + unsigned int * number); + +/* stuff for responding to miniupnpdctl */ +#ifdef USE_MINIUPNPDCTL +void +write_ruleset_details(int s); +#endif + +#endif + + diff --git a/src/contrib/miniupnp/miniupnpd/upnpreplyparse.c b/src/contrib/miniupnp/miniupnpd/upnpreplyparse.c new file mode 100644 index 0000000..5921349 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpreplyparse.c @@ -0,0 +1,196 @@ +/* $Id: upnpreplyparse.c,v 1.20 2017/12/12 11:26:25 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2017 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include + +#include "upnpreplyparse.h" +#include "minixml.h" + +static void +NameValueParserStartElt(void * d, const char * name, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + data->topelt = 1; + if(l>63) + l = 63; + memcpy(data->curelt, name, l); + data->curelt[l] = '\0'; + data->cdata = NULL; + data->cdatalen = 0; +} + +static void +NameValueParserEndElt(void * d, const char * name, int namelen) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + struct NameValue * nv; + (void)name; + (void)namelen; + if(!data->topelt) + return; + if(strcmp(data->curelt, "NewPortListing") != 0) + { + int l; + /* standard case. Limited to n chars strings */ + l = data->cdatalen; + nv = malloc(sizeof(struct NameValue)); + if(nv == NULL) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "NameValueParserEndElt"); +#endif /* DEBUG */ + return; + } + if(l>=(int)sizeof(nv->value)) + l = sizeof(nv->value) - 1; + strncpy(nv->name, data->curelt, 64); + nv->name[63] = '\0'; + if(data->cdata != NULL) + { + memcpy(nv->value, data->cdata, l); + nv->value[l] = '\0'; + } + else + { + nv->value[0] = '\0'; + } + nv->l_next = data->l_head; /* insert in list */ + data->l_head = nv; + } + data->cdata = NULL; + data->cdatalen = 0; + data->topelt = 0; +} + +static void +NameValueParserGetData(void * d, const char * datas, int l) +{ + struct NameValueParserData * data = (struct NameValueParserData *)d; + if(strcmp(data->curelt, "NewPortListing") == 0) + { + /* specific case for NewPortListing which is a XML Document */ + data->portListing = malloc(l + 1); + if(!data->portListing) + { + /* malloc error */ +#ifdef DEBUG + fprintf(stderr, "%s: error allocating memory", + "NameValueParserGetData"); +#endif /* DEBUG */ + return; + } + memcpy(data->portListing, datas, l); + data->portListing[l] = '\0'; + data->portListingLength = l; + } + else + { + /* standard case. */ + data->cdata = datas; + data->cdatalen = l; + } +} + +void +ParseNameValue(const char * buffer, int bufsize, + struct NameValueParserData * data) +{ + struct xmlparser parser; + memset(data, 0, sizeof(struct NameValueParserData)); + /* init xmlparser object */ + parser.xmlstart = buffer; + parser.xmlsize = bufsize; + parser.data = data; + parser.starteltfunc = NameValueParserStartElt; + parser.endeltfunc = NameValueParserEndElt; + parser.datafunc = NameValueParserGetData; + parser.attfunc = 0; + parsexml(&parser); +} + +void +ClearNameValueList(struct NameValueParserData * pdata) +{ + struct NameValue * nv; + if(pdata->portListing) + { + free(pdata->portListing); + pdata->portListing = NULL; + pdata->portListingLength = 0; + } + while((nv = pdata->l_head) != NULL) + { + pdata->l_head = nv->l_next; + free(nv); + } +} + +char * +GetValueFromNameValueList(struct NameValueParserData * pdata, + const char * Name) +{ + struct NameValue * nv; + char * p = NULL; + for(nv = pdata->l_head; + (nv != NULL) && (p == NULL); + nv = nv->l_next) + { + if(strcmp(nv->name, Name) == 0) + p = nv->value; + } + return p; +} + +#if 0 +/* useless now that minixml ignores namespaces by itself */ +char * +GetValueFromNameValueListIgnoreNS(struct NameValueParserData * pdata, + const char * Name) +{ + struct NameValue * nv; + char * p = NULL; + char * pname; + for(nv = pdata->head.lh_first; + (nv != NULL) && (p == NULL); + nv = nv->entries.le_next) + { + pname = strrchr(nv->name, ':'); + if(pname) + pname++; + else + pname = nv->name; + if(strcmp(pname, Name)==0) + p = nv->value; + } + return p; +} +#endif + +/* debug all-in-one function + * do parsing then display to stdout */ +#ifdef DEBUG +void +DisplayNameValueList(char * buffer, int bufsize) +{ + struct NameValueParserData pdata; + struct NameValue * nv; + ParseNameValue(buffer, bufsize, &pdata); + for(nv = pdata.l_head; + nv != NULL; + nv = nv->l_next) + { + printf("%s = %s\n", nv->name, nv->value); + } + ClearNameValueList(&pdata); +} +#endif /* DEBUG */ + diff --git a/src/contrib/miniupnp/miniupnpd/upnpreplyparse.h b/src/contrib/miniupnp/miniupnpd/upnpreplyparse.h new file mode 100644 index 0000000..6badd15 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpreplyparse.h @@ -0,0 +1,63 @@ +/* $Id: upnpreplyparse.h,v 1.19 2014/10/27 16:33:19 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2013 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPREPLYPARSE_H_INCLUDED +#define UPNPREPLYPARSE_H_INCLUDED + +#ifdef __cplusplus +extern "C" { +#endif + +struct NameValue { + struct NameValue * l_next; + char name[64]; + char value[128]; +}; + +struct NameValueParserData { + struct NameValue * l_head; + char curelt[64]; + char * portListing; + int portListingLength; + int topelt; + const char * cdata; + int cdatalen; +}; + +/* ParseNameValue() */ +void +ParseNameValue(const char * buffer, int bufsize, + struct NameValueParserData * data); + +/* ClearNameValueList() */ +void +ClearNameValueList(struct NameValueParserData * pdata); + +/* GetValueFromNameValueList() */ +char * +GetValueFromNameValueList(struct NameValueParserData * pdata, + const char * Name); + +#if 0 +/* GetValueFromNameValueListIgnoreNS() */ +char * +GetValueFromNameValueListIgnoreNS(struct NameValueParserData * pdata, + const char * Name); +#endif + +/* DisplayNameValueList() */ +#ifdef DEBUG +void +DisplayNameValueList(char * buffer, int bufsize); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnpsoap.c b/src/contrib/miniupnp/miniupnpd/upnpsoap.c new file mode 100644 index 0000000..b4f156c --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpsoap.c @@ -0,0 +1,2322 @@ +/* $Id: upnpsoap.c,v 1.151 2018/03/13 10:32:53 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * MiniUPnP project + * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "macros.h" +#include "config.h" +#include "upnpglobalvars.h" +#include "upnphttp.h" +#include "upnpsoap.h" +#include "upnpreplyparse.h" +#include "upnpredirect.h" +#include "upnppinhole.h" +#include "getifaddr.h" +#include "getifstats.h" +#include "getconnstatus.h" +#include "upnpurns.h" +#include "upnputils.h" + +/* utility function */ +static int is_numeric(const char * s) +{ + while(*s) { + if(*s < '0' || *s > '9') return 0; + s++; + } + return 1; +} + +static void +BuildSendAndCloseSoapResp(struct upnphttp * h, + const char * body, int bodylen) +{ + static const char beforebody[] = + "\r\n" + "" + ""; + + static const char afterbody[] = + "" + "\r\n"; + + int r = BuildHeader_upnphttp(h, 200, "OK", sizeof(beforebody) - 1 + + sizeof(afterbody) - 1 + bodylen ); + + if(r >= 0) { + memcpy(h->res_buf + h->res_buflen, beforebody, sizeof(beforebody) - 1); + h->res_buflen += sizeof(beforebody) - 1; + + memcpy(h->res_buf + h->res_buflen, body, bodylen); + h->res_buflen += bodylen; + + memcpy(h->res_buf + h->res_buflen, afterbody, sizeof(afterbody) - 1); + h->res_buflen += sizeof(afterbody) - 1; + } else { + BuildResp2_upnphttp(h, 500, "Internal Server Error", NULL, 0); + } + + SendRespAndClose_upnphttp(h); +} + +static void +GetConnectionTypeInfo(struct upnphttp * h, const char * action, const char * ns) +{ +#if 0 + static const char resp[] = + "" + "IP_Routed" + "IP_Routed" + ""; +#endif + static const char resp[] = + "" + "IP_Routed" + "IP_Routed" + ""; + char body[512]; + int bodylen; + + bodylen = snprintf(body, sizeof(body), resp, + action, ns, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +/* maximum value for a UPNP ui4 type variable */ +#define UPNP_UI4_MAX (4294967295ul) + +static void +GetTotalBytesSent(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + static const char resp[] = + "" + "%lu" + ""; + + char body[512]; + int bodylen; + struct ifdata data; + + r = getifstats(ext_if_name, &data); + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /* was "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1" */ +#ifdef UPNP_STRICT + r<0?0:(data.obytes & UPNP_UI4_MAX), action); +#else /* UPNP_STRICT */ + r<0?0:data.obytes, action); +#endif /* UPNP_STRICT */ + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +GetTotalBytesReceived(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + static const char resp[] = + "" + "%lu" + ""; + + char body[512]; + int bodylen; + struct ifdata data; + + r = getifstats(ext_if_name, &data); + /* TotalBytesReceived + * This variable represents the cumulative counter for total number of + * bytes received downstream across all connection service instances on + * WANDevice. The count rolls over to 0 after it reaching the maximum + * value (2^32)-1. */ + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /* was "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1" */ +#ifdef UPNP_STRICT + r<0?0:(data.ibytes & UPNP_UI4_MAX), action); +#else /* UPNP_STRICT */ + r<0?0:data.ibytes, action); +#endif /* UPNP_STRICT */ + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +GetTotalPacketsSent(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + static const char resp[] = + "" + "%lu" + ""; + + char body[512]; + int bodylen; + struct ifdata data; + + r = getifstats(ext_if_name, &data); + bodylen = snprintf(body, sizeof(body), resp, + action, ns,/*"urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1",*/ +#ifdef UPNP_STRICT + r<0?0:(data.opackets & UPNP_UI4_MAX), action); +#else /* UPNP_STRICT */ + r<0?0:data.opackets, action); +#endif /* UPNP_STRICT */ + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +GetTotalPacketsReceived(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + static const char resp[] = + "" + "%lu" + ""; + + char body[512]; + int bodylen; + struct ifdata data; + + r = getifstats(ext_if_name, &data); + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /* was "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1" */ +#ifdef UPNP_STRICT + r<0?0:(data.ipackets & UPNP_UI4_MAX), action); +#else /* UPNP_STRICT */ + r<0?0:data.ipackets, action); +#endif /* UPNP_STRICT */ + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +GetCommonLinkProperties(struct upnphttp * h, const char * action, const char * ns) +{ + /* WANAccessType : set depending on the hardware : + * DSL, POTS (plain old Telephone service), Cable, Ethernet */ + static const char resp[] = + "" + "%s" + "%lu" + "%lu" + "%s" + ""; + + char body[2048]; + int bodylen; + struct ifdata data; + const char * status = "Up"; /* Up, Down (Required), + * Initializing, Unavailable (Optional) */ + const char * wan_access_type = "Cable"; /* DSL, POTS, Cable, Ethernet */ + char ext_ip_addr[INET_ADDRSTRLEN]; + + if((downstream_bitrate == 0) || (upstream_bitrate == 0)) + { + if(getifstats(ext_if_name, &data) >= 0) + { + if(downstream_bitrate == 0) downstream_bitrate = data.baudrate; + if(upstream_bitrate == 0) upstream_bitrate = data.baudrate; + } + } + if(getifaddr(ext_if_name, ext_ip_addr, INET_ADDRSTRLEN, NULL, NULL) < 0) { + status = "Down"; + } + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /* was "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1" */ + wan_access_type, + upstream_bitrate, downstream_bitrate, + status, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +GetStatusInfo(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "%s" + "ERROR_NONE" + "%ld" + ""; + + char body[512]; + int bodylen; + time_t uptime; + const char * status; + /* ConnectionStatus possible values : + * Unconfigured, Connecting, Connected, PendingDisconnect, + * Disconnecting, Disconnected */ + + status = get_wan_connection_status_str(ext_if_name); + uptime = upnp_get_uptime(); + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /*SERVICE_TYPE_WANIPC,*/ + status, (long)uptime, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +GetNATRSIPStatus(struct upnphttp * h, const char * action, const char * ns) +{ +#if 0 + static const char resp[] = + "" + "0" + "1" + ""; + UNUSED(action); +#endif + static const char resp[] = + "" + "0" + "1" + ""; + char body[512]; + int bodylen; + /* 2.2.9. RSIPAvailable + * This variable indicates if Realm-specific IP (RSIP) is available + * as a feature on the InternetGatewayDevice. RSIP is being defined + * in the NAT working group in the IETF to allow host-NATing using + * a standard set of message exchanges. It also allows end-to-end + * applications that otherwise break if NAT is introduced + * (e.g. IPsec-based VPNs). + * A gateway that does not support RSIP should set this variable to 0. */ + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /*SERVICE_TYPE_WANIPC,*/ + action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +GetExternalIPAddress(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "%s" + ""; + + char body[512]; + int bodylen; + char ext_ip_addr[INET_ADDRSTRLEN]; + /* Does that method need to work with IPv6 ? + * There is usually no NAT with IPv6 */ + +#ifndef MULTIPLE_EXTERNAL_IP + if(use_ext_ip_addr) + { + strncpy(ext_ip_addr, use_ext_ip_addr, INET_ADDRSTRLEN); + ext_ip_addr[INET_ADDRSTRLEN - 1] = '\0'; + } + else if(getifaddr(ext_if_name, ext_ip_addr, INET_ADDRSTRLEN, NULL, NULL) < 0) + { + syslog(LOG_ERR, "Failed to get ip address for interface %s", + ext_if_name); + strncpy(ext_ip_addr, "0.0.0.0", INET_ADDRSTRLEN); + } +#else + struct lan_addr_s * lan_addr; + strncpy(ext_ip_addr, "0.0.0.0", INET_ADDRSTRLEN); + for(lan_addr = lan_addrs.lh_first; lan_addr != NULL; lan_addr = lan_addr->list.le_next) + { + if( (h->clientaddr.s_addr & lan_addr->mask.s_addr) + == (lan_addr->addr.s_addr & lan_addr->mask.s_addr)) + { + strncpy(ext_ip_addr, lan_addr->ext_ip_str, INET_ADDRSTRLEN); + break; + } + } +#endif + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /*SERVICE_TYPE_WANIPC,*/ + ext_ip_addr, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +/* AddPortMapping method of WANIPConnection Service + * Ignored argument : NewEnabled */ +static void +AddPortMapping(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + /*static const char resp[] = + "";*/ + static const char resp[] = + ""; + + char body[512]; + int bodylen; + struct NameValueParserData data; + char * int_ip, * int_port, * ext_port, * protocol, * desc; + char * leaseduration_str; + unsigned int leaseduration; + char * r_host; + unsigned short iport, eport; + + struct hostent *hp; /* getbyhostname() */ + char ** ptr; /* getbyhostname() */ + struct in_addr result_ip;/*unsigned char result_ip[16];*/ /* inet_pton() */ + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + int_ip = GetValueFromNameValueList(&data, "NewInternalClient"); + if (int_ip) { + /* trim */ + while(int_ip[0] == ' ') + int_ip++; + } +#ifdef UPNP_STRICT + if (!int_ip || int_ip[0] == '\0') + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } +#endif + + /* IGD 2 MUST support both wildcard and specific IP address values + * for RemoteHost (only the wildcard value was REQUIRED in release 1.0) */ + r_host = GetValueFromNameValueList(&data, "NewRemoteHost"); +#ifndef SUPPORT_REMOTEHOST +#ifdef UPNP_STRICT + if (r_host && (strlen(r_host) > 0) && (0 != strcmp(r_host, "*"))) + { + ClearNameValueList(&data); + SoapError(h, 726, "RemoteHostOnlySupportsWildcard"); + return; + } +#endif +#endif + +#ifndef UPNP_STRICT + /* if arg is empty, use client address + * see https://github.com/miniupnp/miniupnp/issues/236 */ + if (!int_ip || int_ip[0] == '\0') + { + int_ip = h->clientaddr_str; + memcpy(&result_ip, &(h->clientaddr), sizeof(struct in_addr)); + } + else +#endif + /* if ip not valid assume hostname and convert */ + if (inet_pton(AF_INET, int_ip, &result_ip) <= 0) + { + hp = gethostbyname(int_ip); + if(hp && hp->h_addrtype == AF_INET) + { + for(ptr = hp->h_addr_list; ptr && *ptr; ptr++) + { + int_ip = inet_ntoa(*((struct in_addr *) *ptr)); + result_ip = *((struct in_addr *) *ptr); + /* TODO : deal with more than one ip per hostname */ + break; + } + } + else + { + syslog(LOG_ERR, "Failed to convert hostname '%s' to ip address", int_ip); + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + } + + /* check if NewInternalAddress is the client address */ + if(GETFLAG(SECUREMODEMASK)) + { + if(h->clientaddr.s_addr != result_ip.s_addr) + { + syslog(LOG_INFO, "Client %s tried to redirect port to %s", + inet_ntoa(h->clientaddr), int_ip); + ClearNameValueList(&data); + SoapError(h, 718, "ConflictInMappingEntry"); + return; + } + } + + int_port = GetValueFromNameValueList(&data, "NewInternalPort"); + ext_port = GetValueFromNameValueList(&data, "NewExternalPort"); + protocol = GetValueFromNameValueList(&data, "NewProtocol"); + desc = GetValueFromNameValueList(&data, "NewPortMappingDescription"); + leaseduration_str = GetValueFromNameValueList(&data, "NewLeaseDuration"); + + if (!int_port || !ext_port || !protocol) + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + + eport = (unsigned short)atoi(ext_port); + iport = (unsigned short)atoi(int_port); + + if (strcmp(ext_port, "*") == 0 || eport == 0) + { + ClearNameValueList(&data); + SoapError(h, 716, "Wildcard not permited in ExtPort"); + return; + } + + leaseduration = leaseduration_str ? atoi(leaseduration_str) : 0; +#ifdef IGD_V2 + /* PortMappingLeaseDuration can be either a value between 1 and + * 604800 seconds or the zero value (for infinite lease time). + * Note that an infinite lease time can be only set by out-of-band + * mechanisms like WWW-administration, remote management or local + * management. + * If a control point uses the value 0 to indicate an infinite lease + * time mapping, it is REQUIRED that gateway uses the maximum value + * instead (e.g. 604800 seconds) */ + if(leaseduration == 0 || leaseduration > 604800) + leaseduration = 604800; +#endif + + syslog(LOG_INFO, "%s: ext port %hu to %s:%hu protocol %s for: %s leaseduration=%u rhost=%s", + action, eport, int_ip, iport, protocol, desc, leaseduration, + r_host ? r_host : "NULL"); + + r = upnp_redirect(r_host, eport, int_ip, iport, protocol, desc, leaseduration); + + ClearNameValueList(&data); + + /* possible error codes for AddPortMapping : + * 402 - Invalid Args + * 501 - Action Failed + * 715 - Wildcard not permited in SrcAddr + * 716 - Wildcard not permited in ExtPort + * 718 - ConflictInMappingEntry + * 724 - SamePortValuesRequired (deprecated in IGD v2) + * 725 - OnlyPermanentLeasesSupported + The NAT implementation only supports permanent lease times on + port mappings (deprecated in IGD v2) + * 726 - RemoteHostOnlySupportsWildcard + RemoteHost must be a wildcard and cannot be a specific IP + address or DNS name (deprecated in IGD v2) + * 727 - ExternalPortOnlySupportsWildcard + ExternalPort must be a wildcard and cannot be a specific port + value (deprecated in IGD v2) + * 728 - NoPortMapsAvailable + There are not enough free ports available to complete the mapping + (added in IGD v2) + * 729 - ConflictWithOtherMechanisms (added in IGD v2) */ + switch(r) + { + case 0: /* success */ + bodylen = snprintf(body, sizeof(body), resp, + action, ns/*SERVICE_TYPE_WANIPC*/); + BuildSendAndCloseSoapResp(h, body, bodylen); + break; + case -4: +#ifdef IGD_V2 + SoapError(h, 729, "ConflictWithOtherMechanisms"); + break; +#endif /* IGD_V2 */ + case -2: /* already redirected */ + case -3: /* not permitted */ + SoapError(h, 718, "ConflictInMappingEntry"); + break; + default: + SoapError(h, 501, "ActionFailed"); + } +} + +/* AddAnyPortMapping was added in WANIPConnection v2 */ +static void +AddAnyPortMapping(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + static const char resp[] = + "" + "%hu" + ""; + + char body[512]; + int bodylen; + + struct NameValueParserData data; + const char * int_ip, * int_port, * ext_port, * protocol, * desc; + const char * r_host; + unsigned short iport, eport; + const char * leaseduration_str; + unsigned int leaseduration; + + struct hostent *hp; /* getbyhostname() */ + char ** ptr; /* getbyhostname() */ + struct in_addr result_ip;/*unsigned char result_ip[16];*/ /* inet_pton() */ + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + r_host = GetValueFromNameValueList(&data, "NewRemoteHost"); + ext_port = GetValueFromNameValueList(&data, "NewExternalPort"); + protocol = GetValueFromNameValueList(&data, "NewProtocol"); + int_port = GetValueFromNameValueList(&data, "NewInternalPort"); + int_ip = GetValueFromNameValueList(&data, "NewInternalClient"); + /* NewEnabled */ + desc = GetValueFromNameValueList(&data, "NewPortMappingDescription"); + leaseduration_str = GetValueFromNameValueList(&data, "NewLeaseDuration"); + + leaseduration = leaseduration_str ? atoi(leaseduration_str) : 0; + if(leaseduration == 0) + leaseduration = 604800; + + if (!int_ip || !ext_port || !int_port) + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + + eport = (unsigned short)atoi(ext_port); + iport = (unsigned short)atoi(int_port); + if(iport == 0 || !is_numeric(ext_port)) { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } +#ifndef SUPPORT_REMOTEHOST +#ifdef UPNP_STRICT + if (r_host && (strlen(r_host) > 0) && (0 != strcmp(r_host, "*"))) + { + ClearNameValueList(&data); + SoapError(h, 726, "RemoteHostOnlySupportsWildcard"); + return; + } +#endif +#endif + + /* if ip not valid assume hostname and convert */ + if (inet_pton(AF_INET, int_ip, &result_ip) <= 0) + { + hp = gethostbyname(int_ip); + if(hp && hp->h_addrtype == AF_INET) + { + for(ptr = hp->h_addr_list; ptr && *ptr; ptr++) + { + int_ip = inet_ntoa(*((struct in_addr *) *ptr)); + result_ip = *((struct in_addr *) *ptr); + /* TODO : deal with more than one ip per hostname */ + break; + } + } + else + { + syslog(LOG_ERR, "Failed to convert hostname '%s' to ip address", int_ip); + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + } + + /* check if NewInternalAddress is the client address */ + if(GETFLAG(SECUREMODEMASK)) + { + if(h->clientaddr.s_addr != result_ip.s_addr) + { + syslog(LOG_INFO, "Client %s tried to redirect port to %s", + inet_ntoa(h->clientaddr), int_ip); + ClearNameValueList(&data); + SoapError(h, 606, "Action not authorized"); + return; + } + } + + /* TODO : accept a different external port + * have some smart strategy to choose the port */ + for(;;) { + r = upnp_redirect(r_host, eport, int_ip, iport, protocol, desc, leaseduration); + if(r==-2 && eport < 65535) { + eport++; + } else { + break; + } + } + + ClearNameValueList(&data); + + switch(r) + { + case 0: /* success */ + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /*SERVICE_TYPE_WANIPC,*/ + eport, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + break; + case -2: /* already redirected */ + SoapError(h, 718, "ConflictInMappingEntry"); + break; + case -3: /* not permitted */ + SoapError(h, 606, "Action not authorized"); + break; + default: + SoapError(h, 501, "ActionFailed"); + } +} + +static void +GetSpecificPortMappingEntry(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + static const char resp[] = + "" + "%u" + "%s" + "1" + "%s" + "%u" + ""; + + char body[1024]; + int bodylen; + struct NameValueParserData data; + const char * r_host, * ext_port, * protocol; + unsigned short eport, iport; + char int_ip[32]; + char desc[64]; + unsigned int leaseduration = 0; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + r_host = GetValueFromNameValueList(&data, "NewRemoteHost"); + ext_port = GetValueFromNameValueList(&data, "NewExternalPort"); + protocol = GetValueFromNameValueList(&data, "NewProtocol"); + +#ifdef UPNP_STRICT + if(!ext_port || !protocol || !r_host) +#else + if(!ext_port || !protocol) +#endif + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } +#ifndef SUPPORT_REMOTEHOST +#ifdef UPNP_STRICT + if (r_host && (strlen(r_host) > 0) && (0 != strcmp(r_host, "*"))) + { + ClearNameValueList(&data); + SoapError(h, 726, "RemoteHostOnlySupportsWildcard"); + return; + } +#endif +#endif + + eport = (unsigned short)atoi(ext_port); + if(eport == 0) + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + + /* TODO : add r_host as an input parameter ... + * We prevent several Port Mapping with same external port + * but different remoteHost to be set up, so that is not + * a priority. */ + r = upnp_get_redirection_infos(eport, protocol, &iport, + int_ip, sizeof(int_ip), + desc, sizeof(desc), + NULL, 0, + &leaseduration); + + if(r < 0) + { + SoapError(h, 714, "NoSuchEntryInArray"); + } + else + { + syslog(LOG_INFO, "%s: rhost='%s' %s %s found => %s:%u desc='%s'", + action, + r_host ? r_host : "NULL", ext_port, protocol, int_ip, + (unsigned int)iport, desc); + bodylen = snprintf(body, sizeof(body), resp, + action, ns/*SERVICE_TYPE_WANIPC*/, + (unsigned int)iport, int_ip, desc, leaseduration, + action); + BuildSendAndCloseSoapResp(h, body, bodylen); + } + + ClearNameValueList(&data); +} + +static void +DeletePortMapping(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + /*static const char resp[] = + "" + "";*/ + static const char resp[] = + "" + ""; + + char body[512]; + int bodylen; + struct NameValueParserData data; + const char * ext_port, * protocol; + unsigned short eport; +#ifdef UPNP_STRICT + const char * r_host; +#endif /* UPNP_STRICT */ + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + ext_port = GetValueFromNameValueList(&data, "NewExternalPort"); + protocol = GetValueFromNameValueList(&data, "NewProtocol"); +#ifdef UPNP_STRICT + r_host = GetValueFromNameValueList(&data, "NewRemoteHost"); +#endif /* UPNP_STRICT */ + +#ifdef UPNP_STRICT + if(!ext_port || !protocol || !r_host) +#else + if(!ext_port || !protocol) +#endif /* UPNP_STRICT */ + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } +#ifndef SUPPORT_REMOTEHOST +#ifdef UPNP_STRICT + if (r_host && (strlen(r_host) > 0) && (0 != strcmp(r_host, "*"))) + { + ClearNameValueList(&data); + SoapError(h, 726, "RemoteHostOnlySupportsWildcard"); + return; + } +#endif /* UPNP_STRICT */ +#endif /* SUPPORT_REMOTEHOST */ + + eport = (unsigned short)atoi(ext_port); + if(eport == 0) + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + + syslog(LOG_INFO, "%s: external port: %hu, protocol: %s", + action, eport, protocol); + + /* if in secure mode, check the IP + * Removing a redirection is not a security threat, + * just an annoyance for the user using it. So this is not + * a priority. */ + if(GETFLAG(SECUREMODEMASK)) + { + char int_ip[32]; + struct in_addr int_ip_addr; + unsigned short iport; + unsigned int leaseduration = 0; + r = upnp_get_redirection_infos(eport, protocol, &iport, + int_ip, sizeof(int_ip), + NULL, 0, NULL, 0, + &leaseduration); + if(r >= 0) + { + if(inet_pton(AF_INET, int_ip, &int_ip_addr) > 0) + { + if(h->clientaddr.s_addr != int_ip_addr.s_addr) + { + SoapError(h, 606, "Action not authorized"); + /*SoapError(h, 714, "NoSuchEntryInArray");*/ + ClearNameValueList(&data); + return; + } + } + } + } + + r = upnp_delete_redirection(eport, protocol); + + if(r < 0) + { + SoapError(h, 714, "NoSuchEntryInArray"); + } + else + { + bodylen = snprintf(body, sizeof(body), resp, + action, ns, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + } + + ClearNameValueList(&data); +} + +/* DeletePortMappingRange was added in IGD spec v2 */ +static void +DeletePortMappingRange(struct upnphttp * h, const char * action, const char * ns) +{ + int r = -1; + /*static const char resp[] = + "" + "";*/ + static const char resp[] = + "" + ""; + char body[512]; + int bodylen; + struct NameValueParserData data; + const char * protocol; + const char * startport_s, * endport_s; + unsigned short startport, endport; + /*int manage;*/ + unsigned short * port_list; + unsigned int i, number = 0; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + startport_s = GetValueFromNameValueList(&data, "NewStartPort"); + endport_s = GetValueFromNameValueList(&data, "NewEndPort"); + protocol = GetValueFromNameValueList(&data, "NewProtocol"); + /*manage = atoi(GetValueFromNameValueList(&data, "NewManage"));*/ + if(startport_s == NULL || endport_s == NULL || protocol == NULL || + !is_numeric(startport_s) || !is_numeric(endport_s)) { + SoapError(h, 402, "Invalid Args"); + ClearNameValueList(&data); + return; + } + startport = (unsigned short)atoi(startport_s); + endport = (unsigned short)atoi(endport_s); + + /* possible errors : + 606 - Action not authorized + 730 - PortMappingNotFound + 733 - InconsistentParameter + */ + if(startport > endport) + { + SoapError(h, 733, "InconsistentParameter"); + ClearNameValueList(&data); + return; + } + + syslog(LOG_INFO, "%s: deleting external ports: %hu-%hu, protocol: %s", + action, startport, endport, protocol); + + port_list = upnp_get_portmappings_in_range(startport, endport, + protocol, &number); + if(number == 0) + { + SoapError(h, 730, "PortMappingNotFound"); + ClearNameValueList(&data); + free(port_list); + return; + } + + for(i = 0; i < number; i++) + { + r = upnp_delete_redirection(port_list[i], protocol); + syslog(LOG_INFO, "%s: deleting external port: %hu, protocol: %s: %s", + action, port_list[i], protocol, r < 0 ? "failed" : "ok"); + } + free(port_list); + bodylen = snprintf(body, sizeof(body), resp, + action, ns, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + + ClearNameValueList(&data); +} + +static void +GetGenericPortMappingEntry(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + static const char resp[] = + "" + "%s" + "%u" + "%s" + "%u" + "%s" + "1" + "%s" + "%u" + ""; + + long int index = 0; + unsigned short eport, iport; + const char * m_index; + char * endptr; + char protocol[8], iaddr[32]; + char desc[64]; + char rhost[40]; + unsigned int leaseduration = 0; + struct NameValueParserData data; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + m_index = GetValueFromNameValueList(&data, "NewPortMappingIndex"); + + if(!m_index) + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + errno = 0; /* To distinguish success/failure after call */ + index = strtol(m_index, &endptr, 10); + if((errno == ERANGE && (index == LONG_MAX || index == LONG_MIN)) + || (errno != 0 && index == 0) || (m_index == endptr)) + { + /* should condition (*endptr != '\0') be also an error ? */ + if(m_index == endptr) + syslog(LOG_WARNING, "%s: no digits were found in <%s>", + "GetGenericPortMappingEntry", "NewPortMappingIndex"); + else + syslog(LOG_WARNING, "%s: strtol('%s'): %m", + "GetGenericPortMappingEntry", m_index); + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + + syslog(LOG_INFO, "%s: index=%d", action, (int)index); + + rhost[0] = '\0'; + r = upnp_get_redirection_infos_by_index((int)index, &eport, protocol, &iport, + iaddr, sizeof(iaddr), + desc, sizeof(desc), + rhost, sizeof(rhost), + &leaseduration); + + if(r < 0) + { + SoapError(h, 713, "SpecifiedArrayIndexInvalid"); + } + else + { + int bodylen; + char body[2048]; + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /*SERVICE_TYPE_WANIPC,*/ rhost, + (unsigned int)eport, protocol, (unsigned int)iport, iaddr, desc, + leaseduration, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + } + + ClearNameValueList(&data); +} + +/* GetListOfPortMappings was added in the IGD v2 specification */ +static void +GetListOfPortMappings(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp_start[] = + "" + "" + ""; + + static const char list_start[] = + ""; + static const char list_end[] = + ""; + + static const char entry[] = + "" + "%s" + "%hu" + "%s" + "%hu" + "%s" + "1" + "%s" + "%u" + ""; + + char * body; + size_t bodyalloc; + int bodylen; + + int r = -1; + unsigned short iport; + char int_ip[32]; + char desc[64]; + char rhost[64]; + unsigned int leaseduration = 0; + + struct NameValueParserData data; + const char * startport_s, * endport_s; + unsigned short startport, endport; + const char * protocol; + /*int manage;*/ + const char * number_s; + int number; + unsigned short * port_list; + unsigned int i, list_size = 0; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + startport_s = GetValueFromNameValueList(&data, "NewStartPort"); + endport_s = GetValueFromNameValueList(&data, "NewEndPort"); + protocol = GetValueFromNameValueList(&data, "NewProtocol"); + /*manage_s = GetValueFromNameValueList(&data, "NewManage");*/ + number_s = GetValueFromNameValueList(&data, "NewNumberOfPorts"); + if(startport_s == NULL || endport_s == NULL || protocol == NULL || + number_s == NULL || !is_numeric(number_s) || + !is_numeric(startport_s) || !is_numeric(endport_s)) { + SoapError(h, 402, "Invalid Args"); + ClearNameValueList(&data); + return; + } + + startport = (unsigned short)atoi(startport_s); + endport = (unsigned short)atoi(endport_s); + /*manage = atoi(manage_s);*/ + number = atoi(number_s); + if(number == 0) number = 1000; /* return up to 1000 mappings by default */ + + if(startport > endport) + { + SoapError(h, 733, "InconsistentParameter"); + ClearNameValueList(&data); + return; + } +/* +build the PortMappingList xml document : + + + +202.233.2.1 +2345 +TCP +2345 +192.168.1.137 +1 +dooom +345 + + +*/ + bodyalloc = 4096; + body = malloc(bodyalloc); + if(!body) + { + ClearNameValueList(&data); + SoapError(h, 501, "ActionFailed"); + return; + } + bodylen = snprintf(body, bodyalloc, resp_start, + action, ns/*SERVICE_TYPE_WANIPC*/); + if(bodylen < 0) + { + SoapError(h, 501, "ActionFailed"); + free(body); + return; + } + memcpy(body+bodylen, list_start, sizeof(list_start)); + bodylen += (sizeof(list_start) - 1); + + port_list = upnp_get_portmappings_in_range(startport, endport, + protocol, &list_size); + /* loop through port mappings */ + for(i = 0; number > 0 && i < list_size; i++) + { + /* have a margin of 1024 bytes to store the new entry */ + if((unsigned int)bodylen + 1024 > bodyalloc) + { + char * body_sav = body; + bodyalloc += 4096; + body = realloc(body, bodyalloc); + if(!body) + { + syslog(LOG_CRIT, "realloc(%p, %u) FAILED", body_sav, (unsigned)bodyalloc); + ClearNameValueList(&data); + SoapError(h, 501, "ActionFailed"); + free(body_sav); + free(port_list); + return; + } + } + rhost[0] = '\0'; + r = upnp_get_redirection_infos(port_list[i], protocol, &iport, + int_ip, sizeof(int_ip), + desc, sizeof(desc), + rhost, sizeof(rhost), + &leaseduration); + if(r == 0) + { + bodylen += snprintf(body+bodylen, bodyalloc-bodylen, entry, + rhost, port_list[i], protocol, + iport, int_ip, desc, leaseduration); + number--; + } + } + free(port_list); + port_list = NULL; + + if((bodylen + sizeof(list_end) + 1024) > bodyalloc) + { + char * body_sav = body; + bodyalloc += (sizeof(list_end) + 1024); + body = realloc(body, bodyalloc); + if(!body) + { + syslog(LOG_CRIT, "realloc(%p, %u) FAILED", body_sav, (unsigned)bodyalloc); + ClearNameValueList(&data); + SoapError(h, 501, "ActionFailed"); + free(body_sav); + return; + } + } + memcpy(body+bodylen, list_end, sizeof(list_end)); + bodylen += (sizeof(list_end) - 1); + bodylen += snprintf(body+bodylen, bodyalloc-bodylen, resp_end, + action); + BuildSendAndCloseSoapResp(h, body, bodylen); + free(body); + + ClearNameValueList(&data); +} + +#ifdef ENABLE_L3F_SERVICE +static void +SetDefaultConnectionService(struct upnphttp * h, const char * action, const char * ns) +{ + /*static const char resp[] = + "" + "";*/ + static const char resp[] = + "" + ""; + char body[512]; + int bodylen; + struct NameValueParserData data; + char * p; + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + p = GetValueFromNameValueList(&data, "NewDefaultConnectionService"); + if(p) { + /* 720 InvalidDeviceUUID + * 721 InvalidServiceID + * 723 InvalidConnServiceSelection */ +#ifdef UPNP_STRICT + char * service; + service = strchr(p, ','); + if(0 != memcmp(uuidvalue_wcd, p, sizeof("uuid:00000000-0000-0000-0000-000000000000") - 1)) { + SoapError(h, 720, "InvalidDeviceUUID"); + } else if(service == NULL || 0 != strcmp(service+1, SERVICE_ID_WANIPC)) { + SoapError(h, 721, "InvalidServiceID"); + } else +#endif + { + syslog(LOG_INFO, "%s(%s) : Ignored", action, p); + bodylen = snprintf(body, sizeof(body), resp, + action, ns, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + } + } else { + /* missing argument */ + SoapError(h, 402, "Invalid Args"); + } + ClearNameValueList(&data); +} + +static void +GetDefaultConnectionService(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" +#ifdef IGD_V2 + "%s:WANConnectionDevice:2," +#else + "%s:WANConnectionDevice:1," +#endif + SERVICE_ID_WANIPC "" + ""; + /* example from UPnP_IGD_Layer3Forwarding 1.0.pdf : + * uuid:44f5824f-c57d-418c-a131-f22b34e14111:WANConnectionDevice:1, + * urn:upnp-org:serviceId:WANPPPConn1 */ + char body[1024]; + int bodylen; + + /* namespace : urn:schemas-upnp-org:service:Layer3Forwarding:1 */ + bodylen = snprintf(body, sizeof(body), resp, + action, ns, uuidvalue_wcd, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} +#endif + +/* Added for compliance with WANIPConnection v2 */ +static void +SetConnectionType(struct upnphttp * h, const char * action, const char * ns) +{ +#ifdef UPNP_STRICT + const char * connection_type; +#endif /* UPNP_STRICT */ + struct NameValueParserData data; + UNUSED(action); + UNUSED(ns); + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); +#ifdef UPNP_STRICT + connection_type = GetValueFromNameValueList(&data, "NewConnectionType"); + if(!connection_type) { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } +#endif /* UPNP_STRICT */ + /* Unconfigured, IP_Routed, IP_Bridged */ + ClearNameValueList(&data); + /* always return a ReadOnly error */ + SoapError(h, 731, "ReadOnly"); +} + +/* Added for compliance with WANIPConnection v2 */ +static void +RequestConnection(struct upnphttp * h, const char * action, const char * ns) +{ + UNUSED(action); + UNUSED(ns); + SoapError(h, 606, "Action not authorized"); +} + +/* Added for compliance with WANIPConnection v2 */ +static void +ForceTermination(struct upnphttp * h, const char * action, const char * ns) +{ + UNUSED(action); + UNUSED(ns); + SoapError(h, 606, "Action not authorized"); +} + +/* +If a control point calls QueryStateVariable on a state variable that is not +buffered in memory within (or otherwise available from) the service, +the service must return a SOAP fault with an errorCode of 404 Invalid Var. + +QueryStateVariable remains useful as a limited test tool but may not be +part of some future versions of UPnP. +*/ +static void +QueryStateVariable(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "%s" + ""; + + char body[512]; + int bodylen; + struct NameValueParserData data; + const char * var_name; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + /*var_name = GetValueFromNameValueList(&data, "QueryStateVariable"); */ + /*var_name = GetValueFromNameValueListIgnoreNS(&data, "varName");*/ + var_name = GetValueFromNameValueList(&data, "varName"); + + /*syslog(LOG_INFO, "QueryStateVariable(%.40s)", var_name); */ + + if(!var_name) + { + SoapError(h, 402, "Invalid Args"); + } + else if(strcmp(var_name, "ConnectionStatus") == 0) + { + const char * status; + + status = get_wan_connection_status_str(ext_if_name); + bodylen = snprintf(body, sizeof(body), resp, + action, ns,/*"urn:schemas-upnp-org:control-1-0",*/ + status, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + } +#if 0 + /* not useful */ + else if(strcmp(var_name, "ConnectionType") == 0) + { + bodylen = snprintf(body, sizeof(body), resp, "IP_Routed"); + BuildSendAndCloseSoapResp(h, body, bodylen); + } + else if(strcmp(var_name, "LastConnectionError") == 0) + { + bodylen = snprintf(body, sizeof(body), resp, "ERROR_NONE"); + BuildSendAndCloseSoapResp(h, body, bodylen); + } +#endif + else if(strcmp(var_name, "PortMappingNumberOfEntries") == 0) + { + char strn[10]; + snprintf(strn, sizeof(strn), "%i", + upnp_get_portmapping_number_of_entries()); + bodylen = snprintf(body, sizeof(body), resp, + action, ns,/*"urn:schemas-upnp-org:control-1-0",*/ + strn, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + } + else + { + syslog(LOG_NOTICE, "%s: Unknown: %s", action, var_name?var_name:""); + SoapError(h, 404, "Invalid Var"); + } + + ClearNameValueList(&data); +} + +#ifdef ENABLE_6FC_SERVICE +#ifndef ENABLE_IPV6 +#error "ENABLE_6FC_SERVICE needs ENABLE_IPV6" +#endif +/* WANIPv6FirewallControl actions */ +static void +GetFirewallStatus(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "%d" + "%d" + ""; + + char body[512]; + int bodylen; + + bodylen = snprintf(body, sizeof(body), resp, + action, ns, /*"urn:schemas-upnp-org:service:WANIPv6FirewallControl:1",*/ + GETFLAG(IPV6FCFWDISABLEDMASK) ? 0 : 1, + GETFLAG(IPV6FCINBOUNDDISALLOWEDMASK) ? 0 : 1, + action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static int +CheckStatus(struct upnphttp * h) +{ + if (GETFLAG(IPV6FCFWDISABLEDMASK)) + { + SoapError(h, 702, "FirewallDisabled"); + return 0; + } + else if(GETFLAG(IPV6FCINBOUNDDISALLOWEDMASK)) + { + SoapError(h, 703, "InboundPinholeNotAllowed"); + return 0; + } + else + return 1; +} + +#if 0 +static int connecthostport(const char * host, unsigned short port, char * result) +{ + int s, n; + char hostname[INET6_ADDRSTRLEN]; + char port_str[8], ifname[8], tmp[4]; + struct addrinfo *ai, *p; + struct addrinfo hints; + + memset(&hints, 0, sizeof(hints)); + /* hints.ai_flags = AI_ADDRCONFIG; */ +#ifdef AI_NUMERICSERV + hints.ai_flags = AI_NUMERICSERV; +#endif + hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; /* AF_INET, AF_INET6 or AF_UNSPEC */ + /* hints.ai_protocol = IPPROTO_TCP; */ + snprintf(port_str, sizeof(port_str), "%hu", port); + strcpy(hostname, host); + if(!strncmp(host, "fe80", 4)) + { + printf("Using an linklocal address\n"); + strcpy(ifname, "%"); + snprintf(tmp, sizeof(tmp), "%d", linklocal_index); + strcat(ifname, tmp); + strcat(hostname, ifname); + printf("host: %s\n", hostname); + } + n = getaddrinfo(hostname, port_str, &hints, &ai); + if(n != 0) + { + fprintf(stderr, "getaddrinfo() error : %s\n", gai_strerror(n)); + return -1; + } + s = -1; + for(p = ai; p; p = p->ai_next) + { +#ifdef DEBUG + char tmp_host[256]; + char tmp_service[256]; + printf("ai_family=%d ai_socktype=%d ai_protocol=%d ai_addrlen=%d\n ", + p->ai_family, p->ai_socktype, p->ai_protocol, p->ai_addrlen); + getnameinfo(p->ai_addr, p->ai_addrlen, tmp_host, sizeof(tmp_host), + tmp_service, sizeof(tmp_service), + NI_NUMERICHOST | NI_NUMERICSERV); + printf(" host=%s service=%s\n", tmp_host, tmp_service); +#endif + inet_ntop(AF_INET6, &(((struct sockaddr_in6 *)p->ai_addr)->sin6_addr), result, INET6_ADDRSTRLEN); + return 0; + } + freeaddrinfo(ai); +} +#endif + +/* Check the security policy right */ +static int +PinholeVerification(struct upnphttp * h, char * int_ip, unsigned short int_port) +{ + int n; + char senderAddr[INET6_ADDRSTRLEN]=""; + struct addrinfo hints, *ai, *p; + struct in6_addr result_ip; + + /* Pinhole InternalClient address must correspond to the action sender */ + syslog(LOG_INFO, "Checking internal IP@ and port (Security policy purpose)"); + + hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; + + /* if ip not valid assume hostname and convert */ + if (inet_pton(AF_INET6, int_ip, &result_ip) <= 0) + { + n = getaddrinfo(int_ip, NULL, &hints, &ai); + if(!n && ai->ai_family == AF_INET6) + { + for(p = ai; p; p = p->ai_next) + { + inet_ntop(AF_INET6, (struct in6_addr *) p, int_ip, sizeof(struct in6_addr)); + result_ip = *((struct in6_addr *) p); + /* TODO : deal with more than one ip per hostname */ + break; + } + } + else + { + syslog(LOG_ERR, "Failed to convert hostname '%s' to ip address", int_ip); + SoapError(h, 402, "Invalid Args"); + return -1; + } + freeaddrinfo(p); + } + + if(inet_ntop(AF_INET6, &(h->clientaddr_v6), senderAddr, INET6_ADDRSTRLEN) == NULL) + { + syslog(LOG_ERR, "inet_ntop: %m"); + } +#ifdef DEBUG + printf("\tPinholeVerification:\n\t\tCompare sender @: %s\n\t\t to intClient @: %s\n", senderAddr, int_ip); +#endif + if(strcmp(senderAddr, int_ip) != 0) + if(h->clientaddr_v6.s6_addr != result_ip.s6_addr) + { + syslog(LOG_INFO, "Client %s tried to access pinhole for internal %s and is not authorized to do it", + senderAddr, int_ip); + SoapError(h, 606, "Action not authorized"); + return 0; + } + + /* Pinhole InternalPort must be greater than or equal to 1024 */ + if (int_port < 1024) + { + syslog(LOG_INFO, "Client %s tried to access pinhole with port < 1024 and is not authorized to do it", + senderAddr); + SoapError(h, 606, "Action not authorized"); + return 0; + } + return 1; +} + +static void +AddPinhole(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + static const char resp[] = + "" + "%d" + ""; + char body[512]; + int bodylen; + struct NameValueParserData data; + char * rem_host, * rem_port, * int_ip, * int_port, * protocol, * leaseTime; + int uid = 0; + unsigned short iport, rport; + int ltime; + long proto; + char rem_ip[INET6_ADDRSTRLEN]; + + if(CheckStatus(h)==0) + return; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + rem_host = GetValueFromNameValueList(&data, "RemoteHost"); + rem_port = GetValueFromNameValueList(&data, "RemotePort"); + int_ip = GetValueFromNameValueList(&data, "InternalClient"); + int_port = GetValueFromNameValueList(&data, "InternalPort"); + protocol = GetValueFromNameValueList(&data, "Protocol"); + leaseTime = GetValueFromNameValueList(&data, "LeaseTime"); + + rport = (unsigned short)(rem_port ? atoi(rem_port) : 0); + iport = (unsigned short)(int_port ? atoi(int_port) : 0); + ltime = leaseTime ? atoi(leaseTime) : -1; + errno = 0; + proto = protocol ? strtol(protocol, NULL, 0) : -1; + if(errno != 0 || proto > 65535 || proto < 0) + { + SoapError(h, 402, "Invalid Args"); + goto clear_and_exit; + } + if(iport == 0) + { + SoapError(h, 706, "InternalPortWilcardingNotAllowed"); + goto clear_and_exit; + } + + /* In particular, [IGD2] RECOMMENDS that unauthenticated and + * unauthorized control points are only allowed to invoke + * this action with: + * - InternalPort value greater than or equal to 1024, + * - InternalClient value equals to the control point's IP address. + * It is REQUIRED that InternalClient cannot be one of IPv6 + * addresses used by the gateway. */ + if(!int_ip || 0 == strlen(int_ip) || 0 == strcmp(int_ip, "*")) + { + SoapError(h, 708, "WildCardNotPermittedInSrcIP"); + goto clear_and_exit; + } + /* I guess it is useless to convert int_ip to literal ipv6 address */ + if(rem_host) + { + /* trim */ + while(isspace(rem_host[0])) + rem_host++; + } + /* rem_host should be converted to literal ipv6 : */ + if(rem_host && (rem_host[0] != '\0') && (rem_host[0] != '*')) + { + struct addrinfo *ai, *p; + struct addrinfo hints; + int err; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_INET6; + /*hints.ai_flags = */ + /* hints.ai_protocol = proto; */ + err = getaddrinfo(rem_host, rem_port, &hints, &ai); + if(err == 0) + { + /* take the 1st IPv6 address */ + for(p = ai; p; p = p->ai_next) + { + if(p->ai_family == AF_INET6) + { + inet_ntop(AF_INET6, + &(((struct sockaddr_in6 *)p->ai_addr)->sin6_addr), + rem_ip, sizeof(rem_ip)); + syslog(LOG_INFO, "resolved '%s' to '%s'", rem_host, rem_ip); + rem_host = rem_ip; + break; + } + } + freeaddrinfo(ai); + } + else + { + syslog(LOG_WARNING, "AddPinhole : getaddrinfo(%s) : %s", + rem_host, gai_strerror(err)); +#if 0 + SoapError(h, 402, "Invalid Args"); + goto clear_and_exit; +#endif + } + } + + if(proto == 65535) + { + SoapError(h, 707, "ProtocolWilcardingNotAllowed"); + goto clear_and_exit; + } + if(proto != IPPROTO_UDP && proto != IPPROTO_TCP +#ifdef IPPROTO_UDPITE + && atoi(protocol) != IPPROTO_UDPLITE +#endif + ) + { + SoapError(h, 705, "ProtocolNotSupported"); + goto clear_and_exit; + } + if(ltime < 1 || ltime > 86400) + { + syslog(LOG_WARNING, "%s: LeaseTime=%d not supported, (ip=%s)", + action, ltime, int_ip); + SoapError(h, 402, "Invalid Args"); + goto clear_and_exit; + } + + if(PinholeVerification(h, int_ip, iport) <= 0) + goto clear_and_exit; + + syslog(LOG_INFO, "%s: (inbound) from [%s]:%hu to [%s]:%hu with proto %ld during %d sec", + action, rem_host?rem_host:"any", + rport, int_ip, iport, + proto, ltime); + + /* In cases where the RemoteHost, RemotePort, InternalPort, + * InternalClient and Protocol are the same than an existing pinhole, + * but LeaseTime is different, the device MUST extend the existing + * pinhole's lease time and return the UniqueID of the existing pinhole. */ + r = upnp_add_inboundpinhole(rem_host, rport, int_ip, iport, proto, "IGD2 pinhole", ltime, &uid); + + switch(r) + { + case 1: /* success */ + bodylen = snprintf(body, sizeof(body), + resp, action, + ns/*"urn:schemas-upnp-org:service:WANIPv6FirewallControl:1"*/, + uid, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + break; + case -1: /* not permitted */ + SoapError(h, 701, "PinholeSpaceExhausted"); + break; + default: + SoapError(h, 501, "ActionFailed"); + break; + } + /* 606 Action not authorized + * 701 PinholeSpaceExhausted + * 702 FirewallDisabled + * 703 InboundPinholeNotAllowed + * 705 ProtocolNotSupported + * 706 InternalPortWildcardingNotAllowed + * 707 ProtocolWildcardingNotAllowed + * 708 WildCardNotPermittedInSrcIP */ +clear_and_exit: + ClearNameValueList(&data); +} + +static void +UpdatePinhole(struct upnphttp * h, const char * action, const char * ns) +{ +#if 0 + static const char resp[] = + "" + ""; +#endif + static const char resp[] = + "" + ""; + char body[512]; + int bodylen; + struct NameValueParserData data; + const char * uid_str, * leaseTime; + char iaddr[INET6_ADDRSTRLEN]; + unsigned short iport; + int ltime; + int uid; + int n; + + if(CheckStatus(h)==0) + return; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + uid_str = GetValueFromNameValueList(&data, "UniqueID"); + leaseTime = GetValueFromNameValueList(&data, "NewLeaseTime"); + uid = uid_str ? atoi(uid_str) : -1; + ltime = leaseTime ? atoi(leaseTime) : -1; + ClearNameValueList(&data); + + if(uid < 0 || uid > 65535 || ltime <= 0 || ltime > 86400) + { + SoapError(h, 402, "Invalid Args"); + return; + } + + /* Check that client is not updating an pinhole + * it doesn't have access to, because of its public access */ + n = upnp_get_pinhole_info(uid, NULL, 0, NULL, + iaddr, sizeof(iaddr), &iport, + NULL, /* proto */ + NULL, 0, /* desc, desclen */ + NULL, NULL); + if (n >= 0) + { + if(PinholeVerification(h, iaddr, iport) <= 0) + return; + } + else if(n == -2) + { + SoapError(h, 704, "NoSuchEntry"); + return; + } + else + { + SoapError(h, 501, "ActionFailed"); + return; + } + + syslog(LOG_INFO, "%s: (inbound) updating lease duration to %d for pinhole with ID: %d", + action, ltime, uid); + + n = upnp_update_inboundpinhole(uid, ltime); + if(n == -1) + SoapError(h, 704, "NoSuchEntry"); + else if(n < 0) + SoapError(h, 501, "ActionFailed"); + else { + bodylen = snprintf(body, sizeof(body), resp, + action, ns, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + } +} + +static void +GetOutboundPinholeTimeout(struct upnphttp * h, const char * action, const char * ns) +{ + int r; + + static const char resp[] = + "" + "%d" + ""; + + char body[512]; + int bodylen; + struct NameValueParserData data; + char * int_ip, * int_port, * rem_host, * rem_port, * protocol; + int opt=0; + /*int proto=0;*/ + unsigned short iport, rport; + + if (GETFLAG(IPV6FCFWDISABLEDMASK)) + { + SoapError(h, 702, "FirewallDisabled"); + return; + } + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + int_ip = GetValueFromNameValueList(&data, "InternalClient"); + int_port = GetValueFromNameValueList(&data, "InternalPort"); + rem_host = GetValueFromNameValueList(&data, "RemoteHost"); + rem_port = GetValueFromNameValueList(&data, "RemotePort"); + protocol = GetValueFromNameValueList(&data, "Protocol"); + + rport = (unsigned short)atoi(rem_port); + iport = (unsigned short)atoi(int_port); + /*proto = atoi(protocol);*/ + + syslog(LOG_INFO, "%s: retrieving timeout for outbound pinhole from [%s]:%hu to [%s]:%hu protocol %s", action, int_ip, iport,rem_host, rport, protocol); + + /* TODO */ + r = -1;/*upnp_check_outbound_pinhole(proto, &opt);*/ + + switch(r) + { + case 1: /* success */ + bodylen = snprintf(body, sizeof(body), resp, + action, ns/*"urn:schemas-upnp-org:service:WANIPv6FirewallControl:1"*/, + opt, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + break; + case -5: /* Protocol not supported */ + SoapError(h, 705, "ProtocolNotSupported"); + break; + default: + SoapError(h, 501, "ActionFailed"); + } + ClearNameValueList(&data); +} + +static void +DeletePinhole(struct upnphttp * h, const char * action, const char * ns) +{ + int n; +#if 0 + static const char resp[] = + "" + ""; +#endif + static const char resp[] = + "" + ""; + char body[512]; + int bodylen; + + struct NameValueParserData data; + const char * uid_str; + char iaddr[INET6_ADDRSTRLEN]; + int proto; + unsigned short iport; + unsigned int leasetime; + int uid; + + if(CheckStatus(h)==0) + return; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + uid_str = GetValueFromNameValueList(&data, "UniqueID"); + uid = uid_str ? atoi(uid_str) : -1; + ClearNameValueList(&data); + + if(uid < 0 || uid > 65535) + { + SoapError(h, 402, "Invalid Args"); + return; + } + + /* Check that client is not deleting an pinhole + * it doesn't have access to, because of its public access */ + n = upnp_get_pinhole_info(uid, NULL, 0, NULL, + iaddr, sizeof(iaddr), &iport, + &proto, + NULL, 0, /* desc, desclen */ + &leasetime, NULL); + if (n >= 0) + { + if(PinholeVerification(h, iaddr, iport) <= 0) + return; + } + else if(n == -2) + { + SoapError(h, 704, "NoSuchEntry"); + return; + } + else + { + SoapError(h, 501, "ActionFailed"); + return; + } + + n = upnp_delete_inboundpinhole(uid); + if(n < 0) + { + syslog(LOG_INFO, "%s: (inbound) failed to remove pinhole with ID: %d", + action, uid); + SoapError(h, 501, "ActionFailed"); + return; + } + syslog(LOG_INFO, "%s: (inbound) pinhole with ID %d successfully removed", + action, uid); + bodylen = snprintf(body, sizeof(body), resp, + action, ns, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +CheckPinholeWorking(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "%d" + ""; + char body[512]; + int bodylen; + int r; + struct NameValueParserData data; + const char * uid_str; + int uid; + char iaddr[INET6_ADDRSTRLEN]; + unsigned short iport; + unsigned int packets; + + if(CheckStatus(h)==0) + return; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + uid_str = GetValueFromNameValueList(&data, "UniqueID"); + uid = uid_str ? atoi(uid_str) : -1; + ClearNameValueList(&data); + + if(uid < 0 || uid > 65535) + { + SoapError(h, 402, "Invalid Args"); + return; + } + + /* Check that client is not checking a pinhole + * it doesn't have access to, because of its public access */ + r = upnp_get_pinhole_info(uid, + NULL, 0, NULL, + iaddr, sizeof(iaddr), &iport, + NULL, /* proto */ + NULL, 0, /* desc, desclen */ + NULL, &packets); + if (r >= 0) + { + if(PinholeVerification(h, iaddr, iport) <= 0) + return ; + if(packets == 0) + { + SoapError(h, 709, "NoPacketSent"); + return; + } + bodylen = snprintf(body, sizeof(body), resp, + action, ns/*"urn:schemas-upnp-org:service:WANIPv6FirewallControl:1"*/, + 1, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + } + else if(r == -2) + SoapError(h, 704, "NoSuchEntry"); + else + SoapError(h, 501, "ActionFailed"); +} + +static void +GetPinholePackets(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "%u" + ""; + char body[512]; + int bodylen; + struct NameValueParserData data; + const char * uid_str; + int n; + char iaddr[INET6_ADDRSTRLEN]; + unsigned short iport; + unsigned int packets = 0; + int uid; + int proto; + unsigned int leasetime; + + if(CheckStatus(h)==0) + return; + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + uid_str = GetValueFromNameValueList(&data, "UniqueID"); + uid = uid_str ? atoi(uid_str) : -1; + ClearNameValueList(&data); + + if(uid < 0 || uid > 65535) + { + SoapError(h, 402, "Invalid Args"); + return; + } + + /* Check that client is not getting infos of a pinhole + * it doesn't have access to, because of its public access */ + n = upnp_get_pinhole_info(uid, NULL, 0, NULL, + iaddr, sizeof(iaddr), &iport, + &proto, + NULL, 0, /* desc, desclen */ + &leasetime, &packets); + if (n >= 0) + { + if(PinholeVerification(h, iaddr, iport)<=0) + return ; + } +#if 0 + else if(r == -4 || r == -1) + { + SoapError(h, 704, "NoSuchEntry"); + } +#endif + + bodylen = snprintf(body, sizeof(body), resp, + action, ns/*"urn:schemas-upnp-org:service:WANIPv6FirewallControl:1"*/, + packets, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} +#endif + +#ifdef ENABLE_DP_SERVICE +static void +SendSetupMessage(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "%s" + ""; + char body[1024]; + int bodylen; + struct NameValueParserData data; + const char * ProtocolType; /* string */ + const char * InMessage; /* base64 */ + const char * OutMessage = ""; /* base64 */ + + ParseNameValue(h->req_buf + h->req_contentoff, h->req_contentlen, &data); + ProtocolType = GetValueFromNameValueList(&data, "ProtocolType"); /* string */ + InMessage = GetValueFromNameValueList(&data, "InMessage"); /* base64 */ + + if(ProtocolType == NULL || InMessage == NULL) + { + ClearNameValueList(&data); + SoapError(h, 402, "Invalid Args"); + return; + } + /*if(strcmp(ProtocolType, "DeviceProtection:1") != 0)*/ + if(strcmp(ProtocolType, "WPS") != 0) + { + ClearNameValueList(&data); + SoapError(h, 600, "Argument Value Invalid"); /* 703 ? */ + return; + } + /* TODO : put here code for WPS */ + + bodylen = snprintf(body, sizeof(body), resp, + action, ns/*"urn:schemas-upnp-org:service:DeviceProtection:1"*/, + OutMessage, action); + BuildSendAndCloseSoapResp(h, body, bodylen); + ClearNameValueList(&data); +} + +static void +GetSupportedProtocols(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "" + ""; + char body[1024]; + int bodylen; + const char * ProtocolList = + "\n" + "" + "WPS" + "PKCS5" + ""; + + bodylen = snprintf(body, sizeof(body), resp, + action, ns/*"urn:schemas-upnp-org:service:DeviceProtection:1"*/, + ProtocolList, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} + +static void +GetAssignedRoles(struct upnphttp * h, const char * action, const char * ns) +{ + static const char resp[] = + "" + "%s" + ""; + char body[1024]; + int bodylen; + const char * RoleList = "Public"; /* list of roles separated by spaces */ + +#ifdef ENABLE_HTTPS + if(h->ssl != NULL) { + /* we should get the Roles of the session (based on client certificate) */ + X509 * peercert; + peercert = SSL_get_peer_certificate(h->ssl); + if(peercert != NULL) { + RoleList = "Admin Basic"; + X509_free(peercert); + } + } +#endif + + bodylen = snprintf(body, sizeof(body), resp, + action, ns/*"urn:schemas-upnp-org:service:DeviceProtection:1"*/, + RoleList, action); + BuildSendAndCloseSoapResp(h, body, bodylen); +} +#endif + +/* Windows XP as client send the following requests : + * GetConnectionTypeInfo + * GetNATRSIPStatus + * ? GetTotalBytesSent - WANCommonInterfaceConfig + * ? GetTotalBytesReceived - idem + * ? GetTotalPacketsSent - idem + * ? GetTotalPacketsReceived - idem + * GetCommonLinkProperties - idem + * GetStatusInfo - WANIPConnection + * GetExternalIPAddress + * QueryStateVariable / ConnectionStatus! + */ +static const struct +{ + const char * methodName; + void (*methodImpl)(struct upnphttp *, const char *, const char *); +} +soapMethods[] = +{ + /* WANCommonInterfaceConfig */ + { "QueryStateVariable", QueryStateVariable}, + { "GetTotalBytesSent", GetTotalBytesSent}, + { "GetTotalBytesReceived", GetTotalBytesReceived}, + { "GetTotalPacketsSent", GetTotalPacketsSent}, + { "GetTotalPacketsReceived", GetTotalPacketsReceived}, + { "GetCommonLinkProperties", GetCommonLinkProperties}, + { "GetStatusInfo", GetStatusInfo}, + /* WANIPConnection */ + { "GetConnectionTypeInfo", GetConnectionTypeInfo }, + { "GetNATRSIPStatus", GetNATRSIPStatus}, + { "GetExternalIPAddress", GetExternalIPAddress}, + { "AddPortMapping", AddPortMapping}, + { "DeletePortMapping", DeletePortMapping}, + { "GetGenericPortMappingEntry", GetGenericPortMappingEntry}, + { "GetSpecificPortMappingEntry", GetSpecificPortMappingEntry}, +/* Required in WANIPConnection:2 */ + { "SetConnectionType", SetConnectionType}, + { "RequestConnection", RequestConnection}, + { "ForceTermination", ForceTermination}, + { "AddAnyPortMapping", AddAnyPortMapping}, + { "DeletePortMappingRange", DeletePortMappingRange}, + { "GetListOfPortMappings", GetListOfPortMappings}, +#ifdef ENABLE_L3F_SERVICE + /* Layer3Forwarding */ + { "SetDefaultConnectionService", SetDefaultConnectionService}, + { "GetDefaultConnectionService", GetDefaultConnectionService}, +#endif +#ifdef ENABLE_6FC_SERVICE + /* WANIPv6FirewallControl */ + { "GetFirewallStatus", GetFirewallStatus}, /* Required */ + { "AddPinhole", AddPinhole}, /* Required */ + { "UpdatePinhole", UpdatePinhole}, /* Required */ + { "GetOutboundPinholeTimeout", GetOutboundPinholeTimeout}, /* Optional */ + { "DeletePinhole", DeletePinhole}, /* Required */ + { "CheckPinholeWorking", CheckPinholeWorking}, /* Optional */ + { "GetPinholePackets", GetPinholePackets}, /* Required */ +#endif +#ifdef ENABLE_DP_SERVICE + /* DeviceProtection */ + { "SendSetupMessage", SendSetupMessage}, /* Required */ + { "GetSupportedProtocols", GetSupportedProtocols}, /* Required */ + { "GetAssignedRoles", GetAssignedRoles}, /* Required */ +#endif + { 0, 0 } +}; + +void +ExecuteSoapAction(struct upnphttp * h, const char * action, int n) +{ + char * p; + char * p2; + int i, len, methodlen; + char namespace[256]; + + /* SoapAction example : + * urn:schemas-upnp-org:service:WANIPConnection:1#GetStatusInfo */ + p = strchr(action, '#'); + if(p && (p - action) < n) { + for(i = 0; i < ((int)sizeof(namespace) - 1) && (action + i) < p; i++) + namespace[i] = action[i]; + namespace[i] = '\0'; + p++; + p2 = strchr(p, '"'); + if(p2 && (p2 - action) <= n) + methodlen = p2 - p; + else + methodlen = n - (p - action); + /*syslog(LOG_DEBUG, "SoapMethod: %.*s %d %d %p %p %d", + methodlen, p, methodlen, n, action, p, (int)(p - action));*/ + for(i = 0; soapMethods[i].methodName; i++) { + len = strlen(soapMethods[i].methodName); + if((len == methodlen) && memcmp(p, soapMethods[i].methodName, len) == 0) { +#ifdef DEBUG + syslog(LOG_DEBUG, "Remote Call of SoapMethod '%s' %s", + soapMethods[i].methodName, namespace); +#endif /* DEBUG */ + soapMethods[i].methodImpl(h, soapMethods[i].methodName, namespace); + return; + } + } + syslog(LOG_NOTICE, "SoapMethod: Unknown: %.*s %s", methodlen, p, namespace); + } else { + syslog(LOG_NOTICE, "cannot parse SoapAction"); + } + + SoapError(h, 401, "Invalid Action"); +} + +/* Standard Errors: + * + * errorCode errorDescription Description + * -------- ---------------- ----------- + * 401 Invalid Action No action by that name at this service. + * 402 Invalid Args Could be any of the following: not enough in args, + * too many in args, no in arg by that name, + * one or more in args are of the wrong data type. + * 403 Out of Sync Out of synchronization. + * 501 Action Failed May be returned in current state of service + * prevents invoking that action. + * 600-699 TBD Common action errors. Defined by UPnP Forum + * Technical Committee. + * 700-799 TBD Action-specific errors for standard actions. + * Defined by UPnP Forum working committee. + * 800-899 TBD Action-specific errors for non-standard actions. + * Defined by UPnP vendor. +*/ +void +SoapError(struct upnphttp * h, int errCode, const char * errDesc) +{ + static const char resp[] = + "" + "" + "" + "s:Client" + "UPnPError" + "" + "" + "%d" + "%s" + "" + "" + "" + "" + ""; + + char body[2048]; + int bodylen; + + syslog(LOG_INFO, "Returning UPnPError %d: %s", errCode, errDesc); + bodylen = snprintf(body, sizeof(body), resp, errCode, errDesc); + BuildResp2_upnphttp(h, 500, "Internal Server Error", body, bodylen); + SendRespAndClose_upnphttp(h); +} + diff --git a/src/contrib/miniupnp/miniupnpd/upnpsoap.h b/src/contrib/miniupnp/miniupnpd/upnpsoap.h new file mode 100644 index 0000000..5dda29f --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpsoap.h @@ -0,0 +1,23 @@ +/* $Id: upnpsoap.h,v 1.8 2007/02/07 22:16:19 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPSOAP_H_INCLUDED +#define UPNPSOAP_H_INCLUDED + +/* ExecuteSoapAction(): + * this method executes the requested Soap Action */ +void +ExecuteSoapAction(struct upnphttp *, const char *, int); + +/* SoapError(): + * sends a correct SOAP error with an UPNPError code and + * description */ +void +SoapError(struct upnphttp * h, int errCode, const char * errDesc); + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnpurns.h b/src/contrib/miniupnp/miniupnpd/upnpurns.h new file mode 100644 index 0000000..db10664 --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnpurns.h @@ -0,0 +1,30 @@ +/* $Id: upnpurns.h,v 1.1 2011/05/13 15:32:53 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2011 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPURNS_H_INCLUDED +#define UPNPURNS_H_INCLUDED + +#include "config.h" + +#ifdef IGD_V2 +/* IGD v2 */ +#define DEVICE_TYPE_IGD "urn:schemas-upnp-org:device:InternetGatewayDevice:2" +#define DEVICE_TYPE_WAN "urn:schemas-upnp-org:device:WANDevice:2" +#define DEVICE_TYPE_WANC "urn:schemas-upnp-org:device:WANConnectionDevice:2" +#define SERVICE_TYPE_WANIPC "urn:schemas-upnp-org:service:WANIPConnection:2" +#define SERVICE_ID_WANIPC "urn:upnp-org:serviceId:WANIPConn1" +#else +/* IGD v1 */ +#define DEVICE_TYPE_IGD "urn:schemas-upnp-org:device:InternetGatewayDevice:1" +#define DEVICE_TYPE_WAN "urn:schemas-upnp-org:device:WANDevice:1" +#define DEVICE_TYPE_WANC "urn:schemas-upnp-org:device:WANConnectionDevice:1" +#define SERVICE_TYPE_WANIPC "urn:schemas-upnp-org:service:WANIPConnection:1" +#define SERVICE_ID_WANIPC "urn:upnp-org:serviceId:WANIPConn1" +#endif + +#endif + diff --git a/src/contrib/miniupnp/miniupnpd/upnputils.c b/src/contrib/miniupnp/miniupnpd/upnputils.c new file mode 100644 index 0000000..d0bd5ce --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnputils.c @@ -0,0 +1,238 @@ +/* $Id: upnputils.c,v 1.12 2018/03/13 10:25:20 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2006-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef AF_LINK +#include +#endif +#include + +#include "upnputils.h" +#include "upnpglobalvars.h" +#ifdef ENABLE_IPV6 +#include "getroute.h" +#endif + +int +sockaddr_to_string(const struct sockaddr * addr, char * str, size_t size) +{ + char buffer[64]; + unsigned short port = 0; + int n = -1; + + switch(addr->sa_family) + { +#ifdef AF_INET6 + case AF_INET6: + if(inet_ntop(addr->sa_family, + &((struct sockaddr_in6 *)addr)->sin6_addr, + buffer, sizeof(buffer)) == NULL) { + snprintf(buffer, sizeof(buffer), "inet_ntop: %s", strerror(errno)); + } + port = ntohs(((struct sockaddr_in6 *)addr)->sin6_port); + if(((struct sockaddr_in6 *)addr)->sin6_scope_id > 0) { + char ifname[IF_NAMESIZE]; + if(if_indextoname(((struct sockaddr_in6 *)addr)->sin6_scope_id, ifname) == NULL) + strncpy(ifname, "ERROR", sizeof(ifname)); + n = snprintf(str, size, "[%s%%%s]:%hu", buffer, ifname, port); + } else { + n = snprintf(str, size, "[%s]:%hu", buffer, port); + } + break; +#endif /* AF_INET6 */ + case AF_INET: + if(inet_ntop(addr->sa_family, + &((struct sockaddr_in *)addr)->sin_addr, + buffer, sizeof(buffer)) == NULL) { + snprintf(buffer, sizeof(buffer), "inet_ntop: %s", strerror(errno)); + } + port = ntohs(((struct sockaddr_in *)addr)->sin_port); + n = snprintf(str, size, "%s:%hu", buffer, port); + break; +#ifdef AF_LINK +#if defined(__sun) + /* solaris does not seem to have link_ntoa */ + /* #define link_ntoa _link_ntoa */ +#define link_ntoa(x) "dummy-link_ntoa" +#endif + case AF_LINK: + { + struct sockaddr_dl * sdl = (struct sockaddr_dl *)addr; + n = snprintf(str, size, "index=%hu type=%d %s", + sdl->sdl_index, sdl->sdl_type, + link_ntoa(sdl)); + } + break; +#endif /* AF_LINK */ + default: + n = snprintf(str, size, "unknown address family %d", addr->sa_family); +#if 0 + n = snprintf(str, size, "unknown address family %d " + "%02x %02x %02x %02x %02x %02x %02x %02x", + addr->sa_family, + addr->sa_data[0], addr->sa_data[1], (unsigned)addr->sa_data[2], addr->sa_data[3], + addr->sa_data[4], addr->sa_data[5], (unsigned)addr->sa_data[6], addr->sa_data[7]); +#endif + } + return n; +} + + +int +set_non_blocking(int fd) +{ + int flags = fcntl(fd, F_GETFL); + if(flags < 0) + return 0; + if(fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) + return 0; + return 1; +} + +struct lan_addr_s * +get_lan_for_peer(const struct sockaddr * peer) +{ + struct lan_addr_s * lan_addr = NULL; +#ifdef DEBUG + char dbg_str[64]; +#endif /* DEBUG */ + +#ifdef ENABLE_IPV6 + if(peer->sa_family == AF_INET6) + { + struct sockaddr_in6 * peer6 = (struct sockaddr_in6 *)peer; + if(IN6_IS_ADDR_V4MAPPED(&peer6->sin6_addr)) + { + struct in_addr peer_addr; + memcpy(&peer_addr, &peer6->sin6_addr.s6_addr[12], 4); + for(lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + if( (peer_addr.s_addr & lan_addr->mask.s_addr) + == (lan_addr->addr.s_addr & lan_addr->mask.s_addr)) + break; + } + } + else + { + int index = -1; + if(peer6->sin6_scope_id > 0) + index = (int)peer6->sin6_scope_id; + else + { + if(get_src_for_route_to(peer, NULL, NULL, &index) < 0) + return NULL; + } + syslog(LOG_DEBUG, "%s looking for LAN interface index=%d", + "get_lan_for_peer()", index); + for(lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + syslog(LOG_DEBUG, + "ifname=%s index=%u str=%s addr=%08x mask=%08x", + lan_addr->ifname, lan_addr->index, + lan_addr->str, + ntohl(lan_addr->addr.s_addr), + ntohl(lan_addr->mask.s_addr)); + if(index == (int)lan_addr->index) + break; + } + } + } + else if(peer->sa_family == AF_INET) + { +#endif /* ENABLE_IPV6 */ + for(lan_addr = lan_addrs.lh_first; + lan_addr != NULL; + lan_addr = lan_addr->list.le_next) + { + if( (((const struct sockaddr_in *)peer)->sin_addr.s_addr & lan_addr->mask.s_addr) + == (lan_addr->addr.s_addr & lan_addr->mask.s_addr)) + break; + } +#ifdef ENABLE_IPV6 + } +#endif /* ENABLE_IPV6 */ + +#ifdef DEBUG + sockaddr_to_string(peer, dbg_str, sizeof(dbg_str)); + if(lan_addr) { + syslog(LOG_DEBUG, "%s: %s found in LAN %s %s", + "get_lan_for_peer()", dbg_str, + lan_addr->ifname, lan_addr->str); + } else { + syslog(LOG_DEBUG, "%s: %s not found !", "get_lan_for_peer()", + dbg_str); + } +#endif /* DEBUG */ + return lan_addr; +} + +time_t upnp_time(void) +{ +#if defined(CLOCK_MONOTONIC_FAST) || defined(CLOCK_MONOTONIC) +#if defined(CLOCK_MONOTONIC_FAST) +#define UPNP_CLOCKID CLOCK_MONOTONIC_FAST +#else +#define UPNP_CLOCKID CLOCK_MONOTONIC +#endif + struct timespec ts; + if (clock_gettime(UPNP_CLOCKID, &ts) < 0) + return time(NULL); + else + return ts.tv_sec; +#else + return time(NULL); +#endif +} + +time_t upnp_get_uptime(void) +{ +#if defined(CLOCK_UPTIME_FAST) || defined(CLOCK_UPTIME) +#if defined(CLOCK_UPTIME_FAST) +#define UPNP_CLOCKID_UPTIME CLOCK_UPTIME_FAST +#else +#define UPNP_CLOCKID_UPTIME CLOCK_UPTIME +#endif + if(GETFLAG(SYSUPTIMEMASK)) + { + struct timespec ts; + if (clock_gettime(UPNP_CLOCKID_UPTIME, &ts) >= 0) + return ts.tv_sec; + } +#endif + return upnp_time() - startup_time; +} + +int upnp_gettimeofday(struct timeval * tv) +{ +#if defined(CLOCK_MONOTONIC_FAST) || defined(CLOCK_MONOTONIC) + struct timespec ts; + int ret_code = clock_gettime(UPNP_CLOCKID, &ts); + if (ret_code == 0) + { + tv->tv_sec = ts.tv_sec; + tv->tv_usec = ts.tv_nsec / 1000; + } + return ret_code; +#else + return gettimeofday(tv, NULL); +#endif +} diff --git a/src/contrib/miniupnp/miniupnpd/upnputils.h b/src/contrib/miniupnp/miniupnpd/upnputils.h new file mode 100644 index 0000000..66de75b --- /dev/null +++ b/src/contrib/miniupnp/miniupnpd/upnputils.h @@ -0,0 +1,74 @@ +/* $Id: upnputils.h,v 1.9 2018/03/13 10:25:20 nanard Exp $ */ +/* MiniUPnP project + * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * (c) 2011-2018 Thomas Bernard + * This software is subject to the conditions detailed + * in the LICENCE file provided within the distribution */ + +#ifndef UPNPUTILS_H_INCLUDED +#define UPNPUTILS_H_INCLUDED + +/** + * convert a struct sockaddr to a human readable string. + * [ipv6]:port or ipv4:port + * return the number of characters used (as snprintf) + */ +int +sockaddr_to_string(const struct sockaddr * addr, char * str, size_t size); + +/** + * set the file description as non blocking + * return 0 in case of failure, 1 in case of success + */ +int +set_non_blocking(int fd); + +/** + * get the LAN which the peer belongs to + */ +struct lan_addr_s * +get_lan_for_peer(const struct sockaddr * peer); + +/** + * get the time for upnp (release expiration, etc.) + * Similar to a monotonic time(NULL) + */ +time_t upnp_time(void); + +/** + * return either the machine or the daemon uptime + */ +time_t upnp_get_uptime(void); + +/** + * get the time for upnp + * Similar to a monotonic gettimeofday(tv, NULL) + */ +int upnp_gettimeofday(struct timeval * tv); + +/** + * define portability macros + */ +#if defined(__sun) +static __inline size_t _sa_len(const struct sockaddr *addr) +{ + if (addr->sa_family == AF_INET) + return (sizeof(struct sockaddr_in)); + else if (addr->sa_family == AF_INET6) + return (sizeof(struct sockaddr_in6)); + else + return (sizeof(struct sockaddr)); +} +# define SA_LEN(sa) (_sa_len(sa)) +#else +#if !defined(SA_LEN) +# define SA_LEN(sa) ((sa)->sa_len) +#endif +#endif + +#ifndef MAX +# define MAX(a,b) (((a)>(b))?(a):(b)) +#endif + +#endif + diff --git a/src/contrib/zlib/CMakeLists.txt b/src/contrib/zlib/CMakeLists.txt index 0fe939d..696b9f5 100644 --- a/src/contrib/zlib/CMakeLists.txt +++ b/src/contrib/zlib/CMakeLists.txt @@ -183,10 +183,10 @@ if(MINGW) set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj) endif(MINGW) -add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) +#add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) -set_target_properties(zlib PROPERTIES SOVERSION 1) +# set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) +# set_target_properties(zlib PROPERTIES SOVERSION 1) if(NOT CYGWIN) # This property causes shared libraries on Linux to have the full version @@ -196,22 +196,22 @@ if(NOT CYGWIN) # # This has no effect with MSVC, on that platform the version info for # the DLL comes from the resource file win32/zlib1.rc - set_target_properties(zlib PROPERTIES VERSION ${ZLIB_FULL_VERSION}) +# set_target_properties(zlib PROPERTIES VERSION ${ZLIB_FULL_VERSION}) endif() if(UNIX) # On unix-like platforms the library is almost always called libz - set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z) + set_target_properties( zlibstatic PROPERTIES OUTPUT_NAME z) if(NOT APPLE) - set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") + #set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") endif() elseif(BUILD_SHARED_LIBS AND WIN32) # Creates zlib1.dll when building shared library version - set_target_properties(zlib PROPERTIES SUFFIX "1.dll") + # set_target_properties(zlib PROPERTIES SUFFIX "1.dll") endif() if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL ) - install(TARGETS zlib zlibstatic + install(TARGETS zlibstatic RUNTIME DESTINATION "${INSTALL_BIN_DIR}" ARCHIVE DESTINATION "${INSTALL_LIB_DIR}" LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ) @@ -230,20 +230,20 @@ endif() # Example binaries #============================================================================ -add_executable(example test/example.c) -target_link_libraries(example zlib) -add_test(example example) +#add_executable(example test/example.c) +#target_link_libraries(example zlib) +#add_test(example example) -add_executable(minigzip test/minigzip.c) -target_link_libraries(minigzip zlib) +#add_executable(minigzip test/minigzip.c) +#target_link_libraries(minigzip zlib) if(HAVE_OFF64_T) - add_executable(example64 test/example.c) - target_link_libraries(example64 zlib) - set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") - add_test(example64 example64) +# add_executable(example64 test/example.c) +# target_link_libraries(example64 zlib) +# set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") +# add_test(example64 example64) - add_executable(minigzip64 test/minigzip.c) - target_link_libraries(minigzip64 zlib) - set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") +# add_executable(minigzip64 test/minigzip.c) +# target_link_libraries(minigzip64 zlib) +# set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") endif() diff --git a/src/contrib/zlib/test/infcover.c b/src/contrib/zlib/test/infcover.c index 2be0164..e45b6cb 100644 --- a/src/contrib/zlib/test/infcover.c +++ b/src/contrib/zlib/test/infcover.c @@ -9,7 +9,7 @@ #include #include #include -#include "zlib.h" +#include "../zlib.h" /* get definition of internal structure so we can mess with it (see pull()), and so we can call inflate_trees() (see cover5()) */ diff --git a/src/contrib/zlib/test/minigzip.c b/src/contrib/zlib/test/minigzip.c index e22fb08..93de87b 100644 --- a/src/contrib/zlib/test/minigzip.c +++ b/src/contrib/zlib/test/minigzip.c @@ -15,7 +15,7 @@ /* @(#) $Id$ */ -#include "zlib.h" +#include "../zlib.h" #include #ifdef STDC diff --git a/src/crypto/RIPEMD160.c b/src/crypto/RIPEMD160.c new file mode 100644 index 0000000..0f83230 --- /dev/null +++ b/src/crypto/RIPEMD160.c @@ -0,0 +1,287 @@ +/********************************************************************\ +* +* FILE: rmd160.c +* +* CONTENTS: A sample C-implementation of the RIPEMD-160 +* hash-function. +* TARGET: any computer with an ANSI C compiler +* +* AUTHOR: Antoon Bosselaers, ESAT-COSIC +* DATE: 1 March 1996 +* VERSION: 1.0 +* +* Copyright (c) 1996 Katholieke Universiteit Leuven +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without restriction, +* including without limitation the rights to use, copy, modify, merge, +* publish, distribute, sublicense, and/or sell copies of the Software, +* and to permit persons to whom the Software is furnished to do so, +* subject to the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +\********************************************************************/ + +/* header files */ +#include +#include +#include +#include "RIPEMD160.h" + +/********************************************************************/ + +void MDinit(dword *MDbuf) +{ + MDbuf[0] = 0x67452301UL; + MDbuf[1] = 0xefcdab89UL; + MDbuf[2] = 0x98badcfeUL; + MDbuf[3] = 0x10325476UL; + MDbuf[4] = 0xc3d2e1f0UL; + + return; +} + +/********************************************************************/ + +void compress(dword *MDbuf, dword *X) +{ + dword aa = MDbuf[0], bb = MDbuf[1], cc = MDbuf[2], + dd = MDbuf[3], ee = MDbuf[4]; + dword aaa = MDbuf[0], bbb = MDbuf[1], ccc = MDbuf[2], + ddd = MDbuf[3], eee = MDbuf[4]; + + /* round 1 */ + FF(aa, bb, cc, dd, ee, X[0], 11); + FF(ee, aa, bb, cc, dd, X[1], 14); + FF(dd, ee, aa, bb, cc, X[2], 15); + FF(cc, dd, ee, aa, bb, X[3], 12); + FF(bb, cc, dd, ee, aa, X[4], 5); + FF(aa, bb, cc, dd, ee, X[5], 8); + FF(ee, aa, bb, cc, dd, X[6], 7); + FF(dd, ee, aa, bb, cc, X[7], 9); + FF(cc, dd, ee, aa, bb, X[8], 11); + FF(bb, cc, dd, ee, aa, X[9], 13); + FF(aa, bb, cc, dd, ee, X[10], 14); + FF(ee, aa, bb, cc, dd, X[11], 15); + FF(dd, ee, aa, bb, cc, X[12], 6); + FF(cc, dd, ee, aa, bb, X[13], 7); + FF(bb, cc, dd, ee, aa, X[14], 9); + FF(aa, bb, cc, dd, ee, X[15], 8); + + /* round 2 */ + GG(ee, aa, bb, cc, dd, X[7], 7); + GG(dd, ee, aa, bb, cc, X[4], 6); + GG(cc, dd, ee, aa, bb, X[13], 8); + GG(bb, cc, dd, ee, aa, X[1], 13); + GG(aa, bb, cc, dd, ee, X[10], 11); + GG(ee, aa, bb, cc, dd, X[6], 9); + GG(dd, ee, aa, bb, cc, X[15], 7); + GG(cc, dd, ee, aa, bb, X[3], 15); + GG(bb, cc, dd, ee, aa, X[12], 7); + GG(aa, bb, cc, dd, ee, X[0], 12); + GG(ee, aa, bb, cc, dd, X[9], 15); + GG(dd, ee, aa, bb, cc, X[5], 9); + GG(cc, dd, ee, aa, bb, X[2], 11); + GG(bb, cc, dd, ee, aa, X[14], 7); + GG(aa, bb, cc, dd, ee, X[11], 13); + GG(ee, aa, bb, cc, dd, X[8], 12); + + /* round 3 */ + HH(dd, ee, aa, bb, cc, X[3], 11); + HH(cc, dd, ee, aa, bb, X[10], 13); + HH(bb, cc, dd, ee, aa, X[14], 6); + HH(aa, bb, cc, dd, ee, X[4], 7); + HH(ee, aa, bb, cc, dd, X[9], 14); + HH(dd, ee, aa, bb, cc, X[15], 9); + HH(cc, dd, ee, aa, bb, X[8], 13); + HH(bb, cc, dd, ee, aa, X[1], 15); + HH(aa, bb, cc, dd, ee, X[2], 14); + HH(ee, aa, bb, cc, dd, X[7], 8); + HH(dd, ee, aa, bb, cc, X[0], 13); + HH(cc, dd, ee, aa, bb, X[6], 6); + HH(bb, cc, dd, ee, aa, X[13], 5); + HH(aa, bb, cc, dd, ee, X[11], 12); + HH(ee, aa, bb, cc, dd, X[5], 7); + HH(dd, ee, aa, bb, cc, X[12], 5); + + /* round 4 */ + II(cc, dd, ee, aa, bb, X[1], 11); + II(bb, cc, dd, ee, aa, X[9], 12); + II(aa, bb, cc, dd, ee, X[11], 14); + II(ee, aa, bb, cc, dd, X[10], 15); + II(dd, ee, aa, bb, cc, X[0], 14); + II(cc, dd, ee, aa, bb, X[8], 15); + II(bb, cc, dd, ee, aa, X[12], 9); + II(aa, bb, cc, dd, ee, X[4], 8); + II(ee, aa, bb, cc, dd, X[13], 9); + II(dd, ee, aa, bb, cc, X[3], 14); + II(cc, dd, ee, aa, bb, X[7], 5); + II(bb, cc, dd, ee, aa, X[15], 6); + II(aa, bb, cc, dd, ee, X[14], 8); + II(ee, aa, bb, cc, dd, X[5], 6); + II(dd, ee, aa, bb, cc, X[6], 5); + II(cc, dd, ee, aa, bb, X[2], 12); + + /* round 5 */ + JJ(bb, cc, dd, ee, aa, X[4], 9); + JJ(aa, bb, cc, dd, ee, X[0], 15); + JJ(ee, aa, bb, cc, dd, X[5], 5); + JJ(dd, ee, aa, bb, cc, X[9], 11); + JJ(cc, dd, ee, aa, bb, X[7], 6); + JJ(bb, cc, dd, ee, aa, X[12], 8); + JJ(aa, bb, cc, dd, ee, X[2], 13); + JJ(ee, aa, bb, cc, dd, X[10], 12); + JJ(dd, ee, aa, bb, cc, X[14], 5); + JJ(cc, dd, ee, aa, bb, X[1], 12); + JJ(bb, cc, dd, ee, aa, X[3], 13); + JJ(aa, bb, cc, dd, ee, X[8], 14); + JJ(ee, aa, bb, cc, dd, X[11], 11); + JJ(dd, ee, aa, bb, cc, X[6], 8); + JJ(cc, dd, ee, aa, bb, X[15], 5); + JJ(bb, cc, dd, ee, aa, X[13], 6); + + /* parallel round 1 */ + JJJ(aaa, bbb, ccc, ddd, eee, X[5], 8); + JJJ(eee, aaa, bbb, ccc, ddd, X[14], 9); + JJJ(ddd, eee, aaa, bbb, ccc, X[7], 9); + JJJ(ccc, ddd, eee, aaa, bbb, X[0], 11); + JJJ(bbb, ccc, ddd, eee, aaa, X[9], 13); + JJJ(aaa, bbb, ccc, ddd, eee, X[2], 15); + JJJ(eee, aaa, bbb, ccc, ddd, X[11], 15); + JJJ(ddd, eee, aaa, bbb, ccc, X[4], 5); + JJJ(ccc, ddd, eee, aaa, bbb, X[13], 7); + JJJ(bbb, ccc, ddd, eee, aaa, X[6], 7); + JJJ(aaa, bbb, ccc, ddd, eee, X[15], 8); + JJJ(eee, aaa, bbb, ccc, ddd, X[8], 11); + JJJ(ddd, eee, aaa, bbb, ccc, X[1], 14); + JJJ(ccc, ddd, eee, aaa, bbb, X[10], 14); + JJJ(bbb, ccc, ddd, eee, aaa, X[3], 12); + JJJ(aaa, bbb, ccc, ddd, eee, X[12], 6); + + /* parallel round 2 */ + III(eee, aaa, bbb, ccc, ddd, X[6], 9); + III(ddd, eee, aaa, bbb, ccc, X[11], 13); + III(ccc, ddd, eee, aaa, bbb, X[3], 15); + III(bbb, ccc, ddd, eee, aaa, X[7], 7); + III(aaa, bbb, ccc, ddd, eee, X[0], 12); + III(eee, aaa, bbb, ccc, ddd, X[13], 8); + III(ddd, eee, aaa, bbb, ccc, X[5], 9); + III(ccc, ddd, eee, aaa, bbb, X[10], 11); + III(bbb, ccc, ddd, eee, aaa, X[14], 7); + III(aaa, bbb, ccc, ddd, eee, X[15], 7); + III(eee, aaa, bbb, ccc, ddd, X[8], 12); + III(ddd, eee, aaa, bbb, ccc, X[12], 7); + III(ccc, ddd, eee, aaa, bbb, X[4], 6); + III(bbb, ccc, ddd, eee, aaa, X[9], 15); + III(aaa, bbb, ccc, ddd, eee, X[1], 13); + III(eee, aaa, bbb, ccc, ddd, X[2], 11); + + /* parallel round 3 */ + HHH(ddd, eee, aaa, bbb, ccc, X[15], 9); + HHH(ccc, ddd, eee, aaa, bbb, X[5], 7); + HHH(bbb, ccc, ddd, eee, aaa, X[1], 15); + HHH(aaa, bbb, ccc, ddd, eee, X[3], 11); + HHH(eee, aaa, bbb, ccc, ddd, X[7], 8); + HHH(ddd, eee, aaa, bbb, ccc, X[14], 6); + HHH(ccc, ddd, eee, aaa, bbb, X[6], 6); + HHH(bbb, ccc, ddd, eee, aaa, X[9], 14); + HHH(aaa, bbb, ccc, ddd, eee, X[11], 12); + HHH(eee, aaa, bbb, ccc, ddd, X[8], 13); + HHH(ddd, eee, aaa, bbb, ccc, X[12], 5); + HHH(ccc, ddd, eee, aaa, bbb, X[2], 14); + HHH(bbb, ccc, ddd, eee, aaa, X[10], 13); + HHH(aaa, bbb, ccc, ddd, eee, X[0], 13); + HHH(eee, aaa, bbb, ccc, ddd, X[4], 7); + HHH(ddd, eee, aaa, bbb, ccc, X[13], 5); + + /* parallel round 4 */ + GGG(ccc, ddd, eee, aaa, bbb, X[8], 15); + GGG(bbb, ccc, ddd, eee, aaa, X[6], 5); + GGG(aaa, bbb, ccc, ddd, eee, X[4], 8); + GGG(eee, aaa, bbb, ccc, ddd, X[1], 11); + GGG(ddd, eee, aaa, bbb, ccc, X[3], 14); + GGG(ccc, ddd, eee, aaa, bbb, X[11], 14); + GGG(bbb, ccc, ddd, eee, aaa, X[15], 6); + GGG(aaa, bbb, ccc, ddd, eee, X[0], 14); + GGG(eee, aaa, bbb, ccc, ddd, X[5], 6); + GGG(ddd, eee, aaa, bbb, ccc, X[12], 9); + GGG(ccc, ddd, eee, aaa, bbb, X[2], 12); + GGG(bbb, ccc, ddd, eee, aaa, X[13], 9); + GGG(aaa, bbb, ccc, ddd, eee, X[9], 12); + GGG(eee, aaa, bbb, ccc, ddd, X[7], 5); + GGG(ddd, eee, aaa, bbb, ccc, X[10], 15); + GGG(ccc, ddd, eee, aaa, bbb, X[14], 8); + + /* parallel round 5 */ + FFF(bbb, ccc, ddd, eee, aaa, X[12], 8); + FFF(aaa, bbb, ccc, ddd, eee, X[15], 5); + FFF(eee, aaa, bbb, ccc, ddd, X[10], 12); + FFF(ddd, eee, aaa, bbb, ccc, X[4], 9); + FFF(ccc, ddd, eee, aaa, bbb, X[1], 12); + FFF(bbb, ccc, ddd, eee, aaa, X[5], 5); + FFF(aaa, bbb, ccc, ddd, eee, X[8], 14); + FFF(eee, aaa, bbb, ccc, ddd, X[7], 6); + FFF(ddd, eee, aaa, bbb, ccc, X[6], 8); + FFF(ccc, ddd, eee, aaa, bbb, X[2], 13); + FFF(bbb, ccc, ddd, eee, aaa, X[13], 6); + FFF(aaa, bbb, ccc, ddd, eee, X[14], 5); + FFF(eee, aaa, bbb, ccc, ddd, X[0], 15); + FFF(ddd, eee, aaa, bbb, ccc, X[3], 13); + FFF(ccc, ddd, eee, aaa, bbb, X[9], 11); + FFF(bbb, ccc, ddd, eee, aaa, X[11], 11); + + /* combine results */ + ddd += cc + MDbuf[1]; /* final result for MDbuf[0] */ + MDbuf[1] = MDbuf[2] + dd + eee; + MDbuf[2] = MDbuf[3] + ee + aaa; + MDbuf[3] = MDbuf[4] + aa + bbb; + MDbuf[4] = MDbuf[0] + bb + ccc; + MDbuf[0] = ddd; + + return; +} + +/********************************************************************/ + +void MDfinish(dword *MDbuf, byte *strptr, dword lswlen, dword mswlen) +{ + unsigned int i; /* counter */ + dword X[16]; /* message words */ + + memset(X, 0, 16 * sizeof(dword)); + + /* put bytes from strptr into X */ + for (i = 0; i<(lswlen & 63); i++) { + /* byte i goes into word X[i div 4] at pos. 8*(i mod 4) */ + X[i >> 2] ^= (dword)*strptr++ << (8 * (i & 3)); + } + + /* append the bit m_n == 1 */ + X[(lswlen >> 2) & 15] ^= (dword)1 << (8 * (lswlen & 3) + 7); + + if ((lswlen & 63) > 55) { + /* length goes to next block */ + compress(MDbuf, X); + memset(X, 0, 16 * sizeof(dword)); + } + + /* append length in bits*/ + X[14] = lswlen << 3; + X[15] = (lswlen >> 29) | (mswlen << 3); + compress(MDbuf, X); + + return; +} + +/************************ end of file rmd160.c **********************/ diff --git a/src/crypto/RIPEMD160.h b/src/crypto/RIPEMD160.h new file mode 100644 index 0000000..f0ea97e --- /dev/null +++ b/src/crypto/RIPEMD160.h @@ -0,0 +1,150 @@ +/********************************************************************\ +* +* FILE: rmd160.h +* +* CONTENTS: Header file for a sample C-implementation of the +* RIPEMD-160 hash-function. +* TARGET: any computer with an ANSI C compiler +* +* AUTHOR: Antoon Bosselaers, ESAT-COSIC +* DATE: 1 March 1996 +* VERSION: 1.0 +* +* Copyright (c) 1996 Katholieke Universiteit Leuven +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without restriction, +* including without limitation the rights to use, copy, modify, merge, +* publish, distribute, sublicense, and/or sell copies of the Software, +* and to permit persons to whom the Software is furnished to do so, +* subject to the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +\********************************************************************/ + +#ifndef RMD160H /* make sure this file is read only once */ +#define RMD160H + +/********************************************************************/ +#include +/* typedef 8 and 32 bit types, resp. */ +/* adapt these, if necessary, +for your operating system and compiler */ +typedef unsigned char byte; +typedef uint32_t dword; + +/* if this line causes a compiler error, +adapt the defintion of dword above */ +typedef int the_correct_size_was_chosen[sizeof(dword) == 4 ? 1 : -1]; + +/********************************************************************/ + +/* macro definitions */ + +/* collect four bytes into one word: */ +#define BYTES_TO_DWORD(strptr) \ + (((dword) *((strptr)+3) << 24) | \ + ((dword) *((strptr)+2) << 16) | \ + ((dword) *((strptr)+1) << 8) | \ + ((dword) *(strptr))) + +/* ROL(x, n) cyclically rotates x over n bits to the left */ +/* x must be of an unsigned 32 bits type and 0 <= n < 32. */ +#define ROL(x, n) (((x) << (n)) | ((x) >> (32-(n)))) + +/* the five basic functions F(), G() and H() */ +#define F(x, y, z) ((x) ^ (y) ^ (z)) +#define G(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define H(x, y, z) (((x) | ~(y)) ^ (z)) +#define I(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define J(x, y, z) ((x) ^ ((y) | ~(z))) + +/* the ten basic operations FF() through III() */ +#define FF(a, b, c, d, e, x, s) {\ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define GG(a, b, c, d, e, x, s) {\ + (a) += G((b), (c), (d)) + (x) + 0x5a827999UL;\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define HH(a, b, c, d, e, x, s) {\ + (a) += H((b), (c), (d)) + (x) + 0x6ed9eba1UL;\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define II(a, b, c, d, e, x, s) {\ + (a) += I((b), (c), (d)) + (x) + 0x8f1bbcdcUL;\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define JJ(a, b, c, d, e, x, s) {\ + (a) += J((b), (c), (d)) + (x) + 0xa953fd4eUL;\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define FFF(a, b, c, d, e, x, s) {\ + (a) += F((b), (c), (d)) + (x);\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define GGG(a, b, c, d, e, x, s) {\ + (a) += G((b), (c), (d)) + (x) + 0x7a6d76e9UL;\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define HHH(a, b, c, d, e, x, s) {\ + (a) += H((b), (c), (d)) + (x) + 0x6d703ef3UL;\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define III(a, b, c, d, e, x, s) {\ + (a) += I((b), (c), (d)) + (x) + 0x5c4dd124UL;\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } +#define JJJ(a, b, c, d, e, x, s) {\ + (a) += J((b), (c), (d)) + (x) + 0x50a28be6UL;\ + (a) = ROL((a), (s)) + (e);\ + (c) = ROL((c), 10);\ + } + +/********************************************************************/ + +/* function prototypes */ + +void MDinit(dword *MDbuf); +/* +* initializes MDbuffer to "magic constants" +*/ + +void compress(dword *MDbuf, dword *X); +/* +* the compression function. +* transforms MDbuf using message bytes X[0] through X[15] +*/ + +void MDfinish(dword *MDbuf, byte *strptr, dword lswlen, dword mswlen); +/* +* puts bytes from strptr into X and pad out; appends length +* and finally, compresses the last block(s) +* note: length in bits == 8 * (lswlen + 2^32 mswlen). +* note: there are (lswlen mod 64) bytes left in strptr. +*/ + +#endif /* RMD160H */ + +/*********************** end of file rmd160.h ***********************/ diff --git a/src/crypto/RIPEMD160_helper.cpp b/src/crypto/RIPEMD160_helper.cpp new file mode 100644 index 0000000..1386540 --- /dev/null +++ b/src/crypto/RIPEMD160_helper.cpp @@ -0,0 +1,67 @@ +// Copyright (c) 2020 The Zano developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "RIPEMD160_helper.h" +#include "auto_val_init.h" +extern "C" { +#include "RIPEMD160.h" +} + +#define RMDsize 160 + +namespace crypto { + + void RIPEMD160_hash(const void *data, size_t length_size_t, hash160 &h) + { + + dword MDbuf[RMDsize / 32] = {0}; /* contains (A, B, C, D(, E)) */ + byte* hashcode = (byte*)&h; /* hashcode[RMDsize / 8]; /* for final hash-value */ + dword X[16] = {0}; /* current 16-word chunk */ + unsigned int i = 0; /* counter */ + dword length = static_cast(length_size_t); /* length in bytes of message */ + dword nbytes = 0; /* # of bytes not yet processed */ + byte* message = (byte*)data; + + /* initialize */ + MDinit(MDbuf); + //length = (dword)strlen((char *)message); + + /* process message in 16-word chunks */ + for (nbytes = length; nbytes > 63; nbytes -= 64) { + for (i = 0; i < 16; i++) { + X[i] = BYTES_TO_DWORD(message); + message += 4; + } + compress(MDbuf, X); + }/* length mod 64 bytes left */ + + /* finish: */ + MDfinish(MDbuf, message, length, 0); + + for (i = 0; i < RMDsize / 8; i += 4) { + hashcode[i] = (byte)MDbuf[i >> 2]; /* implicit cast to byte */ + hashcode[i + 1] = (byte)(MDbuf[i >> 2] >> 8); /* extracts the 8 least */ + hashcode[i + 2] = (byte)(MDbuf[i >> 2] >> 16); /* significant bits. */ + hashcode[i + 3] = (byte)(MDbuf[i >> 2] >> 24); + } + } + + hash160 RIPEMD160_hash(const void *data, size_t length) + { + hash160 h = AUTO_VAL_INIT(h); + RIPEMD160_hash(data, length, h); + return h; + } + + hash RIPEMD160_hash_256(const void *data, size_t length) + { + hash160 h = RIPEMD160_hash(data, length); + hash h256 = AUTO_VAL_INIT(h256); + memcpy(&h256, &h, sizeof(h)); + return h256; + } + +} diff --git a/src/crypto/RIPEMD160_helper.h b/src/crypto/RIPEMD160_helper.h new file mode 100644 index 0000000..2257e2b --- /dev/null +++ b/src/crypto/RIPEMD160_helper.h @@ -0,0 +1,27 @@ +// Copyright (c) 2020 The Zano developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "hash.h" + + +namespace crypto { + +#pragma pack(push, 1) + POD_CLASS hash160{ + char data[20]; + }; +#pragma pack(pop) + + void RIPEMD160_hash(const void *data, size_t length, hash160 &h); + hash160 RIPEMD160_hash(const void *data, size_t length); + hash RIPEMD160_hash_256(const void *data, size_t length); + +} + +POD_MAKE_HASHABLE(crypto, hash160) + + + diff --git a/src/crypto/bitcoin/byteswap.h b/src/crypto/bitcoin/byteswap.h new file mode 100644 index 0000000..27ef1a1 --- /dev/null +++ b/src/crypto/bitcoin/byteswap.h @@ -0,0 +1,59 @@ +// Copyright (c) 2014-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMPAT_BYTESWAP_H +#define BITCOIN_COMPAT_BYTESWAP_H + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include + +#if defined(HAVE_BYTESWAP_H) +#include +#endif + +#if defined(MAC_OSX) + +#include +#define bswap_16(x) OSSwapInt16(x) +#define bswap_32(x) OSSwapInt32(x) +#define bswap_64(x) OSSwapInt64(x) + +#else +// Non-MacOS / non-Darwin + +#if HAVE_DECL_BSWAP_16 == 0 +inline uint16_t bswap_16(uint16_t x) +{ + return (x >> 8) | (x << 8); +} +#endif // HAVE_DECL_BSWAP16 == 0 + +#if HAVE_DECL_BSWAP_32 == 0 +inline uint32_t bswap_32(uint32_t x) +{ + return (((x & 0xff000000U) >> 24) | ((x & 0x00ff0000U) >> 8) | + ((x & 0x0000ff00U) << 8) | ((x & 0x000000ffU) << 24)); +} +#endif // HAVE_DECL_BSWAP32 == 0 + +#if HAVE_DECL_BSWAP_64 == 0 +inline uint64_t bswap_64(uint64_t x) +{ + return (((x & 0xff00000000000000ull) >> 56) + | ((x & 0x00ff000000000000ull) >> 40) + | ((x & 0x0000ff0000000000ull) >> 24) + | ((x & 0x000000ff00000000ull) >> 8) + | ((x & 0x00000000ff000000ull) << 8) + | ((x & 0x0000000000ff0000ull) << 24) + | ((x & 0x000000000000ff00ull) << 40) + | ((x & 0x00000000000000ffull) << 56)); +} +#endif // HAVE_DECL_BSWAP64 == 0 + +#endif // defined(MAC_OSX) + +#endif // BITCOIN_COMPAT_BYTESWAP_H diff --git a/src/crypto/bitcoin/common.h b/src/crypto/bitcoin/common.h new file mode 100644 index 0000000..afaf0e1 --- /dev/null +++ b/src/crypto/bitcoin/common.h @@ -0,0 +1,110 @@ +// Copyright (c) 2014-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_COMMON_H +#define BITCOIN_CRYPTO_COMMON_H + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include + +#include "endian.h" + +uint16_t static inline ReadLE16(const unsigned char* ptr) +{ + uint16_t x; + memcpy((char*)&x, ptr, 2); + return le16toh(x); +} + +uint32_t static inline ReadLE32(const unsigned char* ptr) +{ + uint32_t x; + memcpy((char*)&x, ptr, 4); + return le32toh(x); +} + +uint64_t static inline ReadLE64(const unsigned char* ptr) +{ + uint64_t x; + memcpy((char*)&x, ptr, 8); + return le64toh(x); +} + +void static inline WriteLE16(unsigned char* ptr, uint16_t x) +{ + uint16_t v = htole16(x); + memcpy(ptr, (char*)&v, 2); +} + +void static inline WriteLE32(unsigned char* ptr, uint32_t x) +{ + uint32_t v = htole32(x); + memcpy(ptr, (char*)&v, 4); +} + +void static inline WriteLE64(unsigned char* ptr, uint64_t x) +{ + uint64_t v = htole64(x); + memcpy(ptr, (char*)&v, 8); +} + +uint16_t static inline ReadBE16(const unsigned char* ptr) +{ + uint16_t x; + memcpy((char*)&x, ptr, 2); + return be16toh(x); +} + +uint32_t static inline ReadBE32(const unsigned char* ptr) +{ + uint32_t x; + memcpy((char*)&x, ptr, 4); + return be32toh(x); +} + +uint64_t static inline ReadBE64(const unsigned char* ptr) +{ + uint64_t x; + memcpy((char*)&x, ptr, 8); + return be64toh(x); +} + +void static inline WriteBE32(unsigned char* ptr, uint32_t x) +{ + uint32_t v = htobe32(x); + memcpy(ptr, (char*)&v, 4); +} + +void static inline WriteBE64(unsigned char* ptr, uint64_t x) +{ + uint64_t v = htobe64(x); + memcpy(ptr, (char*)&v, 8); +} + +/** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ +uint64_t static inline CountBits(uint64_t x) +{ +#if HAVE_BUILTIN_CLZL + if (sizeof(unsigned long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; + } +#endif +#if HAVE_BUILTIN_CLZLL + if (sizeof(unsigned long long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; + } +#endif + int ret = 0; + while (x) { + x >>= 1; + ++ret; + } + return ret; +} + +#endif // BITCOIN_CRYPTO_COMMON_H \ No newline at end of file diff --git a/src/crypto/bitcoin/cpuid.h b/src/crypto/bitcoin/cpuid.h new file mode 100644 index 0000000..0877ad4 --- /dev/null +++ b/src/crypto/bitcoin/cpuid.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMPAT_CPUID_H +#define BITCOIN_COMPAT_CPUID_H + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#define HAVE_GETCPUID + +#include + +// We can't use cpuid.h's __get_cpuid as it does not support subleafs. +void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + +#endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#endif // BITCOIN_COMPAT_CPUID_H diff --git a/src/crypto/bitcoin/endian.h b/src/crypto/bitcoin/endian.h new file mode 100644 index 0000000..354c637 --- /dev/null +++ b/src/crypto/bitcoin/endian.h @@ -0,0 +1,241 @@ +// Copyright (c) 2014-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMPAT_ENDIAN_H +#define BITCOIN_COMPAT_ENDIAN_H + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include "byteswap.h" + +#include + +#if defined(HAVE_ENDIAN_H) +#include +#elif defined(HAVE_SYS_ENDIAN_H) +#include +#endif + +#ifndef HAVE_CONFIG_H +// While not technically a supported configuration, defaulting to defining these +// DECLs when we were compiled without autotools makes it easier for other build +// systems to build things like libbitcoinconsensus for strange targets. +#ifdef htobe16 +#define HAVE_DECL_HTOBE16 1 +#endif +#ifdef htole16 +#define HAVE_DECL_HTOLE16 1 +#endif +#ifdef be16toh +#define HAVE_DECL_BE16TOH 1 +#endif +#ifdef le16toh +#define HAVE_DECL_LE16TOH 1 +#endif + +#ifdef htobe32 +#define HAVE_DECL_HTOBE32 1 +#endif +#ifdef htole32 +#define HAVE_DECL_HTOLE32 1 +#endif +#ifdef be32toh +#define HAVE_DECL_BE32TOH 1 +#endif +#ifdef le32toh +#define HAVE_DECL_LE32TOH 1 +#endif + +#ifdef htobe64 +#define HAVE_DECL_HTOBE64 1 +#endif +#ifdef htole64 +#define HAVE_DECL_HTOLE64 1 +#endif +#ifdef be64toh +#define HAVE_DECL_BE64TOH 1 +#endif +#ifdef le64toh +#define HAVE_DECL_LE64TOH 1 +#endif + +#endif // HAVE_CONFIG_H + +#if defined(WORDS_BIGENDIAN) + +#if HAVE_DECL_HTOBE16 == 0 +inline uint16_t htobe16(uint16_t host_16bits) +{ + return host_16bits; +} +#endif // HAVE_DECL_HTOBE16 + +#if HAVE_DECL_HTOLE16 == 0 +inline uint16_t htole16(uint16_t host_16bits) +{ + return bswap_16(host_16bits); +} +#endif // HAVE_DECL_HTOLE16 + +#if HAVE_DECL_BE16TOH == 0 +inline uint16_t be16toh(uint16_t big_endian_16bits) +{ + return big_endian_16bits; +} +#endif // HAVE_DECL_BE16TOH + +#if HAVE_DECL_LE16TOH == 0 +inline uint16_t le16toh(uint16_t little_endian_16bits) +{ + return bswap_16(little_endian_16bits); +} +#endif // HAVE_DECL_LE16TOH + +#if HAVE_DECL_HTOBE32 == 0 +inline uint32_t htobe32(uint32_t host_32bits) +{ + return host_32bits; +} +#endif // HAVE_DECL_HTOBE32 + +#if HAVE_DECL_HTOLE32 == 0 +inline uint32_t htole32(uint32_t host_32bits) +{ + return bswap_32(host_32bits); +} +#endif // HAVE_DECL_HTOLE32 + +#if HAVE_DECL_BE32TOH == 0 +inline uint32_t be32toh(uint32_t big_endian_32bits) +{ + return big_endian_32bits; +} +#endif // HAVE_DECL_BE32TOH + +#if HAVE_DECL_LE32TOH == 0 +inline uint32_t le32toh(uint32_t little_endian_32bits) +{ + return bswap_32(little_endian_32bits); +} +#endif // HAVE_DECL_LE32TOH + +#if HAVE_DECL_HTOBE64 == 0 +inline uint64_t htobe64(uint64_t host_64bits) +{ + return host_64bits; +} +#endif // HAVE_DECL_HTOBE64 + +#if HAVE_DECL_HTOLE64 == 0 +inline uint64_t htole64(uint64_t host_64bits) +{ + return bswap_64(host_64bits); +} +#endif // HAVE_DECL_HTOLE64 + +#if HAVE_DECL_BE64TOH == 0 +inline uint64_t be64toh(uint64_t big_endian_64bits) +{ + return big_endian_64bits; +} +#endif // HAVE_DECL_BE64TOH + +#if HAVE_DECL_LE64TOH == 0 +inline uint64_t le64toh(uint64_t little_endian_64bits) +{ + return bswap_64(little_endian_64bits); +} +#endif // HAVE_DECL_LE64TOH + +#else // WORDS_BIGENDIAN + +#if HAVE_DECL_HTOBE16 == 0 +inline uint16_t htobe16(uint16_t host_16bits) +{ + return bswap_16(host_16bits); +} +#endif // HAVE_DECL_HTOBE16 + +#if HAVE_DECL_HTOLE16 == 0 +inline uint16_t htole16(uint16_t host_16bits) +{ + return host_16bits; +} +#endif // HAVE_DECL_HTOLE16 + +#if HAVE_DECL_BE16TOH == 0 +inline uint16_t be16toh(uint16_t big_endian_16bits) +{ + return bswap_16(big_endian_16bits); +} +#endif // HAVE_DECL_BE16TOH + +#if HAVE_DECL_LE16TOH == 0 +inline uint16_t le16toh(uint16_t little_endian_16bits) +{ + return little_endian_16bits; +} +#endif // HAVE_DECL_LE16TOH + +#if HAVE_DECL_HTOBE32 == 0 +inline uint32_t htobe32(uint32_t host_32bits) +{ + return bswap_32(host_32bits); +} +#endif // HAVE_DECL_HTOBE32 + +#if HAVE_DECL_HTOLE32 == 0 +inline uint32_t htole32(uint32_t host_32bits) +{ + return host_32bits; +} +#endif // HAVE_DECL_HTOLE32 + +#if HAVE_DECL_BE32TOH == 0 +inline uint32_t be32toh(uint32_t big_endian_32bits) +{ + return bswap_32(big_endian_32bits); +} +#endif // HAVE_DECL_BE32TOH + +#if HAVE_DECL_LE32TOH == 0 +inline uint32_t le32toh(uint32_t little_endian_32bits) +{ + return little_endian_32bits; +} +#endif // HAVE_DECL_LE32TOH + +#if HAVE_DECL_HTOBE64 == 0 +inline uint64_t htobe64(uint64_t host_64bits) +{ + return bswap_64(host_64bits); +} +#endif // HAVE_DECL_HTOBE64 + +#if HAVE_DECL_HTOLE64 == 0 +inline uint64_t htole64(uint64_t host_64bits) +{ + return host_64bits; +} +#endif // HAVE_DECL_HTOLE64 + +#if HAVE_DECL_BE64TOH == 0 +inline uint64_t be64toh(uint64_t big_endian_64bits) +{ + return bswap_64(big_endian_64bits); +} +#endif // HAVE_DECL_BE64TOH + +#if HAVE_DECL_LE64TOH == 0 +inline uint64_t le64toh(uint64_t little_endian_64bits) +{ + return little_endian_64bits; +} +#endif // HAVE_DECL_LE64TOH + +#endif // WORDS_BIGENDIAN + +#endif // BITCOIN_COMPAT_ENDIAN_H diff --git a/src/crypto/bitcoin/sha256.cpp b/src/crypto/bitcoin/sha256.cpp new file mode 100644 index 0000000..5626695 --- /dev/null +++ b/src/crypto/bitcoin/sha256.cpp @@ -0,0 +1,719 @@ +// Copyright (c) 2014-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "sha256.h" +#include "common.h" + +#include +#include + +//#include + +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) +#if defined(USE_ASM) +namespace sha256_sse4 +{ + void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); +} +#endif +#endif + +namespace sha256d64_sse41 +{ + void Transform_4way(unsigned char* out, const unsigned char* in); +} + +namespace sha256d64_avx2 +{ + void Transform_8way(unsigned char* out, const unsigned char* in); +} + +namespace sha256d64_shani +{ + void Transform_2way(unsigned char* out, const unsigned char* in); +} + +namespace sha256_shani +{ + void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); +} + +// Internal implementation code. +namespace +{ + /// Internal SHA-256 implementation. + namespace sha256 + { + uint32_t inline Ch(uint32_t x, uint32_t y, uint32_t z) { return z ^ (x & (y ^ z)); } + uint32_t inline Maj(uint32_t x, uint32_t y, uint32_t z) { return (x & y) | (z & (x | y)); } + uint32_t inline Sigma0(uint32_t x) { return (x >> 2 | x << 30) ^ (x >> 13 | x << 19) ^ (x >> 22 | x << 10); } + uint32_t inline Sigma1(uint32_t x) { return (x >> 6 | x << 26) ^ (x >> 11 | x << 21) ^ (x >> 25 | x << 7); } + uint32_t inline sigma0(uint32_t x) { return (x >> 7 | x << 25) ^ (x >> 18 | x << 14) ^ (x >> 3); } + uint32_t inline sigma1(uint32_t x) { return (x >> 17 | x << 15) ^ (x >> 19 | x << 13) ^ (x >> 10); } + + /** One round of SHA-256. */ + void inline Round(uint32_t a, uint32_t b, uint32_t c, uint32_t& d, uint32_t e, uint32_t f, uint32_t g, uint32_t& h, uint32_t k) + { + uint32_t t1 = h + Sigma1(e) + Ch(e, f, g) + k; + uint32_t t2 = Sigma0(a) + Maj(a, b, c); + d += t1; + h = t1 + t2; + } + + /** Initialize SHA-256 state. */ + void inline Initialize(uint32_t* s) + { + s[0] = 0x6a09e667ul; + s[1] = 0xbb67ae85ul; + s[2] = 0x3c6ef372ul; + s[3] = 0xa54ff53aul; + s[4] = 0x510e527ful; + s[5] = 0x9b05688cul; + s[6] = 0x1f83d9abul; + s[7] = 0x5be0cd19ul; + } + + /** Perform a number of SHA-256 transformations, processing 64-byte chunks. */ + void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) + { + while (blocks--) { + uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; + uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; + + Round(a, b, c, d, e, f, g, h, 0x428a2f98 + (w0 = ReadBE32(chunk + 0))); + Round(h, a, b, c, d, e, f, g, 0x71374491 + (w1 = ReadBE32(chunk + 4))); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf + (w2 = ReadBE32(chunk + 8))); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5 + (w3 = ReadBE32(chunk + 12))); + Round(e, f, g, h, a, b, c, d, 0x3956c25b + (w4 = ReadBE32(chunk + 16))); + Round(d, e, f, g, h, a, b, c, 0x59f111f1 + (w5 = ReadBE32(chunk + 20))); + Round(c, d, e, f, g, h, a, b, 0x923f82a4 + (w6 = ReadBE32(chunk + 24))); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5 + (w7 = ReadBE32(chunk + 28))); + Round(a, b, c, d, e, f, g, h, 0xd807aa98 + (w8 = ReadBE32(chunk + 32))); + Round(h, a, b, c, d, e, f, g, 0x12835b01 + (w9 = ReadBE32(chunk + 36))); + Round(g, h, a, b, c, d, e, f, 0x243185be + (w10 = ReadBE32(chunk + 40))); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3 + (w11 = ReadBE32(chunk + 44))); + Round(e, f, g, h, a, b, c, d, 0x72be5d74 + (w12 = ReadBE32(chunk + 48))); + Round(d, e, f, g, h, a, b, c, 0x80deb1fe + (w13 = ReadBE32(chunk + 52))); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7 + (w14 = ReadBE32(chunk + 56))); + Round(b, c, d, e, f, g, h, a, 0xc19bf174 + (w15 = ReadBE32(chunk + 60))); + + Round(a, b, c, d, e, f, g, h, 0xe49b69c1 + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0xefbe4786 + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x0fc19dc6 + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x240ca1cc + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x2de92c6f + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x4a7484aa + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x76f988da + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0x983e5152 + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0xa831c66d + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0xb00327c8 + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0xbf597fc7 + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0xc6e00bf3 + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xd5a79147 + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0x06ca6351 + (w14 += sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0x14292967 + (w15 += sigma1(w13) + w8 + sigma0(w0))); + + Round(a, b, c, d, e, f, g, h, 0x27b70a85 + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0x2e1b2138 + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x53380d13 + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x650a7354 + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x766a0abb + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x81c2c92e + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x92722c85 + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1 + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0xa81a664b + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0xc24b8b70 + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0xc76c51a3 + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0xd192e819 + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xd6990624 + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0xf40e3585 + (w14 += sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0x106aa070 + (w15 += sigma1(w13) + w8 + sigma0(w0))); + + Round(a, b, c, d, e, f, g, h, 0x19a4c116 + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0x1e376c08 + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x2748774c + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x34b0bcb5 + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x391c0cb3 + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x5b9cca4f + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x682e6ff3 + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0x748f82ee + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0x78a5636f + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0x84c87814 + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0x8cc70208 + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0x90befffa + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xa4506ceb + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0xbef9a3f7 + (w14 + sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0xc67178f2 + (w15 + sigma1(w13) + w8 + sigma0(w0))); + + s[0] += a; + s[1] += b; + s[2] += c; + s[3] += d; + s[4] += e; + s[5] += f; + s[6] += g; + s[7] += h; + chunk += 64; + } + } + + void TransformD64(unsigned char* out, const unsigned char* in) + { + // Transform 1 + uint32_t a = 0x6a09e667ul; + uint32_t b = 0xbb67ae85ul; + uint32_t c = 0x3c6ef372ul; + uint32_t d = 0xa54ff53aul; + uint32_t e = 0x510e527ful; + uint32_t f = 0x9b05688cul; + uint32_t g = 0x1f83d9abul; + uint32_t h = 0x5be0cd19ul; + + uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; + + Round(a, b, c, d, e, f, g, h, 0x428a2f98ul + (w0 = ReadBE32(in + 0))); + Round(h, a, b, c, d, e, f, g, 0x71374491ul + (w1 = ReadBE32(in + 4))); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcful + (w2 = ReadBE32(in + 8))); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5ul + (w3 = ReadBE32(in + 12))); + Round(e, f, g, h, a, b, c, d, 0x3956c25bul + (w4 = ReadBE32(in + 16))); + Round(d, e, f, g, h, a, b, c, 0x59f111f1ul + (w5 = ReadBE32(in + 20))); + Round(c, d, e, f, g, h, a, b, 0x923f82a4ul + (w6 = ReadBE32(in + 24))); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5ul + (w7 = ReadBE32(in + 28))); + Round(a, b, c, d, e, f, g, h, 0xd807aa98ul + (w8 = ReadBE32(in + 32))); + Round(h, a, b, c, d, e, f, g, 0x12835b01ul + (w9 = ReadBE32(in + 36))); + Round(g, h, a, b, c, d, e, f, 0x243185beul + (w10 = ReadBE32(in + 40))); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3ul + (w11 = ReadBE32(in + 44))); + Round(e, f, g, h, a, b, c, d, 0x72be5d74ul + (w12 = ReadBE32(in + 48))); + Round(d, e, f, g, h, a, b, c, 0x80deb1feul + (w13 = ReadBE32(in + 52))); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7ul + (w14 = ReadBE32(in + 56))); + Round(b, c, d, e, f, g, h, a, 0xc19bf174ul + (w15 = ReadBE32(in + 60))); + Round(a, b, c, d, e, f, g, h, 0xe49b69c1ul + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0xefbe4786ul + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x0fc19dc6ul + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x240ca1ccul + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x2de92c6ful + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x4a7484aaul + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x5cb0a9dcul + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x76f988daul + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0x983e5152ul + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0xa831c66dul + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0xb00327c8ul + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0xbf597fc7ul + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0xc6e00bf3ul + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xd5a79147ul + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0x06ca6351ul + (w14 += sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0x14292967ul + (w15 += sigma1(w13) + w8 + sigma0(w0))); + Round(a, b, c, d, e, f, g, h, 0x27b70a85ul + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0x2e1b2138ul + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x4d2c6dfcul + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x53380d13ul + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x650a7354ul + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x766a0abbul + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x81c2c92eul + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x92722c85ul + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1ul + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0xa81a664bul + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0xc24b8b70ul + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0xc76c51a3ul + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0xd192e819ul + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xd6990624ul + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0xf40e3585ul + (w14 += sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0x106aa070ul + (w15 += sigma1(w13) + w8 + sigma0(w0))); + Round(a, b, c, d, e, f, g, h, 0x19a4c116ul + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0x1e376c08ul + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x2748774cul + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x34b0bcb5ul + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x391c0cb3ul + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x4ed8aa4aul + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x5b9cca4ful + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x682e6ff3ul + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0x748f82eeul + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0x78a5636ful + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0x84c87814ul + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0x8cc70208ul + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0x90befffaul + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xa4506cebul + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0xbef9a3f7ul + (w14 + sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0xc67178f2ul + (w15 + sigma1(w13) + w8 + sigma0(w0))); + + a += 0x6a09e667ul; + b += 0xbb67ae85ul; + c += 0x3c6ef372ul; + d += 0xa54ff53aul; + e += 0x510e527ful; + f += 0x9b05688cul; + g += 0x1f83d9abul; + h += 0x5be0cd19ul; + + uint32_t t0 = a, t1 = b, t2 = c, t3 = d, t4 = e, t5 = f, t6 = g, t7 = h; + + // Transform 2 + Round(a, b, c, d, e, f, g, h, 0xc28a2f98ul); + Round(h, a, b, c, d, e, f, g, 0x71374491ul); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcful); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5ul); + Round(e, f, g, h, a, b, c, d, 0x3956c25bul); + Round(d, e, f, g, h, a, b, c, 0x59f111f1ul); + Round(c, d, e, f, g, h, a, b, 0x923f82a4ul); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5ul); + Round(a, b, c, d, e, f, g, h, 0xd807aa98ul); + Round(h, a, b, c, d, e, f, g, 0x12835b01ul); + Round(g, h, a, b, c, d, e, f, 0x243185beul); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3ul); + Round(e, f, g, h, a, b, c, d, 0x72be5d74ul); + Round(d, e, f, g, h, a, b, c, 0x80deb1feul); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7ul); + Round(b, c, d, e, f, g, h, a, 0xc19bf374ul); + Round(a, b, c, d, e, f, g, h, 0x649b69c1ul); + Round(h, a, b, c, d, e, f, g, 0xf0fe4786ul); + Round(g, h, a, b, c, d, e, f, 0x0fe1edc6ul); + Round(f, g, h, a, b, c, d, e, 0x240cf254ul); + Round(e, f, g, h, a, b, c, d, 0x4fe9346ful); + Round(d, e, f, g, h, a, b, c, 0x6cc984beul); + Round(c, d, e, f, g, h, a, b, 0x61b9411eul); + Round(b, c, d, e, f, g, h, a, 0x16f988faul); + Round(a, b, c, d, e, f, g, h, 0xf2c65152ul); + Round(h, a, b, c, d, e, f, g, 0xa88e5a6dul); + Round(g, h, a, b, c, d, e, f, 0xb019fc65ul); + Round(f, g, h, a, b, c, d, e, 0xb9d99ec7ul); + Round(e, f, g, h, a, b, c, d, 0x9a1231c3ul); + Round(d, e, f, g, h, a, b, c, 0xe70eeaa0ul); + Round(c, d, e, f, g, h, a, b, 0xfdb1232bul); + Round(b, c, d, e, f, g, h, a, 0xc7353eb0ul); + Round(a, b, c, d, e, f, g, h, 0x3069bad5ul); + Round(h, a, b, c, d, e, f, g, 0xcb976d5ful); + Round(g, h, a, b, c, d, e, f, 0x5a0f118ful); + Round(f, g, h, a, b, c, d, e, 0xdc1eeefdul); + Round(e, f, g, h, a, b, c, d, 0x0a35b689ul); + Round(d, e, f, g, h, a, b, c, 0xde0b7a04ul); + Round(c, d, e, f, g, h, a, b, 0x58f4ca9dul); + Round(b, c, d, e, f, g, h, a, 0xe15d5b16ul); + Round(a, b, c, d, e, f, g, h, 0x007f3e86ul); + Round(h, a, b, c, d, e, f, g, 0x37088980ul); + Round(g, h, a, b, c, d, e, f, 0xa507ea32ul); + Round(f, g, h, a, b, c, d, e, 0x6fab9537ul); + Round(e, f, g, h, a, b, c, d, 0x17406110ul); + Round(d, e, f, g, h, a, b, c, 0x0d8cd6f1ul); + Round(c, d, e, f, g, h, a, b, 0xcdaa3b6dul); + Round(b, c, d, e, f, g, h, a, 0xc0bbbe37ul); + Round(a, b, c, d, e, f, g, h, 0x83613bdaul); + Round(h, a, b, c, d, e, f, g, 0xdb48a363ul); + Round(g, h, a, b, c, d, e, f, 0x0b02e931ul); + Round(f, g, h, a, b, c, d, e, 0x6fd15ca7ul); + Round(e, f, g, h, a, b, c, d, 0x521afacaul); + Round(d, e, f, g, h, a, b, c, 0x31338431ul); + Round(c, d, e, f, g, h, a, b, 0x6ed41a95ul); + Round(b, c, d, e, f, g, h, a, 0x6d437890ul); + Round(a, b, c, d, e, f, g, h, 0xc39c91f2ul); + Round(h, a, b, c, d, e, f, g, 0x9eccabbdul); + Round(g, h, a, b, c, d, e, f, 0xb5c9a0e6ul); + Round(f, g, h, a, b, c, d, e, 0x532fb63cul); + Round(e, f, g, h, a, b, c, d, 0xd2c741c6ul); + Round(d, e, f, g, h, a, b, c, 0x07237ea3ul); + Round(c, d, e, f, g, h, a, b, 0xa4954b68ul); + Round(b, c, d, e, f, g, h, a, 0x4c191d76ul); + + w0 = t0 + a; + w1 = t1 + b; + w2 = t2 + c; + w3 = t3 + d; + w4 = t4 + e; + w5 = t5 + f; + w6 = t6 + g; + w7 = t7 + h; + + // Transform 3 + a = 0x6a09e667ul; + b = 0xbb67ae85ul; + c = 0x3c6ef372ul; + d = 0xa54ff53aul; + e = 0x510e527ful; + f = 0x9b05688cul; + g = 0x1f83d9abul; + h = 0x5be0cd19ul; + + Round(a, b, c, d, e, f, g, h, 0x428a2f98ul + w0); + Round(h, a, b, c, d, e, f, g, 0x71374491ul + w1); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcful + w2); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5ul + w3); + Round(e, f, g, h, a, b, c, d, 0x3956c25bul + w4); + Round(d, e, f, g, h, a, b, c, 0x59f111f1ul + w5); + Round(c, d, e, f, g, h, a, b, 0x923f82a4ul + w6); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5ul + w7); + Round(a, b, c, d, e, f, g, h, 0x5807aa98ul); + Round(h, a, b, c, d, e, f, g, 0x12835b01ul); + Round(g, h, a, b, c, d, e, f, 0x243185beul); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3ul); + Round(e, f, g, h, a, b, c, d, 0x72be5d74ul); + Round(d, e, f, g, h, a, b, c, 0x80deb1feul); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7ul); + Round(b, c, d, e, f, g, h, a, 0xc19bf274ul); + Round(a, b, c, d, e, f, g, h, 0xe49b69c1ul + (w0 += sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0xefbe4786ul + (w1 += 0xa00000ul + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x0fc19dc6ul + (w2 += sigma1(w0) + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x240ca1ccul + (w3 += sigma1(w1) + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x2de92c6ful + (w4 += sigma1(w2) + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x4a7484aaul + (w5 += sigma1(w3) + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x5cb0a9dcul + (w6 += sigma1(w4) + 0x100ul + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x76f988daul + (w7 += sigma1(w5) + w0 + 0x11002000ul)); + Round(a, b, c, d, e, f, g, h, 0x983e5152ul + (w8 = 0x80000000ul + sigma1(w6) + w1)); + Round(h, a, b, c, d, e, f, g, 0xa831c66dul + (w9 = sigma1(w7) + w2)); + Round(g, h, a, b, c, d, e, f, 0xb00327c8ul + (w10 = sigma1(w8) + w3)); + Round(f, g, h, a, b, c, d, e, 0xbf597fc7ul + (w11 = sigma1(w9) + w4)); + Round(e, f, g, h, a, b, c, d, 0xc6e00bf3ul + (w12 = sigma1(w10) + w5)); + Round(d, e, f, g, h, a, b, c, 0xd5a79147ul + (w13 = sigma1(w11) + w6)); + Round(c, d, e, f, g, h, a, b, 0x06ca6351ul + (w14 = sigma1(w12) + w7 + 0x400022ul)); + Round(b, c, d, e, f, g, h, a, 0x14292967ul + (w15 = 0x100ul + sigma1(w13) + w8 + sigma0(w0))); + Round(a, b, c, d, e, f, g, h, 0x27b70a85ul + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0x2e1b2138ul + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x4d2c6dfcul + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x53380d13ul + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x650a7354ul + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x766a0abbul + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x81c2c92eul + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x92722c85ul + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1ul + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0xa81a664bul + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0xc24b8b70ul + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0xc76c51a3ul + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0xd192e819ul + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xd6990624ul + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0xf40e3585ul + (w14 += sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0x106aa070ul + (w15 += sigma1(w13) + w8 + sigma0(w0))); + Round(a, b, c, d, e, f, g, h, 0x19a4c116ul + (w0 += sigma1(w14) + w9 + sigma0(w1))); + Round(h, a, b, c, d, e, f, g, 0x1e376c08ul + (w1 += sigma1(w15) + w10 + sigma0(w2))); + Round(g, h, a, b, c, d, e, f, 0x2748774cul + (w2 += sigma1(w0) + w11 + sigma0(w3))); + Round(f, g, h, a, b, c, d, e, 0x34b0bcb5ul + (w3 += sigma1(w1) + w12 + sigma0(w4))); + Round(e, f, g, h, a, b, c, d, 0x391c0cb3ul + (w4 += sigma1(w2) + w13 + sigma0(w5))); + Round(d, e, f, g, h, a, b, c, 0x4ed8aa4aul + (w5 += sigma1(w3) + w14 + sigma0(w6))); + Round(c, d, e, f, g, h, a, b, 0x5b9cca4ful + (w6 += sigma1(w4) + w15 + sigma0(w7))); + Round(b, c, d, e, f, g, h, a, 0x682e6ff3ul + (w7 += sigma1(w5) + w0 + sigma0(w8))); + Round(a, b, c, d, e, f, g, h, 0x748f82eeul + (w8 += sigma1(w6) + w1 + sigma0(w9))); + Round(h, a, b, c, d, e, f, g, 0x78a5636ful + (w9 += sigma1(w7) + w2 + sigma0(w10))); + Round(g, h, a, b, c, d, e, f, 0x84c87814ul + (w10 += sigma1(w8) + w3 + sigma0(w11))); + Round(f, g, h, a, b, c, d, e, 0x8cc70208ul + (w11 += sigma1(w9) + w4 + sigma0(w12))); + Round(e, f, g, h, a, b, c, d, 0x90befffaul + (w12 += sigma1(w10) + w5 + sigma0(w13))); + Round(d, e, f, g, h, a, b, c, 0xa4506cebul + (w13 += sigma1(w11) + w6 + sigma0(w14))); + Round(c, d, e, f, g, h, a, b, 0xbef9a3f7ul + (w14 + sigma1(w12) + w7 + sigma0(w15))); + Round(b, c, d, e, f, g, h, a, 0xc67178f2ul + (w15 + sigma1(w13) + w8 + sigma0(w0))); + + // Output + WriteBE32(out + 0, a + 0x6a09e667ul); + WriteBE32(out + 4, b + 0xbb67ae85ul); + WriteBE32(out + 8, c + 0x3c6ef372ul); + WriteBE32(out + 12, d + 0xa54ff53aul); + WriteBE32(out + 16, e + 0x510e527ful); + WriteBE32(out + 20, f + 0x9b05688cul); + WriteBE32(out + 24, g + 0x1f83d9abul); + WriteBE32(out + 28, h + 0x5be0cd19ul); + } + + } // namespace sha256 + + typedef void(*TransformType)(uint32_t*, const unsigned char*, size_t); + typedef void(*TransformD64Type)(unsigned char*, const unsigned char*); + + template + void TransformD64Wrapper(unsigned char* out, const unsigned char* in) + { + uint32_t s[8]; + static const unsigned char padding1[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 + }; + unsigned char buffer2[64] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 + }; + sha256::Initialize(s); + tr(s, in, 1); + tr(s, padding1, 1); + WriteBE32(buffer2 + 0, s[0]); + WriteBE32(buffer2 + 4, s[1]); + WriteBE32(buffer2 + 8, s[2]); + WriteBE32(buffer2 + 12, s[3]); + WriteBE32(buffer2 + 16, s[4]); + WriteBE32(buffer2 + 20, s[5]); + WriteBE32(buffer2 + 24, s[6]); + WriteBE32(buffer2 + 28, s[7]); + sha256::Initialize(s); + tr(s, buffer2, 1); + WriteBE32(out + 0, s[0]); + WriteBE32(out + 4, s[1]); + WriteBE32(out + 8, s[2]); + WriteBE32(out + 12, s[3]); + WriteBE32(out + 16, s[4]); + WriteBE32(out + 20, s[5]); + WriteBE32(out + 24, s[6]); + WriteBE32(out + 28, s[7]); + } + + TransformType Transform = sha256::Transform; + TransformD64Type TransformD64 = sha256::TransformD64; + TransformD64Type TransformD64_2way = nullptr; + TransformD64Type TransformD64_4way = nullptr; + TransformD64Type TransformD64_8way = nullptr; + + bool SelfTest() { + // Input state (equal to the initial SHA256 state) + static const uint32_t init[8] = { + 0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul + }; + // Some random input data to test with + static const unsigned char data[641] = "-" // Intentionally not aligned + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua. Et m" + "olestie ac feugiat sed lectus vestibulum mattis ullamcorper. Mor" + "bi blandit cursus risus at ultrices mi tempus imperdiet nulla. N" + "unc congue nisi vita suscipit tellus mauris. Imperdiet proin fer" + "mentum leo vel orci. Massa tempor nec feugiat nisl pretium fusce" + " id velit. Telus in metus vulputate eu scelerisque felis. Mi tem" + "pus imperdiet nulla malesuada pellentesque. Tristique magna sit."; + // Expected output state for hashing the i*64 first input bytes above (excluding SHA256 padding). + static const uint32_t result[9][8] = { + { 0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul }, + { 0x91f8ec6bul, 0x4da10fe3ul, 0x1c9c292cul, 0x45e18185ul, 0x435cc111ul, 0x3ca26f09ul, 0xeb954caeul, 0x402a7069ul }, + { 0xcabea5acul, 0x374fb97cul, 0x182ad996ul, 0x7bd69cbful, 0x450ff900ul, 0xc1d2be8aul, 0x6a41d505ul, 0xe6212dc3ul }, + { 0xbcff09d6ul, 0x3e76f36eul, 0x3ecb2501ul, 0x78866e97ul, 0xe1c1e2fdul, 0x32f4eafful, 0x8aa6c4e5ul, 0xdfc024bcul }, + { 0xa08c5d94ul, 0x0a862f93ul, 0x6b7f2f40ul, 0x8f9fae76ul, 0x6d40439ful, 0x79dcee0cul, 0x3e39ff3aul, 0xdc3bdbb1ul }, + { 0x216a0895ul, 0x9f1a3662ul, 0xe99946f9ul, 0x87ba4364ul, 0x0fb5db2cul, 0x12bed3d3ul, 0x6689c0c7ul, 0x292f1b04ul }, + { 0xca3067f8ul, 0xbc8c2656ul, 0x37cb7e0dul, 0x9b6b8b0ful, 0x46dc380bul, 0xf1287f57ul, 0xc42e4b23ul, 0x3fefe94dul }, + { 0x3e4c4039ul, 0xbb6fca8cul, 0x6f27d2f7ul, 0x301e44a4ul, 0x8352ba14ul, 0x5769ce37ul, 0x48a1155ful, 0xc0e1c4c6ul }, + { 0xfe2fa9ddul, 0x69d0862bul, 0x1ae0db23ul, 0x471f9244ul, 0xf55c0145ul, 0xc30f9c3bul, 0x40a84ea0ul, 0x5b8a266cul }, + }; + // Expected output for each of the individual 8 64-byte messages under full double SHA256 (including padding). + static const unsigned char result_d64[256] = { + 0x09, 0x3a, 0xc4, 0xd0, 0x0f, 0xf7, 0x57, 0xe1, 0x72, 0x85, 0x79, 0x42, 0xfe, 0xe7, 0xe0, 0xa0, + 0xfc, 0x52, 0xd7, 0xdb, 0x07, 0x63, 0x45, 0xfb, 0x53, 0x14, 0x7d, 0x17, 0x22, 0x86, 0xf0, 0x52, + 0x48, 0xb6, 0x11, 0x9e, 0x6e, 0x48, 0x81, 0x6d, 0xcc, 0x57, 0x1f, 0xb2, 0x97, 0xa8, 0xd5, 0x25, + 0x9b, 0x82, 0xaa, 0x89, 0xe2, 0xfd, 0x2d, 0x56, 0xe8, 0x28, 0x83, 0x0b, 0xe2, 0xfa, 0x53, 0xb7, + 0xd6, 0x6b, 0x07, 0x85, 0x83, 0xb0, 0x10, 0xa2, 0xf5, 0x51, 0x3c, 0xf9, 0x60, 0x03, 0xab, 0x45, + 0x6c, 0x15, 0x6e, 0xef, 0xb5, 0xac, 0x3e, 0x6c, 0xdf, 0xb4, 0x92, 0x22, 0x2d, 0xce, 0xbf, 0x3e, + 0xe9, 0xe5, 0xf6, 0x29, 0x0e, 0x01, 0x4f, 0xd2, 0xd4, 0x45, 0x65, 0xb3, 0xbb, 0xf2, 0x4c, 0x16, + 0x37, 0x50, 0x3c, 0x6e, 0x49, 0x8c, 0x5a, 0x89, 0x2b, 0x1b, 0xab, 0xc4, 0x37, 0xd1, 0x46, 0xe9, + 0x3d, 0x0e, 0x85, 0xa2, 0x50, 0x73, 0xa1, 0x5e, 0x54, 0x37, 0xd7, 0x94, 0x17, 0x56, 0xc2, 0xd8, + 0xe5, 0x9f, 0xed, 0x4e, 0xae, 0x15, 0x42, 0x06, 0x0d, 0x74, 0x74, 0x5e, 0x24, 0x30, 0xce, 0xd1, + 0x9e, 0x50, 0xa3, 0x9a, 0xb8, 0xf0, 0x4a, 0x57, 0x69, 0x78, 0x67, 0x12, 0x84, 0x58, 0xbe, 0xc7, + 0x36, 0xaa, 0xee, 0x7c, 0x64, 0xa3, 0x76, 0xec, 0xff, 0x55, 0x41, 0x00, 0x2a, 0x44, 0x68, 0x4d, + 0xb6, 0x53, 0x9e, 0x1c, 0x95, 0xb7, 0xca, 0xdc, 0x7f, 0x7d, 0x74, 0x27, 0x5c, 0x8e, 0xa6, 0x84, + 0xb5, 0xac, 0x87, 0xa9, 0xf3, 0xff, 0x75, 0xf2, 0x34, 0xcd, 0x1a, 0x3b, 0x82, 0x2c, 0x2b, 0x4e, + 0x6a, 0x46, 0x30, 0xa6, 0x89, 0x86, 0x23, 0xac, 0xf8, 0xa5, 0x15, 0xe9, 0x0a, 0xaa, 0x1e, 0x9a, + 0xd7, 0x93, 0x6b, 0x28, 0xe4, 0x3b, 0xfd, 0x59, 0xc6, 0xed, 0x7c, 0x5f, 0xa5, 0x41, 0xcb, 0x51 + }; + + + // Test Transform() for 0 through 8 transformations. + for (size_t i = 0; i <= 8; ++i) { + uint32_t state[8]; + std::copy(init, init + 8, state); + Transform(state, data + 1, i); + if (!std::equal(state, state + 8, result[i])) return false; + } + + // Test TransformD64 + unsigned char out[32]; + TransformD64(out, data + 1); + if (!std::equal(out, out + 32, result_d64)) return false; + + // Test TransformD64_2way, if available. + if (TransformD64_2way) { + unsigned char out[64]; + TransformD64_2way(out, data + 1); + if (!std::equal(out, out + 64, result_d64)) return false; + } + + // Test TransformD64_4way, if available. + if (TransformD64_4way) { + unsigned char out[128]; + TransformD64_4way(out, data + 1); + if (!std::equal(out, out + 128, result_d64)) return false; + } + + // Test TransformD64_8way, if available. + if (TransformD64_8way) { + unsigned char out[256]; + TransformD64_8way(out, data + 1); + if (!std::equal(out, out + 256, result_d64)) return false; + } + + return true; + } + +#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) + /** Check whether the OS has enabled AVX registers. */ + bool AVXEnabled() + { + uint32_t a, d; + __asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(0)); + return (a & 6) == 6; + } +#endif +} // namespace + + +std::string SHA256AutoDetect() +{ + std::string ret = "standard"; +#if defined(USE_ASM) && defined(HAVE_GETCPUID) + bool have_sse4 = false; + bool have_xsave = false; + bool have_avx = false; + bool have_avx2 = false; + bool have_shani = false; + bool enabled_avx = false; + + (void)AVXEnabled; + (void)have_sse4; + (void)have_avx; + (void)have_xsave; + (void)have_avx2; + (void)have_shani; + (void)enabled_avx; + + uint32_t eax, ebx, ecx, edx; + GetCPUID(1, 0, eax, ebx, ecx, edx); + have_sse4 = (ecx >> 19) & 1; + have_xsave = (ecx >> 27) & 1; + have_avx = (ecx >> 28) & 1; + if (have_xsave && have_avx) { + enabled_avx = AVXEnabled(); + } + if (have_sse4) { + GetCPUID(7, 0, eax, ebx, ecx, edx); + have_avx2 = (ebx >> 5) & 1; + have_shani = (ebx >> 29) & 1; + } + +#if defined(ENABLE_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) + if (have_shani) { + Transform = sha256_shani::Transform; + TransformD64 = TransformD64Wrapper; + TransformD64_2way = sha256d64_shani::Transform_2way; + ret = "shani(1way,2way)"; + have_sse4 = false; // Disable SSE4/AVX2; + have_avx2 = false; + } +#endif + + if (have_sse4) { +#if defined(__x86_64__) || defined(__amd64__) + Transform = sha256_sse4::Transform; + TransformD64 = TransformD64Wrapper; + ret = "sse4(1way)"; +#endif +#if defined(ENABLE_SSE41) && !defined(BUILD_BITCOIN_INTERNAL) + TransformD64_4way = sha256d64_sse41::Transform_4way; + ret += ",sse41(4way)"; +#endif + } + +#if defined(ENABLE_AVX2) && !defined(BUILD_BITCOIN_INTERNAL) + if (have_avx2 && have_avx && enabled_avx) { + TransformD64_8way = sha256d64_avx2::Transform_8way; + ret += ",avx2(8way)"; + } +#endif +#endif + + assert(SelfTest()); + return ret; +} + +////// SHA-256 + +CSHA256::CSHA256() : bytes(0) +{ + sha256::Initialize(s); +} + +CSHA256& CSHA256::Write(const unsigned char* data, size_t len) +{ + const unsigned char* end = data + len; + size_t bufsize = bytes % 64; + if (bufsize && bufsize + len >= 64) { + // Fill the buffer, and process it. + memcpy(buf + bufsize, data, 64 - bufsize); + bytes += 64 - bufsize; + data += 64 - bufsize; + Transform(s, buf, 1); + bufsize = 0; + } + if (end - data >= 64) { + size_t blocks = (end - data) / 64; + Transform(s, data, blocks); + data += 64 * blocks; + bytes += 64 * blocks; + } + if (end > data) { + // Fill the buffer with what remains. + memcpy(buf + bufsize, data, end - data); + bytes += end - data; + } + return *this; +} + +void CSHA256::Finalize(unsigned char hash[OUTPUT_SIZE]) +{ + static const unsigned char pad[64] = { 0x80 }; + unsigned char sizedesc[8]; + WriteBE64(sizedesc, bytes << 3); + Write(pad, 1 + ((119 - (bytes % 64)) % 64)); + Write(sizedesc, 8); + WriteBE32(hash, s[0]); + WriteBE32(hash + 4, s[1]); + WriteBE32(hash + 8, s[2]); + WriteBE32(hash + 12, s[3]); + WriteBE32(hash + 16, s[4]); + WriteBE32(hash + 20, s[5]); + WriteBE32(hash + 24, s[6]); + WriteBE32(hash + 28, s[7]); +} + +CSHA256& CSHA256::Reset() +{ + bytes = 0; + sha256::Initialize(s); + return *this; +} + +void SHA256D64(unsigned char* out, const unsigned char* in, size_t blocks) +{ + if (TransformD64_8way) { + while (blocks >= 8) { + TransformD64_8way(out, in); + out += 256; + in += 512; + blocks -= 8; + } + } + if (TransformD64_4way) { + while (blocks >= 4) { + TransformD64_4way(out, in); + out += 128; + in += 256; + blocks -= 4; + } + } + if (TransformD64_2way) { + while (blocks >= 2) { + TransformD64_2way(out, in); + out += 64; + in += 128; + blocks -= 2; + } + } + while (blocks) { + TransformD64(out, in); + out += 32; + in += 64; + --blocks; + } +} \ No newline at end of file diff --git a/src/crypto/bitcoin/sha256.h b/src/crypto/bitcoin/sha256.h new file mode 100644 index 0000000..80d3625 --- /dev/null +++ b/src/crypto/bitcoin/sha256.h @@ -0,0 +1,41 @@ +// Copyright (c) 2014-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_SHA256_H +#define BITCOIN_CRYPTO_SHA256_H + +#include +#include +#include + +/** A hasher class for SHA-256. */ +class CSHA256 +{ +private: + uint32_t s[8]; + unsigned char buf[64]; + uint64_t bytes; + +public: + static const size_t OUTPUT_SIZE = 32; + + CSHA256(); + CSHA256& Write(const unsigned char* data, size_t len); + void Finalize(unsigned char hash[OUTPUT_SIZE]); + CSHA256& Reset(); +}; + +/** Autodetect the best available SHA256 implementation. +* Returns the name of the implementation. +*/ +std::string SHA256AutoDetect(); + +/** Compute multiple double-SHA256's of 64-byte blobs. +* output: pointer to a blocks*32 byte output buffer +* input: pointer to a blocks*64 byte input buffer +* blocks: the number of hashes to compute. +*/ +void SHA256D64(unsigned char* output, const unsigned char* input, size_t blocks); + +#endif // BITCOIN_CRYPTO_SHA256_H \ No newline at end of file diff --git a/src/crypto/bitcoin/sha256_helper.h b/src/crypto/bitcoin/sha256_helper.h new file mode 100644 index 0000000..f376913 --- /dev/null +++ b/src/crypto/bitcoin/sha256_helper.h @@ -0,0 +1,27 @@ +// Copyright (c) 2020 The Zano developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "crypto/hash.h" +#include "sha256.h" + + +namespace crypto { + + inline void sha256_hash(const void *data, std::size_t length, hash &h) + { + CSHA256 sh; + sh.Write((const unsigned char*)data, length); + sh.Finalize((unsigned char* )&h); + } + + inline hash sha256_hash(const void *data, std::size_t length) + { + hash h; + sha256_hash(data, length, h); + return h; + } + +} diff --git a/src/crypto/blake2-impl.h b/src/crypto/blake2-impl.h new file mode 100644 index 0000000..c1df82e --- /dev/null +++ b/src/crypto/blake2-impl.h @@ -0,0 +1,160 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_IMPL_H +#define BLAKE2_IMPL_H + +#include +#include + +#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) + #if defined(_MSC_VER) + #define BLAKE2_INLINE __inline + #elif defined(__GNUC__) + #define BLAKE2_INLINE __inline__ + #else + #define BLAKE2_INLINE + #endif +#else + #define BLAKE2_INLINE inline +#endif + +static BLAKE2_INLINE uint32_t load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint32_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8) | + (( uint32_t )( p[2] ) << 16) | + (( uint32_t )( p[3] ) << 24) ; +#endif +} + +static BLAKE2_INLINE uint64_t load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) | + (( uint64_t )( p[6] ) << 48) | + (( uint64_t )( p[7] ) << 56) ; +#endif +} + +static BLAKE2_INLINE uint16_t load16( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint16_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return ( uint16_t )((( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8)); +#endif +} + +static BLAKE2_INLINE void store16( void *dst, uint16_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static BLAKE2_INLINE void store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +#endif +} + +static BLAKE2_INLINE void store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); + p[6] = (uint8_t)(w >> 48); + p[7] = (uint8_t)(w >> 56); +#endif +} + +static BLAKE2_INLINE uint64_t load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) ; +} + +static BLAKE2_INLINE void store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); +} + +static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) +{ + static void *(*const volatile memset_v)(void *, int, size_t) = &memset; + memset_v(v, 0, n); +} + +#endif diff --git a/src/crypto/blake2.h b/src/crypto/blake2.h new file mode 100644 index 0000000..4949c65 --- /dev/null +++ b/src/crypto/blake2.h @@ -0,0 +1,200 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_H +#define BLAKE2_H + +#include +#include + +#if defined(_MSC_VER) +#define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop)) +#else +#define BLAKE2_PACKED(x) x __attribute__((packed)) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + + typedef struct blake2s_state__ + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2s_state; + + typedef struct blake2b_state__ + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2b_state; + + typedef struct blake2sp_state__ + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2sp_state; + + typedef struct blake2bp_state__ + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2bp_state; + + + BLAKE2_PACKED(struct blake2s_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint16_t xof_length; /* 14 */ + uint8_t node_depth; /* 15 */ + uint8_t inner_length; /* 16 */ + /* uint8_t reserved[0]; */ + uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ + uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ + }); + + typedef struct blake2s_param__ blake2s_param; + + BLAKE2_PACKED(struct blake2b_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint32_t xof_length; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ + }); + + typedef struct blake2b_param__ blake2b_param; + + typedef struct blake2xs_state__ + { + blake2s_state S[1]; + blake2s_param P[1]; + } blake2xs_state; + + typedef struct blake2xb_state__ + { + blake2b_state S[1]; + blake2b_param P[1]; + } blake2xb_state; + + + /* Padded structs result in a compile-time error */ + + enum { + BLAKE2_DUMMY_1 = 1/(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), + BLAKE2_DUMMY_2 = 1/(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) + }; +// static_assert(sizeof(blake2s_param) == BLAKE2S_OUTBYTES, "Wrong size of blake2s_param"); +// static_assert(sizeof(blake2b_param) == BLAKE2B_OUTBYTES, "Wrong size of blake2b_param"); + + + /* Streaming API */ + int blake2s_init( blake2s_state *S, size_t outlen ); + int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); + int blake2s_final( blake2s_state *S, void *out, size_t outlen ); + + int blake2b_init( blake2b_state *S, size_t outlen ); + int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); + int blake2b_final( blake2b_state *S, void *out, size_t outlen ); + + int blake2sp_init( blake2sp_state *S, size_t outlen ); + int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); + int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); + + int blake2bp_init( blake2bp_state *S, size_t outlen ); + int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); + int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); + + /* Variable output length API */ + int blake2xs_init( blake2xs_state *S, const size_t outlen ); + int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); + int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); + + int blake2xb_init( blake2xb_state *S, const size_t outlen ); + int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); + int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); + + /* Simple API */ + int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + /* This is simply an alias for blake2b */ + int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/src/crypto/blake2b-ref.c b/src/crypto/blake2b-ref.c new file mode 100644 index 0000000..cd38b1b --- /dev/null +++ b/src/crypto/blake2b-ref.c @@ -0,0 +1,379 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ + +#include +#include +#include + +#include "blake2.h" +#include "blake2-impl.h" + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +static void blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = (uint64_t)-1; +} + +/* Some helper functions, not necessarily useful */ +static int blake2b_is_lastblock( const blake2b_state *S ) +{ + return S->f[0] != 0; +} + +static void blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = (uint64_t)-1; +} + +static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); +} + +static void blake2b_init0( blake2b_state *S ) +{ + size_t i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + const uint8_t *p = ( const uint8_t * )( P ); + size_t i; + + blake2b_init0( S ); + + /* IV XOR ParamBlock */ + for( i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + S->outlen = P->digest_length; + return 0; +} + + + +int blake2b_init( blake2b_state *S, size_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = (uint8_t)keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) + +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + +static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + size_t i; + + for( i = 0; i < 16; ++i ) { + m[i] = load64( block + i * sizeof( m[i] ) ); + } + + for( i = 0; i < 8; ++i ) { + v[i] = S->h[i]; + } + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } +} + +#undef G +#undef ROUND + +int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) +{ + const unsigned char * in = (const unsigned char *)pin; + if( inlen > 0 ) + { + size_t left = S->buflen; + size_t fill = BLAKE2B_BLOCKBYTES - left; + if( inlen > fill ) + { + S->buflen = 0; + memcpy( S->buf + left, in, fill ); /* Fill buffer */ + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); /* Compress */ + in += fill; inlen -= fill; + while(inlen > BLAKE2B_BLOCKBYTES) { + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress( S, in ); + in += BLAKE2B_BLOCKBYTES; + inlen -= BLAKE2B_BLOCKBYTES; + } + } + memcpy( S->buf + S->buflen, in, inlen ); + S->buflen += inlen; + } + return 0; +} + +int blake2b_final( blake2b_state *S, void *out, size_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; + size_t i; + + if( out == NULL || outlen < S->outlen ) + return -1; + + if( blake2b_is_lastblock( S ) ) + return -1; + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, S->outlen ); + secure_zero_memory(buffer, sizeof(buffer)); + return 0; +} + +/* inlen, at least, should be uint64_t. Others can be size_t. */ +int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + + if( keylen > BLAKE2B_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + blake2b_update( S, ( const uint8_t * )in, inlen ); + blake2b_final( S, out, outlen ); + return 0; +} + +int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { + return blake2b(out, outlen, in, inlen, key, keylen); +} + +#if defined(SUPERCOP) +int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) +{ + return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); +} +#endif + +#if defined(BLAKE2B_SELFTEST) +#include +#include "blake2-kat.h" +int main( void ) +{ + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[BLAKE2_KAT_LENGTH]; + size_t i, step; + + for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) + key[i] = ( uint8_t )i; + + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + buf[i] = ( uint8_t )i; + + /* Test simple API */ + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); + + if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + { + goto fail; + } + } + + /* Test streaming API */ + for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b_state S; + uint8_t * p = buf; + size_t mlen = i; + int err = 0; + + if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { + goto fail; + } + + while (mlen >= step) { + if ( (err = blake2b_update(&S, p, step)) < 0 ) { + goto fail; + } + mlen -= step; + p += step; + } + if ( (err = blake2b_update(&S, p, mlen)) < 0) { + goto fail; + } + if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { + goto fail; + } + + if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { + goto fail; + } + } + } + + puts( "ok" ); + return 0; +fail: + puts("error"); + return -1; +} +#endif diff --git a/src/crypto/chacha8.h b/src/crypto/chacha8.h index 98e1428..cebc6ab 100644 --- a/src/crypto/chacha8.h +++ b/src/crypto/chacha8.h @@ -78,10 +78,7 @@ namespace crypto { buff = decrypted_buff; return true; } -// inline bool chacha_decrypt(std::string& buff, const std::string& pass) -// { -// return chacha_crypt(buff, pass); -// } + template inline bool chacha_crypt(pod_to_encrypt& crypt, const pod_pass& pass) { @@ -91,6 +88,7 @@ namespace crypto { memcpy(&crypt, buff.data(), sizeof(crypt)); return true; } + template inline bool chacha_crypt(std::string& buff, const pod_pass& pass) { @@ -101,6 +99,15 @@ namespace crypto { return true; } + template + inline std::string chacha_crypt(const std::string& input, const pod_pass& pass) + { + std::string result; + result.resize(input.size()); + if (chacha_crypt(input.data(), input.size(), (void*)result.data(), &pass, sizeof pass)) + return result; + return ""; + } } diff --git a/src/crypto/chacha8_stream.c b/src/crypto/chacha8_stream.c new file mode 100644 index 0000000..94d9d81 --- /dev/null +++ b/src/crypto/chacha8_stream.c @@ -0,0 +1,115 @@ +/* +chacha-ref.c version 20080118 +D. J. Bernstein +Public domain. +*/ + +#include "ecrypt-sync.h" + +#define ROTATE(v,c) (ROTL32(v,c)) +#define XOR(v,w) ((v) ^ (w)) +#define PLUS(v,w) (U32V((v) + (w))) +#define PLUSONE(v) (PLUS((v),1)) + +#define QUARTERROUND(a,b,c,d) \ + x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]),16); \ + x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]),12); \ + x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]), 8); \ + x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]), 7); + +static void salsa20_wordtobyte(u8 output[64], const u32 input[16]) +{ + u32 x[16]; + int i; + + for (i = 0; i < 16; ++i) x[i] = input[i]; + for (i = 8; i > 0; i -= 2) { + QUARTERROUND(0, 4, 8, 12) + QUARTERROUND(1, 5, 9, 13) + QUARTERROUND(2, 6, 10, 14) + QUARTERROUND(3, 7, 11, 15) + QUARTERROUND(0, 5, 10, 15) + QUARTERROUND(1, 6, 11, 12) + QUARTERROUND(2, 7, 8, 13) + QUARTERROUND(3, 4, 9, 14) + } + for (i = 0; i < 16; ++i) x[i] = PLUS(x[i], input[i]); + for (i = 0; i < 16; ++i) U32TO8_LITTLE(output + 4 * i, x[i]); +} + +void ECRYPT_init(void) +{ + return; +} + +static const char sigma[16] = "expand 32-byte k"; +static const char tau[16] = "expand 16-byte k"; + +void ECRYPT_keysetup(ECRYPT_ctx *x, const u8 *k, u32 kbits, u32 ivbits) +{ + const char *constants; + + x->input[4] = U8TO32_LITTLE(k + 0); + x->input[5] = U8TO32_LITTLE(k + 4); + x->input[6] = U8TO32_LITTLE(k + 8); + x->input[7] = U8TO32_LITTLE(k + 12); + if (kbits == 256) { /* recommended */ + k += 16; + constants = sigma; + } + else { /* kbits == 128 */ + constants = tau; + } + x->input[8] = U8TO32_LITTLE(k + 0); + x->input[9] = U8TO32_LITTLE(k + 4); + x->input[10] = U8TO32_LITTLE(k + 8); + x->input[11] = U8TO32_LITTLE(k + 12); + x->input[0] = U8TO32_LITTLE(constants + 0); + x->input[1] = U8TO32_LITTLE(constants + 4); + x->input[2] = U8TO32_LITTLE(constants + 8); + x->input[3] = U8TO32_LITTLE(constants + 12); +} + +void ECRYPT_ivsetup(ECRYPT_ctx *x, const u8 *iv) +{ + x->input[12] = 0; + x->input[13] = 0; + x->input[14] = U8TO32_LITTLE(iv + 0); + x->input[15] = U8TO32_LITTLE(iv + 4); +} + +void ECRYPT_encrypt_bytes(ECRYPT_ctx *x, const u8 *m, u8 *c, u32 bytes) +{ + u8 output[64]; + int i; + + if (!bytes) return; + for (;;) { + salsa20_wordtobyte(output, x->input); + x->input[12] = PLUSONE(x->input[12]); + if (!x->input[12]) { + x->input[13] = PLUSONE(x->input[13]); + /* stopping at 2^70 bytes per nonce is user's responsibility */ + } + if (bytes <= 64) { + for (i = 0; i < bytes; ++i) c[i] = m[i] ^ output[i]; + return; + } + for (i = 0; i < 64; ++i) c[i] = m[i] ^ output[i]; + bytes -= 64; + c += 64; + m += 64; + } +} + +void ECRYPT_decrypt_bytes(ECRYPT_ctx *x, const u8 *c, u8 *m, u32 bytes) +{ + ECRYPT_encrypt_bytes(x, c, m, bytes); +} + +void ECRYPT_keystream_bytes(ECRYPT_ctx *x, u8 *stream, u32 bytes) +{ + u32 i; + for (i = 0; i < bytes; ++i) stream[i] = 0; + ECRYPT_encrypt_bytes(x, stream, stream, bytes); +} \ No newline at end of file diff --git a/src/crypto/chacha8_stream.h b/src/crypto/chacha8_stream.h new file mode 100644 index 0000000..a2d0b07 --- /dev/null +++ b/src/crypto/chacha8_stream.h @@ -0,0 +1,289 @@ +/* ecrypt-sync.h */ + +/* +* Header file for synchronous stream ciphers without authentication +* mechanism. +* +* *** Please only edit parts marked with "[edit]". *** +*/ + +#ifndef ECRYPT_SYNC +#define ECRYPT_SYNC + +#if defined(__cplusplus) +extern "C" { +#endif + + +#include "ecrypt-portable.h" + + /* ------------------------------------------------------------------------- */ + + /* Cipher parameters */ + + /* + * The name of your cipher. + */ +#define ECRYPT_NAME "ChaCha8" +#define ECRYPT_PROFILE "_____" + + /* + * Specify which key and IV sizes are supported by your cipher. A user + * should be able to enumerate the supported sizes by running the + * following code: + * + * for (i = 0; ECRYPT_KEYSIZE(i) <= ECRYPT_MAXKEYSIZE; ++i) + * { + * keysize = ECRYPT_KEYSIZE(i); + * + * ... + * } + * + * All sizes are in bits. + */ + +#define ECRYPT_MAXKEYSIZE 256 /* [edit] */ +#define ECRYPT_KEYSIZE(i) (128 + (i)*128) /* [edit] */ + +#define ECRYPT_MAXIVSIZE 64 /* [edit] */ +#define ECRYPT_IVSIZE(i) (64 + (i)*64) /* [edit] */ + + /* ------------------------------------------------------------------------- */ + + /* Data structures */ + + /* + * ECRYPT_ctx is the structure containing the representation of the + * internal state of your cipher. + */ + + typedef struct + { + u32 input[16]; /* could be compressed */ + /* + * [edit] + * + * Put here all state variable needed during the encryption process. + */ + } ECRYPT_ctx; + + /* ------------------------------------------------------------------------- */ + + /* Mandatory functions */ + + /* + * Key and message independent initialization. This function will be + * called once when the program starts (e.g., to build expanded S-box + * tables). + */ + void ECRYPT_init(); + + /* + * Key setup. It is the user's responsibility to select the values of + * keysize and ivsize from the set of supported values specified + * above. + */ + void ECRYPT_keysetup( + ECRYPT_ctx* ctx, + const u8* key, + u32 keysize, /* Key size in bits. */ + u32 ivsize); /* IV size in bits. */ + + /* + * IV setup. After having called ECRYPT_keysetup(), the user is + * allowed to call ECRYPT_ivsetup() different times in order to + * encrypt/decrypt different messages with the same key but different + * IV's. + */ + void ECRYPT_ivsetup( + ECRYPT_ctx* ctx, + const u8* iv); + + /* + * Encryption/decryption of arbitrary length messages. + * + * For efficiency reasons, the API provides two types of + * encrypt/decrypt functions. The ECRYPT_encrypt_bytes() function + * (declared here) encrypts byte strings of arbitrary length, while + * the ECRYPT_encrypt_blocks() function (defined later) only accepts + * lengths which are multiples of ECRYPT_BLOCKLENGTH. + * + * The user is allowed to make multiple calls to + * ECRYPT_encrypt_blocks() to incrementally encrypt a long message, + * but he is NOT allowed to make additional encryption calls once he + * has called ECRYPT_encrypt_bytes() (unless he starts a new message + * of course). For example, this sequence of calls is acceptable: + * + * ECRYPT_keysetup(); + * + * ECRYPT_ivsetup(); + * ECRYPT_encrypt_blocks(); + * ECRYPT_encrypt_blocks(); + * ECRYPT_encrypt_bytes(); + * + * ECRYPT_ivsetup(); + * ECRYPT_encrypt_blocks(); + * ECRYPT_encrypt_blocks(); + * + * ECRYPT_ivsetup(); + * ECRYPT_encrypt_bytes(); + * + * The following sequence is not: + * + * ECRYPT_keysetup(); + * ECRYPT_ivsetup(); + * ECRYPT_encrypt_blocks(); + * ECRYPT_encrypt_bytes(); + * ECRYPT_encrypt_blocks(); + */ + + void ECRYPT_encrypt_bytes( + ECRYPT_ctx* ctx, + const u8* plaintext, + u8* ciphertext, + u32 msglen); /* Message length in bytes. */ + + void ECRYPT_decrypt_bytes( + ECRYPT_ctx* ctx, + const u8* ciphertext, + u8* plaintext, + u32 msglen); /* Message length in bytes. */ + + /* ------------------------------------------------------------------------- */ + + /* Optional features */ + + /* + * For testing purposes it can sometimes be useful to have a function + * which immediately generates keystream without having to provide it + * with a zero plaintext. If your cipher cannot provide this function + * (e.g., because it is not strictly a synchronous cipher), please + * reset the ECRYPT_GENERATES_KEYSTREAM flag. + */ + +#define ECRYPT_GENERATES_KEYSTREAM +#ifdef ECRYPT_GENERATES_KEYSTREAM + + void ECRYPT_keystream_bytes( + ECRYPT_ctx* ctx, + u8* keystream, + u32 length); /* Length of keystream in bytes. */ + +#endif + +/* ------------------------------------------------------------------------- */ + +/* Optional optimizations */ + +/* +* By default, the functions in this section are implemented using +* calls to functions declared above. However, you might want to +* implement them differently for performance reasons. +*/ + +/* +* All-in-one encryption/decryption of (short) packets. +* +* The default definitions of these functions can be found in +* "ecrypt-sync.c". If you want to implement them differently, please +* undef the ECRYPT_USES_DEFAULT_ALL_IN_ONE flag. +*/ +#define ECRYPT_USES_DEFAULT_ALL_IN_ONE /* [edit] */ + + void ECRYPT_encrypt_packet( + ECRYPT_ctx* ctx, + const u8* iv, + const u8* plaintext, + u8* ciphertext, + u32 msglen); + + void ECRYPT_decrypt_packet( + ECRYPT_ctx* ctx, + const u8* iv, + const u8* ciphertext, + u8* plaintext, + u32 msglen); + + /* + * Encryption/decryption of blocks. + * + * By default, these functions are defined as macros. If you want to + * provide a different implementation, please undef the + * ECRYPT_USES_DEFAULT_BLOCK_MACROS flag and implement the functions + * declared below. + */ + +#define ECRYPT_BLOCKLENGTH 64 /* [edit] */ + +#define ECRYPT_USES_DEFAULT_BLOCK_MACROS /* [edit] */ +#ifdef ECRYPT_USES_DEFAULT_BLOCK_MACROS + +#define ECRYPT_encrypt_blocks(ctx, plaintext, ciphertext, blocks) \ + ECRYPT_encrypt_bytes(ctx, plaintext, ciphertext, \ + (blocks) * ECRYPT_BLOCKLENGTH) + +#define ECRYPT_decrypt_blocks(ctx, ciphertext, plaintext, blocks) \ + ECRYPT_decrypt_bytes(ctx, ciphertext, plaintext, \ + (blocks) * ECRYPT_BLOCKLENGTH) + +#ifdef ECRYPT_GENERATES_KEYSTREAM + +#define ECRYPT_keystream_blocks(ctx, keystream, blocks) \ + ECRYPT_keystream_bytes(ctx, keystream, \ + (blocks) * ECRYPT_BLOCKLENGTH) + +#endif + +#else + + void ECRYPT_encrypt_blocks( + ECRYPT_ctx* ctx, + const u8* plaintext, + u8* ciphertext, + u32 blocks); /* Message length in blocks. */ + + void ECRYPT_decrypt_blocks( + ECRYPT_ctx* ctx, + const u8* ciphertext, + u8* plaintext, + u32 blocks); /* Message length in blocks. */ + +#ifdef ECRYPT_GENERATES_KEYSTREAM + + void ECRYPT_keystream_blocks( + ECRYPT_ctx* ctx, + const u8* keystream, + u32 blocks); /* Keystream length in blocks. */ + +#endif + +#endif + +/* +* If your cipher can be implemented in different ways, you can use +* the ECRYPT_VARIANT parameter to allow the user to choose between +* them at compile time (e.g., gcc -DECRYPT_VARIANT=3 ...). Please +* only use this possibility if you really think it could make a +* significant difference and keep the number of variants +* (ECRYPT_MAXVARIANT) as small as possible (definitely not more than +* 10). Note also that all variants should have exactly the same +* external interface (i.e., the same ECRYPT_BLOCKLENGTH, etc.). +*/ +#define ECRYPT_MAXVARIANT 1 /* [edit] */ + +#ifndef ECRYPT_VARIANT +#define ECRYPT_VARIANT 1 +#endif + +#if (ECRYPT_VARIANT > ECRYPT_MAXVARIANT) +#error this variant does not exist +#endif + +/* ------------------------------------------------------------------------- */ + +#if defined(__cplusplus) +//extern "C" { +} +#endif + +#endif \ No newline at end of file diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 97e7df5..caf20a0 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -1,25 +1,50 @@ +// Copyright (c) 2018-2021 Zano Project +// Copyright (c) 2020-2021 sowle (val@zano.org, crypto.sowle@gmail.com) // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// This file contains pieces of code from libsodium project. +// libsodium is licensed under the ISC License: +/* + * ISC License + * + * Copyright (c) 2013-2020 + * Frank Denis + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ #include #include - +#include +#include #include "warnings.h" #include "crypto-ops.h" +#include "hash-ops.h" // for cn_fast_hash DISABLE_VS_WARNINGS(4146 4244) /* Predeclarations */ -static void fe_mul(fe, const fe, const fe); -static void fe_sq(fe, const fe); -static void fe_tobytes(unsigned char *, const fe); +void fe_mul(fe, const fe, const fe); +void fe_sq(fe, const fe); +void fe_tobytes(unsigned char *, const fe); static void ge_madd(ge_p1p1 *, const ge_p3 *, const ge_precomp *); static void ge_msub(ge_p1p1 *, const ge_p3 *, const ge_precomp *); static void ge_p2_0(ge_p2 *); static void ge_p3_dbl(ge_p1p1 *, const ge_p3 *); -static void ge_sub(ge_p1p1 *, const ge_p3 *, const ge_cached *); +void ge_sub(ge_p1p1 *, const ge_p3 *, const ge_cached *); static void fe_divpowm1(fe, const fe, const fe); /* Common functions */ @@ -48,7 +73,7 @@ static uint64_t load_4(const unsigned char *in) h = 0 */ -static void fe_0(fe h) { +void fe_0(fe h) { h[0] = 0; h[1] = 0; h[2] = 0; @@ -232,7 +257,7 @@ static void fe_copy(fe h, const fe f) { /* From fe_invert.c */ -static void fe_invert(fe out, const fe z) { +void fe_invert(fe out, const fe z) { fe t0; fe t1; fe t2; @@ -308,7 +333,7 @@ static int fe_isnegative(const fe f) { /* From fe_isnonzero.c, modified */ -static int fe_isnonzero(const fe f) { +int fe_isnonzero(const fe f) { unsigned char s[32]; fe_tobytes(s, f); return (((int) (s[0] | s[1] | s[2] | s[3] | s[4] | s[5] | s[6] | s[7] | s[8] | @@ -317,6 +342,16 @@ static int fe_isnonzero(const fe f) { s[27] | s[28] | s[29] | s[30] | s[31]) - 1) >> 8) + 1; } +int fe_cmp(const fe a, const fe b) +{ + for (size_t i = 9; i != SIZE_MAX; --i) + { + if ((const uint32_t)a[i] < (const uint32_t)b[i]) return -1; + if ((const uint32_t)a[i] > (const uint32_t)b[i]) return 1; + } + return 0; +} + /* From fe_mul.c */ /* @@ -351,7 +386,7 @@ Can get away with 11 carries, but then data flow is much deeper. With tighter constraints on inputs can squeeze carries into int32. */ -static void fe_mul(fe h, const fe f, const fe g) { +void fe_mul(fe h, const fe f, const fe g) { int32_t f0 = f[0]; int32_t f1 = f[1]; int32_t f2 = f[2]; @@ -631,7 +666,7 @@ Postconditions: See fe_mul.c for discussion of implementation strategy. */ -static void fe_sq(fe h, const fe f) { +void fe_sq(fe h, const fe f) { int32_t f0 = f[0]; int32_t f1 = f[1]; int32_t f2 = f[2]; @@ -1005,7 +1040,7 @@ Proof: so floor(2^(-255)(h + 19 2^(-25) h9 + 2^(-1))) = q. */ -static void fe_tobytes(unsigned char *s, const fe h) { +void fe_tobytes(unsigned char *s, const fe h) { int32_t h0 = h[0]; int32_t h1 = h[1]; int32_t h2 = h[2]; @@ -1210,6 +1245,96 @@ void ge_double_scalarmult_base_vartime(ge_p2 *r, const unsigned char *a, const g } } +/* +r = a * A + b * B +where a = a[0]+256*a[1]+...+256^31 a[31]. +and b = b[0]+256*b[1]+...+256^31 b[31]. +B is the Ed25519 base point (x,4/5) with x positive. +*/ + +void ge_double_scalarmult_base_vartime_p3(ge_p3 *r, const unsigned char *a, const ge_p3 *A, const unsigned char *b) { + signed char aslide[256]; + signed char bslide[256]; + ge_dsmp Ai; /* A, 3A, 5A, 7A, 9A, 11A, 13A, 15A */ + ge_p1p1 t; + ge_p3 u; + ge_p2 r_p2; + int i; + + slide(aslide, a); + slide(bslide, b); + ge_dsm_precomp(Ai, A); + + ge_p2_0(&r_p2); + + for (i = 255; i >= 0; --i) { + if (aslide[i] || bslide[i]) break; + } + + for (; i >= 0; --i) { + ge_p2_dbl(&t, &r_p2); + + if (aslide[i] > 0) { + ge_p1p1_to_p3(&u, &t); + ge_add(&t, &u, &Ai[aslide[i]/2]); + } else if (aslide[i] < 0) { + ge_p1p1_to_p3(&u, &t); + ge_sub(&t, &u, &Ai[(-aslide[i])/2]); + } + + if (bslide[i] > 0) { + ge_p1p1_to_p3(&u, &t); + ge_madd(&t, &u, &ge_Bi[bslide[i]/2]); + } else if (bslide[i] < 0) { + ge_p1p1_to_p3(&u, &t); + ge_msub(&t, &u, &ge_Bi[(-bslide[i])/2]); + } + + if (i != 0) + ge_p1p1_to_p2(&r_p2, &t); + else + ge_p1p1_to_p3(r, &t); // last step + } +} + +void ge_scalarmult_vartime_p3(ge_p3 *r, const unsigned char *a, const ge_p3 *A) { + signed char aslide[256]; + ge_dsmp Ai; /* A, 3A, 5A, 7A, 9A, 11A, 13A, 15A */ + ge_p1p1 t; + ge_p3 u; + ge_p2 r_p2; + int i; + + slide(aslide, a); + ge_dsm_precomp(Ai, A); + + ge_p2_0(&r_p2); + ge_p3_0(r); + + for (i = 255; i >= 0; --i) { + if (aslide[i]) break; + } + + for (; i >= 0; --i) { + ge_p2_dbl(&t, &r_p2); + + if (aslide[i] > 0) { + ge_p1p1_to_p3(&u, &t); + ge_add(&t, &u, &Ai[aslide[i] / 2]); + } + else if (aslide[i] < 0) { + ge_p1p1_to_p3(&u, &t); + ge_sub(&t, &u, &Ai[(-aslide[i]) / 2]); + } + + if (i != 0) + ge_p1p1_to_p2(&r_p2, &t); + else + ge_p1p1_to_p3(r, &t); // last step + } +} + + /* From ge_frombytes.c, modified */ int ge_frombytes_vartime(ge_p3 *h, const unsigned char *s) { @@ -1398,7 +1523,7 @@ void ge_p2_dbl(ge_p1p1 *r, const ge_p2 *p) { /* From ge_p3_0.c */ -static void ge_p3_0(ge_p3 *h) { +void ge_p3_0(ge_p3 *h) { fe_0(h->X); fe_1(h->Y); fe_1(h->Z); @@ -1565,7 +1690,7 @@ void ge_scalarmult_base(ge_p3 *h, const unsigned char *a) { r = p - q */ -static void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q) { +void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q) { fe t0; fe_add(r->X, p->Y, p->X); fe_sub(r->Y, p->Y, p->X); @@ -1976,6 +2101,72 @@ void ge_scalarmult(ge_p2 *r, const unsigned char *a, const ge_p3 *A) { } } +/* Assumes that a[31] <= 127 */ +void ge_scalarmult_p3(ge_p3 *result, const unsigned char *a, const ge_p3 *A) { + signed char e[64]; + int carry, carry2, i; + ge_cached Ai[8]; /* 1 * A, 2 * A, ..., 8 * A */ + ge_p1p1 t; + ge_p3 u; + ge_p2 r_tmp; + ge_p2* r = &r_tmp; + + carry = 0; /* 0..1 */ + for (i = 0; i < 31; i++) { + carry += a[i]; /* 0..256 */ + carry2 = (carry + 8) >> 4; /* 0..16 */ + e[2 * i] = carry - (carry2 << 4); /* -8..7 */ + carry = (carry2 + 8) >> 4; /* 0..1 */ + e[2 * i + 1] = carry2 - (carry << 4); /* -8..7 */ + } + carry += a[31]; /* 0..128 */ + carry2 = (carry + 8) >> 4; /* 0..8 */ + e[62] = carry - (carry2 << 4); /* -8..7 */ + e[63] = carry2; /* 0..8 */ + + ge_p3_to_cached(&Ai[0], A); + for (i = 0; i < 7; i++) { + ge_add(&t, A, &Ai[i]); + ge_p1p1_to_p3(&u, &t); + ge_p3_to_cached(&Ai[i + 1], &u); + } + + ge_p2_0(r); + for (i = 63; i >= 0; i--) { + signed char b = e[i]; + unsigned char bnegative = negative(b); + unsigned char babs = b - (((-bnegative) & b) << 1); + ge_cached cur, minuscur; + ge_p2_dbl(&t, r); + ge_p1p1_to_p2(r, &t); + ge_p2_dbl(&t, r); + ge_p1p1_to_p2(r, &t); + ge_p2_dbl(&t, r); + ge_p1p1_to_p2(r, &t); + ge_p2_dbl(&t, r); + ge_p1p1_to_p3(&u, &t); + ge_cached_0(&cur); + ge_cached_cmov(&cur, &Ai[0], equal(babs, 1)); + ge_cached_cmov(&cur, &Ai[1], equal(babs, 2)); + ge_cached_cmov(&cur, &Ai[2], equal(babs, 3)); + ge_cached_cmov(&cur, &Ai[3], equal(babs, 4)); + ge_cached_cmov(&cur, &Ai[4], equal(babs, 5)); + ge_cached_cmov(&cur, &Ai[5], equal(babs, 6)); + ge_cached_cmov(&cur, &Ai[6], equal(babs, 7)); + ge_cached_cmov(&cur, &Ai[7], equal(babs, 8)); + fe_copy(minuscur.YplusX, cur.YminusX); + fe_copy(minuscur.YminusX, cur.YplusX); + fe_copy(minuscur.Z, cur.Z); + fe_neg(minuscur.T2d, cur.T2d); + ge_cached_cmov(&cur, &minuscur, bnegative); + ge_add(&t, &u, &cur); + if (i > 0) + ge_p1p1_to_p2(r, &t); + else + ge_p1p1_to_p3(result, &t); // last iteration + } +} + void ge_double_scalarmult_precomp_vartime(ge_p2 *r, const unsigned char *a, const ge_p3 *A, const unsigned char *b, const ge_dsmp Bi) { signed char aslide[256]; signed char bslide[256]; @@ -2078,17 +2269,17 @@ void ge_fromfe_frombytes_vartime(ge_p2 *r, const unsigned char *s) { /* End fe_frombytes.c */ - fe_sq2(v, u); /* 2 * u^2 */ - fe_1(w); - fe_add(w, v, w); /* w = 2 * u^2 + 1 */ - fe_sq(x, w); /* w^2 */ - fe_mul(y, fe_ma2, v); /* -2 * A^2 * u^2 */ - fe_add(x, x, y); /* x = w^2 - 2 * A^2 * u^2 */ - fe_divpowm1(r->X, w, x); /* (w / x)^(m + 1) */ - fe_sq(y, r->X); - fe_mul(x, y, x); - fe_sub(y, w, x); - fe_copy(z, fe_ma); + fe_sq2(v, u); /* v = 2 * u^2 */ + fe_1(w); /* w = 1 */ + fe_add(w, v, w); /* w = 2 * u^2 + 1 */ + fe_sq(x, w); /* x = w^2 */ + fe_mul(y, fe_ma2, v); /* y = -A^2 * (2 * u^2) = -2 * A^2 * u^2 n.b. A = 2 * (1 - d) / (1 + d) = 486662*/ + fe_add(x, x, y); /* x = w^2 - 2 * A^2 * u^2 */ + fe_divpowm1(r->X, w, x); /* r->X = (w / x)^(m + 1) */ + fe_sq(y, r->X); /* y = (w / x)^(2*(m + 1)) */ + fe_mul(x, y, x); /* x = (w / x)^(2*(m + 1)) * (w^2 - 2 * A^2 * u^2) */ + fe_sub(y, w, x); /* y = 2 * u^2 + 1 - (w / x)^(2*(m + 1)) * (w^2 - 2 * A^2 * u^2) */ + fe_copy(z, fe_ma); /* z = -A */ if (fe_isnonzero(y)) { fe_add(y, w, x); if (fe_isnonzero(y)) { @@ -2897,3 +3088,1225 @@ int sc_isnonzero(const unsigned char *s) { s[18] | s[19] | s[20] | s[21] | s[22] | s[23] | s[24] | s[25] | s[26] | s[27] | s[28] | s[29] | s[30] | s[31]) - 1) >> 8) + 1; } + +// see implmentation of ge_frombytes_vartime above +void fe_frombytes(fe h, const unsigned char *s) +{ + /* From fe_frombytes.c */ + + int64_t h0 = load_4(s); + int64_t h1 = load_3(s + 4) << 6; + int64_t h2 = load_3(s + 7) << 5; + int64_t h3 = load_3(s + 10) << 3; + int64_t h4 = load_3(s + 13) << 2; + int64_t h5 = load_4(s + 16); + int64_t h6 = load_3(s + 20) << 7; + int64_t h7 = load_3(s + 23) << 5; + int64_t h8 = load_3(s + 26) << 4; + int64_t h9 = (load_3(s + 29) & 8388607) << 2; + int64_t carry0; + int64_t carry1; + int64_t carry2; + int64_t carry3; + int64_t carry4; + int64_t carry5; + int64_t carry6; + int64_t carry7; + int64_t carry8; + int64_t carry9; + + carry9 = (h9 + (int64_t)(1 << 24)) >> 25; h0 += carry9 * 19; h9 -= carry9 << 25; + carry1 = (h1 + (int64_t)(1 << 24)) >> 25; h2 += carry1; h1 -= carry1 << 25; + carry3 = (h3 + (int64_t)(1 << 24)) >> 25; h4 += carry3; h3 -= carry3 << 25; + carry5 = (h5 + (int64_t)(1 << 24)) >> 25; h6 += carry5; h5 -= carry5 << 25; + carry7 = (h7 + (int64_t)(1 << 24)) >> 25; h8 += carry7; h7 -= carry7 << 25; + + carry0 = (h0 + (int64_t)(1 << 25)) >> 26; h1 += carry0; h0 -= carry0 << 26; + carry2 = (h2 + (int64_t)(1 << 25)) >> 26; h3 += carry2; h2 -= carry2 << 26; + carry4 = (h4 + (int64_t)(1 << 25)) >> 26; h5 += carry4; h4 -= carry4 << 26; + carry6 = (h6 + (int64_t)(1 << 25)) >> 26; h7 += carry6; h6 -= carry6 << 26; + carry8 = (h8 + (int64_t)(1 << 25)) >> 26; h9 += carry8; h8 -= carry8 << 26; + + h[0] = h0; + h[1] = h1; + h[2] = h2; + h[3] = h3; + h[4] = h4; + h[5] = h5; + h[6] = h6; + h[7] = h7; + h[8] = h8; + h[9] = h9; +} + +/* +// Old naive and slow implementation using sc_mulsub +void sc_mul(unsigned char *s, const unsigned char *a, const unsigned char *b) +{ + unsigned char c[32]; + unsigned char neg_a[32]; + sc_0(c); + sc_sub(neg_a, c, a); + // s = c - ab + sc_mulsub(s, neg_a, b, c); +} +*/ + +/* libsodium/crypto_core/ed25519/ref10/ed25519_ref10.c */ +/* + Input: + a[0]+256*a[1]+...+256^31*a[31] = a + b[0]+256*b[1]+...+256^31*b[31] = b + * + Output: + s[0]+256*s[1]+...+256^31*s[31] = (ab) mod l + where l = 2^252 + 27742317777372353535851937790883648493. + */ +void sc_mul(unsigned char* s, const unsigned char* a, const unsigned char* b) +{ + int64_t a0 = 2097151 & load_3(a); + int64_t a1 = 2097151 & (load_4(a + 2) >> 5); + int64_t a2 = 2097151 & (load_3(a + 5) >> 2); + int64_t a3 = 2097151 & (load_4(a + 7) >> 7); + int64_t a4 = 2097151 & (load_4(a + 10) >> 4); + int64_t a5 = 2097151 & (load_3(a + 13) >> 1); + int64_t a6 = 2097151 & (load_4(a + 15) >> 6); + int64_t a7 = 2097151 & (load_3(a + 18) >> 3); + int64_t a8 = 2097151 & load_3(a + 21); + int64_t a9 = 2097151 & (load_4(a + 23) >> 5); + int64_t a10 = 2097151 & (load_3(a + 26) >> 2); + int64_t a11 = (load_4(a + 28) >> 7); + + int64_t b0 = 2097151 & load_3(b); + int64_t b1 = 2097151 & (load_4(b + 2) >> 5); + int64_t b2 = 2097151 & (load_3(b + 5) >> 2); + int64_t b3 = 2097151 & (load_4(b + 7) >> 7); + int64_t b4 = 2097151 & (load_4(b + 10) >> 4); + int64_t b5 = 2097151 & (load_3(b + 13) >> 1); + int64_t b6 = 2097151 & (load_4(b + 15) >> 6); + int64_t b7 = 2097151 & (load_3(b + 18) >> 3); + int64_t b8 = 2097151 & load_3(b + 21); + int64_t b9 = 2097151 & (load_4(b + 23) >> 5); + int64_t b10 = 2097151 & (load_3(b + 26) >> 2); + int64_t b11 = (load_4(b + 28) >> 7); + + int64_t s0; + int64_t s1; + int64_t s2; + int64_t s3; + int64_t s4; + int64_t s5; + int64_t s6; + int64_t s7; + int64_t s8; + int64_t s9; + int64_t s10; + int64_t s11; + int64_t s12; + int64_t s13; + int64_t s14; + int64_t s15; + int64_t s16; + int64_t s17; + int64_t s18; + int64_t s19; + int64_t s20; + int64_t s21; + int64_t s22; + int64_t s23; + + int64_t carry0; + int64_t carry1; + int64_t carry2; + int64_t carry3; + int64_t carry4; + int64_t carry5; + int64_t carry6; + int64_t carry7; + int64_t carry8; + int64_t carry9; + int64_t carry10; + int64_t carry11; + int64_t carry12; + int64_t carry13; + int64_t carry14; + int64_t carry15; + int64_t carry16; + int64_t carry17; + int64_t carry18; + int64_t carry19; + int64_t carry20; + int64_t carry21; + int64_t carry22; + + s0 = a0 * b0; + s1 = a0 * b1 + a1 * b0; + s2 = a0 * b2 + a1 * b1 + a2 * b0; + s3 = a0 * b3 + a1 * b2 + a2 * b1 + a3 * b0; + s4 = a0 * b4 + a1 * b3 + a2 * b2 + a3 * b1 + a4 * b0; + s5 = a0 * b5 + a1 * b4 + a2 * b3 + a3 * b2 + a4 * b1 + a5 * b0; + s6 = a0 * b6 + a1 * b5 + a2 * b4 + a3 * b3 + a4 * b2 + a5 * b1 + a6 * b0; + s7 = a0 * b7 + a1 * b6 + a2 * b5 + a3 * b4 + a4 * b3 + a5 * b2 + + a6 * b1 + a7 * b0; + s8 = a0 * b8 + a1 * b7 + a2 * b6 + a3 * b5 + a4 * b4 + a5 * b3 + + a6 * b2 + a7 * b1 + a8 * b0; + s9 = a0 * b9 + a1 * b8 + a2 * b7 + a3 * b6 + a4 * b5 + a5 * b4 + + a6 * b3 + a7 * b2 + a8 * b1 + a9 * b0; + s10 = a0 * b10 + a1 * b9 + a2 * b8 + a3 * b7 + a4 * b6 + a5 * b5 + + a6 * b4 + a7 * b3 + a8 * b2 + a9 * b1 + a10 * b0; + s11 = a0 * b11 + a1 * b10 + a2 * b9 + a3 * b8 + a4 * b7 + a5 * b6 + + a6 * b5 + a7 * b4 + a8 * b3 + a9 * b2 + a10 * b1 + a11 * b0; + s12 = a1 * b11 + a2 * b10 + a3 * b9 + a4 * b8 + a5 * b7 + a6 * b6 + + a7 * b5 + a8 * b4 + a9 * b3 + a10 * b2 + a11 * b1; + s13 = a2 * b11 + a3 * b10 + a4 * b9 + a5 * b8 + a6 * b7 + a7 * b6 + + a8 * b5 + a9 * b4 + a10 * b3 + a11 * b2; + s14 = a3 * b11 + a4 * b10 + a5 * b9 + a6 * b8 + a7 * b7 + a8 * b6 + + a9 * b5 + a10 * b4 + a11 * b3; + s15 = a4 * b11 + a5 * b10 + a6 * b9 + a7 * b8 + a8 * b7 + a9 * b6 + + a10 * b5 + a11 * b4; + s16 = + a5 * b11 + a6 * b10 + a7 * b9 + a8 * b8 + a9 * b7 + a10 * b6 + a11 * b5; + s17 = a6 * b11 + a7 * b10 + a8 * b9 + a9 * b8 + a10 * b7 + a11 * b6; + s18 = a7 * b11 + a8 * b10 + a9 * b9 + a10 * b8 + a11 * b7; + s19 = a8 * b11 + a9 * b10 + a10 * b9 + a11 * b8; + s20 = a9 * b11 + a10 * b10 + a11 * b9; + s21 = a10 * b11 + a11 * b10; + s22 = a11 * b11; + s23 = 0; + + carry0 = (s0 + (int64_t)(1L << 20)) >> 21; + s1 += carry0; + s0 -= carry0 * ((uint64_t)1L << 21); + carry2 = (s2 + (int64_t)(1L << 20)) >> 21; + s3 += carry2; + s2 -= carry2 * ((uint64_t)1L << 21); + carry4 = (s4 + (int64_t)(1L << 20)) >> 21; + s5 += carry4; + s4 -= carry4 * ((uint64_t)1L << 21); + carry6 = (s6 + (int64_t)(1L << 20)) >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry8 = (s8 + (int64_t)(1L << 20)) >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry10 = (s10 + (int64_t)(1L << 20)) >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + carry12 = (s12 + (int64_t)(1L << 20)) >> 21; + s13 += carry12; + s12 -= carry12 * ((uint64_t)1L << 21); + carry14 = (s14 + (int64_t)(1L << 20)) >> 21; + s15 += carry14; + s14 -= carry14 * ((uint64_t)1L << 21); + carry16 = (s16 + (int64_t)(1L << 20)) >> 21; + s17 += carry16; + s16 -= carry16 * ((uint64_t)1L << 21); + carry18 = (s18 + (int64_t)(1L << 20)) >> 21; + s19 += carry18; + s18 -= carry18 * ((uint64_t)1L << 21); + carry20 = (s20 + (int64_t)(1L << 20)) >> 21; + s21 += carry20; + s20 -= carry20 * ((uint64_t)1L << 21); + carry22 = (s22 + (int64_t)(1L << 20)) >> 21; + s23 += carry22; + s22 -= carry22 * ((uint64_t)1L << 21); + + carry1 = (s1 + (int64_t)(1L << 20)) >> 21; + s2 += carry1; + s1 -= carry1 * ((uint64_t)1L << 21); + carry3 = (s3 + (int64_t)(1L << 20)) >> 21; + s4 += carry3; + s3 -= carry3 * ((uint64_t)1L << 21); + carry5 = (s5 + (int64_t)(1L << 20)) >> 21; + s6 += carry5; + s5 -= carry5 * ((uint64_t)1L << 21); + carry7 = (s7 + (int64_t)(1L << 20)) >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry9 = (s9 + (int64_t)(1L << 20)) >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry11 = (s11 + (int64_t)(1L << 20)) >> 21; + s12 += carry11; + s11 -= carry11 * ((uint64_t)1L << 21); + carry13 = (s13 + (int64_t)(1L << 20)) >> 21; + s14 += carry13; + s13 -= carry13 * ((uint64_t)1L << 21); + carry15 = (s15 + (int64_t)(1L << 20)) >> 21; + s16 += carry15; + s15 -= carry15 * ((uint64_t)1L << 21); + carry17 = (s17 + (int64_t)(1L << 20)) >> 21; + s18 += carry17; + s17 -= carry17 * ((uint64_t)1L << 21); + carry19 = (s19 + (int64_t)(1L << 20)) >> 21; + s20 += carry19; + s19 -= carry19 * ((uint64_t)1L << 21); + carry21 = (s21 + (int64_t)(1L << 20)) >> 21; + s22 += carry21; + s21 -= carry21 * ((uint64_t)1L << 21); + + s11 += s23 * 666643; + s12 += s23 * 470296; + s13 += s23 * 654183; + s14 -= s23 * 997805; + s15 += s23 * 136657; + s16 -= s23 * 683901; + + s10 += s22 * 666643; + s11 += s22 * 470296; + s12 += s22 * 654183; + s13 -= s22 * 997805; + s14 += s22 * 136657; + s15 -= s22 * 683901; + + s9 += s21 * 666643; + s10 += s21 * 470296; + s11 += s21 * 654183; + s12 -= s21 * 997805; + s13 += s21 * 136657; + s14 -= s21 * 683901; + + s8 += s20 * 666643; + s9 += s20 * 470296; + s10 += s20 * 654183; + s11 -= s20 * 997805; + s12 += s20 * 136657; + s13 -= s20 * 683901; + + s7 += s19 * 666643; + s8 += s19 * 470296; + s9 += s19 * 654183; + s10 -= s19 * 997805; + s11 += s19 * 136657; + s12 -= s19 * 683901; + + s6 += s18 * 666643; + s7 += s18 * 470296; + s8 += s18 * 654183; + s9 -= s18 * 997805; + s10 += s18 * 136657; + s11 -= s18 * 683901; + + carry6 = (s6 + (int64_t)(1L << 20)) >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry8 = (s8 + (int64_t)(1L << 20)) >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry10 = (s10 + (int64_t)(1L << 20)) >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + carry12 = (s12 + (int64_t)(1L << 20)) >> 21; + s13 += carry12; + s12 -= carry12 * ((uint64_t)1L << 21); + carry14 = (s14 + (int64_t)(1L << 20)) >> 21; + s15 += carry14; + s14 -= carry14 * ((uint64_t)1L << 21); + carry16 = (s16 + (int64_t)(1L << 20)) >> 21; + s17 += carry16; + s16 -= carry16 * ((uint64_t)1L << 21); + + carry7 = (s7 + (int64_t)(1L << 20)) >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry9 = (s9 + (int64_t)(1L << 20)) >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry11 = (s11 + (int64_t)(1L << 20)) >> 21; + s12 += carry11; + s11 -= carry11 * ((uint64_t)1L << 21); + carry13 = (s13 + (int64_t)(1L << 20)) >> 21; + s14 += carry13; + s13 -= carry13 * ((uint64_t)1L << 21); + carry15 = (s15 + (int64_t)(1L << 20)) >> 21; + s16 += carry15; + s15 -= carry15 * ((uint64_t)1L << 21); + + s5 += s17 * 666643; + s6 += s17 * 470296; + s7 += s17 * 654183; + s8 -= s17 * 997805; + s9 += s17 * 136657; + s10 -= s17 * 683901; + + s4 += s16 * 666643; + s5 += s16 * 470296; + s6 += s16 * 654183; + s7 -= s16 * 997805; + s8 += s16 * 136657; + s9 -= s16 * 683901; + + s3 += s15 * 666643; + s4 += s15 * 470296; + s5 += s15 * 654183; + s6 -= s15 * 997805; + s7 += s15 * 136657; + s8 -= s15 * 683901; + + s2 += s14 * 666643; + s3 += s14 * 470296; + s4 += s14 * 654183; + s5 -= s14 * 997805; + s6 += s14 * 136657; + s7 -= s14 * 683901; + + s1 += s13 * 666643; + s2 += s13 * 470296; + s3 += s13 * 654183; + s4 -= s13 * 997805; + s5 += s13 * 136657; + s6 -= s13 * 683901; + + s0 += s12 * 666643; + s1 += s12 * 470296; + s2 += s12 * 654183; + s3 -= s12 * 997805; + s4 += s12 * 136657; + s5 -= s12 * 683901; + s12 = 0; + + carry0 = (s0 + (int64_t)(1L << 20)) >> 21; + s1 += carry0; + s0 -= carry0 * ((uint64_t)1L << 21); + carry2 = (s2 + (int64_t)(1L << 20)) >> 21; + s3 += carry2; + s2 -= carry2 * ((uint64_t)1L << 21); + carry4 = (s4 + (int64_t)(1L << 20)) >> 21; + s5 += carry4; + s4 -= carry4 * ((uint64_t)1L << 21); + carry6 = (s6 + (int64_t)(1L << 20)) >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry8 = (s8 + (int64_t)(1L << 20)) >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry10 = (s10 + (int64_t)(1L << 20)) >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + + carry1 = (s1 + (int64_t)(1L << 20)) >> 21; + s2 += carry1; + s1 -= carry1 * ((uint64_t)1L << 21); + carry3 = (s3 + (int64_t)(1L << 20)) >> 21; + s4 += carry3; + s3 -= carry3 * ((uint64_t)1L << 21); + carry5 = (s5 + (int64_t)(1L << 20)) >> 21; + s6 += carry5; + s5 -= carry5 * ((uint64_t)1L << 21); + carry7 = (s7 + (int64_t)(1L << 20)) >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry9 = (s9 + (int64_t)(1L << 20)) >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry11 = (s11 + (int64_t)(1L << 20)) >> 21; + s12 += carry11; + s11 -= carry11 * ((uint64_t)1L << 21); + + s0 += s12 * 666643; + s1 += s12 * 470296; + s2 += s12 * 654183; + s3 -= s12 * 997805; + s4 += s12 * 136657; + s5 -= s12 * 683901; + s12 = 0; + + carry0 = s0 >> 21; + s1 += carry0; + s0 -= carry0 * ((uint64_t)1L << 21); + carry1 = s1 >> 21; + s2 += carry1; + s1 -= carry1 * ((uint64_t)1L << 21); + carry2 = s2 >> 21; + s3 += carry2; + s2 -= carry2 * ((uint64_t)1L << 21); + carry3 = s3 >> 21; + s4 += carry3; + s3 -= carry3 * ((uint64_t)1L << 21); + carry4 = s4 >> 21; + s5 += carry4; + s4 -= carry4 * ((uint64_t)1L << 21); + carry5 = s5 >> 21; + s6 += carry5; + s5 -= carry5 * ((uint64_t)1L << 21); + carry6 = s6 >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry7 = s7 >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry8 = s8 >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry9 = s9 >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry10 = s10 >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + carry11 = s11 >> 21; + s12 += carry11; + s11 -= carry11 * ((uint64_t)1L << 21); + + s0 += s12 * 666643; + s1 += s12 * 470296; + s2 += s12 * 654183; + s3 -= s12 * 997805; + s4 += s12 * 136657; + s5 -= s12 * 683901; + + carry0 = s0 >> 21; + s1 += carry0; + s0 -= carry0 * ((uint64_t)1L << 21); + carry1 = s1 >> 21; + s2 += carry1; + s1 -= carry1 * ((uint64_t)1L << 21); + carry2 = s2 >> 21; + s3 += carry2; + s2 -= carry2 * ((uint64_t)1L << 21); + carry3 = s3 >> 21; + s4 += carry3; + s3 -= carry3 * ((uint64_t)1L << 21); + carry4 = s4 >> 21; + s5 += carry4; + s4 -= carry4 * ((uint64_t)1L << 21); + carry5 = s5 >> 21; + s6 += carry5; + s5 -= carry5 * ((uint64_t)1L << 21); + carry6 = s6 >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry7 = s7 >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry8 = s8 >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry9 = s9 >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry10 = s10 >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + + s[0] = s0 >> 0; + s[1] = s0 >> 8; + s[2] = (s0 >> 16) | (s1 * ((uint64_t)1 << 5)); + s[3] = s1 >> 3; + s[4] = s1 >> 11; + s[5] = (s1 >> 19) | (s2 * ((uint64_t)1 << 2)); + s[6] = s2 >> 6; + s[7] = (s2 >> 14) | (s3 * ((uint64_t)1 << 7)); + s[8] = s3 >> 1; + s[9] = s3 >> 9; + s[10] = (s3 >> 17) | (s4 * ((uint64_t)1 << 4)); + s[11] = s4 >> 4; + s[12] = s4 >> 12; + s[13] = (s4 >> 20) | (s5 * ((uint64_t)1 << 1)); + s[14] = s5 >> 7; + s[15] = (s5 >> 15) | (s6 * ((uint64_t)1 << 6)); + s[16] = s6 >> 2; + s[17] = s6 >> 10; + s[18] = (s6 >> 18) | (s7 * ((uint64_t)1 << 3)); + s[19] = s7 >> 5; + s[20] = s7 >> 13; + s[21] = s8 >> 0; + s[22] = s8 >> 8; + s[23] = (s8 >> 16) | (s9 * ((uint64_t)1 << 5)); + s[24] = s9 >> 3; + s[25] = s9 >> 11; + s[26] = (s9 >> 19) | (s10 * ((uint64_t)1 << 2)); + s[27] = s10 >> 6; + s[28] = (s10 >> 14) | (s11 * ((uint64_t)1 << 7)); + s[29] = s11 >> 1; + s[30] = s11 >> 9; + s[31] = s11 >> 17; +} + +/* libsodium/crypto_core/ed25519/ref10/ed25519_ref10.c */ +/* + Input: + a[0]+256*a[1]+...+256^31*a[31] = a + b[0]+256*b[1]+...+256^31*b[31] = b + c[0]+256*c[1]+...+256^31*c[31] = c + * + Output: + s[0]+256*s[1]+...+256^31*s[31] = (ab+c) mod l + where l = 2^252 + 27742317777372353535851937790883648493. + */ +void sc_muladd(unsigned char* s, const unsigned char* a, + const unsigned char* b, const unsigned char* c) +{ + int64_t a0 = 2097151 & load_3(a); + int64_t a1 = 2097151 & (load_4(a + 2) >> 5); + int64_t a2 = 2097151 & (load_3(a + 5) >> 2); + int64_t a3 = 2097151 & (load_4(a + 7) >> 7); + int64_t a4 = 2097151 & (load_4(a + 10) >> 4); + int64_t a5 = 2097151 & (load_3(a + 13) >> 1); + int64_t a6 = 2097151 & (load_4(a + 15) >> 6); + int64_t a7 = 2097151 & (load_3(a + 18) >> 3); + int64_t a8 = 2097151 & load_3(a + 21); + int64_t a9 = 2097151 & (load_4(a + 23) >> 5); + int64_t a10 = 2097151 & (load_3(a + 26) >> 2); + int64_t a11 = (load_4(a + 28) >> 7); + + int64_t b0 = 2097151 & load_3(b); + int64_t b1 = 2097151 & (load_4(b + 2) >> 5); + int64_t b2 = 2097151 & (load_3(b + 5) >> 2); + int64_t b3 = 2097151 & (load_4(b + 7) >> 7); + int64_t b4 = 2097151 & (load_4(b + 10) >> 4); + int64_t b5 = 2097151 & (load_3(b + 13) >> 1); + int64_t b6 = 2097151 & (load_4(b + 15) >> 6); + int64_t b7 = 2097151 & (load_3(b + 18) >> 3); + int64_t b8 = 2097151 & load_3(b + 21); + int64_t b9 = 2097151 & (load_4(b + 23) >> 5); + int64_t b10 = 2097151 & (load_3(b + 26) >> 2); + int64_t b11 = (load_4(b + 28) >> 7); + + int64_t c0 = 2097151 & load_3(c); + int64_t c1 = 2097151 & (load_4(c + 2) >> 5); + int64_t c2 = 2097151 & (load_3(c + 5) >> 2); + int64_t c3 = 2097151 & (load_4(c + 7) >> 7); + int64_t c4 = 2097151 & (load_4(c + 10) >> 4); + int64_t c5 = 2097151 & (load_3(c + 13) >> 1); + int64_t c6 = 2097151 & (load_4(c + 15) >> 6); + int64_t c7 = 2097151 & (load_3(c + 18) >> 3); + int64_t c8 = 2097151 & load_3(c + 21); + int64_t c9 = 2097151 & (load_4(c + 23) >> 5); + int64_t c10 = 2097151 & (load_3(c + 26) >> 2); + int64_t c11 = (load_4(c + 28) >> 7); + + int64_t s0; + int64_t s1; + int64_t s2; + int64_t s3; + int64_t s4; + int64_t s5; + int64_t s6; + int64_t s7; + int64_t s8; + int64_t s9; + int64_t s10; + int64_t s11; + int64_t s12; + int64_t s13; + int64_t s14; + int64_t s15; + int64_t s16; + int64_t s17; + int64_t s18; + int64_t s19; + int64_t s20; + int64_t s21; + int64_t s22; + int64_t s23; + + int64_t carry0; + int64_t carry1; + int64_t carry2; + int64_t carry3; + int64_t carry4; + int64_t carry5; + int64_t carry6; + int64_t carry7; + int64_t carry8; + int64_t carry9; + int64_t carry10; + int64_t carry11; + int64_t carry12; + int64_t carry13; + int64_t carry14; + int64_t carry15; + int64_t carry16; + int64_t carry17; + int64_t carry18; + int64_t carry19; + int64_t carry20; + int64_t carry21; + int64_t carry22; + + s0 = c0 + a0 * b0; + s1 = c1 + a0 * b1 + a1 * b0; + s2 = c2 + a0 * b2 + a1 * b1 + a2 * b0; + s3 = c3 + a0 * b3 + a1 * b2 + a2 * b1 + a3 * b0; + s4 = c4 + a0 * b4 + a1 * b3 + a2 * b2 + a3 * b1 + a4 * b0; + s5 = c5 + a0 * b5 + a1 * b4 + a2 * b3 + a3 * b2 + a4 * b1 + a5 * b0; + s6 = c6 + a0 * b6 + a1 * b5 + a2 * b4 + a3 * b3 + a4 * b2 + a5 * b1 + + a6 * b0; + s7 = c7 + a0 * b7 + a1 * b6 + a2 * b5 + a3 * b4 + a4 * b3 + a5 * b2 + + a6 * b1 + a7 * b0; + s8 = c8 + a0 * b8 + a1 * b7 + a2 * b6 + a3 * b5 + a4 * b4 + a5 * b3 + + a6 * b2 + a7 * b1 + a8 * b0; + s9 = c9 + a0 * b9 + a1 * b8 + a2 * b7 + a3 * b6 + a4 * b5 + a5 * b4 + + a6 * b3 + a7 * b2 + a8 * b1 + a9 * b0; + s10 = c10 + a0 * b10 + a1 * b9 + a2 * b8 + a3 * b7 + a4 * b6 + a5 * b5 + + a6 * b4 + a7 * b3 + a8 * b2 + a9 * b1 + a10 * b0; + s11 = c11 + a0 * b11 + a1 * b10 + a2 * b9 + a3 * b8 + a4 * b7 + a5 * b6 + + a6 * b5 + a7 * b4 + a8 * b3 + a9 * b2 + a10 * b1 + a11 * b0; + s12 = a1 * b11 + a2 * b10 + a3 * b9 + a4 * b8 + a5 * b7 + a6 * b6 + + a7 * b5 + a8 * b4 + a9 * b3 + a10 * b2 + a11 * b1; + s13 = a2 * b11 + a3 * b10 + a4 * b9 + a5 * b8 + a6 * b7 + a7 * b6 + + a8 * b5 + a9 * b4 + a10 * b3 + a11 * b2; + s14 = a3 * b11 + a4 * b10 + a5 * b9 + a6 * b8 + a7 * b7 + a8 * b6 + + a9 * b5 + a10 * b4 + a11 * b3; + s15 = a4 * b11 + a5 * b10 + a6 * b9 + a7 * b8 + a8 * b7 + a9 * b6 + + a10 * b5 + a11 * b4; + s16 = + a5 * b11 + a6 * b10 + a7 * b9 + a8 * b8 + a9 * b7 + a10 * b6 + a11 * b5; + s17 = a6 * b11 + a7 * b10 + a8 * b9 + a9 * b8 + a10 * b7 + a11 * b6; + s18 = a7 * b11 + a8 * b10 + a9 * b9 + a10 * b8 + a11 * b7; + s19 = a8 * b11 + a9 * b10 + a10 * b9 + a11 * b8; + s20 = a9 * b11 + a10 * b10 + a11 * b9; + s21 = a10 * b11 + a11 * b10; + s22 = a11 * b11; + s23 = 0; + + carry0 = (s0 + (int64_t)(1L << 20)) >> 21; + s1 += carry0; + s0 -= carry0 * ((uint64_t)1L << 21); + carry2 = (s2 + (int64_t)(1L << 20)) >> 21; + s3 += carry2; + s2 -= carry2 * ((uint64_t)1L << 21); + carry4 = (s4 + (int64_t)(1L << 20)) >> 21; + s5 += carry4; + s4 -= carry4 * ((uint64_t)1L << 21); + carry6 = (s6 + (int64_t)(1L << 20)) >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry8 = (s8 + (int64_t)(1L << 20)) >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry10 = (s10 + (int64_t)(1L << 20)) >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + carry12 = (s12 + (int64_t)(1L << 20)) >> 21; + s13 += carry12; + s12 -= carry12 * ((uint64_t)1L << 21); + carry14 = (s14 + (int64_t)(1L << 20)) >> 21; + s15 += carry14; + s14 -= carry14 * ((uint64_t)1L << 21); + carry16 = (s16 + (int64_t)(1L << 20)) >> 21; + s17 += carry16; + s16 -= carry16 * ((uint64_t)1L << 21); + carry18 = (s18 + (int64_t)(1L << 20)) >> 21; + s19 += carry18; + s18 -= carry18 * ((uint64_t)1L << 21); + carry20 = (s20 + (int64_t)(1L << 20)) >> 21; + s21 += carry20; + s20 -= carry20 * ((uint64_t)1L << 21); + carry22 = (s22 + (int64_t)(1L << 20)) >> 21; + s23 += carry22; + s22 -= carry22 * ((uint64_t)1L << 21); + + carry1 = (s1 + (int64_t)(1L << 20)) >> 21; + s2 += carry1; + s1 -= carry1 * ((uint64_t)1L << 21); + carry3 = (s3 + (int64_t)(1L << 20)) >> 21; + s4 += carry3; + s3 -= carry3 * ((uint64_t)1L << 21); + carry5 = (s5 + (int64_t)(1L << 20)) >> 21; + s6 += carry5; + s5 -= carry5 * ((uint64_t)1L << 21); + carry7 = (s7 + (int64_t)(1L << 20)) >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry9 = (s9 + (int64_t)(1L << 20)) >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry11 = (s11 + (int64_t)(1L << 20)) >> 21; + s12 += carry11; + s11 -= carry11 * ((uint64_t)1L << 21); + carry13 = (s13 + (int64_t)(1L << 20)) >> 21; + s14 += carry13; + s13 -= carry13 * ((uint64_t)1L << 21); + carry15 = (s15 + (int64_t)(1L << 20)) >> 21; + s16 += carry15; + s15 -= carry15 * ((uint64_t)1L << 21); + carry17 = (s17 + (int64_t)(1L << 20)) >> 21; + s18 += carry17; + s17 -= carry17 * ((uint64_t)1L << 21); + carry19 = (s19 + (int64_t)(1L << 20)) >> 21; + s20 += carry19; + s19 -= carry19 * ((uint64_t)1L << 21); + carry21 = (s21 + (int64_t)(1L << 20)) >> 21; + s22 += carry21; + s21 -= carry21 * ((uint64_t)1L << 21); + + s11 += s23 * 666643; + s12 += s23 * 470296; + s13 += s23 * 654183; + s14 -= s23 * 997805; + s15 += s23 * 136657; + s16 -= s23 * 683901; + + s10 += s22 * 666643; + s11 += s22 * 470296; + s12 += s22 * 654183; + s13 -= s22 * 997805; + s14 += s22 * 136657; + s15 -= s22 * 683901; + + s9 += s21 * 666643; + s10 += s21 * 470296; + s11 += s21 * 654183; + s12 -= s21 * 997805; + s13 += s21 * 136657; + s14 -= s21 * 683901; + + s8 += s20 * 666643; + s9 += s20 * 470296; + s10 += s20 * 654183; + s11 -= s20 * 997805; + s12 += s20 * 136657; + s13 -= s20 * 683901; + + s7 += s19 * 666643; + s8 += s19 * 470296; + s9 += s19 * 654183; + s10 -= s19 * 997805; + s11 += s19 * 136657; + s12 -= s19 * 683901; + + s6 += s18 * 666643; + s7 += s18 * 470296; + s8 += s18 * 654183; + s9 -= s18 * 997805; + s10 += s18 * 136657; + s11 -= s18 * 683901; + + carry6 = (s6 + (int64_t)(1L << 20)) >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry8 = (s8 + (int64_t)(1L << 20)) >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry10 = (s10 + (int64_t)(1L << 20)) >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + carry12 = (s12 + (int64_t)(1L << 20)) >> 21; + s13 += carry12; + s12 -= carry12 * ((uint64_t)1L << 21); + carry14 = (s14 + (int64_t)(1L << 20)) >> 21; + s15 += carry14; + s14 -= carry14 * ((uint64_t)1L << 21); + carry16 = (s16 + (int64_t)(1L << 20)) >> 21; + s17 += carry16; + s16 -= carry16 * ((uint64_t)1L << 21); + + carry7 = (s7 + (int64_t)(1L << 20)) >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry9 = (s9 + (int64_t)(1L << 20)) >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry11 = (s11 + (int64_t)(1L << 20)) >> 21; + s12 += carry11; + s11 -= carry11 * ((uint64_t)1L << 21); + carry13 = (s13 + (int64_t)(1L << 20)) >> 21; + s14 += carry13; + s13 -= carry13 * ((uint64_t)1L << 21); + carry15 = (s15 + (int64_t)(1L << 20)) >> 21; + s16 += carry15; + s15 -= carry15 * ((uint64_t)1L << 21); + + s5 += s17 * 666643; + s6 += s17 * 470296; + s7 += s17 * 654183; + s8 -= s17 * 997805; + s9 += s17 * 136657; + s10 -= s17 * 683901; + + s4 += s16 * 666643; + s5 += s16 * 470296; + s6 += s16 * 654183; + s7 -= s16 * 997805; + s8 += s16 * 136657; + s9 -= s16 * 683901; + + s3 += s15 * 666643; + s4 += s15 * 470296; + s5 += s15 * 654183; + s6 -= s15 * 997805; + s7 += s15 * 136657; + s8 -= s15 * 683901; + + s2 += s14 * 666643; + s3 += s14 * 470296; + s4 += s14 * 654183; + s5 -= s14 * 997805; + s6 += s14 * 136657; + s7 -= s14 * 683901; + + s1 += s13 * 666643; + s2 += s13 * 470296; + s3 += s13 * 654183; + s4 -= s13 * 997805; + s5 += s13 * 136657; + s6 -= s13 * 683901; + + s0 += s12 * 666643; + s1 += s12 * 470296; + s2 += s12 * 654183; + s3 -= s12 * 997805; + s4 += s12 * 136657; + s5 -= s12 * 683901; + s12 = 0; + + carry0 = (s0 + (int64_t)(1L << 20)) >> 21; + s1 += carry0; + s0 -= carry0 * ((uint64_t)1L << 21); + carry2 = (s2 + (int64_t)(1L << 20)) >> 21; + s3 += carry2; + s2 -= carry2 * ((uint64_t)1L << 21); + carry4 = (s4 + (int64_t)(1L << 20)) >> 21; + s5 += carry4; + s4 -= carry4 * ((uint64_t)1L << 21); + carry6 = (s6 + (int64_t)(1L << 20)) >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry8 = (s8 + (int64_t)(1L << 20)) >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry10 = (s10 + (int64_t)(1L << 20)) >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + + carry1 = (s1 + (int64_t)(1L << 20)) >> 21; + s2 += carry1; + s1 -= carry1 * ((uint64_t)1L << 21); + carry3 = (s3 + (int64_t)(1L << 20)) >> 21; + s4 += carry3; + s3 -= carry3 * ((uint64_t)1L << 21); + carry5 = (s5 + (int64_t)(1L << 20)) >> 21; + s6 += carry5; + s5 -= carry5 * ((uint64_t)1L << 21); + carry7 = (s7 + (int64_t)(1L << 20)) >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry9 = (s9 + (int64_t)(1L << 20)) >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry11 = (s11 + (int64_t)(1L << 20)) >> 21; + s12 += carry11; + s11 -= carry11 * ((uint64_t)1L << 21); + + s0 += s12 * 666643; + s1 += s12 * 470296; + s2 += s12 * 654183; + s3 -= s12 * 997805; + s4 += s12 * 136657; + s5 -= s12 * 683901; + s12 = 0; + + carry0 = s0 >> 21; + s1 += carry0; + s0 -= carry0 * ((uint64_t)1L << 21); + carry1 = s1 >> 21; + s2 += carry1; + s1 -= carry1 * ((uint64_t)1L << 21); + carry2 = s2 >> 21; + s3 += carry2; + s2 -= carry2 * ((uint64_t)1L << 21); + carry3 = s3 >> 21; + s4 += carry3; + s3 -= carry3 * ((uint64_t)1L << 21); + carry4 = s4 >> 21; + s5 += carry4; + s4 -= carry4 * ((uint64_t)1L << 21); + carry5 = s5 >> 21; + s6 += carry5; + s5 -= carry5 * ((uint64_t)1L << 21); + carry6 = s6 >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry7 = s7 >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry8 = s8 >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry9 = s9 >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry10 = s10 >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + carry11 = s11 >> 21; + s12 += carry11; + s11 -= carry11 * ((uint64_t)1L << 21); + + s0 += s12 * 666643; + s1 += s12 * 470296; + s2 += s12 * 654183; + s3 -= s12 * 997805; + s4 += s12 * 136657; + s5 -= s12 * 683901; + + carry0 = s0 >> 21; + s1 += carry0; + s0 -= carry0 * ((uint64_t)1L << 21); + carry1 = s1 >> 21; + s2 += carry1; + s1 -= carry1 * ((uint64_t)1L << 21); + carry2 = s2 >> 21; + s3 += carry2; + s2 -= carry2 * ((uint64_t)1L << 21); + carry3 = s3 >> 21; + s4 += carry3; + s3 -= carry3 * ((uint64_t)1L << 21); + carry4 = s4 >> 21; + s5 += carry4; + s4 -= carry4 * ((uint64_t)1L << 21); + carry5 = s5 >> 21; + s6 += carry5; + s5 -= carry5 * ((uint64_t)1L << 21); + carry6 = s6 >> 21; + s7 += carry6; + s6 -= carry6 * ((uint64_t)1L << 21); + carry7 = s7 >> 21; + s8 += carry7; + s7 -= carry7 * ((uint64_t)1L << 21); + carry8 = s8 >> 21; + s9 += carry8; + s8 -= carry8 * ((uint64_t)1L << 21); + carry9 = s9 >> 21; + s10 += carry9; + s9 -= carry9 * ((uint64_t)1L << 21); + carry10 = s10 >> 21; + s11 += carry10; + s10 -= carry10 * ((uint64_t)1L << 21); + + s[0] = s0 >> 0; + s[1] = s0 >> 8; + s[2] = (s0 >> 16) | (s1 * ((uint64_t)1 << 5)); + s[3] = s1 >> 3; + s[4] = s1 >> 11; + s[5] = (s1 >> 19) | (s2 * ((uint64_t)1 << 2)); + s[6] = s2 >> 6; + s[7] = (s2 >> 14) | (s3 * ((uint64_t)1 << 7)); + s[8] = s3 >> 1; + s[9] = s3 >> 9; + s[10] = (s3 >> 17) | (s4 * ((uint64_t)1 << 4)); + s[11] = s4 >> 4; + s[12] = s4 >> 12; + s[13] = (s4 >> 20) | (s5 * ((uint64_t)1 << 1)); + s[14] = s5 >> 7; + s[15] = (s5 >> 15) | (s6 * ((uint64_t)1 << 6)); + s[16] = s6 >> 2; + s[17] = s6 >> 10; + s[18] = (s6 >> 18) | (s7 * ((uint64_t)1 << 3)); + s[19] = s7 >> 5; + s[20] = s7 >> 13; + s[21] = s8 >> 0; + s[22] = s8 >> 8; + s[23] = (s8 >> 16) | (s9 * ((uint64_t)1 << 5)); + s[24] = s9 >> 3; + s[25] = s9 >> 11; + s[26] = (s9 >> 19) | (s10 * ((uint64_t)1 << 2)); + s[27] = s10 >> 6; + s[28] = (s10 >> 14) | (s11 * ((uint64_t)1 << 7)); + s[29] = s11 >> 1; + s[30] = s11 >> 9; + s[31] = s11 >> 17; +} + +/* libsodium/crypto_core/ed25519/ref10/ed25519_ref10.c */ +void sc_sqmul(unsigned char s[32], const int n, const unsigned char a[32]) +{ + int i; + for (i = 0; i < n; ++i) + sc_mul(s, s, s); + sc_mul(s, s, a); +} + +/* libsodium/crypto_core/ed25519/ref10/ed25519_ref10.c */ +void sc_invert(unsigned char* recip, const unsigned char* s) +{ + unsigned char _10[32], _100[32], _1000[32], _10000[32], _100000[32], + _1000000[32], _10010011[32], _10010111[32], _100110[32], _1010[32], + _1010000[32], _1010011[32], _1011[32], _10110[32], _10111101[32], + _11[32], _1100011[32], _1100111[32], _11010011[32], _1101011[32], + _11100111[32], _11101011[32], _11110101[32]; + + sc_mul(_10, s, s); + sc_mul(_11, s, _10); + sc_mul(_100, s, _11); + sc_mul(_1000, _100, _100); + sc_mul(_1010, _10, _1000); + sc_mul(_1011, s, _1010); + sc_mul(_10000, _1000, _1000); + sc_mul(_10110, _1011, _1011); + sc_mul(_100000, _1010, _10110); + sc_mul(_100110, _10000, _10110); + sc_mul(_1000000, _100000, _100000); + sc_mul(_1010000, _10000, _1000000); + sc_mul(_1010011, _11, _1010000); + sc_mul(_1100011, _10000, _1010011); + sc_mul(_1100111, _100, _1100011); + sc_mul(_1101011, _100, _1100111); + sc_mul(_10010011, _1000000, _1010011); + sc_mul(_10010111, _100, _10010011); + sc_mul(_10111101, _100110, _10010111); + sc_mul(_11010011, _10110, _10111101); + sc_mul(_11100111, _1010000, _10010111); + sc_mul(_11101011, _100, _11100111); + sc_mul(_11110101, _1010, _11101011); + + sc_mul(recip, _1011, _11110101); + + sc_sqmul(recip, 126, _1010011); + + sc_sqmul(recip, 9, _10); + sc_mul(recip, recip, _11110101); + sc_sqmul(recip, 7, _1100111); + sc_sqmul(recip, 9, _11110101); + sc_sqmul(recip, 11, _10111101); + sc_sqmul(recip, 8, _11100111); + sc_sqmul(recip, 9, _1101011); + sc_sqmul(recip, 6, _1011); + sc_sqmul(recip, 14, _10010011); + sc_sqmul(recip, 10, _1100011); + sc_sqmul(recip, 9, _10010111); + sc_sqmul(recip, 10, _11110101); + sc_sqmul(recip, 8, _11010011); + sc_sqmul(recip, 8, _11101011); +} + +/* + In: t (x, y, z) + Out: r (x, t, z, t) + + Note: expensive conversion because of fe_invert +*/ +void ge_p2_to_p3(ge_p3 *r, const ge_p2 *t) +{ + fe_copy(r->X, t->X); + fe_copy(r->Y, t->Y); + fe_copy(r->Z, t->Z); + fe_invert(r->T, t->Z); + fe_mul(r->T, r->T, t->Y); + fe_mul(r->T, r->T, t->X); +} + + +/* + In: data -- points to 'size' bytes of data + Out: res = Hp(data) + where Hp = 8 * ge_fromfe_frombytes_vartime(cn_fast_hash(data)) +*/ +void ge_bytes_hash_to_ec(ge_p3 *res, const void *data, size_t size) +{ + unsigned char h[HASH_SIZE]; + ge_p2 point; + ge_p1p1 point2; + + cn_fast_hash(data, size, (char*)h); + ge_fromfe_frombytes_vartime(&point, &h[0]); + /*ge_p2_to_p3(res, &point); -- can be used to avoid multiplication by 8 for debugging */ + ge_mul8(&point2, &point); + ge_p1p1_to_p3(res, &point2); +} + +void ge_bytes_hash_to_ec_32(ge_p3 *res, const unsigned char *ge_bytes) +{ + ge_bytes_hash_to_ec(res, ge_bytes, 32); +} + +void ge_mul8_p3(ge_p3 *r, const ge_p3 *t) +{ + ge_p1p1 p1; + ge_p2 p2; + + // TODO: consider removing the following copy, replace &p2 by (ge_p2*)t as it's ugly but possible + ge_p3_to_p2(&p2, t); // copying + + ge_p2_dbl(&p1, &p2); // 3 fe_sq, 1 fe_sq2, 5 fe_add/sub + ge_p1p1_to_p2(&p2, &p1); // 3 fe_mul + ge_p2_dbl(&p1, &p2); // 3 fe_sq, 1 fe_sq2, 5 fe_add/sub + ge_p1p1_to_p2(&p2, &p1); // 3 fe_mul + ge_p2_dbl(&p1, &p2); // 3 fe_sq, 1 fe_sq2, 5 fe_add/sub + + ge_p1p1_to_p3(r, &p1); // 4 fe_mul +} + + +// returns the most non-zero index of r +int slide_v2(signed char *r, const unsigned char *a) +{ + int i; + int b; + int k; + int nzi = 0; + + for (i = 0; i < 256; ++i) { + r[i] = 1 & (a[i >> 3] >> (i & 7)); + } + + for (i = 0; i < 256; ++i) { + if (r[i]) { + for (b = 1; b <= 6 && i + b < 256; ++b) { + if (r[i + b]) { + if (r[i] + (r[i + b] << b) <= 15) { + r[i] += r[i + b] << b; r[i + b] = 0; + } + else if (r[i] - (r[i + b] << b) >= -15) { + r[i] -= r[i + b] << b; + for (k = i + b; k < 256; ++k) { + if (!r[k]) { + r[k] = 1; + break; + } + r[k] = 0; + } + } + else + break; + } + } + if (r[i]) + nzi = i; + } + } + + return nzi; +} + +void ge_scalarmult_vartime_p3_v2(ge_p3 *r, const unsigned char *a, const ge_p3 *A) +{ + signed char aslide[256]; + ge_dsmp Ai; /* A, 3A, 5A, 7A, 9A, 11A, 13A, 15A */ + ge_p1p1 t; + ge_p3 u; + ge_p2 r_p2; + int i; + + i = slide_v2(aslide, a); + + if (i == 0) + { + ge_p3_0(r); + return; + } + + ge_dsm_precomp(Ai, A); + ge_p2_0(&r_p2); + + for (; i >= 0; --i) + { + ge_p2_dbl(&t, &r_p2); + if (aslide[i] > 0) + { + ge_p1p1_to_p3(&u, &t); + ge_add(&t, &u, &Ai[aslide[i] / 2]); + } + else if (aslide[i] < 0) + { + ge_p1p1_to_p3(&u, &t); + ge_sub(&t, &u, &Ai[(-aslide[i]) / 2]); + } + if (i != 0) + ge_p1p1_to_p2(&r_p2, &t); + else + ge_p1p1_to_p3(r, &t); + } +} diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index 9d07fc8..6fb6917 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -1,8 +1,10 @@ +// Copyright (c) 2018-2021 Zano Project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#include // size_t /* From fe.h */ @@ -101,19 +103,43 @@ void sc_reduce(unsigned char *); /* New code */ void ge_scalarmult(ge_p2 *, const unsigned char *, const ge_p3 *); +void ge_scalarmult_p3(ge_p3 *, const unsigned char *, const ge_p3 *); void ge_double_scalarmult_precomp_vartime(ge_p2 *, const unsigned char *, const ge_p3 *, const unsigned char *, const ge_dsmp); void ge_mul8(ge_p1p1 *, const ge_p2 *); +void ge_mul8_p3(ge_p3 *, const ge_p3 *); +void ge_fromfe_frombytes_vartime(ge_p2 *, const unsigned char *); +void ge_p2_to_p3(ge_p3 *r, const ge_p2 *t); +void ge_bytes_hash_to_ec(ge_p3 *, const void *, size_t); +void ge_bytes_hash_to_ec_32(ge_p3 *, const unsigned char *); + +void ge_p3_0(ge_p3 *h); +void ge_sub(ge_p1p1 *, const ge_p3 *, const ge_cached *); +void ge_double_scalarmult_base_vartime_p3(ge_p3 *r, const unsigned char *a, const ge_p3 *A, const unsigned char *b); +void ge_scalarmult_vartime_p3(ge_p3 *r, const unsigned char *a, const ge_p3 *A); +void ge_scalarmult_vartime_p3_v2(ge_p3 *r, const unsigned char *a, const ge_p3 *A); + extern const fe fe_ma2; extern const fe fe_ma; extern const fe fe_fffb1; extern const fe fe_fffb2; extern const fe fe_fffb3; extern const fe fe_fffb4; -void ge_fromfe_frombytes_vartime(ge_p2 *, const unsigned char *); + void sc_0(unsigned char *); void sc_reduce32(unsigned char *); void sc_add(unsigned char *, const unsigned char *, const unsigned char *); void sc_sub(unsigned char *, const unsigned char *, const unsigned char *); void sc_mulsub(unsigned char *, const unsigned char *, const unsigned char *, const unsigned char *); +void sc_mul(unsigned char *, const unsigned char *, const unsigned char *); +void sc_muladd(unsigned char* s, const unsigned char* a, const unsigned char* b, const unsigned char* c); int sc_check(const unsigned char *); int sc_isnonzero(const unsigned char *); /* Doesn't normalize */ +void sc_invert(unsigned char*, const unsigned char*); + +void fe_sq(fe h, const fe f); +int fe_isnonzero(const fe f); +int fe_cmp(const fe a, const fe b); +void fe_mul(fe, const fe, const fe); +void fe_frombytes(fe h, const unsigned char *s); +void fe_invert(fe out, const fe z); +void fe_tobytes(unsigned char *s, const fe h); diff --git a/src/crypto/crypto-sugar.cpp b/src/crypto/crypto-sugar.cpp new file mode 100644 index 0000000..4774e9b --- /dev/null +++ b/src/crypto/crypto-sugar.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2020-2021 Zano Project +// Copyright (c) 2020-2021 sowle (val@zano.org, crypto.sowle@gmail.com) +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// Note: This file originates from tests/functional_tests/crypto_tests.cpp + +#include "crypto-sugar.h" + +namespace crypto +{ + + const point_g_t c_point_G; + + const scalar_t c_scalar_1 = { 1 }; + const scalar_t c_scalar_L = { 0x5812631a5cf5d3ed, 0x14def9dea2f79cd6, 0x0, 0x1000000000000000 }; + const scalar_t c_scalar_Lm1 = { 0x5812631a5cf5d3ec, 0x14def9dea2f79cd6, 0x0, 0x1000000000000000 }; + const scalar_t c_scalar_P = { 0xffffffffffffffed, 0xffffffffffffffff, 0xffffffffffffffff, 0x7fffffffffffffff }; + const scalar_t c_scalar_Pm1 = { 0xffffffffffffffec, 0xffffffffffffffff, 0xffffffffffffffff, 0x7fffffffffffffff }; + const scalar_t c_scalar_256m1 = { 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff }; + const scalar_t c_scalar_1div8 = { 0x6106e529e2dc2f79, 0x07d39db37d1cdad0, 0x0, 0x0600000000000000 }; + + const point_t c_point_H = { 0x05087c1f5b9b32d6, 0x00547595f445c3b5, 0x764df64578552f2a, 0x8a49a651e0e0da45 }; // == Hp(G), this is being checked in bpp_basics + const point_t c_point_0 = point_t(point_t::tag_zero()); + +} // namespace crypto diff --git a/src/crypto/crypto-sugar.h b/src/crypto/crypto-sugar.h new file mode 100644 index 0000000..43e69c4 --- /dev/null +++ b/src/crypto/crypto-sugar.h @@ -0,0 +1,904 @@ +// Copyright (c) 2020-2021 Zano Project +// Copyright (c) 2020-2021 sowle (val@zano.org, crypto.sowle@gmail.com) +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// Note: This file originates from tests/functional_tests/crypto_tests.cpp +#pragma once +#include +#include +#include "crypto.h" + +namespace crypto +{ + extern "C" + { +#include "crypto/crypto-ops.h" + } // extern "C" + + + + // + // Helpers + // + + template + std::string pod_to_hex_reversed(const pod_t &h) + { + constexpr char hexmap[] = "0123456789abcdef"; + const unsigned char* data = reinterpret_cast(&h); + size_t len = sizeof h; + + std::string s(len * 2, ' '); + for (size_t i = 0; i < len; ++i) { + s[2 * i] = hexmap[data[len - 1 - i] >> 4]; + s[2 * i + 1] = hexmap[data[len - 1 - i] & 0x0F]; + } + + return s; + } + + template + std::string pod_to_hex(const pod_t &h) + { + constexpr char hexmap[] = "0123456789abcdef"; + const unsigned char* data = reinterpret_cast(&h); + size_t len = sizeof h; + + std::string s(len * 2, ' '); + for (size_t i = 0; i < len; ++i) { + s[2 * i] = hexmap[data[i] >> 4]; + s[2 * i + 1] = hexmap[data[i] & 0x0F]; + } + + return s; + } + + template + std::string pod_to_hex_comma_separated_bytes(const pod_t &h) + { + std::stringstream ss; + ss << std::hex << std::setfill('0'); + size_t len = sizeof h; + const unsigned char* p = (const unsigned char*)&h; + for (size_t i = 0; i < len; ++i) + { + ss << "0x" << std::setw(2) << static_cast(p[i]); + if (i + 1 != len) + ss << ", "; + } + return ss.str(); + } + + template + std::string pod_to_hex_comma_separated_uint64(const pod_t &h) + { + static_assert((sizeof h) % 8 == 0, "size of h should be a multiple of 64 bit"); + size_t len = (sizeof h) / 8; + std::stringstream ss; + ss << std::hex << std::setfill('0'); + const uint64_t* p = (const uint64_t*)&h; + for (size_t i = 0; i < len; ++i) + { + ss << "0x" << std::setw(16) << static_cast(p[i]); + if (i + 1 != len) + ss << ", "; + } + return ss.str(); + } + + template + bool parse_tpod_from_hex_string(const std::string& hex_str, t_pod_type& t_pod) + { + static const int16_t char_map[256] = { // 0-9, a-f, A-F is only allowed + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x00 - 0x1F + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, // 0x20 - 0x3F + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x40 - 0x5F + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x60 - 0x7F + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0x80 - 0x9F + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xA0 - 0xBF + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 0xC0 - 0xDF + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; // 0xE0 - 0xFF + + size_t pod_size = sizeof t_pod; + uint8_t *p = reinterpret_cast(&t_pod); + + if (hex_str.size() != 2 * pod_size) + return false; + + for (size_t i = 0; i < pod_size; ++i) + { + int16_t hi = char_map[static_cast(hex_str[2 * i])]; + int16_t lo = char_map[static_cast(hex_str[2 * i + 1])]; + if (hi < 0 || lo < 0) + { + // invalid characters in hex_str + memset(p, 0, pod_size); + return false; + } + p[i] = static_cast(hi * 16 + lo); // write byte to pod + } + return true; + } + + template + t_pod_type parse_tpod_from_hex_string(const std::string& hex_str) + { + t_pod_type t_pod = AUTO_VAL_INIT(t_pod); + parse_tpod_from_hex_string(hex_str, t_pod); + return t_pod; + } + + // + // scalar_t - holds a 256-bit scalar, normally in [0..L-1] + // + struct alignas(32) scalar_t + { + union + { + uint64_t m_u64[4]; + unsigned char m_s[32]; + }; + + scalar_t() + {} + + // won't check scalar range validity (< L) + scalar_t(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3) + { + m_u64[0] = a0; + m_u64[1] = a1; + m_u64[2] = a2; + m_u64[3] = a3; + } + + // won't check scalar range validity (< L) + scalar_t(const unsigned char(&v)[32]) + { + memcpy(m_s, v, 32); + } + + // won't check secret key validity (sk < L) + scalar_t(const crypto::secret_key& sk) + { + from_secret_key(sk); + } + + // copy data and reduce + scalar_t(const crypto::hash& hash) + { + m_u64[0] = ((uint64_t*)&hash)[0]; + m_u64[1] = ((uint64_t*)&hash)[1]; + m_u64[2] = ((uint64_t*)&hash)[2]; + m_u64[3] = ((uint64_t*)&hash)[3]; + sc_reduce32(&m_s[0]); + } + + scalar_t(uint64_t v) + { + zero(); + m_u64[0] = v; + // do not need to call reduce as 2^64 < L + } + + // copy at most 256 bits (32 bytes) and reduce + template + explicit scalar_t(const boost::multiprecision::number& bigint) + { + zero(); + unsigned int bytes_to_copy = bigint.backend().size() * bigint.backend().limb_bits / 8; + if (bytes_to_copy > sizeof *this) + bytes_to_copy = sizeof *this; + memcpy(&m_s[0], bigint.backend().limbs(), bytes_to_copy); + sc_reduce32(&m_s[0]); + } + + unsigned char* data() + { + return &m_s[0]; + } + + const unsigned char* data() const + { + return &m_s[0]; + } + + crypto::secret_key &as_secret_key() + { + return *(crypto::secret_key*)&m_s[0]; + } + + const crypto::secret_key& as_secret_key() const + { + return *(const crypto::secret_key*)&m_s[0]; + } + + operator crypto::secret_key() const + { + crypto::secret_key result; + memcpy(result.data, &m_s, sizeof result.data); + return result; + } + + void from_secret_key(const crypto::secret_key& sk) + { + uint64_t *p_sk64 = (uint64_t*)&sk; + m_u64[0] = p_sk64[0]; + m_u64[1] = p_sk64[1]; + m_u64[2] = p_sk64[2]; + m_u64[3] = p_sk64[3]; + // assuming secret key is correct (< L), so we don't need to call reduce here + } + + void zero() + { + m_u64[0] = 0; + m_u64[1] = 0; + m_u64[2] = 0; + m_u64[3] = 0; + } + + // genrate 0 <= x < L + static scalar_t random() + { + scalar_t result; + result.make_random(); + return result; + } + + // genrate 0 <= x < L + void make_random() + { + unsigned char tmp[64]; + crypto::generate_random_bytes(64, tmp); + sc_reduce(tmp); + memcpy(&m_s, tmp, sizeof m_s); + + /* // for tests + int x[8] = { rand() }; + crypto::cn_fast_hash(&x, sizeof x, *(crypto::hash*)this); + sc_reduce32(m_s); + */ + } + + bool is_zero() const + { + return sc_isnonzero(&m_s[0]) == 0; + } + + bool is_reduced() const + { + return sc_check(&m_s[0]) == 0; + } + + void reduce() + { + sc_reduce32(&m_s[0]); + } + + scalar_t operator+(const scalar_t& v) const + { + scalar_t result; + sc_add(&result.m_s[0], &m_s[0], &v.m_s[0]); + return result; + } + + scalar_t& operator+=(const scalar_t& v) + { + sc_add(&m_s[0], &m_s[0], &v.m_s[0]); + return *this; + } + + scalar_t operator-(const scalar_t& v) const + { + scalar_t result; + sc_sub(&result.m_s[0], &m_s[0], &v.m_s[0]); + return result; + } + + scalar_t& operator-=(const scalar_t& v) + { + sc_sub(&m_s[0], &m_s[0], &v.m_s[0]); + return *this; + } + + scalar_t operator*(const scalar_t& v) const + { + scalar_t result; + sc_mul(result.m_s, m_s, v.m_s); + return result; + } + + scalar_t& operator*=(const scalar_t& v) + { + sc_mul(m_s, m_s, v.m_s); + return *this; + } + + /* + I think it has bad symantic (operator-like), consider rename/reimplement -- sowle + */ + // returns this * b + c + scalar_t muladd(const scalar_t& b, const scalar_t& c) const + { + scalar_t result; + sc_muladd(result.m_s, m_s, b.m_s, c.m_s); + return result; + } + + // returns this = a * b + c + scalar_t& assign_muladd(const scalar_t& a, const scalar_t& b, const scalar_t& c) + { + sc_muladd(m_s, a.m_s, b.m_s, c.m_s); + return *this; + } + + scalar_t reciprocal() const + { + scalar_t result; + sc_invert(result.m_s, m_s); + return result; + } + + scalar_t operator/(const scalar_t& v) const + { + return operator*(v.reciprocal()); + } + + scalar_t& operator/=(const scalar_t& v) + { + scalar_t reciprocal; + sc_invert(&reciprocal.m_s[0], &v.m_s[0]); + sc_mul(&m_s[0], &m_s[0], &reciprocal.m_s[0]); + return *this; + } + + bool operator==(const scalar_t& rhs) const + { + return + m_u64[0] == rhs.m_u64[0] && + m_u64[1] == rhs.m_u64[1] && + m_u64[2] == rhs.m_u64[2] && + m_u64[3] == rhs.m_u64[3]; + } + + bool operator!=(const scalar_t& rhs) const + { + return + m_u64[0] != rhs.m_u64[0] || + m_u64[1] != rhs.m_u64[1] || + m_u64[2] != rhs.m_u64[2] || + m_u64[3] != rhs.m_u64[3]; + } + + bool operator<(const scalar_t& rhs) const + { + if (m_u64[3] < rhs.m_u64[3]) return true; + if (m_u64[3] > rhs.m_u64[3]) return false; + if (m_u64[2] < rhs.m_u64[2]) return true; + if (m_u64[2] > rhs.m_u64[2]) return false; + if (m_u64[1] < rhs.m_u64[1]) return true; + if (m_u64[1] > rhs.m_u64[1]) return false; + if (m_u64[0] < rhs.m_u64[0]) return true; + if (m_u64[0] > rhs.m_u64[0]) return false; + return false; + } + + bool operator>(const scalar_t& rhs) const + { + if (m_u64[3] < rhs.m_u64[3]) return false; + if (m_u64[3] > rhs.m_u64[3]) return true; + if (m_u64[2] < rhs.m_u64[2]) return false; + if (m_u64[2] > rhs.m_u64[2]) return true; + if (m_u64[1] < rhs.m_u64[1]) return false; + if (m_u64[1] > rhs.m_u64[1]) return true; + if (m_u64[0] < rhs.m_u64[0]) return false; + if (m_u64[0] > rhs.m_u64[0]) return true; + return false; + } + + friend std::ostream& operator<<(std::ostream& ss, const scalar_t &v) + { + return ss << pod_to_hex(v); + } + + std::string to_string_as_hex_number() const + { + return pod_to_hex_reversed(*this); + } + + std::string to_string_as_secret_key() const + { + return pod_to_hex(*this); + } + + template + MP_type as_boost_mp_type() const + { + MP_type result = 0; + static_assert(sizeof result >= sizeof *this, "size missmatch"); // to avoid using types less than uint256_t + unsigned int sz = sizeof *this / sizeof(boost::multiprecision::limb_type); + result.backend().resize(sz, sz); + memcpy(result.backend().limbs(), &m_s[0], sizeof *this); + result.backend().normalize(); + return result; + } + + }; // struct scalar_t + + + // + // + // + struct point_t + { + struct tag_zero {}; + + // A point(x, y) is represented in extended homogeneous coordinates (X, Y, Z, T) + // with x = X / Z, y = Y / Z, x * y = T / Z. + ge_p3 m_p3; + + point_t() + { + } + + explicit point_t(const crypto::public_key& pk) + { + if (!from_public_key(pk)) + zero(); + } + + point_t(const unsigned char(&v)[32]) + { + static_assert(sizeof(crypto::public_key) == sizeof v, "size missmatch"); + if (!from_public_key(*(const crypto::public_key*)v)) + zero(); + } + + point_t(const uint64_t(&v)[4]) + { + static_assert(sizeof(crypto::public_key) == sizeof v, "size missmatch"); + if (!from_public_key(*(const crypto::public_key*)v)) + zero(); + } + + point_t(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3) + { + crypto::public_key pk; + ((uint64_t*)&pk)[0] = a0; + ((uint64_t*)&pk)[1] = a1; + ((uint64_t*)&pk)[2] = a2; + ((uint64_t*)&pk)[3] = a3; + + if (!from_public_key(pk)) + zero(); + } + + explicit point_t(tag_zero&&) + { + zero(); + } + + void zero() + { + ge_p3_0(&m_p3); + } + + bool is_zero() const + { + // (0, 1) ~ (0, z, z, 0) + return fe_isnonzero(m_p3.X) * fe_cmp(m_p3.Y, m_p3.Z) == 0; + } + + bool from_public_key(const crypto::public_key& pk) + { + return ge_frombytes_vartime(&m_p3, reinterpret_cast(&pk)) == 0; + } + + bool from_key_image(const crypto::key_image& ki) + { + return ge_frombytes_vartime(&m_p3, reinterpret_cast(&ki)) == 0; + } + + bool from_string(const std::string& str) + { + crypto::public_key pk; + if (!parse_tpod_from_hex_string(str, pk)) + return false; + return from_public_key(pk); + } + + crypto::public_key to_public_key() const + { + crypto::public_key result; + ge_p3_tobytes((unsigned char*)&result, &m_p3); + return result; + } + + void to_public_key(crypto::public_key& result) const + { + ge_p3_tobytes((unsigned char*)&result, &m_p3); + } + + crypto::key_image to_key_image() const + { + crypto::key_image result; + ge_p3_tobytes((unsigned char*)&result, &m_p3); + return result; + } + + point_t operator+(const point_t& rhs) const + { + point_t result; + ge_cached rhs_c; + ge_p1p1 t; + ge_p3_to_cached(&rhs_c, &rhs.m_p3); + ge_add(&t, &m_p3, &rhs_c); + ge_p1p1_to_p3(&result.m_p3, &t); + return result; + } + + point_t& operator+=(const point_t& rhs) + { + ge_cached rhs_c; + ge_p1p1 t; + ge_p3_to_cached(&rhs_c, &rhs.m_p3); + ge_add(&t, &m_p3, &rhs_c); + ge_p1p1_to_p3(&m_p3, &t); + return *this; + } + + point_t operator-(const point_t& rhs) const + { + point_t result; + ge_cached rhs_c; + ge_p1p1 t; + ge_p3_to_cached(&rhs_c, &rhs.m_p3); + ge_sub(&t, &m_p3, &rhs_c); + ge_p1p1_to_p3(&result.m_p3, &t); + return result; + } + + point_t& operator-=(const point_t& rhs) + { + ge_cached rhs_c; + ge_p1p1 t; + ge_p3_to_cached(&rhs_c, &rhs.m_p3); + ge_sub(&t, &m_p3, &rhs_c); + ge_p1p1_to_p3(&m_p3, &t); + return *this; + } + + friend point_t operator*(const scalar_t& lhs, const point_t& rhs) + { + point_t result; + ge_scalarmult_p3(&result.m_p3, lhs.m_s, &rhs.m_p3); + return result; + } + + point_t& operator*=(const scalar_t& rhs) + { + // TODO: ge_scalarmult_vartime_p3 + ge_scalarmult_p3(&m_p3, rhs.m_s, &m_p3); + return *this; + } + + friend point_t operator/(const point_t& lhs, const scalar_t& rhs) + { + point_t result; + scalar_t reciprocal; + sc_invert(&reciprocal.m_s[0], &rhs.m_s[0]); + ge_scalarmult_p3(&result.m_p3, &reciprocal.m_s[0], &lhs.m_p3); + return result; + } + + point_t& modify_mul8() + { + ge_mul8_p3(&m_p3, &m_p3); + return *this; + } + + // returns a * this + G + point_t mul_plus_G(const scalar_t& a) const + { + static const unsigned char one[32] = { 1 }; + static_assert(sizeof one == sizeof(crypto::ec_scalar), "size missmatch"); + + point_t result; + ge_double_scalarmult_base_vartime_p3(&result.m_p3, &a.m_s[0], &m_p3, &one[0]); + return result; + } + + // returns a * this + b * G + point_t mul_plus_G(const scalar_t& a, const scalar_t& b) const + { + point_t result; + ge_double_scalarmult_base_vartime_p3(&result.m_p3, &a.m_s[0], &m_p3, &b.m_s[0]); + return result; + } + + // *this = a * A + b * G + void assign_mul_plus_G(const scalar_t& a, const point_t& A, const scalar_t& b) + { + ge_double_scalarmult_base_vartime_p3(&m_p3, &a.m_s[0], &A.m_p3, &b.m_s[0]); + } + + friend bool operator==(const point_t& lhs, const point_t& rhs) + { + // convert to xy form, then compare components (because (x, y, z, t) representation is not unique) + fe lrecip, lx, ly; + fe rrecip, rx, ry; + + fe_invert(lrecip, lhs.m_p3.Z); + fe_invert(rrecip, rhs.m_p3.Z); + + fe_mul(lx, lhs.m_p3.X, lrecip); + fe_mul(rx, rhs.m_p3.X, rrecip); + if (memcmp(&lx, &rx, sizeof lx) != 0) + return false; + + fe_mul(ly, lhs.m_p3.Y, lrecip); + fe_mul(ry, rhs.m_p3.Y, rrecip); + if (memcmp(&ly, &ry, sizeof ly) != 0) + return false; + + return true; + }; + + friend std::ostream& operator<<(std::ostream& ss, const point_t &v) + { + crypto::public_key pk = v.to_public_key(); + return ss << pod_to_hex(pk); + } + + operator std::string() const + { + crypto::public_key pk = to_public_key(); + return pod_to_hex(pk); + } + + std::string to_string() const + { + crypto::public_key pk = to_public_key(); + return pod_to_hex(pk); + } + + std::string to_hex_comma_separated_bytes_str() const + { + crypto::public_key pk = to_public_key(); + return pod_to_hex_comma_separated_bytes(pk); + } + + std::string to_hex_comma_separated_uint64_str() const + { + crypto::public_key pk = to_public_key(); + return pod_to_hex_comma_separated_uint64(pk); + } + + }; // struct point_t + + + // + // point_g_t -- special type for curve's base point + // + struct point_g_t : public point_t + { + point_g_t() + { + scalar_t one(1); + ge_scalarmult_base(&m_p3, &one.m_s[0]); + } + + friend point_t operator*(const scalar_t& lhs, const point_g_t&) + { + point_t result; + ge_scalarmult_base(&result.m_p3, &lhs.m_s[0]); + return result; + } + + friend point_t operator/(const point_g_t&, const scalar_t& rhs) + { + point_t result; + scalar_t reciprocal; + sc_invert(&reciprocal.m_s[0], &rhs.m_s[0]); + ge_scalarmult_base(&result.m_p3, &reciprocal.m_s[0]); + return result; + } + + static_assert(sizeof(crypto::public_key) == 32, "size error"); + + }; // struct point_g_t + + + // + // Global constants + // + + extern const point_g_t c_point_G; + + extern const scalar_t c_scalar_1; + extern const scalar_t c_scalar_L; + extern const scalar_t c_scalar_Lm1; + extern const scalar_t c_scalar_P; + extern const scalar_t c_scalar_Pm1; + extern const scalar_t c_scalar_256m1; + extern const scalar_t c_scalar_1div8; + + extern const point_t c_point_H; + extern const point_t c_point_0; + + // + // hash functions' helper + // + struct hash_helper_t + { + static scalar_t hs(const scalar_t& s) + { + return scalar_t(crypto::cn_fast_hash(s.data(), sizeof s)); // will reduce mod L + } + + static scalar_t hs(const void* data, size_t size) + { + return scalar_t(crypto::cn_fast_hash(data, size)); // will reduce mod L + } + + static scalar_t hs(const std::string& str) + { + return scalar_t(crypto::cn_fast_hash(str.c_str(), str.size())); // will reduce mod L + } + + struct hs_t + { + hs_t() + { + static_assert(sizeof(scalar_t) == sizeof(crypto::public_key), "unexpected size of data"); + } + + void reserve(size_t elements_count) + { + m_elements.reserve(elements_count); + } + + void resize(size_t elements_count) + { + m_elements.resize(elements_count); + } + + void clear() + { + m_elements.clear(); + } + + void add_scalar(const scalar_t& scalar) + { + m_elements.emplace_back(scalar); + } + + void add_point(const point_t& point) + { + m_elements.emplace_back(point.to_public_key()); + + // faster? + /* static_assert(sizeof point.m_p3 == 5 * sizeof(item_t), "size missmatch"); + const item_t *p = (item_t*)&point.m_p3; + m_elements.emplace_back(p[0]); + m_elements.emplace_back(p[1]); + m_elements.emplace_back(p[2]); + m_elements.emplace_back(p[3]); + m_elements.emplace_back(p[4]); */ + } + + void add_pub_key(const crypto::public_key& pk) + { + m_elements.emplace_back(pk); + } + + scalar_t& access_scalar(size_t index) + { + return m_elements[index].scalar; + } + + public_key& access_public_key(size_t index) + { + return m_elements[index].pk; + } + + void add_points_array(const std::vector& points_array) + { + for (size_t i = 0, size = points_array.size(); i < size; ++i) + add_point(points_array[i]); + } + + void add_pub_keys_array(const std::vector& pub_keys_array) + { + for (size_t i = 0, size = pub_keys_array.size(); i < size; ++i) + m_elements.emplace_back(pub_keys_array[i]); + } + + void add_key_images_array(const std::vector& key_image_array) + { + for (size_t i = 0, size = key_image_array.size(); i < size; ++i) + m_elements.emplace_back(key_image_array[i]); + } + + scalar_t calc_hash(bool clear = true) + { + size_t data_size_bytes = m_elements.size() * sizeof(item_t); + crypto::hash hash; + crypto::cn_fast_hash(m_elements.data(), data_size_bytes, hash); + if (clear) + this->clear(); + return scalar_t(hash); // this will reduce to L + } + + void assign_calc_hash(scalar_t& result, bool clear = true) + { + static_assert(sizeof result == sizeof(crypto::hash), "size missmatch"); + size_t data_size_bytes = m_elements.size() * sizeof(item_t); + crypto::cn_fast_hash(m_elements.data(), data_size_bytes, (crypto::hash&)result); + result.reduce(); + if (clear) + this->clear(); + } + + union item_t + { + item_t() {} + item_t(const scalar_t& scalar) : scalar(scalar) {} + item_t(const crypto::public_key& pk) : pk(pk) {} + item_t(const crypto::key_image& ki) : ki(ki) {} + scalar_t scalar; + crypto::public_key pk; + crypto::key_image ki; + }; + + std::vector m_elements; + }; + + static scalar_t hs(const scalar_t& s, const std::vector& ps0, const std::vector& ps1) + { + hs_t hs_calculator; + hs_calculator.add_scalar(s); + hs_calculator.add_points_array(ps0); + hs_calculator.add_points_array(ps1); + return hs_calculator.calc_hash(); + } + + static scalar_t hs(const crypto::hash& s, const std::vector& ps0, const std::vector& ps1) + { + static_assert(sizeof(crypto::hash) == sizeof(scalar_t), "size missmatch"); + hs_t hs_calculator; + hs_calculator.add_scalar(*reinterpret_cast(&s)); + hs_calculator.add_pub_keys_array(ps0); + hs_calculator.add_key_images_array(ps1); + return hs_calculator.calc_hash(); + } + + static scalar_t hs(const std::vector& ps0, const std::vector& ps1) + { + hs_t hs_calculator; + hs_calculator.add_points_array(ps0); + hs_calculator.add_points_array(ps1); + return hs_calculator.calc_hash(); + } + + static point_t hp(const point_t& p) + { + point_t result; + crypto::public_key pk = p.to_public_key(); + + ge_bytes_hash_to_ec_32(&result.m_p3, (const unsigned char*)&pk); + + return result; + } + + static point_t hp(const crypto::public_key& p) + { + point_t result; + ge_bytes_hash_to_ec_32(&result.m_p3, (const unsigned char*)&p); + return result; + } + }; // hash_helper_t struct + + +} // namespace crypto diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index df6c31e..a1e0c64 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying @@ -18,12 +18,20 @@ #include "crypto.h" #include "hash.h" +#if !defined(NDEBUG) +# define crypto_assert(expression) assert(expression) +#else +# define crypto_assert(expression) ((void)0) +#endif + namespace crypto { DISABLE_GCC_AND_CLANG_WARNING(strict-aliasing) + const unsigned char Z_[32] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; const unsigned char I_[32] = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; const unsigned char L_[32] = { 0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 }; + const key_image Z = *reinterpret_cast(&Z_); const key_image I = *reinterpret_cast(&I_); const key_image L = *reinterpret_cast(&L_); @@ -77,34 +85,31 @@ namespace crypto { memcpy(&res, tmp, 32); } - void crypto_ops::keys_from_default(unsigned char* a_part, public_key &pub, secret_key &sec, size_t brain_wallet_seed_size) - { - unsigned char tmp[64] = { 0 }; - - if (!(sizeof(tmp) >= brain_wallet_seed_size)) - { - throw std::runtime_error("size mismatch"); - } - - memcpy(tmp, a_part, brain_wallet_seed_size); - - cn_fast_hash(tmp, 32, (char*)&tmp[32]); - - sc_reduce(tmp); - memcpy(&sec, tmp, 32); - ge_p3 point; - ge_scalarmult_base(&point, &sec); - ge_p3_tobytes(&pub, &point); - } - - void crypto_ops::generate_brain_keys(public_key &pub, secret_key &sec, std::string& seed, size_t brain_wallet_seed_size) + void crypto_ops::keys_from_default(const unsigned char* a_part, public_key &pub, secret_key &sec, size_t keys_seed_binary_size) { - std::vector tmp_vector; - tmp_vector.resize(brain_wallet_seed_size, 0); - unsigned char *tmp = &tmp_vector[0]; - generate_random_bytes(brain_wallet_seed_size, tmp); - seed.assign((const char*)tmp, brain_wallet_seed_size); - keys_from_default(tmp, pub, sec, brain_wallet_seed_size); + unsigned char tmp[64] = { 0 }; + + if (!(sizeof(tmp) >= keys_seed_binary_size)) + { + throw std::runtime_error("size mismatch"); + } + + memcpy(tmp, a_part, keys_seed_binary_size); + + cn_fast_hash(tmp, 32, (char*)&tmp[32]); + + sc_reduce(tmp); + memcpy(&sec, tmp, 32); + ge_p3 point; + ge_scalarmult_base(&point, &sec); + ge_p3_tobytes(&pub, &point); + } + + void crypto_ops::generate_seed_keys(public_key &pub, secret_key &sec, std::vector& keys_seed_binary, size_t keys_seed_binary_size) + { + keys_seed_binary.resize(keys_seed_binary_size, 0); + generate_random_bytes(keys_seed_binary_size, keys_seed_binary.data()); + keys_from_default(keys_seed_binary.data(), pub, sec, keys_seed_binary_size); } static inline void hash_to_scalar(const void *data, size_t length, ec_scalar &res) @@ -141,8 +146,8 @@ namespace crypto { { ge_p3 A = ge_p3(); ge_p2 R = ge_p2(); - // maybe use assert instead? - ge_frombytes_vartime(&A, reinterpret_cast(&P)); + if (ge_frombytes_vartime(&A, reinterpret_cast(&P)) != 0) + return Z; ge_scalarmult(&R, reinterpret_cast(&a), &A); key_image a_p = key_image(); ge_tobytes(reinterpret_cast(&a_p), &R); @@ -152,9 +157,8 @@ namespace crypto { bool crypto_ops::validate_key_image(const key_image& ki) { if (!(scalarmult_key(ki, L) == I)) - { return false; - } + return true; } @@ -172,7 +176,7 @@ namespace crypto { ge_p3 point; ge_p2 point2; ge_p1p1 point3; - assert(sc_check(&key2) == 0); + crypto_assert(sc_check(&key2) == 0); if (ge_frombytes_vartime(&point, &key1) != 0) { return false; } @@ -191,7 +195,11 @@ namespace crypto { char *end = buf.output_index; buf.derivation = derivation; tools::write_varint(end, output_index); - assert(end <= buf.output_index + sizeof buf.output_index); + if (!(end <= buf.output_index + sizeof buf.output_index)) + { + crypto_assert(false); + return; + } hash_to_scalar(&buf, end - reinterpret_cast(&buf), res); } @@ -218,7 +226,7 @@ namespace crypto { void crypto_ops::derive_secret_key(const key_derivation &derivation, size_t output_index, const secret_key &base, secret_key &derived_key) { ec_scalar scalar; - assert(sc_check(&base) == 0); + crypto_assert(sc_check(&base) == 0); derivation_to_scalar(derivation, output_index, scalar); sc_add(&derived_key, &base, &scalar); } @@ -238,10 +246,10 @@ namespace crypto { { ge_p3 t; public_key t2; - assert(sc_check(&sec) == 0); + crypto_assert(sc_check(&sec) == 0); ge_scalarmult_base(&t, &sec); ge_p3_tobytes(&t2, &t); - assert(pub == t2); + crypto_assert(pub == t2); } #endif buf.h = prefix_hash; @@ -258,7 +266,7 @@ namespace crypto { ge_p3 tmp3; ec_scalar c; s_comm buf; - assert(check_key(pub)); + crypto_assert(check_key(pub)); buf.h = prefix_hash; buf.key = pub; if (ge_frombytes_vartime(&tmp3, &pub) != 0) { @@ -287,23 +295,25 @@ namespace crypto { void crypto_ops::generate_key_image(const public_key &pub, const secret_key &sec, key_image &image) { ge_p3 point; ge_p2 point2; - assert(sc_check(&sec) == 0); + crypto_assert(sc_check(&sec) == 0); hash_to_ec(pub, point); ge_scalarmult(&point2, &sec, &point); ge_tobytes(&image, &point2); } -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4200) -struct rs_comm_entry -{ - ec_point a, b; -}; - struct rs_comm { + struct rs_comm_entry + { + ec_point a, b; + }; + + struct rs_comm + { hash h; struct rs_comm_entry ab[]; }; -POP_WARNINGS +POP_VS_WARNINGS static inline size_t rs_comm_size(size_t pubs_count) { return sizeof(rs_comm)+pubs_count * sizeof(rs_comm_entry); @@ -319,20 +329,24 @@ POP_WARNINGS ge_dsmp image_pre; ec_scalar sum, k, h; rs_comm *const buf = reinterpret_cast(alloca(rs_comm_size(pubs_count))); - assert(sec_index < pubs_count); + if (!(sec_index < pubs_count)) + { + crypto_assert(false); + return; + } #if !defined(NDEBUG) { ge_p3 t; public_key t2; key_image t3; - assert(sc_check(&sec) == 0); + crypto_assert(sc_check(&sec) == 0); ge_scalarmult_base(&t, &sec); ge_p3_tobytes(&t2, &t); - assert(*pubs[sec_index] == t2); + crypto_assert(*pubs[sec_index] == t2); generate_key_image(*pubs[sec_index], sec, t3); - assert(image == t3); + crypto_assert(image == t3); for (i = 0; i < pubs_count; i++) { - assert(check_key(*pubs[i])); + crypto_assert(check_key(*pubs[i])); } } #endif @@ -381,7 +395,7 @@ POP_WARNINGS rs_comm *const buf = reinterpret_cast(alloca(rs_comm_size(pubs_count))); #if !defined(NDEBUG) for (i = 0; i < pubs_count; i++) { - assert(check_key(*pubs[i])); + crypto_assert(check_key(*pubs[i])); } #endif if (ge_frombytes_vartime(&image_unp, &image) != 0) { @@ -399,10 +413,10 @@ POP_WARNINGS if (ge_frombytes_vartime(&tmp3, &*pubs[i]) != 0) { return false; } - ge_double_scalarmult_base_vartime(&tmp2, &sig[i].c, &tmp3, &sig[i].r); + ge_double_scalarmult_base_vartime(&tmp2, &sig[i].c, &tmp3, &sig[i].r); // L_i = r_i * G + c_i * P_i ge_tobytes(&buf->ab[i].a, &tmp2); hash_to_ec(*pubs[i], tmp3); - ge_double_scalarmult_precomp_vartime(&tmp2, &sig[i].r, &tmp3, &sig[i].c, image_pre); + ge_double_scalarmult_precomp_vartime(&tmp2, &sig[i].r, &tmp3, &sig[i].c, image_pre); // R_i = r_i * Hp(P_i) + c_i * I ge_tobytes(&buf->ab[i].b, &tmp2); sc_add(&sum, &sum, &sig[i].c); } @@ -410,4 +424,5 @@ POP_WARNINGS sc_sub(&h, &h, &sum); return sc_isnonzero(&h) == 0; } -} + +} // namespace crypto diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index d039eee..64df244 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "common/pod-class.h" #include "generic-ops.h" @@ -17,7 +18,7 @@ #include "warnings.h" -PUSH_WARNINGS +PUSH_GCC_WARNINGS DISABLE_CLANG_WARNING(unused-private-field) @@ -73,10 +74,10 @@ namespace crypto { static void generate_keys(public_key &, secret_key &); friend void generate_keys(public_key &, secret_key &); - static void generate_brain_keys(public_key &, secret_key &, std::string& seed, size_t brain_wallet_seed_size); - friend void generate_brain_keys(public_key &, secret_key &, std::string& seed, size_t brain_wallet_seed_size); - static void keys_from_default(unsigned char* a_part, public_key &pub, secret_key &sec, size_t brain_wallet_seed_size); - friend void keys_from_default(unsigned char* a_part, public_key &pub, secret_key &sec, size_t brain_wallet_seed_size); + static void generate_seed_keys(public_key &pub, secret_key &sec, std::vector& keys_seed_binary, size_t keys_seed_binary_size); + friend void generate_seed_keys(public_key &pub, secret_key &sec, std::vector& keys_seed_binary, size_t keys_seed_binary_size); + static void keys_from_default(const unsigned char* a_part, public_key &pub, secret_key &sec, size_t keys_seed_binary_size); + friend void keys_from_default(const unsigned char* a_part, public_key &pub, secret_key &sec, size_t keys_seed_binary_size); static void dependent_key(const secret_key& first, secret_key& second); friend void dependent_key(const secret_key& first, secret_key& second); static bool check_key(const public_key &); @@ -135,14 +136,14 @@ namespace crypto { crypto_ops::generate_keys(pub, sec); } - inline void generate_brain_keys(public_key &pub, secret_key &sec, std::string& seed, size_t brain_wallet_seed_size) { - crypto_ops::generate_brain_keys(pub, sec, seed, brain_wallet_seed_size); + inline void generate_seed_keys(public_key &pub, secret_key &sec, std::vector& keys_seed_binary, size_t keys_seed_binary_size) + { + crypto_ops::generate_seed_keys(pub, sec, keys_seed_binary, keys_seed_binary_size); } - - inline void keys_from_default(unsigned char* a_part, public_key &pub, secret_key &sec, size_t brain_wallet_seed_size) + inline void keys_from_default(const unsigned char* a_part, public_key &pub, secret_key &sec, size_t keys_seed_binary_size) { - crypto_ops::keys_from_default(a_part, pub, sec, brain_wallet_seed_size); + crypto_ops::keys_from_default(a_part, pub, sec, keys_seed_binary_size); } inline void dependent_key(const secret_key& first, secret_key& second){ @@ -226,13 +227,76 @@ namespace crypto { return check_ring_signature(prefix_hash, image, pubs.data(), pubs.size(), sig); } -} + class stream_cn_hash + { + public: + static constexpr size_t DATA_BLOCK_SIZE = 1024 * 1024; + stream_cn_hash() + : m_buffer(HASH_SIZE + DATA_BLOCK_SIZE, '\0') + , m_p_hash(const_cast(reinterpret_cast(m_buffer.data()))) + , m_p_data(const_cast(reinterpret_cast(m_buffer.data())) + HASH_SIZE) + , m_ready(false) + , m_data_used(0) + { + m_ready = true; + } -POD_MAKE_COMPARABLE(crypto, public_key) + bool update(const void* data, size_t size) + { + if (!m_ready) + return false; + + const uint8_t* p_source_data = reinterpret_cast(data); + + while(size > 0) + { + // fill the buffer up + size_t bytes_to_copy = std::min(size, DATA_BLOCK_SIZE - m_data_used); + memcpy(m_p_data + m_data_used, p_source_data, bytes_to_copy); + m_data_used += bytes_to_copy; + p_source_data += bytes_to_copy; + size -= bytes_to_copy; + + if (m_data_used == DATA_BLOCK_SIZE) + { + // calc imtermediate hash of the whole buffer and put the result into the beginning of the buffer + *m_p_hash = cn_fast_hash(m_buffer.data(), HASH_SIZE + m_data_used); + // clear data buffer for new bytes + memset(m_p_data, 0, DATA_BLOCK_SIZE); + m_data_used = 0; + } + + // repeat if there are source bytes left + } + + return true; + } + + hash calculate_hash() + { + if (m_data_used == 0) + return *m_p_hash; + + m_ready = false; + return cn_fast_hash(m_buffer.data(), HASH_SIZE + m_data_used); + } + + private: + const std::string m_buffer; + hash* const m_p_hash; + uint8_t* const m_p_data; + size_t m_data_used; + bool m_ready; + }; // class stream_cn_hash + +} // namespace crypto + +POD_MAKE_HASHABLE(crypto, public_key) POD_MAKE_COMPARABLE(crypto, secret_key) POD_MAKE_HASHABLE(crypto, key_image) POD_MAKE_COMPARABLE(crypto, signature) POD_MAKE_COMPARABLE(crypto, key_derivation) POD_MAKE_LESS_OPERATOR(crypto, hash) POD_MAKE_LESS_OPERATOR(crypto, key_image) +POP_GCC_WARNINGS \ No newline at end of file diff --git a/src/crypto/ecrypt-config.h b/src/crypto/ecrypt-config.h new file mode 100644 index 0000000..9176de1 --- /dev/null +++ b/src/crypto/ecrypt-config.h @@ -0,0 +1,272 @@ +/* ecrypt-config.h */ + +/* *** Normally, it should not be necessary to edit this file. *** */ + +#ifndef ECRYPT_CONFIG +#define ECRYPT_CONFIG + +/* ------------------------------------------------------------------------- */ + +/* Guess the endianness of the target architecture. */ + +/* +* The LITTLE endian machines: +*/ +#if defined(__ultrix) /* Older MIPS */ +#define ECRYPT_LITTLE_ENDIAN +#elif defined(__alpha) /* Alpha */ +#define ECRYPT_LITTLE_ENDIAN +#elif defined(i386) /* x86 (gcc) */ +#define ECRYPT_LITTLE_ENDIAN +#elif defined(__i386) /* x86 (gcc) */ +#define ECRYPT_LITTLE_ENDIAN +#elif defined(_M_IX86) /* x86 (MSC, Borland) */ +#define ECRYPT_LITTLE_ENDIAN +#elif defined(_MSC_VER) /* x86 (surely MSC) */ +#define ECRYPT_LITTLE_ENDIAN +#elif defined(__INTEL_COMPILER) /* x86 (surely Intel compiler icl.exe) */ +#define ECRYPT_LITTLE_ENDIAN + +/* +* The BIG endian machines: +*/ +#elif defined(sun) /* Newer Sparc's */ +#define ECRYPT_BIG_ENDIAN +#elif defined(__ppc__) /* PowerPC */ +#define ECRYPT_BIG_ENDIAN + +/* +* Finally machines with UNKNOWN endianness: +*/ +#elif defined (_AIX) /* RS6000 */ +#define ECRYPT_UNKNOWN +#elif defined(__hpux) /* HP-PA */ +#define ECRYPT_UNKNOWN +#elif defined(__aux) /* 68K */ +#define ECRYPT_UNKNOWN +#elif defined(__dgux) /* 88K (but P6 in latest boxes) */ +#define ECRYPT_UNKNOWN +#elif defined(__sgi) /* Newer MIPS */ +#define ECRYPT_UNKNOWN +#else /* Any other processor */ +#define ECRYPT_UNKNOWN +#endif + +/* ------------------------------------------------------------------------- */ + +/* +* Find minimal-width types to store 8-bit, 16-bit, 32-bit, and 64-bit +* integers. +* +* Note: to enable 64-bit types on 32-bit compilers, it might be +* necessary to switch from ISO C90 mode to ISO C99 mode (e.g., gcc +* -std=c99). +*/ + +#include + +/* --- check char --- */ + +#if (UCHAR_MAX / 0xFU > 0xFU) +#ifndef I8T +#define I8T char +#define U8C(v) (v##U) + +#if (UCHAR_MAX == 0xFFU) +#define ECRYPT_I8T_IS_BYTE +#endif + +#endif + +#if (UCHAR_MAX / 0xFFU > 0xFFU) +#ifndef I16T +#define I16T char +#define U16C(v) (v##U) +#endif + +#if (UCHAR_MAX / 0xFFFFU > 0xFFFFU) +#ifndef I32T +#define I32T char +#define U32C(v) (v##U) +#endif + +#if (UCHAR_MAX / 0xFFFFFFFFU > 0xFFFFFFFFU) +#ifndef I64T +#define I64T char +#define U64C(v) (v##U) +#define ECRYPT_NATIVE64 +#endif + +#endif +#endif +#endif +#endif + +/* --- check short --- */ + +#if (USHRT_MAX / 0xFU > 0xFU) +#ifndef I8T +#define I8T short +#define U8C(v) (v##U) + +#if (USHRT_MAX == 0xFFU) +#define ECRYPT_I8T_IS_BYTE +#endif + +#endif + +#if (USHRT_MAX / 0xFFU > 0xFFU) +#ifndef I16T +#define I16T short +#define U16C(v) (v##U) +#endif + +#if (USHRT_MAX / 0xFFFFU > 0xFFFFU) +#ifndef I32T +#define I32T short +#define U32C(v) (v##U) +#endif + +#if (USHRT_MAX / 0xFFFFFFFFU > 0xFFFFFFFFU) +#ifndef I64T +#define I64T short +#define U64C(v) (v##U) +#define ECRYPT_NATIVE64 +#endif + +#endif +#endif +#endif +#endif + +/* --- check int --- */ + +#if (UINT_MAX / 0xFU > 0xFU) +#ifndef I8T +#define I8T int +#define U8C(v) (v##U) + +#if (ULONG_MAX == 0xFFU) +#define ECRYPT_I8T_IS_BYTE +#endif + +#endif + +#if (UINT_MAX / 0xFFU > 0xFFU) +#ifndef I16T +#define I16T int +#define U16C(v) (v##U) +#endif + +#if (UINT_MAX / 0xFFFFU > 0xFFFFU) +#ifndef I32T +#define I32T int +#define U32C(v) (v##U) +#endif + +#if (UINT_MAX / 0xFFFFFFFFU > 0xFFFFFFFFU) +#ifndef I64T +#define I64T int +#define U64C(v) (v##U) +#define ECRYPT_NATIVE64 +#endif + +#endif +#endif +#endif +#endif + +/* --- check long --- */ + +#if (ULONG_MAX / 0xFUL > 0xFUL) +#ifndef I8T +#define I8T long +#define U8C(v) (v##UL) + +#if (ULONG_MAX == 0xFFUL) +#define ECRYPT_I8T_IS_BYTE +#endif + +#endif + +#if (ULONG_MAX / 0xFFUL > 0xFFUL) +#ifndef I16T +#define I16T long +#define U16C(v) (v##UL) +#endif + +#if (ULONG_MAX / 0xFFFFUL > 0xFFFFUL) +#ifndef I32T +#define I32T long +#define U32C(v) (v##UL) +#endif + +#if (ULONG_MAX / 0xFFFFFFFFUL > 0xFFFFFFFFUL) +#ifndef I64T +#define I64T long +#define U64C(v) (v##UL) +#define ECRYPT_NATIVE64 +#endif + +#endif +#endif +#endif +#endif + +/* --- check long long --- */ + +#ifdef ULLONG_MAX + +#if (ULLONG_MAX / 0xFULL > 0xFULL) +#ifndef I8T +#define I8T long long +#define U8C(v) (v##ULL) + +#if (ULLONG_MAX == 0xFFULL) +#define ECRYPT_I8T_IS_BYTE +#endif + +#endif + +#if (ULLONG_MAX / 0xFFULL > 0xFFULL) +#ifndef I16T +#define I16T long long +#define U16C(v) (v##ULL) +#endif + +#if (ULLONG_MAX / 0xFFFFULL > 0xFFFFULL) +#ifndef I32T +#define I32T long long +#define U32C(v) (v##ULL) +#endif + +#if (ULLONG_MAX / 0xFFFFFFFFULL > 0xFFFFFFFFULL) +#ifndef I64T +#define I64T long long +#define U64C(v) (v##ULL) +#endif + +#endif +#endif +#endif +#endif + +#endif + +/* --- check __int64 --- */ + +#ifdef _UI64_MAX + +#if (_UI64_MAX / 0xFFFFFFFFui64 > 0xFFFFFFFFui64) +#ifndef I64T +#define I64T __int64 +#define U64C(v) (v##ui64) +#endif + +#endif + +#endif + +/* ------------------------------------------------------------------------- */ + +#endif \ No newline at end of file diff --git a/src/crypto/ecrypt-machine.h b/src/crypto/ecrypt-machine.h new file mode 100644 index 0000000..83356b1 --- /dev/null +++ b/src/crypto/ecrypt-machine.h @@ -0,0 +1,46 @@ +/* ecrypt-machine.h */ + +/* +* This file is included by 'ecrypt-portable.h'. It allows to override +* the default macros for specific platforms. Please carefully check +* the machine code generated by your compiler (with optimisations +* turned on) before deciding to edit this file. +*/ + +/* ------------------------------------------------------------------------- */ + +#if (defined(ECRYPT_DEFAULT_ROT) && !defined(ECRYPT_MACHINE_ROT)) + +#define ECRYPT_MACHINE_ROT + +#if (defined(WIN32) && defined(_MSC_VER)) + +#undef ROTL32 +#undef ROTR32 +#undef ROTL64 +#undef ROTR64 + +#include + +#define ROTL32(v, n) _lrotl(v, n) +#define ROTR32(v, n) _lrotr(v, n) +#define ROTL64(v, n) _rotl64(v, n) +#define ROTR64(v, n) _rotr64(v, n) + +#endif + +#endif + +/* ------------------------------------------------------------------------- */ + +#if (defined(ECRYPT_DEFAULT_SWAP) && !defined(ECRYPT_MACHINE_SWAP)) + +#define ECRYPT_MACHINE_SWAP + +/* +* If you want to overwrite the default swap macros, put it here. And so on. +*/ + +#endif + +/* ------------------------------------------------------------------------- */ \ No newline at end of file diff --git a/src/crypto/ecrypt-portable.h b/src/crypto/ecrypt-portable.h new file mode 100644 index 0000000..4baa4d6 --- /dev/null +++ b/src/crypto/ecrypt-portable.h @@ -0,0 +1,303 @@ +/* ecrypt-portable.h */ + +/* +* WARNING: the conversions defined below are implemented as macros, +* and should be used carefully. They should NOT be used with +* parameters which perform some action. E.g., the following two lines +* are not equivalent: +* +* 1) ++x; y = ROTL32(x, n); +* 2) y = ROTL32(++x, n); +*/ + +/* +* *** Please do not edit this file. *** +* +* The default macros can be overridden for specific architectures by +* editing 'ecrypt-machine.h'. +*/ + +#ifndef ECRYPT_PORTABLE +#define ECRYPT_PORTABLE + +#include "ecrypt-config.h" + +/* ------------------------------------------------------------------------- */ + +/* +* The following types are defined (if available): +* +* u8: unsigned integer type, at least 8 bits +* u16: unsigned integer type, at least 16 bits +* u32: unsigned integer type, at least 32 bits +* u64: unsigned integer type, at least 64 bits +* +* s8, s16, s32, s64 -> signed counterparts of u8, u16, u32, u64 +* +* The selection of minimum-width integer types is taken care of by +* 'ecrypt-config.h'. Note: to enable 64-bit types on 32-bit +* compilers, it might be necessary to switch from ISO C90 mode to ISO +* C99 mode (e.g., gcc -std=c99). +*/ + +#ifdef I8T +typedef signed I8T s8; +typedef unsigned I8T u8; +#endif + +#ifdef I16T +typedef signed I16T s16; +typedef unsigned I16T u16; +#endif + +#ifdef I32T +typedef signed I32T s32; +typedef unsigned I32T u32; +#endif + +#ifdef I64T +typedef signed I64T s64; +typedef unsigned I64T u64; +#endif + +/* +* The following macros are used to obtain exact-width results. +*/ + +#define U8V(v) ((u8)(v) & U8C(0xFF)) +#define U16V(v) ((u16)(v) & U16C(0xFFFF)) +#define U32V(v) ((u32)(v) & U32C(0xFFFFFFFF)) +#define U64V(v) ((u64)(v) & U64C(0xFFFFFFFFFFFFFFFF)) + +/* ------------------------------------------------------------------------- */ + +/* +* The following macros return words with their bits rotated over n +* positions to the left/right. +*/ + +#define ECRYPT_DEFAULT_ROT + +#define ROTL8(v, n) \ + (U8V((v) << (n)) | ((v) >> (8 - (n)))) + +#define ROTL16(v, n) \ + (U16V((v) << (n)) | ((v) >> (16 - (n)))) + +#define ROTL32(v, n) \ + (U32V((v) << (n)) | ((v) >> (32 - (n)))) + +#define ROTL64(v, n) \ + (U64V((v) << (n)) | ((v) >> (64 - (n)))) + +#define ROTR8(v, n) ROTL8(v, 8 - (n)) +#define ROTR16(v, n) ROTL16(v, 16 - (n)) +#define ROTR32(v, n) ROTL32(v, 32 - (n)) +#define ROTR64(v, n) ROTL64(v, 64 - (n)) + +#include "ecrypt-machine.h" + +/* ------------------------------------------------------------------------- */ + +/* +* The following macros return a word with bytes in reverse order. +*/ + +#define ECRYPT_DEFAULT_SWAP + +#define SWAP16(v) \ + ROTL16(v, 8) + +#define SWAP32(v) \ + ((ROTL32(v, 8) & U32C(0x00FF00FF)) | \ + (ROTL32(v, 24) & U32C(0xFF00FF00))) + +#ifdef ECRYPT_NATIVE64 +#define SWAP64(v) \ + ((ROTL64(v, 8) & U64C(0x000000FF000000FF)) | \ + (ROTL64(v, 24) & U64C(0x0000FF000000FF00)) | \ + (ROTL64(v, 40) & U64C(0x00FF000000FF0000)) | \ + (ROTL64(v, 56) & U64C(0xFF000000FF000000))) +#else +#define SWAP64(v) \ + (((u64)SWAP32(U32V(v)) << 32) | (u64)SWAP32(U32V(v >> 32))) +#endif + +#include "ecrypt-machine.h" + +#define ECRYPT_DEFAULT_WTOW + +#ifdef ECRYPT_LITTLE_ENDIAN +#define U16TO16_LITTLE(v) (v) +#define U32TO32_LITTLE(v) (v) +#define U64TO64_LITTLE(v) (v) + +#define U16TO16_BIG(v) SWAP16(v) +#define U32TO32_BIG(v) SWAP32(v) +#define U64TO64_BIG(v) SWAP64(v) +#endif + +#ifdef ECRYPT_BIG_ENDIAN +#define U16TO16_LITTLE(v) SWAP16(v) +#define U32TO32_LITTLE(v) SWAP32(v) +#define U64TO64_LITTLE(v) SWAP64(v) + +#define U16TO16_BIG(v) (v) +#define U32TO32_BIG(v) (v) +#define U64TO64_BIG(v) (v) +#endif + +#include "ecrypt-machine.h" + +/* +* The following macros load words from an array of bytes with +* different types of endianness, and vice versa. +*/ + +#define ECRYPT_DEFAULT_BTOW + +#if (!defined(ECRYPT_UNKNOWN) && defined(ECRYPT_I8T_IS_BYTE)) + +#define U8TO16_LITTLE(p) U16TO16_LITTLE(((u16*)(p))[0]) +#define U8TO32_LITTLE(p) U32TO32_LITTLE(((u32*)(p))[0]) +#define U8TO64_LITTLE(p) U64TO64_LITTLE(((u64*)(p))[0]) + +#define U8TO16_BIG(p) U16TO16_BIG(((u16*)(p))[0]) +#define U8TO32_BIG(p) U32TO32_BIG(((u32*)(p))[0]) +#define U8TO64_BIG(p) U64TO64_BIG(((u64*)(p))[0]) + +#define U16TO8_LITTLE(p, v) (((u16*)(p))[0] = U16TO16_LITTLE(v)) +#define U32TO8_LITTLE(p, v) (((u32*)(p))[0] = U32TO32_LITTLE(v)) +#define U64TO8_LITTLE(p, v) (((u64*)(p))[0] = U64TO64_LITTLE(v)) + +#define U16TO8_BIG(p, v) (((u16*)(p))[0] = U16TO16_BIG(v)) +#define U32TO8_BIG(p, v) (((u32*)(p))[0] = U32TO32_BIG(v)) +#define U64TO8_BIG(p, v) (((u64*)(p))[0] = U64TO64_BIG(v)) + +#else + +#define U8TO16_LITTLE(p) \ + (((u16)((p)[0]) ) | \ + ((u16)((p)[1]) << 8)) + +#define U8TO32_LITTLE(p) \ + (((u32)((p)[0]) ) | \ + ((u32)((p)[1]) << 8) | \ + ((u32)((p)[2]) << 16) | \ + ((u32)((p)[3]) << 24)) + +#ifdef ECRYPT_NATIVE64 +#define U8TO64_LITTLE(p) \ + (((u64)((p)[0]) ) | \ + ((u64)((p)[1]) << 8) | \ + ((u64)((p)[2]) << 16) | \ + ((u64)((p)[3]) << 24) | \ + ((u64)((p)[4]) << 32) | \ + ((u64)((p)[5]) << 40) | \ + ((u64)((p)[6]) << 48) | \ + ((u64)((p)[7]) << 56)) +#else +#define U8TO64_LITTLE(p) \ + ((u64)U8TO32_LITTLE(p) | ((u64)U8TO32_LITTLE((p) + 4) << 32)) +#endif + +#define U8TO16_BIG(p) \ + (((u16)((p)[0]) << 8) | \ + ((u16)((p)[1]) )) + +#define U8TO32_BIG(p) \ + (((u32)((p)[0]) << 24) | \ + ((u32)((p)[1]) << 16) | \ + ((u32)((p)[2]) << 8) | \ + ((u32)((p)[3]) )) + +#ifdef ECRYPT_NATIVE64 +#define U8TO64_BIG(p) \ + (((u64)((p)[0]) << 56) | \ + ((u64)((p)[1]) << 48) | \ + ((u64)((p)[2]) << 40) | \ + ((u64)((p)[3]) << 32) | \ + ((u64)((p)[4]) << 24) | \ + ((u64)((p)[5]) << 16) | \ + ((u64)((p)[6]) << 8) | \ + ((u64)((p)[7]) )) +#else +#define U8TO64_BIG(p) \ + (((u64)U8TO32_BIG(p) << 32) | (u64)U8TO32_BIG((p) + 4)) +#endif + +#define U16TO8_LITTLE(p, v) \ + do { \ + (p)[0] = U8V((v) ); \ + (p)[1] = U8V((v) >> 8); \ + } while (0) + +#define U32TO8_LITTLE(p, v) \ + do { \ + (p)[0] = U8V((v) ); \ + (p)[1] = U8V((v) >> 8); \ + (p)[2] = U8V((v) >> 16); \ + (p)[3] = U8V((v) >> 24); \ + } while (0) + +#ifdef ECRYPT_NATIVE64 +#define U64TO8_LITTLE(p, v) \ + do { \ + (p)[0] = U8V((v) ); \ + (p)[1] = U8V((v) >> 8); \ + (p)[2] = U8V((v) >> 16); \ + (p)[3] = U8V((v) >> 24); \ + (p)[4] = U8V((v) >> 32); \ + (p)[5] = U8V((v) >> 40); \ + (p)[6] = U8V((v) >> 48); \ + (p)[7] = U8V((v) >> 56); \ + } while (0) +#else +#define U64TO8_LITTLE(p, v) \ + do { \ + U32TO8_LITTLE((p), U32V((v) )); \ + U32TO8_LITTLE((p) + 4, U32V((v) >> 32)); \ + } while (0) +#endif + +#define U16TO8_BIG(p, v) \ + do { \ + (p)[0] = U8V((v) ); \ + (p)[1] = U8V((v) >> 8); \ + } while (0) + +#define U32TO8_BIG(p, v) \ + do { \ + (p)[0] = U8V((v) >> 24); \ + (p)[1] = U8V((v) >> 16); \ + (p)[2] = U8V((v) >> 8); \ + (p)[3] = U8V((v) ); \ + } while (0) + +#ifdef ECRYPT_NATIVE64 +#define U64TO8_BIG(p, v) \ + do { \ + (p)[0] = U8V((v) >> 56); \ + (p)[1] = U8V((v) >> 48); \ + (p)[2] = U8V((v) >> 40); \ + (p)[3] = U8V((v) >> 32); \ + (p)[4] = U8V((v) >> 24); \ + (p)[5] = U8V((v) >> 16); \ + (p)[6] = U8V((v) >> 8); \ + (p)[7] = U8V((v) ); \ + } while (0) +#else +#define U64TO8_BIG(p, v) \ + do { \ + U32TO8_BIG((p), U32V((v) >> 32)); \ + U32TO8_BIG((p) + 4, U32V((v) )); \ + } while (0) +#endif + +#endif + +#include "ecrypt-machine.h" + +/* ------------------------------------------------------------------------- */ + +#endif \ No newline at end of file diff --git a/src/crypto/ecrypt-sync.h b/src/crypto/ecrypt-sync.h new file mode 100644 index 0000000..cd5f5d4 --- /dev/null +++ b/src/crypto/ecrypt-sync.h @@ -0,0 +1,258 @@ +/* ecrypt-sync.h */ + +/* +* Header file for synchronous stream ciphers without authentication +* mechanism. +* +* *** Please only edit parts marked with "[edit]". *** +*/ + +#ifndef ECRYPT_SYNC_AE +#define ECRYPT_SYNC_AE + +#include "ecrypt-portable.h" + +/* ------------------------------------------------------------------------- */ + +/* Cipher parameters */ + +/* +* The name of your cipher. +*/ +#define ECRYPT_NAME "Salsa20 stream cipher" /* [edit] */ + +/* +* Specify which key and IV sizes are supported by your cipher. A user +* should be able to enumerate the supported sizes by running the +* following code: +* +* for (i = 0; ECRYPT_KEYSIZE(i) <= ECRYPT_MAXKEYSIZE; ++i) +* { +* keysize = ECRYPT_KEYSIZE(i); +* +* ... +* } +* +* All sizes are in bits. +*/ + +#define ECRYPT_MAXKEYSIZE 256 /* [edit] */ +#define ECRYPT_KEYSIZE(i) (128 + (i)*128) /* [edit] */ + +#define ECRYPT_MAXIVSIZE 64 /* [edit] */ +#define ECRYPT_IVSIZE(i) (64 + (i)*64) /* [edit] */ + +/* ------------------------------------------------------------------------- */ + +/* Data structures */ + +/* +* ECRYPT_ctx is the structure containing the representation of the +* internal state of your cipher. +*/ + +typedef struct +{ + u32 input[16]; /* could be compressed */ + /* + * [edit] + * + * Put here all state variable needed during the encryption process. + */ +} ECRYPT_ctx; + +/* ------------------------------------------------------------------------- */ + +/* Mandatory functions */ + +/* +* Key and message independent initialization. This function will be +* called once when the program starts (e.g., to build expanded S-box +* tables). +*/ +void ECRYPT_init(); + +/* +* Key setup. It is the user's responsibility to select the values of +* keysize and ivsize from the set of supported values specified +* above. +*/ +void ECRYPT_keysetup( + ECRYPT_ctx* ctx, + const u8* key, + u32 keysize, /* Key size in bits. */ + u32 ivsize); /* IV size in bits. */ + + /* + * IV setup. After having called ECRYPT_keysetup(), the user is + * allowed to call ECRYPT_ivsetup() different times in order to + * encrypt/decrypt different messages with the same key but different + * IV's. + */ +void ECRYPT_ivsetup( + ECRYPT_ctx* ctx, + const u8* iv); + +/* +* Encryption/decryption of arbitrary length messages. +* +* For efficiency reasons, the API provides two types of +* encrypt/decrypt functions. The ECRYPT_encrypt_bytes() function +* (declared here) encrypts byte strings of arbitrary length, while +* the ECRYPT_encrypt_blocks() function (defined later) only accepts +* lengths which are multiples of ECRYPT_BLOCKLENGTH. +* +* The user is allowed to make multiple calls to +* ECRYPT_encrypt_blocks() to incrementally encrypt a long message, +* but he is NOT allowed to make additional encryption calls once he +* has called ECRYPT_encrypt_bytes() (unless he starts a new message +* of course). For example, this sequence of calls is acceptable: +* +* ECRYPT_keysetup(); +* +* ECRYPT_ivsetup(); +* ECRYPT_encrypt_blocks(); +* ECRYPT_encrypt_blocks(); +* ECRYPT_encrypt_bytes(); +* +* ECRYPT_ivsetup(); +* ECRYPT_encrypt_blocks(); +* ECRYPT_encrypt_blocks(); +* +* ECRYPT_ivsetup(); +* ECRYPT_encrypt_bytes(); +* +* The following sequence is not: +* +* ECRYPT_keysetup(); +* ECRYPT_ivsetup(); +* ECRYPT_encrypt_blocks(); +* ECRYPT_encrypt_bytes(); +* ECRYPT_encrypt_blocks(); +*/ + +void ECRYPT_encrypt_bytes( + ECRYPT_ctx* ctx, + const u8* plaintext, + u8* ciphertext, + u32 msglen); /* Message length in bytes. */ + +void ECRYPT_decrypt_bytes( + ECRYPT_ctx* ctx, + const u8* ciphertext, + u8* plaintext, + u32 msglen); /* Message length in bytes. */ + + /* ------------------------------------------------------------------------- */ + + /* Optional features */ + + /* + * For testing purposes it can sometimes be useful to have a function + * which immediately generates keystream without having to provide it + * with a zero plaintext. If your cipher cannot provide this function + * (e.g., because it is not strictly a synchronous cipher), please + * reset the ECRYPT_GENERATES_KEYSTREAM flag. + */ + +#define ECRYPT_GENERATES_KEYSTREAM +#ifdef ECRYPT_GENERATES_KEYSTREAM + +void ECRYPT_keystream_bytes( + ECRYPT_ctx* ctx, + u8* keystream, + u32 length); /* Length of keystream in bytes. */ + +#endif + + /* ------------------------------------------------------------------------- */ + + /* Optional optimizations */ + + /* + * By default, the functions in this section are implemented using + * calls to functions declared above. However, you might want to + * implement them differently for performance reasons. + */ + + /* + * All-in-one encryption/decryption of (short) packets. + * + * The default definitions of these functions can be found in + * "ecrypt-sync.c". If you want to implement them differently, please + * undef the ECRYPT_USES_DEFAULT_ALL_IN_ONE flag. + */ +#define ECRYPT_USES_DEFAULT_ALL_IN_ONE /* [edit] */ + +void ECRYPT_encrypt_packet( + ECRYPT_ctx* ctx, + const u8* iv, + const u8* plaintext, + u8* ciphertext, + u32 msglen); + +void ECRYPT_decrypt_packet( + ECRYPT_ctx* ctx, + const u8* iv, + const u8* ciphertext, + u8* plaintext, + u32 msglen); + +/* +* Encryption/decryption of blocks. +* +* By default, these functions are defined as macros. If you want to +* provide a different implementation, please undef the +* ECRYPT_USES_DEFAULT_BLOCK_MACROS flag and implement the functions +* declared below. +*/ + +#define ECRYPT_BLOCKLENGTH 64 /* [edit] */ + +#define ECRYPT_USES_DEFAULT_BLOCK_MACROS /* [edit] */ +#ifdef ECRYPT_USES_DEFAULT_BLOCK_MACROS + +#define ECRYPT_encrypt_blocks(ctx, plaintext, ciphertext, blocks) \ + ECRYPT_encrypt_bytes(ctx, plaintext, ciphertext, \ + (blocks) * ECRYPT_BLOCKLENGTH) + +#define ECRYPT_decrypt_blocks(ctx, ciphertext, plaintext, blocks) \ + ECRYPT_decrypt_bytes(ctx, ciphertext, plaintext, \ + (blocks) * ECRYPT_BLOCKLENGTH) + +#ifdef ECRYPT_GENERATES_KEYSTREAM + +#define ECRYPT_keystream_blocks(ctx, keystream, blocks) \ + ECRYPT_AE_keystream_bytes(ctx, keystream, \ + (blocks) * ECRYPT_BLOCKLENGTH) + +#endif + +#else + +void ECRYPT_encrypt_blocks( + ECRYPT_ctx* ctx, + const u8* plaintext, + u8* ciphertext, + u32 blocks); /* Message length in blocks. */ + +void ECRYPT_decrypt_blocks( + ECRYPT_ctx* ctx, + const u8* ciphertext, + u8* plaintext, + u32 blocks); /* Message length in blocks. */ + +#ifdef ECRYPT_GENERATES_KEYSTREAM + +void ECRYPT_keystream_blocks( + ECRYPT_AE_ctx* ctx, + const u8* keystream, + u32 blocks); /* Keystream length in blocks. */ + +#endif + +#endif + + /* ------------------------------------------------------------------------- */ + +#endif \ No newline at end of file diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index b199700..fcc06d1 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -22,7 +22,7 @@ static inline const void *cpadd(const void *p, size_t i) { return (const char *) p + i; } -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4267) static_assert(sizeof(size_t) == 4 || sizeof(size_t) == 8, "size_t must be 4 or 8 bytes long"); static inline void place_length(uint8_t *buffer, size_t bufsize, size_t length) { @@ -32,7 +32,7 @@ static inline void place_length(uint8_t *buffer, size_t bufsize, size_t length) *(uint64_t *) padd(buffer, bufsize - 8) = swap64be(length); } } -POP_WARNINGS +POP_VS_WARNINGS #pragma pack(push, 1) union hash_state { @@ -52,8 +52,8 @@ void hash_process(union hash_state *state, const uint8_t *buf, size_t count); #define HASH_DATA_AREA 136 +void cn_fast_hash_old(const void *data, size_t length, char *hash); void cn_fast_hash(const void *data, size_t length, char *hash); -//void cn_slow_hash(const void *data, size_t length, char *hash); void hash_extra_blake(const void *data, size_t length, char *hash); void hash_extra_groestl(const void *data, size_t length, char *hash); diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 219c060..0d399f4 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -1,3 +1,4 @@ +// Copyright (c) 2020-2021 Zano project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,8 +18,14 @@ void hash_process(union hash_state *state, const uint8_t *buf, size_t count) { keccak1600(buf, (int)count, (uint8_t*)state); } -void cn_fast_hash(const void *data, size_t length, char *hash) { +void cn_fast_hash_old(const void *data, size_t length, char *hash) +{ union hash_state state; hash_process(&state, data, length); memcpy(hash, &state, HASH_SIZE); } + +void cn_fast_hash(const void *data, size_t length, char *hash) +{ + keccak(data, (int)length, (uint8_t*)hash, HASH_SIZE); +} diff --git a/src/crypto/hash.h b/src/crypto/hash.h index d803b6e..7fb0573 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -8,7 +8,11 @@ #include "common/pod-class.h" #include "generic-ops.h" - +#include "warnings.h" +PUSH_VS_WARNINGS +DISABLE_VS_WARNINGS(4804) +#include "blake2.h" +POP_VS_WARNINGS namespace crypto { extern "C" { @@ -41,6 +45,12 @@ namespace crypto { tree_hash(reinterpret_cast(hashes), count, reinterpret_cast(&root_hash)); } + inline hash blake2_hash(const void *data, std::size_t length) { + hash h; + blake2(&h, sizeof(h), data, length, nullptr, 0); + return h; + } + } POD_MAKE_HASHABLE(crypto, hash) diff --git a/src/crypto/random.c b/src/crypto/random.c index b16d39c..bdac1e7 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -10,14 +10,14 @@ //#include "initializer.h" #include "random.h" -static void generate_system_random_bytes(size_t n, void *result); +static_assert(RANDOM_STATE_SIZE >= HASH_DATA_AREA, "Invalid RANDOM_STATE_SIZE"); #if defined(_WIN32) #include #include -static void generate_system_random_bytes(size_t n, void *result) { +void generate_system_random_bytes(size_t n, void *result) { HCRYPTPROV prov; #define must_succeed(x) do if (!(x)) assert(0); while (0) if(!CryptAcquireContext(&prov, NULL, NULL, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) @@ -40,7 +40,7 @@ static void generate_system_random_bytes(size_t n, void *result) { #include #include -static void generate_system_random_bytes(size_t n, void *result) { +void generate_system_random_bytes(size_t n, void *result) { int fd; if ((fd = open("/dev/urandom", O_RDONLY | O_NOCTTY | O_CLOEXEC)) < 0) { err(EXIT_FAILURE, "open /dev/urandom"); @@ -68,7 +68,7 @@ static void generate_system_random_bytes(size_t n, void *result) { #endif -/* static */ union hash_state state; // NOTE: 'static' is commented out here to be able to store/load global random generator state in tests (see also random_helper.h) +static union hash_state state; #if !defined(NDEBUG) static volatile int curstate; /* To catch thread safety problems. */ @@ -105,6 +105,57 @@ void grant_random_initialize(void) } } +void random_prng_initialize_with_seed(uint64_t seed) +{ + grant_random_initialize(); +#if !defined(NDEBUG) + assert(curstate == 1); + curstate = 3; +#endif + memset(&state, 0, sizeof state); + memcpy(&state, &seed, sizeof seed); + for(size_t i = 0, count = seed & 31; i < count; ++i) + hash_permutation(&state); +#if !defined(NDEBUG) + assert(curstate == 3); + curstate = 1; +#endif +} + +void random_prng_get_state(void *state_buffer, const size_t buffer_size) +{ + grant_random_initialize(); +#if !defined(NDEBUG) + assert(curstate == 1); + curstate = 4; +#endif + + assert(sizeof state == buffer_size); + memcpy(state_buffer, &state, buffer_size); + +#if !defined(NDEBUG) + assert(curstate == 4); + curstate = 1; +#endif +} + +void random_prng_set_state(const void *state_buffer, const size_t buffer_size) +{ + grant_random_initialize(); +#if !defined(NDEBUG) + assert(curstate == 1); + curstate = 5; +#endif + + assert(sizeof state == buffer_size); + memcpy(&state, state_buffer, buffer_size); + +#if !defined(NDEBUG) + assert(curstate == 5); + curstate = 1; +#endif +} + void generate_random_bytes(size_t n, void *result) { grant_random_initialize(); #if !defined(NDEBUG) diff --git a/src/crypto/random.h b/src/crypto/random.h index 587d730..dbc7930 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -1,3 +1,5 @@ +// Copyright (c) 2018-2019 Zano Project +// Copyright (c) 2014-2018 The Boolberry developers // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -5,6 +7,31 @@ #pragma once #include +#include + +// use the cryptographically secure Pseudo-Random Number Generator provided by the operating system +void generate_system_random_bytes(size_t n, void *result); void generate_random_bytes(size_t n, void *result); -void grant_random_initialize(void); \ No newline at end of file + +// checks if PRNG is initialized and initializes it if necessary +void grant_random_initialize(void); + +#define RANDOM_STATE_SIZE 200 + +// explicitly define USE_INSECURE_RANDOM_RPNG_ROUTINES for using random_initialize_with_seed +#ifdef USE_INSECURE_RANDOM_RPNG_ROUTINES +// reinitializes PRNG with the given seed +// !!!ATTENTION!!!! Improper use of this routine may lead to SECURITY BREACH! +// Use with care and ONLY for tests or debug purposes! +void random_prng_initialize_with_seed(uint64_t seed); + +// gets internal RPNG state (state_buffer should be 200 bytes long) +void random_prng_get_state(void *state_buffer, const size_t buffer_size); + +// sets internal RPNG state (state_buffer should be 200 bytes long) +// !!!ATTENTION!!!! Improper use of this routine may lead to SECURITY BREACH! +// Use with care and ONLY for tests or debug purposes! +void random_prng_set_state(const void *state_buffer, const size_t buffer_size); + +#endif // #ifdef USE_INSECURE_RANDOM_RPNG_ROUTINES diff --git a/src/currency_core/account.cpp b/src/currency_core/account.cpp index a291feb..97d9480 100644 --- a/src/currency_core/account.cpp +++ b/src/currency_core/account.cpp @@ -17,34 +17,38 @@ using namespace std; -DISABLE_VS_WARNINGS(4244 4345) - - +//DISABLE_VS_WARNINGS(4244 4345) namespace currency { - - - //----------------------------------------------------------------- account_base::account_base() - :m_keys{} - ,m_creation_timestamp{} - ,m_seed{} { set_null(); } //----------------------------------------------------------------- void account_base::set_null() { + // fill sensitive data with random bytes + crypto::generate_random_bytes(sizeof m_keys.spend_secret_key, &m_keys.spend_secret_key); + crypto::generate_random_bytes(sizeof m_keys.view_secret_key, &m_keys.view_secret_key); + if (m_keys_seed_binary.size()) + crypto::generate_random_bytes(m_keys_seed_binary.size(), &m_keys_seed_binary[0]); + + // clear m_keys = account_keys(); + m_creation_timestamp = 0; + m_keys_seed_binary.clear(); } //----------------------------------------------------------------- - void account_base::generate() + void account_base::generate(bool auditable /* = false */) { - generate_brain_keys(m_keys.m_account_address.m_spend_public_key, m_keys.m_spend_secret_key, m_seed, BRAINWALLET_DEFAULT_SEED_SIZE); - dependent_key(m_keys.m_spend_secret_key, m_keys.m_view_secret_key); - if (!crypto::secret_key_to_public_key(m_keys.m_view_secret_key, m_keys.m_account_address.m_view_public_key)) + if (auditable) + m_keys.account_address.flags = ACCOUNT_PUBLIC_ADDRESS_FLAG_AUDITABLE; + + crypto::generate_seed_keys(m_keys.account_address.spend_public_key, m_keys.spend_secret_key, m_keys_seed_binary, BRAINWALLET_DEFAULT_SEED_SIZE); + crypto::dependent_key(m_keys.spend_secret_key, m_keys.view_secret_key); + if (!crypto::secret_key_to_public_key(m_keys.view_secret_key, m_keys.account_address.view_public_key)) throw std::runtime_error("Failed to create public view key"); @@ -56,81 +60,258 @@ namespace currency return m_keys; } //----------------------------------------------------------------- - std::string account_base::get_restore_data() const + void crypt_with_pass(const void* scr_data, std::size_t src_length, void* dst_data, const std::string& password) { - return m_seed; + crypto::chacha8_key key = AUTO_VAL_INIT(key); + crypto::generate_chacha8_key(password, key); + crypto::hash pass_hash = crypto::cn_fast_hash(password.data(), password.size()); + crypto::chacha8_iv iv = AUTO_VAL_INIT(iv); + CHECK_AND_ASSERT_THROW_MES(sizeof(pass_hash) >= sizeof(iv), "Invalid configuration: hash size is less than keys_file_data.iv"); + iv = *((crypto::chacha8_iv*)&pass_hash); + crypto::chacha8(scr_data, src_length, key, iv, (char*)dst_data); } - //----------------------------------------------------------------- - std::string account_base::get_restore_braindata() const + + std::string account_base::get_seed_phrase(const std::string& password) const { - std::string restore_buff = get_restore_data(); - std::vector v; - v.assign((unsigned char*)restore_buff.data(), (unsigned char*)restore_buff.data() + restore_buff.size()); - std::string seed_brain_data = tools::mnemonic_encoding::binary2text(v); - std::string timestamp_word = currency::get_word_from_timstamp(m_creation_timestamp); - seed_brain_data = seed_brain_data + timestamp_word; - return seed_brain_data; + if (m_keys_seed_binary.empty()) + return ""; + + std::vector processed_seed_binary = m_keys_seed_binary; + if (!password.empty()) + { + //encrypt seed phrase binary data + crypt_with_pass(&m_keys_seed_binary[0], m_keys_seed_binary.size(), &processed_seed_binary[0], password); + } + + std::string keys_seed_text = tools::mnemonic_encoding::binary2text(processed_seed_binary); + std::string timestamp_word = currency::get_word_from_timstamp(m_creation_timestamp, !password.empty()); + + // floor creation time to WALLET_BRAIN_DATE_QUANTUM to make checksum calculation stable + bool self_check_is_password_used = false; + uint64_t creation_timestamp_rounded = get_timstamp_from_word(timestamp_word, self_check_is_password_used); + CHECK_AND_ASSERT_THROW_MES(self_check_is_password_used == !password.empty(), "Account seed phrase internal error: password flag encoded wrong"); + + constexpr uint16_t checksum_max = tools::mnemonic_encoding::NUMWORDS >> 1; // maximum value of checksum + std::string binary_for_check_sum((const char*)&m_keys_seed_binary[0], m_keys_seed_binary.size()); + binary_for_check_sum.append(password); + crypto::hash h = crypto::cn_fast_hash(binary_for_check_sum.data(), binary_for_check_sum.size()); + *reinterpret_cast(&h) = creation_timestamp_rounded; + h = crypto::cn_fast_hash(&h, sizeof h); + uint64_t h_64 = *reinterpret_cast(&h); + uint16_t checksum = h_64 % (checksum_max + 1); + + uint8_t auditable_flag = 0; + if (m_keys.account_address.flags & ACCOUNT_PUBLIC_ADDRESS_FLAG_AUDITABLE) + auditable_flag = 1; + + uint32_t auditable_flag_and_checksum = (auditable_flag & 1) | (checksum << 1); + std::string auditable_flag_and_checksum_word = tools::mnemonic_encoding::word_by_num(auditable_flag_and_checksum); + + return keys_seed_text + " " + timestamp_word + " " + auditable_flag_and_checksum_word; } //----------------------------------------------------------------- - bool account_base::restore_keys(const std::string& restore_data) + std::string account_base::get_tracking_seed() const { - //CHECK_AND_ASSERT_MES(restore_data.size() == ACCOUNT_RESTORE_DATA_SIZE, false, "wrong restore data size"); - if (restore_data.size() == BRAINWALLET_DEFAULT_SEED_SIZE) - { - crypto::keys_from_default((unsigned char*)restore_data.data(), m_keys.m_account_address.m_spend_public_key, m_keys.m_spend_secret_key, BRAINWALLET_DEFAULT_SEED_SIZE); - } - else - { - LOG_ERROR("wrong restore data size=" << restore_data.size()); - return false; - } - m_seed = restore_data; - crypto::dependent_key(m_keys.m_spend_secret_key, m_keys.m_view_secret_key); - bool r = crypto::secret_key_to_public_key(m_keys.m_view_secret_key, m_keys.m_account_address.m_view_public_key); + return get_public_address_str() + ":" + + epee::string_tools::pod_to_hex(m_keys.view_secret_key) + + (m_creation_timestamp ? ":" : "") + (m_creation_timestamp ? epee::string_tools::num_to_string_fast(m_creation_timestamp) : ""); + } + //----------------------------------------------------------------- + bool account_base::restore_keys(const std::vector& keys_seed_binary) + { + CHECK_AND_ASSERT_MES(keys_seed_binary.size() == BRAINWALLET_DEFAULT_SEED_SIZE, false, "wrong restore data size: " << keys_seed_binary.size()); + crypto::keys_from_default(keys_seed_binary.data(), m_keys.account_address.spend_public_key, m_keys.spend_secret_key, keys_seed_binary.size()); + crypto::dependent_key(m_keys.spend_secret_key, m_keys.view_secret_key); + bool r = crypto::secret_key_to_public_key(m_keys.view_secret_key, m_keys.account_address.view_public_key); CHECK_AND_ASSERT_MES(r, false, "failed to secret_key_to_public_key for view key"); - set_createtime(0); return true; } //----------------------------------------------------------------- - bool account_base::restore_keys_from_braindata(const std::string& restore_data_) + bool account_base::restore_from_seed_phrase(const std::string& seed_phrase, const std::string& seed_password) { //cut the last timestamp word from restore_dats std::list words; - boost::split(words, restore_data_, boost::is_space()); - CHECK_AND_ASSERT_MES(words.size() == BRAINWALLET_DEFAULT_WORDS_COUNT, false, "Words count missmatch: " << words.size()); - - std::string timestamp_word = words.back(); - words.erase(--words.end()); - - std::string restore_data_local = boost::algorithm::join(words, " "); + boost::split(words, seed_phrase, boost::is_space()); - std::vector bin = tools::mnemonic_encoding::text2binary(restore_data_local); - if (!bin.size()) + std::string keys_seed_text, timestamp_word, auditable_flag_and_checksum_word; + if (words.size() == SEED_PHRASE_V1_WORDS_COUNT) + { + // 24 seed words + one timestamp word = 25 total + timestamp_word = words.back(); + words.erase(--words.end()); + keys_seed_text = boost::algorithm::join(words, " "); + } + else if (words.size() == SEED_PHRASE_V2_WORDS_COUNT) + { + // 24 seed words + one timestamp word + one flags & checksum = 26 total + auditable_flag_and_checksum_word = words.back(); + words.erase(--words.end()); + timestamp_word = words.back(); + words.erase(--words.end()); + keys_seed_text = boost::algorithm::join(words, " "); + } + else + { + LOG_ERROR("Invalid seed words count: " << words.size()); return false; + } + + uint64_t auditable_flag_and_checksum = UINT64_MAX; + if (!auditable_flag_and_checksum_word.empty()) + { + try { + auditable_flag_and_checksum = tools::mnemonic_encoding::num_by_word(auditable_flag_and_checksum_word); + } + catch (...) + { + return false; + } + + } + - std::string restore_buff((const char*)&bin[0], bin.size()); - bool r = restore_keys(restore_buff); + std::vector keys_seed_binary = tools::mnemonic_encoding::text2binary(keys_seed_text); + std::vector keys_seed_processed_binary = keys_seed_binary; + + + bool has_password = false; + try { + m_creation_timestamp = get_timstamp_from_word(timestamp_word, has_password); + } + catch (...) + { + return false; + } + //double check is password setting from timestamp word match with passed parameters + CHECK_AND_ASSERT_MES(has_password != seed_password.empty(), false, "Seed phrase password wrong interpretation"); + if (has_password) + { + CHECK_AND_ASSERT_MES(!seed_password.empty(), false, "Seed phrase password wrong interpretation: internal error"); + crypt_with_pass(&keys_seed_binary[0], keys_seed_binary.size(), &keys_seed_processed_binary[0], seed_password); + } + + CHECK_AND_ASSERT_MES(keys_seed_processed_binary.size(), false, "text2binary failed to convert the given text"); // don't prints event incorrect seed into the log for security + + bool auditable_flag = false; + + // check the checksum if checksum word provided + if (auditable_flag_and_checksum != UINT64_MAX) + { + auditable_flag = (auditable_flag_and_checksum & 1) != 0; // auditable flag is the lower 1 bit + uint16_t checksum = static_cast(auditable_flag_and_checksum >> 1); // checksum -- everything else + constexpr uint16_t checksum_max = tools::mnemonic_encoding::NUMWORDS >> 1; // maximum value of checksum + std::string binary_for_check_sum((const char*)&keys_seed_processed_binary[0], keys_seed_processed_binary.size()); + binary_for_check_sum.append(seed_password); + crypto::hash h = crypto::cn_fast_hash(binary_for_check_sum.data(), binary_for_check_sum.size()); + *reinterpret_cast(&h) = m_creation_timestamp; + h = crypto::cn_fast_hash(&h, sizeof h); + uint64_t h_64 = *reinterpret_cast(&h); + uint16_t checksum_calculated = h_64 % (checksum_max + 1); + if (checksum != checksum_calculated) + { + LOG_PRINT_L0("seed phase has invalid checksum: " << checksum_calculated << ", while " << checksum << " is expected, check your words"); + return false; + } + } + + bool r = restore_keys(keys_seed_processed_binary); CHECK_AND_ASSERT_MES(r, false, "restore_keys failed"); - m_creation_timestamp = get_timstamp_from_word(timestamp_word); + + m_keys_seed_binary = keys_seed_processed_binary; + + if (auditable_flag) + m_keys.account_address.flags |= ACCOUNT_PUBLIC_ADDRESS_FLAG_AUDITABLE; + return true; } //----------------------------------------------------------------- + bool account_base::is_seed_tracking(const std::string& seed_phrase) + { + return seed_phrase.find(':') != std::string::npos; + } + //----------------------------------------------------------------- + bool account_base::is_seed_password_protected(const std::string& seed_phrase, bool& is_password_protected) + { + //cut the last timestamp word from restore_dats + std::list words; + boost::split(words, seed_phrase, boost::is_space()); + + //let's validate each word + for (const auto& w: words) + { + if (!tools::mnemonic_encoding::valid_word(w)) + return false; + } + + std::string timestamp_word; + if (words.size() == SEED_PHRASE_V1_WORDS_COUNT) + { + // 24 seed words + one timestamp word = 25 total + timestamp_word = words.back(); + } + else if (words.size() == SEED_PHRASE_V2_WORDS_COUNT) + { + // 24 seed words + one timestamp word + one flags & checksum = 26 total + words.erase(--words.end()); + timestamp_word = words.back(); + } + else + { + return false; + } + + get_timstamp_from_word(timestamp_word, is_password_protected); + + return true; + } + //----------------------------------------------------------------- + bool account_base::restore_from_tracking_seed(const std::string& tracking_seed) + { + set_null(); + bool r = parse_tracking_seed(tracking_seed, m_keys.account_address, m_keys.view_secret_key, m_creation_timestamp); + return r; + } + //----------------------------------------------------------------- std::string account_base::get_public_address_str() const { //TODO: change this code into base 58 - return get_account_address_as_str(m_keys.m_account_address); + return get_account_address_as_str(m_keys.account_address); + } + //----------------------------------------------------------------- + void account_base::make_account_watch_only() + { + // keep only: + // timestamp + // view pub & spend pub + flags (public address) + // view sec + + // store to local tmp + uint64_t local_ts = m_creation_timestamp; + account_public_address local_addr = m_keys.account_address; + crypto::secret_key local_view_sec = m_keys.view_secret_key; + + // clear + set_null(); + + // restore + m_creation_timestamp = local_ts; + m_keys.account_address = local_addr; + m_keys.view_secret_key = local_view_sec; } //----------------------------------------------------------------- std::string transform_addr_to_str(const account_public_address& addr) { return get_account_address_as_str(addr); } - + //----------------------------------------------------------------- account_public_address transform_str_to_addr(const std::string& str) { account_public_address ad = AUTO_VAL_INIT(ad); - get_account_address_from_str(ad, str); + if (!get_account_address_from_str(ad, str)) + { + CHECK_AND_ASSERT_THROW_MES(false, "cannot parse address from string: " << str); + } return ad; } } \ No newline at end of file diff --git a/src/currency_core/account.h b/src/currency_core/account.h index 656c497..305b02e 100644 --- a/src/currency_core/account.h +++ b/src/currency_core/account.h @@ -12,7 +12,8 @@ #define BRAINWALLET_DEFAULT_SEED_SIZE 32 #define ACCOUNT_RESTORE_DATA_SIZE BRAINWALLET_DEFAULT_SEED_SIZE -#define BRAINWALLET_DEFAULT_WORDS_COUNT 25 +#define SEED_PHRASE_V1_WORDS_COUNT 25 +#define SEED_PHRASE_V2_WORDS_COUNT 26 @@ -29,14 +30,14 @@ namespace currency struct account_keys { - account_public_address m_account_address; - crypto::secret_key m_spend_secret_key; - crypto::secret_key m_view_secret_key; + account_public_address account_address; + crypto::secret_key spend_secret_key; + crypto::secret_key view_secret_key; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(m_account_address) - KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_spend_secret_key) - KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_view_secret_key) + KV_SERIALIZE_N(account_address, "m_account_address") + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(spend_secret_key, "m_spend_secret_key") + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(view_secret_key, "m_view_secret_key") END_KV_SERIALIZE_MAP() }; @@ -47,15 +48,15 @@ namespace currency { public: account_base(); - void generate(); + void generate(bool auditable = false); const account_keys& get_keys() const; - const account_public_address& get_public_address() const { return m_keys.m_account_address; }; + const account_public_address& get_public_address() const { return m_keys.account_address; }; std::string get_public_address_str() const; - std::string get_restore_data() const; - std::string get_restore_braindata() const; - - bool restore_keys(const std::string& restore_data); - bool restore_keys_from_braindata(const std::string& restore_data); + + std::string get_seed_phrase(const std::string& seed_password) const; + std::string get_tracking_seed() const; + bool restore_from_seed_phrase(const std::string& seed_phrase, const std::string& seed_password); + bool restore_from_tracking_seed(const std::string& tracking_seed); uint64_t get_createtime() const { return m_creation_timestamp; } void set_createtime(uint64_t val) { m_creation_timestamp = val; } @@ -63,25 +64,37 @@ namespace currency bool load(const std::string& file_path); bool store(const std::string& file_path); + void make_account_watch_only(); + bool is_watch_only() const { return m_keys.spend_secret_key == currency::null_skey; } + bool is_auditable() const { return m_keys.account_address.is_auditable(); } + template inline void serialize(t_archive &a, const unsigned int /*ver*/) { a & m_keys; a & m_creation_timestamp; - a & m_seed; + a & m_keys_seed_binary; } + static std::string vector_of_chars_to_string(const std::vector& v) { return std::string(v.begin(), v.end()); } + static std::vector string_to_vector_of_chars(const std::string& v) { return std::vector(v.begin(), v.end()); } + static bool is_seed_password_protected(const std::string& seed_phrase, bool& is_password_protected); + static bool is_seed_tracking(const std::string& seed_phrase); + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(m_keys) KV_SERIALIZE(m_creation_timestamp) - KV_SERIALIZE(m_seed) + KV_SERIALIZE_CUSTOM_N(m_keys_seed_binary, std::string, vector_of_chars_to_string, string_to_vector_of_chars, "m_seed") END_KV_SERIALIZE_MAP() private: void set_null(); + bool restore_keys(const std::vector& keys_seed_binary); + account_keys m_keys; uint64_t m_creation_timestamp; - std::string m_seed; + + std::vector m_keys_seed_binary; }; @@ -90,9 +103,9 @@ namespace currency inline bool operator==(const account_keys& lhs, const account_keys& rhs) { - return lhs.m_account_address == rhs.m_account_address && - lhs.m_spend_secret_key == rhs.m_spend_secret_key && - lhs.m_view_secret_key == rhs.m_view_secret_key; + return lhs.account_address == rhs.account_address && + lhs.spend_secret_key == rhs.spend_secret_key && + lhs.view_secret_key == rhs.view_secret_key; } inline bool operator!=(const account_keys& lhs, const account_keys& rhs) { diff --git a/src/currency_core/account_boost_serialization.h b/src/currency_core/account_boost_serialization.h index f4d3c61..f0f2d63 100644 --- a/src/currency_core/account_boost_serialization.h +++ b/src/currency_core/account_boost_serialization.h @@ -17,9 +17,9 @@ namespace boost template inline void serialize(Archive &a, currency::account_keys &x, const boost::serialization::version_type ver) { - a & x.m_account_address; - a & x.m_spend_secret_key; - a & x.m_view_secret_key; + a & x.account_address; + a & x.spend_secret_key; + a & x.view_secret_key; } } diff --git a/src/currency_core/alias_helper.h b/src/currency_core/alias_helper.h index cf027eb..3bc5b33 100644 --- a/src/currency_core/alias_helper.h +++ b/src/currency_core/alias_helper.h @@ -37,7 +37,7 @@ namespace tools if (!cb(req_alias_info, alias_info)) return false; - if (alias_info.status != CORE_RPC_STATUS_OK || !alias_info.alias_details.address.size()) + if (alias_info.status != API_RETURN_CODE_OK || !alias_info.alias_details.address.size()) return false; addr_str_local = alias_info.alias_details.address; diff --git a/src/currency_core/basic_kv_structs.h b/src/currency_core/basic_kv_structs.h new file mode 100644 index 0000000..1ff520c --- /dev/null +++ b/src/currency_core/basic_kv_structs.h @@ -0,0 +1,27 @@ +// Copyright (c) 2014-2018 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include "warnings.h" + +PUSH_VS_WARNINGS +DISABLE_VS_WARNINGS(4100) +DISABLE_VS_WARNINGS(4503) +#include "serialization/keyvalue_serialization.h" +#include "storages/portable_storage_template_helper.h" +POP_VS_WARNINGS + +namespace currency +{ + template + struct struct_with_one_t_type + { + t_type v; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(v) + END_KV_SERIALIZE_MAP() + }; +} diff --git a/src/currency_core/basic_pow_helpers.cpp b/src/currency_core/basic_pow_helpers.cpp index 6b3e53f..ddb8791 100644 --- a/src/currency_core/basic_pow_helpers.cpp +++ b/src/currency_core/basic_pow_helpers.cpp @@ -22,21 +22,35 @@ namespace currency { //-------------------------------------------------------------- - //global object -// crypto::ethash::cache_manager cache; -// void ethash_set_use_dag(bool use_dag) -// { -// cache.set_use_dag(use_dag); -// } -// //------------------------------------------------------------------ -// const uint8_t* ethash_get_dag(uint64_t epoch, uint64_t& dag_size) -// { -// return cache.get_dag(epoch, dag_size); -// } + int ethash_custom_log_get_level() + { + return epee::log_space::get_set_log_detalisation_level(); + } + //-------------------------------------------------------------- + void ethash_custom_log(const std::string& m, bool add_callstack) + { + std::string msg = epee::log_space::log_singletone::get_prefix_entry() + "[ETHASH]" + m; + if (add_callstack) + msg = msg + "callstask: " + epee::misc_utils::get_callstack(); + + epee::log_space::log_singletone::do_log_message(msg, LOG_LEVEL_0, epee::log_space::console_color_default, true, LOG_DEFAULT_TARGET); + } + //-------------------------------------------------------------- + void init_ethash_log_if_necessary() + { + static bool inited = false; + if (inited) + return; + + ethash::access_custom_log_level_function() = ðash_custom_log_get_level; + ethash::access_custom_log_function() = ðash_custom_log; + + inited = true; + } //------------------------------------------------------------------ int ethash_height_to_epoch(uint64_t height) { - return height / ETHASH_EPOCH_LENGTH; + return static_cast(height / ETHASH_EPOCH_LENGTH); } //-------------------------------------------------------------- crypto::hash ethash_epoch_to_seed(int epoch) @@ -49,9 +63,11 @@ namespace currency //-------------------------------------------------------------- crypto::hash get_block_longhash(uint64_t height, const crypto::hash& block_header_hash, uint64_t nonce) { + init_ethash_log_if_necessary(); int epoch = ethash_height_to_epoch(height); - const auto& context = progpow::get_global_epoch_context_full(static_cast(epoch)); - auto res_eth = progpow::hash(context, height, *(ethash::hash256*)&block_header_hash, nonce); + std::shared_ptr p_context = progpow::get_global_epoch_context_full(static_cast(epoch)); + CHECK_AND_ASSERT_THROW_MES(p_context, "progpow::get_global_epoch_context_full returned null"); + auto res_eth = progpow::hash(*p_context, static_cast(height), *(ethash::hash256*)&block_header_hash, nonce); crypto::hash result = currency::null_hash; memcpy(&result.data, &res_eth.final_hash, sizeof(res_eth.final_hash)); return result; diff --git a/src/currency_core/bc_attachments_service_manager.h b/src/currency_core/bc_attachments_service_manager.h index bd4f264..632126a 100644 --- a/src/currency_core/bc_attachments_service_manager.h +++ b/src/currency_core/bc_attachments_service_manager.h @@ -27,7 +27,7 @@ namespace currency class bc_attachment_services_manager { public: - bc_attachment_services_manager(i_core_event_handler* pcore_event_handler) : m_pcore_event_handler(pcore_event_handler), m_core_runtime_config(get_default_core_runtime_config()) + bc_attachment_services_manager(/* i_core_event_handler* pcore_event_handler*/) : /*m_pcore_event_handler(pcore_event_handler),*/ m_core_runtime_config(get_default_core_runtime_config()) {} @@ -43,7 +43,7 @@ namespace currency private: std::map m_services; - i_core_event_handler* m_pcore_event_handler; + //i_core_event_handler* m_pcore_event_handler; core_runtime_config m_core_runtime_config; }; diff --git a/src/currency_core/bc_escrow_service.h b/src/currency_core/bc_escrow_service.h index 6a84b56..c9e00ae 100644 --- a/src/currency_core/bc_escrow_service.h +++ b/src/currency_core/bc_escrow_service.h @@ -91,6 +91,9 @@ namespace bc_services } +#define CURRENCY_CPD_ARCHIVE_VER 1 +BOOST_CLASS_VERSION(bc_services::contract_private_details, CURRENCY_CPD_ARCHIVE_VER) + namespace boost { namespace serialization @@ -100,8 +103,20 @@ namespace boost { ar & cpd.title; ar & cpd.comment; - ar & cpd.a_addr; // usually buyer - ar & cpd.b_addr; // usually seller + if (version == 0 && !archive_t::is_saving::value) + { + // loading from version 0 (pre-auditable address format) + currency::account_public_address_old old_addr = AUTO_VAL_INIT(old_addr); + ar & old_addr; + cpd.a_addr = currency::account_public_address::from_old(old_addr); + ar & old_addr; + cpd.b_addr = currency::account_public_address::from_old(old_addr); + } + else + { + ar & cpd.a_addr; // usually buyer + ar & cpd.b_addr; // usually seller + } ar & cpd.amount_to_pay; ar & cpd.amount_a_pledge; ar & cpd.amount_b_pledge; diff --git a/src/currency_core/bc_offers_service.cpp b/src/currency_core/bc_offers_service.cpp index 44e1de3..5906463 100644 --- a/src/currency_core/bc_offers_service.cpp +++ b/src/currency_core/bc_offers_service.cpp @@ -31,8 +31,10 @@ namespace bc_services //------------------------------------------------------------------ bc_offers_service::~bc_offers_service() { + TRY_ENTRY(); if (!m_deinitialized) deinit(); + CATCH_ENTRY_NO_RETURN(); } //------------------------------------------------------------------ bool bc_offers_service::init(const std::string& config_folder, const boost::program_options::variables_map& vm) diff --git a/src/currency_core/blockchain_storage.cpp b/src/currency_core/blockchain_storage.cpp index 038e326..e60e5b6 100644 --- a/src/currency_core/blockchain_storage.cpp +++ b/src/currency_core/blockchain_storage.cpp @@ -6,6 +6,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include +#include #include #include #include @@ -14,7 +16,7 @@ #include "include_base_utils.h" -#include "common/db_backend_lmdb.h" +#include "common/db_backend_selector.h" #include "common/command_line.h" #include "blockchain_storage.h" @@ -30,9 +32,11 @@ #include "crypto/hash.h" #include "miner_common.h" #include "storages/portable_storage_template_helper.h" -#include "common/db_backend_lmdb.h" #include "basic_pow_helpers.h" #include "version.h" +#include "tx_semantic_validation.h" +#include "crypto/RIPEMD160_helper.h" +#include "crypto/bitcoin/sha256_helper.h" #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "core" @@ -58,11 +62,19 @@ using namespace currency; #define BLOCKCHAIN_STORAGE_OPTIONS_ID_CURRENT_BLOCK_CUMUL_SZ_LIMIT 0 #define BLOCKCHAIN_STORAGE_OPTIONS_ID_CURRENT_PRUNED_RS_HEIGHT 1 #define BLOCKCHAIN_STORAGE_OPTIONS_ID_LAST_WORKED_VERSION 2 -#define BLOCKCHAIN_STORAGE_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION 3 //mismatch here means full resync +#define BLOCKCHAIN_STORAGE_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION 3 //DON'T CHANGE THIS, if you need to resync db change BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION #define BLOCKCHAIN_STORAGE_OPTIONS_ID_STORAGE_MINOR_COMPATIBILITY_VERSION 4 //mismatch here means some reinitializations #define TARGETDATA_CACHE_SIZE DIFFICULTY_WINDOW + 10 +#ifndef TESTNET +#define BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION 57000 +#else +#define BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION 18000 +#endif +#define BLOCK_POS_STRICT_SEQUENCE_LIMIT 20 + + DISABLE_VS_WARNINGS(4267) @@ -73,7 +85,7 @@ namespace } //------------------------------------------------------------------ -blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::shared_ptr(new tools::db::lmdb_db_backend), m_rw_lock), +blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(nullptr, m_rw_lock), m_db_blocks(m_db), m_db_blocks_index(m_db), m_db_transactions(m_db), @@ -96,12 +108,14 @@ blockchain_storage::blockchain_storage(tx_memory_pool& tx_pool) :m_db(std::share m_core_runtime_config(get_default_core_runtime_config()), //m_bei_stub(AUTO_VAL_INIT(m_bei_stub)), m_event_handler(&m_event_handler_stub), - m_services_mgr(nullptr), m_interprocess_locker_file(0), m_current_fee_median(0), m_current_fee_median_effective_index(0), m_is_reorganize_in_process(false), - m_deinit_is_done(false) + m_deinit_is_done(false), + m_cached_next_pow_difficulty(0), + m_cached_next_pos_difficulty(0), + m_blockchain_launch_timestamp(0) { @@ -146,7 +160,6 @@ void blockchain_storage::init_options(boost::program_options::options_descriptio command_line::add_arg(desc, arg_db_cache_l2); } //------------------------------------------------------------------ - uint64_t blockchain_storage::get_block_h_older_then(uint64_t timestamp) const { // get avarage block position @@ -199,83 +212,226 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro { // CRITICAL_REGION_LOCAL(m_read_lock); + tools::db::db_backend_selector dbbs; + dbbs.init(vm); + auto p_backend = dbbs.create_backend(); + if (!p_backend) + { + LOG_PRINT_RED_L0("Failed to create db engine"); + return false; + } + m_db.reset_backend(p_backend); + LOG_PRINT_L0("DB ENGINE USED BY CORE: " << m_db.get_backend()->name()); + if (!validate_instance(config_folder)) { LOG_ERROR("Failed to initialize instance"); return false; } - uint64_t cache_size = CACHE_SIZE; + uint64_t cache_size_l1 = CACHE_SIZE; if (command_line::has_arg(vm, arg_db_cache_l1)) { - cache_size = command_line::get_arg(vm, arg_db_cache_l1); + cache_size_l1 = command_line::get_arg(vm, arg_db_cache_l1); } - LOG_PRINT_GREEN("Using db file cache size(L1): " << cache_size, LOG_LEVEL_0); - + LOG_PRINT_GREEN("Using db file cache size(L1): " << cache_size_l1, LOG_LEVEL_0); m_config_folder = config_folder; - LOG_PRINT_L0("Loading blockchain..."); - const std::string folder_name = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME; - tools::create_directories_if_necessary(folder_name); - bool res = m_db.open(folder_name, cache_size); - CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << folder_name); - res = m_db_blocks.init(BLOCKCHAIN_STORAGE_CONTAINER_BLOCKS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_blocks_index.init(BLOCKCHAIN_STORAGE_CONTAINER_BLOCKS_INDEX); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_transactions.init(BLOCKCHAIN_STORAGE_CONTAINER_TRANSACTIONS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_spent_keys.init(BLOCKCHAIN_STORAGE_CONTAINER_SPENT_KEYS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_outputs.init(BLOCKCHAIN_STORAGE_CONTAINER_OUTPUTS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_multisig_outs.init(BLOCKCHAIN_STORAGE_CONTAINER_MULTISIG_OUTS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_solo_options.init(BLOCKCHAIN_STORAGE_CONTAINER_SOLO_OPTIONS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_aliases.init(BLOCKCHAIN_STORAGE_CONTAINER_ALIASES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_addr_to_alias.init(BLOCKCHAIN_STORAGE_CONTAINER_ADDR_TO_ALIAS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_per_block_gindex_incs.init(BLOCKCHAIN_STORAGE_CONTAINER_GINDEX_INCS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - - if (command_line::has_arg(vm, arg_db_cache_l2)) + // remove old incompatible DB + const std::string old_db_folder_path = m_config_folder + "/" CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD; + if (boost::filesystem::exists(epee::string_encoding::utf8_to_wstring(old_db_folder_path))) { - uint64_t cache_size = command_line::get_arg(vm, arg_db_cache_l2); - LOG_PRINT_GREEN("Using db items cache size(L2): " << cache_size, LOG_LEVEL_0); - m_db_blocks_index.set_cache_size(cache_size); - m_db_blocks.set_cache_size(cache_size); - m_db_blocks_index.set_cache_size(cache_size); - m_db_transactions.set_cache_size(cache_size); - m_db_spent_keys.set_cache_size(cache_size); - //m_db_outputs.set_cache_size(cache_size); - m_db_multisig_outs.set_cache_size(cache_size); - m_db_solo_options.set_cache_size(cache_size); - m_db_aliases.set_cache_size(cache_size); - m_db_addr_to_alias.set_cache_size(cache_size); + LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0); + boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path)); } - bool need_reinit = false; - bool need_reinit_medians = false; + const std::string db_folder_path = dbbs.get_db_folder_path(); + LOG_PRINT_L0("Loading blockchain from " << db_folder_path); + + bool db_opened_okay = false; + for(size_t loading_attempt_no = 0; loading_attempt_no < 2; ++loading_attempt_no) + { + bool res = m_db.open(db_folder_path, cache_size_l1); + if (!res) + { + // if DB could not be opened -- try to remove the whole folder and re-open DB + LOG_PRINT_YELLOW("Failed to initialize database in folder: " << db_folder_path << ", first attempt", LOG_LEVEL_0); + boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(db_folder_path)); + res = m_db.open(db_folder_path, cache_size_l1); + CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << db_folder_path << ", second attempt"); + } + + res = m_db_blocks.init(BLOCKCHAIN_STORAGE_CONTAINER_BLOCKS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_blocks_index.init(BLOCKCHAIN_STORAGE_CONTAINER_BLOCKS_INDEX); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_transactions.init(BLOCKCHAIN_STORAGE_CONTAINER_TRANSACTIONS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_spent_keys.init(BLOCKCHAIN_STORAGE_CONTAINER_SPENT_KEYS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_outputs.init(BLOCKCHAIN_STORAGE_CONTAINER_OUTPUTS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_multisig_outs.init(BLOCKCHAIN_STORAGE_CONTAINER_MULTISIG_OUTS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_solo_options.init(BLOCKCHAIN_STORAGE_CONTAINER_SOLO_OPTIONS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_aliases.init(BLOCKCHAIN_STORAGE_CONTAINER_ALIASES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_addr_to_alias.init(BLOCKCHAIN_STORAGE_CONTAINER_ADDR_TO_ALIAS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_per_block_gindex_incs.init(BLOCKCHAIN_STORAGE_CONTAINER_GINDEX_INCS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + + if (command_line::has_arg(vm, arg_db_cache_l2)) + { + uint64_t cache_size = command_line::get_arg(vm, arg_db_cache_l2); + LOG_PRINT_GREEN("Using db items cache size(L2): " << cache_size, LOG_LEVEL_0); + m_db_blocks_index.set_cache_size(cache_size); + m_db_blocks.set_cache_size(cache_size); + m_db_blocks_index.set_cache_size(cache_size); + m_db_transactions.set_cache_size(cache_size); + m_db_spent_keys.set_cache_size(cache_size); + //m_db_outputs.set_cache_size(cache_size); + m_db_multisig_outs.set_cache_size(cache_size); + m_db_solo_options.set_cache_size(cache_size); + m_db_aliases.set_cache_size(cache_size); + m_db_addr_to_alias.set_cache_size(cache_size); + } + + bool need_reinit = false; + if (m_db_blocks.size() != 0) + { +#ifndef TESTNET + if ((m_db_storage_major_compatibility_version == 93 || m_db_storage_major_compatibility_version == 94) && BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION == 95) + { + // migrate DB to rebuild aliases container + LOG_PRINT_MAGENTA("Migrating DB: " << m_db_storage_major_compatibility_version << " -> " << BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION, LOG_LEVEL_0); + + res = m_db_aliases.deinit(); + CHECK_AND_ASSERT_MES(res, false, "Unable to deinit m_db_aliases"); + res = m_db_addr_to_alias.deinit(); + CHECK_AND_ASSERT_MES(res, false, "Unable to deinit m_db_addr_to_alias"); + + typedef tools::db::cached_key_value_accessor, true, true> aliases_container_old; + aliases_container_old db_aliases_old(m_db); + res = db_aliases_old.init(BLOCKCHAIN_STORAGE_CONTAINER_ALIASES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db_aliases_old"); + + typedef tools::db::cached_key_value_accessor, true, false> address_to_aliases_container_old; + address_to_aliases_container_old db_addr_to_alias_old(m_db); + res = db_addr_to_alias_old.init(BLOCKCHAIN_STORAGE_CONTAINER_ADDR_TO_ALIAS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db_addr_to_alias_old"); + + // temporary set db compatibility version to zero during migration in order to trigger db reinit on the next lanunch in case the process stops in the middle + m_db.begin_transaction(); + uint64_t tmp_db_maj_version = m_db_storage_major_compatibility_version; + m_db_storage_major_compatibility_version = 0; + m_db.commit_transaction(); + + typedef std::vector>> tmp_container_t; + tmp_container_t temp_container; + db_aliases_old.enumerate_items([&temp_container](uint64_t i, const std::string& alias, const std::list& alias_entries) + { + std::pair> p(alias, std::list()); + for(auto& entry : alias_entries) + p.second.push_back(static_cast(entry)); // here conversion to the new format goes + temp_container.emplace_back(p); + return true; + }); + + typedef std::vector>> add_to_alias_container_t; + add_to_alias_container_t addr_to_alias_container; + db_addr_to_alias_old.enumerate_items([&addr_to_alias_container](uint64_t n, const account_public_address_old& addr_old, const std::set& aliases){ + addr_to_alias_container.emplace_back(std::make_pair(account_public_address::from_old(addr_old), aliases)); + return true; + }); + + // clear and close old format container + m_db.begin_transaction(); + db_aliases_old.clear(); + db_addr_to_alias_old.clear(); + m_db.commit_transaction(); + db_aliases_old.deinit(); + db_addr_to_alias_old.deinit(); + + res = m_db_aliases.init(BLOCKCHAIN_STORAGE_CONTAINER_ALIASES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init m_db_aliases"); + res = m_db_addr_to_alias.init(BLOCKCHAIN_STORAGE_CONTAINER_ADDR_TO_ALIAS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init m_db_addr_to_alias"); + + // re-populate all alias entries back + m_db.begin_transaction(); + for(auto& el : temp_container) + m_db_aliases.set(el.first, el.second); + + for(auto& el : addr_to_alias_container) + m_db_addr_to_alias.set(el.first, el.second); + m_db.commit_transaction(); + + // restore db maj compartibility + m_db.begin_transaction(); + m_db_storage_major_compatibility_version = tmp_db_maj_version; + m_db.commit_transaction(); + + LOG_PRINT_MAGENTA("Migrating DB: successfully done", LOG_LEVEL_0); + } + else if (m_db_storage_major_compatibility_version == 93 && BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION == 94) + { + // do not reinit db if moving from version 93 to version 94 + LOG_PRINT_MAGENTA("DB storage does not need reinit because moving from v93 to v94", LOG_LEVEL_0); + } +#else + // TESTNET + if (m_db_storage_major_compatibility_version == 95 && BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION == 96) + { + // do not reinit TESTNET db if moving from version 95 to version 96 + LOG_PRINT_MAGENTA("DB storage does not need reinit because moving from v95 to v96", LOG_LEVEL_0); + } +#endif + else if (m_db_storage_major_compatibility_version != BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION) + { + need_reinit = true; + LOG_PRINT_MAGENTA("DB storage needs reinit because it has major compatibility ver " << m_db_storage_major_compatibility_version << ", expected : " << BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION, LOG_LEVEL_0); + } + else if (m_db_storage_minor_compatibility_version > BLOCKCHAIN_STORAGE_MINOR_COMPATIBILITY_VERSION) + { + // reinit db only if minor version in the DB is greather (i.e. newer) than minor version in the code + need_reinit = true; + LOG_PRINT_MAGENTA("DB storage needs reinit because it has minor compatibility ver " << m_db_storage_minor_compatibility_version << " that is greater than BLOCKCHAIN_STORAGE_MINOR_COMPATIBILITY_VERSION: " << BLOCKCHAIN_STORAGE_MINOR_COMPATIBILITY_VERSION, LOG_LEVEL_0); + } + } + + if (need_reinit) + { + LOG_PRINT_L1("DB at " << db_folder_path << " is about to be deleted and re-created..."); + m_db_blocks.deinit(); + m_db_blocks_index.deinit(); + m_db_transactions.deinit(); + m_db_spent_keys.deinit(); + m_db_outputs.deinit(); + m_db_multisig_outs.deinit(); + m_db_solo_options.deinit(); + m_db_aliases.deinit(); + m_db_addr_to_alias.deinit(); + m_db_per_block_gindex_incs.deinit(); + m_db.close(); + size_t files_removed = boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(db_folder_path)); + LOG_PRINT_L1(files_removed << " files at " << db_folder_path << " removed"); + + // try to re-create DB and re-init containers + continue; + } + + db_opened_okay = true; + break; + } + + CHECK_AND_ASSERT_MES(db_opened_okay, false, "All attempts to open DB at " << db_folder_path << " failed"); + if (!m_db_blocks.size()) { - need_reinit = true; - } - else if (m_db_storage_major_compatibility_version != BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION) - { - need_reinit = true; - LOG_PRINT_MAGENTA("DB storage needs reinit because it has major compatibility ver " << m_db_storage_major_compatibility_version << ", expected : " << BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION, LOG_LEVEL_0); - } - else if (m_db_storage_minor_compatibility_version != BLOCKCHAIN_STORAGE_MINOR_COMPATIBILITY_VERSION) - { - if (m_db_storage_minor_compatibility_version < 1) - need_reinit_medians = true; - } - if (need_reinit) - { - clear(); + // empty DB: generate and add genesis block block bl = boost::value_initialized(); block_verification_context bvc = boost::value_initialized(); generate_genesis_block(bl); @@ -283,14 +439,8 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro CHECK_AND_ASSERT_MES(!bvc.m_verification_failed, false, "Failed to add genesis block to blockchain"); LOG_PRINT_MAGENTA("Storage initialized with genesis", LOG_LEVEL_0); } - if (need_reinit_medians) - { - bool r = rebuild_tx_fee_medians(); - CHECK_AND_ASSERT_MES(r, false, "failed to rebuild_tx_fee_medians()"); - } - - initialize_db_solo_options_values(); + store_db_solo_options_values(); m_services_mgr.init(config_folder, vm); @@ -299,19 +449,78 @@ bool blockchain_storage::init(const std::string& config_folder, const boost::pro if(!m_db_blocks.back()->bl.timestamp) timestamp_diff = m_core_runtime_config.get_core_time() - 1341378000; + m_db.begin_transaction(); + set_lost_tx_unmixable(); + m_db.commit_transaction(); + LOG_PRINT_GREEN("Blockchain initialized. (v:" << m_db_storage_major_compatibility_version << ") last block: " << m_db_blocks.size() - 1 << ENDL << "genesis: " << get_block_hash(m_db_blocks[0]->bl) << ENDL << "last block: " << m_db_blocks.size() - 1 << ", " << misc_utils::get_time_interval_string(timestamp_diff) << " time ago" << ENDL << "current pos difficulty: " << get_next_diff_conditional(true) << ENDL << "current pow difficulty: " << get_next_diff_conditional(false) << ENDL - << "total tansactions: " << m_db_transactions.size(), + << "total transactions: " << m_db_transactions.size(), LOG_LEVEL_0); - return true; } + //------------------------------------------------------------------ -void blockchain_storage::initialize_db_solo_options_values() +bool blockchain_storage::set_lost_tx_unmixable_for_height(uint64_t height) +{ +#ifndef TESTNET + if (height == 75738) + return set_lost_tx_unmixable(); +#endif + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::set_lost_tx_unmixable() +{ +#ifndef TESTNET + if (m_db_blocks.size() > 75738) + { + crypto::hash tx_id_1 = epee::string_tools::parse_tpod_from_hex_string("c2a2229d614e7c026433efbcfdbd0be1f68d9b419220336df3e2c209f5d57314"); + crypto::hash tx_id_2 = epee::string_tools::parse_tpod_from_hex_string("647f936c6ffbd136f5c95d9a90ad554bdb4c01541c6eb5755ad40b984d80da67"); + + auto tx_ptr_1 = m_db_transactions.find(tx_id_1); + CHECK_AND_ASSERT_MES(tx_ptr_1, false, "Internal error: filed to find lost tx"); + transaction_chain_entry tx1_local_entry(*tx_ptr_1); + for (size_t i = 0; i != tx1_local_entry.m_spent_flags.size(); i++) + { + tx1_local_entry.m_spent_flags[i] = true; + } + m_db_transactions.set(tx_id_1, tx1_local_entry); + + auto tx_ptr_2 = m_db_transactions.find(tx_id_2); + transaction_chain_entry tx2_local_entry(*tx_ptr_2); + CHECK_AND_ASSERT_MES(tx_ptr_1, false, "Internal error: filed to find lost tx"); + for (size_t i = 0; i != tx2_local_entry.m_spent_flags.size(); i++) + { + tx2_local_entry.m_spent_flags[i] = true; + } + m_db_transactions.set(tx_id_2, tx2_local_entry); + } +#endif + return true; +} +//------------------------------------------------------------------ +void blockchain_storage::patch_out_if_needed(txout_to_key& out, const crypto::hash& tx_id, uint64_t n) const +{ +#ifndef TESTNET + static crypto::hash tx_id_1 = epee::string_tools::parse_tpod_from_hex_string("c2a2229d614e7c026433efbcfdbd0be1f68d9b419220336df3e2c209f5d57314"); + static crypto::hash tx_id_2 = epee::string_tools::parse_tpod_from_hex_string("647f936c6ffbd136f5c95d9a90ad554bdb4c01541c6eb5755ad40b984d80da67"); + + if (tx_id == tx_id_1 && n == 12) + { + out.mix_attr = CURRENCY_TO_KEY_OUT_FORCED_NO_MIX; + }else if(tx_id == tx_id_2 && n == 5) + { + out.mix_attr = CURRENCY_TO_KEY_OUT_FORCED_NO_MIX; + } +#endif +} +//------------------------------------------------------------------ +void blockchain_storage::store_db_solo_options_values() { m_db.begin_transaction(); m_db_storage_major_compatibility_version = BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION; @@ -328,7 +537,7 @@ bool blockchain_storage::deinit() return true; } //------------------------------------------------------------------ -bool blockchain_storage::pop_block_from_blockchain() +bool blockchain_storage::pop_block_from_blockchain(transactions_map& onboard_transactions) { CRITICAL_REGION_LOCAL(m_read_lock); @@ -338,7 +547,7 @@ bool blockchain_storage::pop_block_from_blockchain() CHECK_AND_ASSERT_MES(bei_ptr.get(), false, "pop_block_from_blockchain: can't pop from blockchain"); uint64_t fee_total = 0; - bool r = purge_block_data_from_blockchain(bei_ptr->bl, bei_ptr->bl.tx_hashes.size(), fee_total); + bool r = purge_block_data_from_blockchain(bei_ptr->bl, bei_ptr->bl.tx_hashes.size(), fee_total, onboard_transactions); CHECK_AND_ASSERT_MES(r, false, "Failed to purge_block_data_from_blockchain for block " << get_block_hash(bei_ptr->bl) << " on height " << h); pop_block_from_per_block_increments(bei_ptr->height); @@ -417,17 +626,22 @@ bool blockchain_storage::prune_ring_signatures_and_attachments_if_need() { CRITICAL_REGION_LOCAL(m_read_lock); - if(m_db_blocks.size() && m_checkpoints.get_top_checkpoint_height() && m_checkpoints.get_top_checkpoint_height() > m_db_current_pruned_rs_height) + if (m_db_blocks.size() > 1 && m_checkpoints.get_top_checkpoint_height() && m_checkpoints.get_top_checkpoint_height() > m_db_current_pruned_rs_height) { - LOG_PRINT_CYAN("Starting pruning ring signatues and attachments...", LOG_LEVEL_0); - uint64_t tx_count = 0, sig_count = 0, attach_count = 0; - for(uint64_t height = m_db_current_pruned_rs_height; height < m_db_blocks.size() && height <= m_checkpoints.get_top_checkpoint_height(); height++) + uint64_t pruning_last_height = std::min(m_db_blocks.size() - 1, m_checkpoints.get_top_checkpoint_height()); + if (pruning_last_height > m_db_current_pruned_rs_height) { - bool res = prune_ring_signatures_and_attachments(height, tx_count, sig_count, attach_count); - CHECK_AND_ASSERT_MES(res, false, "failed to prune_ring_signatures_and_attachments for height = " << height); + LOG_PRINT_CYAN("Starting pruning ring signatues and attachments from height " << m_db_current_pruned_rs_height + 1 << " to height " << pruning_last_height + << " (" << pruning_last_height - m_db_current_pruned_rs_height << " blocks)", LOG_LEVEL_0); + uint64_t tx_count = 0, sig_count = 0, attach_count = 0; + for(uint64_t height = m_db_current_pruned_rs_height + 1; height <= pruning_last_height; height++) + { + bool res = prune_ring_signatures_and_attachments(height, tx_count, sig_count, attach_count); + CHECK_AND_ASSERT_MES(res, false, "failed to prune_ring_signatures_and_attachments for height = " << height); + } + m_db_current_pruned_rs_height = pruning_last_height; + LOG_PRINT_CYAN("Transaction pruning finished: " << sig_count << " signatures and " << attach_count << " attachments released in " << tx_count << " transactions.", LOG_LEVEL_0); } - m_db_current_pruned_rs_height = m_checkpoints.get_top_checkpoint_height(); - LOG_PRINT_CYAN("Transaction pruning finished: " << sig_count << " signatures and " << attach_count << " attachments released in " << tx_count << " transactions.", LOG_LEVEL_0); } return true; } @@ -442,7 +656,7 @@ bool blockchain_storage::clear() m_db_transactions.clear(); m_db_spent_keys.clear(); m_db_solo_options.clear(); - initialize_db_solo_options_values(); + store_db_solo_options_values(); m_db_outputs.clear(); m_db_multisig_outs.clear(); m_db_aliases.clear(); @@ -528,9 +742,13 @@ bool blockchain_storage::purge_transaction_keyimages_from_blockchain(const trans } return true; } + bool operator()(const txin_htlc& inp) const + { + return this->operator()(static_cast(inp)); + } }; - BOOST_FOREACH(const txin_v& in, tx.vin) + for(const txin_v& in : tx.vin) { bool r = boost::apply_visitor(purge_transaction_visitor(*this, m_db_spent_keys, strict_check), in); CHECK_AND_ASSERT_MES(!strict_check || r, false, "failed to process purge_transaction_visitor"); @@ -538,7 +756,7 @@ bool blockchain_storage::purge_transaction_keyimages_from_blockchain(const trans return true; } //------------------------------------------------------------------ -bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee) +bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee, transaction& tx_) { fee = 0; CRITICAL_REGION_LOCAL(m_read_lock); @@ -546,6 +764,7 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t auto tx_res_ptr = m_db_transactions.find(tx_id); CHECK_AND_ASSERT_MES(tx_res_ptr != m_db_transactions.end(), false, "transaction " << tx_id << " is not found in blockchain index!!"); const transaction& tx = tx_res_ptr->tx; + tx_ = tx; fee = get_tx_fee(tx_res_ptr->tx); purge_transaction_keyimages_from_blockchain(tx, true); @@ -576,10 +795,11 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t bool blockchain_storage::purge_block_data_from_blockchain(const block& b, size_t processed_tx_count) { uint64_t total_fee = 0; - return purge_block_data_from_blockchain(b, processed_tx_count, total_fee); + transactions_map onboard_transactions; + return purge_block_data_from_blockchain(b, processed_tx_count, total_fee, onboard_transactions); } //------------------------------------------------------------------ -bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_t processed_tx_count, uint64_t& fee_total) +bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_t processed_tx_count, uint64_t& fee_total, transactions_map& onboard_transactions) { CRITICAL_REGION_LOCAL(m_read_lock); fee_total = 0; @@ -588,11 +808,13 @@ bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_ CHECK_AND_ASSERT_MES(processed_tx_count <= bl.tx_hashes.size(), false, "wrong processed_tx_count in purge_block_data_from_blockchain"); for(size_t count = 0; count != processed_tx_count; count++) { - res = purge_transaction_from_blockchain(bl.tx_hashes[(processed_tx_count -1)- count], fee) && res; + transaction tx = AUTO_VAL_INIT(tx); + res = purge_transaction_from_blockchain(bl.tx_hashes[(processed_tx_count -1)- count], fee, tx) && res; fee_total += fee; + onboard_transactions[bl.tx_hashes[(processed_tx_count - 1) - count]] = tx; } - - res = purge_transaction_from_blockchain(get_transaction_hash(bl.miner_tx), fee) && res; + transaction tx = AUTO_VAL_INIT(tx); + res = purge_transaction_from_blockchain(get_transaction_hash(bl.miner_tx), fee, tx) && res; return res; } //------------------------------------------------------------------ @@ -755,20 +977,22 @@ bool blockchain_storage::get_block_by_height(uint64_t h, block &blk) const // invalid.push_back(v.first); // } //------------------------------------------------------------------ -bool blockchain_storage::rollback_blockchain_switching(std::list& original_chain, size_t rollback_height) +bool blockchain_storage::rollback_blockchain_switching(std::list& original_chain, size_t rollback_height) { CRITICAL_REGION_LOCAL(m_read_lock); //remove failed subchain for(size_t i = m_db_blocks.size()-1; i >=rollback_height; i--) { - bool r = pop_block_from_blockchain(); + transactions_map ot; + bool r = pop_block_from_blockchain(ot); CHECK_AND_ASSERT_MES(r, false, "PANIC!!! failed to remove block while chain switching during the rollback!"); } //return back original chain - BOOST_FOREACH(auto& bl, original_chain) + BOOST_FOREACH(auto& oce, original_chain) { block_verification_context bvc = boost::value_initialized(); - bool r = handle_block_to_main_chain(bl, bvc); + bvc.m_onboard_transactions.swap(oce.onboard_transactions); + bool r = handle_block_to_main_chain(oce.b, bvc); CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC!!! failed to add (again) block while chain switching during the rollback!"); } @@ -831,13 +1055,15 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch ); //disconnecting old chain - std::list disconnected_chain; + std::list disconnected_chain; for(size_t i = m_db_blocks.size()-1; i >=split_height; i--) { - block b = m_db_blocks[i]->bl; - bool r = pop_block_from_blockchain(); - CHECK_AND_ASSERT_MES(r, false, "failed to remove block " << get_block_hash(b) << " @ " << get_block_height(b) << " on chain switching"); - disconnected_chain.push_front(b); + disconnected_chain.push_front(block_ws_txs()); + block_ws_txs& bwt = disconnected_chain.front(); + bwt.b = m_db_blocks[i]->bl; + bool r = pop_block_from_blockchain(bwt.onboard_transactions); + CHECK_AND_ASSERT_MES(r, false, "failed to remove block " << get_block_hash(bwt.b) << " @ " << get_block_height(bwt.b) << " on chain switching"); + CHECK_AND_ASSERT_MES(validate_blockchain_prev_links(), false, "EPIC FAIL!"); } @@ -846,6 +1072,7 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch { auto ch_ent = *alt_ch_iter; block_verification_context bvc = boost::value_initialized(); + bvc.m_onboard_transactions = ch_ent->second.onboard_transactions; bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc); if(!r || !bvc.m_added_to_main_chain) { @@ -867,13 +1094,20 @@ bool blockchain_storage::switch_to_alternative_blockchain(alt_chain_type& alt_ch for(auto& old_ch_ent : disconnected_chain) { block_verification_context bvc = boost::value_initialized(); - bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc); + bvc.m_onboard_transactions.swap(old_ch_ent.onboard_transactions); + bool r = handle_alternative_block(old_ch_ent.b, get_block_hash(old_ch_ent.b), bvc); if(!r) { LOG_ERROR("Failed to push ex-main chain blocks to alternative chain "); rollback_blockchain_switching(disconnected_chain, split_height); CHECK_AND_ASSERT_MES(validate_blockchain_prev_links(), false, "EPIC FAIL!"); - return false; + + //can't do return false here, because of the risc to get stuck in "PANIC" mode when nor of + //new chain nor altchain can be inserted into main chain. Got core caught in this trap when + //when machine time was wrongly set for a few hours back, then blocks which was detached from main chain + //couldn't be added as alternative due to timestamps validation(timestamps assumed as from future) + //thanks @Gigabyted for reporting this problem + break; } } @@ -896,9 +1130,10 @@ wide_difficulty_type blockchain_storage::get_next_diff_conditional(bool pos) con return DIFFICULTY_STARTER; //skip genesis timestamp TIME_MEASURE_START_PD(target_calculating_enum_blocks); + CRITICAL_REGION_BEGIN(m_targetdata_cache_lock); std::list>& targetdata_cache = pos ? m_pos_targetdata_cache : m_pow_targetdata_cache; - if (targetdata_cache.empty()) - load_targetdata_cache(pos); + //if (targetdata_cache.empty()) + load_targetdata_cache(pos); size_t count = 0; for (auto it = targetdata_cache.rbegin(); it != targetdata_cache.rend() && count < DIFFICULTY_WINDOW; it++) @@ -907,16 +1142,26 @@ wide_difficulty_type blockchain_storage::get_next_diff_conditional(bool pos) con commulative_difficulties.push_back(it->first); ++count; } + CRITICAL_REGION_END(); wide_difficulty_type& dif = pos ? m_cached_next_pos_difficulty : m_cached_next_pow_difficulty; TIME_MEASURE_FINISH_PD(target_calculating_enum_blocks); TIME_MEASURE_START_PD(target_calculating_calc); - dif = next_difficulty(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + if (m_db_blocks.size() > m_core_runtime_config.hard_fork_01_starts_after_height) + { + dif = next_difficulty_2(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + } + else + { + dif = next_difficulty_1(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + } + + TIME_MEASURE_FINISH_PD(target_calculating_calc); return dif; } //------------------------------------------------------------------ -wide_difficulty_type blockchain_storage::get_next_diff_conditional2(bool pos, const alt_chain_type& alt_chain, uint64_t split_height) const +wide_difficulty_type blockchain_storage::get_next_diff_conditional2(bool pos, const alt_chain_type& alt_chain, uint64_t split_height, const alt_block_extended_info& abei) const { CRITICAL_REGION_LOCAL(m_read_lock); std::vector timestamps; @@ -939,7 +1184,13 @@ wide_difficulty_type blockchain_storage::get_next_diff_conditional2(bool pos, co return true; }; enum_blockchain(cb, alt_chain, split_height); - return next_difficulty(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + + wide_difficulty_type diff = 0; + if(abei.height > m_core_runtime_config.hard_fork_01_starts_after_height) + diff = next_difficulty_2(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + else + diff = next_difficulty_1(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET : DIFFICULTY_POW_TARGET); + return diff; } //------------------------------------------------------------------ wide_difficulty_type blockchain_storage::get_cached_next_difficulty(bool pos) const @@ -998,7 +1249,7 @@ wide_difficulty_type blockchain_storage::get_next_difficulty_for_alternative_cha commulative_difficulties.push_back(m_db_blocks[i]->cumulative_diff_precise); } - return next_difficulty(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET:DIFFICULTY_POW_TARGET); + return next_difficulty_1(timestamps, commulative_difficulties, pos ? DIFFICULTY_POS_TARGET:DIFFICULTY_POW_TARGET); } //------------------------------------------------------------------ bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height, bool pos) const @@ -1015,9 +1266,29 @@ bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t CHECK_AND_ASSERT_MES(b.miner_tx.vin[1].type() == typeid(txin_to_key), false, "coinstake transaction in the block has the wrong type"); } - CHECK_AND_ASSERT_MES(get_tx_unlock_time(b.miner_tx) == height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW, - false, - "coinbase transaction has wrong unlock time: " << get_tx_unlock_time(b.miner_tx) << ", expected: " << height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + if (height > m_core_runtime_config.hard_fork_01_starts_after_height) + { + // new rules that allow different unlock time in coinbase outputs + uint64_t max_unlock_time = 0; + uint64_t min_unlock_time = 0; + bool r = get_tx_max_min_unlock_time(b.miner_tx, max_unlock_time, min_unlock_time); + CHECK_AND_ASSERT_MES(r && min_unlock_time >= height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW, + false, + "coinbase transaction has wrong min_unlock_time: " << min_unlock_time << ", expected: " << height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + } + else + { + //------------------------------------------------------------------ + //bool blockchain_storage:: + // pre-hard fork rules that don't allow different unlock time in coinbase outputs + uint64_t max_unlock_time = 0; + uint64_t min_unlock_time = 0; + bool r = get_tx_max_min_unlock_time(b.miner_tx, max_unlock_time, min_unlock_time); + CHECK_AND_ASSERT_MES(r && max_unlock_time == min_unlock_time && min_unlock_time == height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW, + false, + "coinbase transaction has wrong min_unlock_time: " << min_unlock_time << ", expected: " << height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + } + //check outs overflow if(!check_outs_overflow(b.miner_tx)) @@ -1133,10 +1404,46 @@ bool blockchain_storage::create_block_template(block& b, const pos_entry& pe, fill_block_template_func_t custom_fill_block_template_func /* = nullptr */) const { + create_block_template_params params = AUTO_VAL_INIT(params); + params.miner_address = miner_address; + params.stakeholder_address = stakeholder_address; + params.ex_nonce = ex_nonce; + params.pos = pos; + params.pe = pe; + params.pcustom_fill_block_template_func = custom_fill_block_template_func; + create_block_template_response resp = AUTO_VAL_INIT(resp); + bool r = create_block_template(params, resp); + b = resp.b; + diffic = resp.diffic; + height = resp.height; + return r; +} + +bool blockchain_storage::create_block_template(const create_block_template_params& params, create_block_template_response& resp) const +{ + const account_public_address& miner_address = params.miner_address; + const account_public_address& stakeholder_address = params.stakeholder_address; + const blobdata& ex_nonce = params.ex_nonce; + bool pos = params.pos; + const pos_entry& pe = params.pe; + fill_block_template_func_t* pcustom_fill_block_template_func = params.pcustom_fill_block_template_func; + + uint64_t& height = resp.height; + block& b = resp.b; + wide_difficulty_type& diffic = resp.diffic; + + size_t median_size; boost::multiprecision::uint128_t already_generated_coins; CRITICAL_REGION_BEGIN(m_read_lock); - b.major_version = CURRENT_BLOCK_MAJOR_VERSION; + height = m_db_blocks.size(); + if(height <= m_core_runtime_config.hard_fork_01_starts_after_height) + b.major_version = BLOCK_MAJOR_VERSION_INITIAL; + else if(height <= m_core_runtime_config.hard_fork_03_starts_after_height) + b.major_version = HF1_BLOCK_MAJOR_VERSION; + else + b.major_version = CURRENT_BLOCK_MAJOR_VERSION; + b.minor_version = CURRENT_BLOCK_MINOR_VERSION; b.prev_id = get_top_block_id(); b.timestamp = m_core_runtime_config.get_core_time(); @@ -1153,20 +1460,20 @@ bool blockchain_storage::create_block_template(block& b, CHECK_AND_ASSERT_MES(diffic, false, "difficulty owverhead."); - height = m_db_blocks.size(); + median_size = m_db_current_block_cumul_sz_limit / 2; already_generated_coins = m_db_blocks.back()->already_generated_coins; CRITICAL_REGION_END(); - size_t txs_size; - uint64_t fee; + size_t txs_size = 0; + uint64_t fee = 0; bool block_filled = false; - if (custom_fill_block_template_func == nullptr) - block_filled = m_tx_pool.fill_block_template(b, pos, median_size, already_generated_coins, txs_size, fee, height); + if (pcustom_fill_block_template_func == nullptr) + block_filled = m_tx_pool.fill_block_template(b, pos, median_size, already_generated_coins, txs_size, fee, height, params.explicit_txs); else - block_filled = (*custom_fill_block_template_func)(b, pos, median_size, already_generated_coins, txs_size, fee, height); + block_filled = (*pcustom_fill_block_template_func)(b, pos, median_size, already_generated_coins, txs_size, fee, height); if (!block_filled) return false; @@ -1273,7 +1580,7 @@ std::string blockchain_storage::print_alt_chain(alt_chain_type alt_chain) return ss.str(); } //------------------------------------------------------------------ -bool blockchain_storage::append_altblock_keyimages_to_big_heap(const crypto::hash& block_id, const std::set& alt_block_keyimages) +bool blockchain_storage::append_altblock_keyimages_to_big_heap(const crypto::hash& block_id, const std::unordered_set& alt_block_keyimages) { for (auto& ki : alt_block_keyimages) m_altblocks_keyimages[ki].push_back(block_id); @@ -1320,15 +1627,14 @@ bool blockchain_storage::purge_altblock_keyimages_from_big_heap(const block& b, std::shared_ptr tx_ptr; if (!get_transaction_from_pool_or_db(tx_id, tx_ptr)) { - LOG_ERROR("failed to get alt block tx " << tx_id << " on block detach from alts"); continue; } transaction& tx = *tx_ptr; for (size_t n = 0; n < tx.vin.size(); ++n) { - if (tx.vin[n].type() == typeid(txin_to_key)) + if (tx.vin[n].type() == typeid(txin_to_key) || tx.vin[n].type() == typeid(txin_htlc)) { - purge_keyimage_from_big_heap(boost::get(tx.vin[n]).k_image, id); + purge_keyimage_from_big_heap(get_to_key_input_from_txin_v(tx.vin[n]).k_image, id); } } } @@ -1414,6 +1720,36 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: alt_block_extended_info abei = AUTO_VAL_INIT(abei); abei.bl = b; + abei.onboard_transactions.swap(bvc.m_onboard_transactions); + //for altblocks we should be sure that all transactions kept in onboard_transactions + for (auto& h: abei.bl.tx_hashes) + { + if (abei.onboard_transactions.count(h) == 0) + { + //need to take if from pool + transaction tx = AUTO_VAL_INIT(tx); + bool r = m_tx_pool.get_transaction(h, tx); + if (!r) + { + //transaction could be in main chain + auto tx_ptr = m_db_transactions.find(h); + if (!tx_ptr) + { + LOG_ERROR("Transaction " << h << " for altblock " << get_block_hash(abei.bl) << " not found"); + } + else + { + abei.onboard_transactions[h] = tx_ptr->tx; + } + } + else + { + abei.onboard_transactions[h] = tx; + } + } + } + + abei.timestamp = m_core_runtime_config.get_core_time(); abei.height = alt_chain.size() ? it_prev->second.height + 1 : *ptr_main_prev + 1; CHECK_AND_ASSERT_MES_CUSTOM(coinbase_height == abei.height, false, bvc.m_verification_failed = true, "block coinbase height doesn't match with altchain height, declined"); uint64_t connection_height = alt_chain.size() ? alt_chain.front()->second.height:abei.height; @@ -1439,8 +1775,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: CHECK_AND_ASSERT_MES_CUSTOM(!(pos_block && abei.height < m_core_runtime_config.pos_minimum_heigh), false, bvc.m_verification_failed = true, "PoS block is not allowed on this height"); - //wide_difficulty_type current_diff = get_next_difficulty_for_alternative_chain(alt_chain, bei, pos_block); - wide_difficulty_type current_diff = get_next_diff_conditional2(pos_block, alt_chain, connection_height); + wide_difficulty_type current_diff = get_next_diff_conditional2(pos_block, alt_chain, connection_height, abei); CHECK_AND_ASSERT_MES_CUSTOM(current_diff, false, bvc.m_verification_failed = true, "!!!!!!! DIFFICULTY OVERHEAD !!!!!!!"); @@ -1477,7 +1812,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: bvc.m_verification_failed = true; return false; } - std::set alt_block_keyimages; + std::unordered_set alt_block_keyimages; uint64_t ki_lookup_total = 0; if (!validate_alt_block_txs(b, id, alt_block_keyimages, abei, alt_chain, connection_height, ki_lookup_total)) { @@ -1499,10 +1834,23 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: if (abei.height >= m_core_runtime_config.pos_minimum_heigh) cumulative_diff_delta = correct_difficulty_with_sequence_factor(sequence_factor, cumulative_diff_delta); - abei.cumulative_diff_adjusted += cumulative_diff_delta; + if (abei.height > BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION && abei.height <= m_core_runtime_config.hard_fork_01_starts_after_height && pos_block && sequence_factor > BLOCK_POS_STRICT_SEQUENCE_LIMIT) + { + LOG_PRINT_RED_L0("Alternative block " << id << " @ " << abei.height << " has too big sequence factor: " << sequence_factor << ", rejected"); + bvc.m_verification_failed = true; + return false; + } - abei.cumulative_diff_precise = get_last_alt_x_block_cumulative_precise_difficulty(alt_chain, abei.height, pos_block); + abei.cumulative_diff_adjusted += cumulative_diff_delta; + wide_difficulty_type last_x_cumul_dif_precise_adj = 0; + abei.cumulative_diff_precise = get_last_alt_x_block_cumulative_precise_difficulty(alt_chain, abei.height-1, pos_block, last_x_cumul_dif_precise_adj); abei.cumulative_diff_precise += current_diff; + ////////////////////////////////////////////////////////////////////////// + + wide_difficulty_type diff_precise_adj = correct_difficulty_with_sequence_factor(sequence_factor, current_diff); + abei.cumulative_diff_precise_adjusted = last_x_cumul_dif_precise_adj + diff_precise_adj; + + ////////////////////////////////////////////////////////////////////////// #ifdef _DEBUG auto i_dres = m_alternative_chains.find(id); @@ -1515,7 +1863,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: alt_chain.push_back(i_res.first); //check if difficulty bigger then in main chain - bvc.height_difference = get_top_block_height() >= abei.height ? get_top_block_height() - abei.height : 0; + bvc.m_height_difference = get_top_block_height() >= abei.height ? get_top_block_height() - abei.height : 0; crypto::hash proof = null_hash; std::stringstream ss_pow_pos_info; @@ -1537,7 +1885,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: << ENDL << "HEIGHT " << abei.height << ", difficulty: " << abei.difficulty << ", cumul_diff_precise: " << abei.cumulative_diff_precise << ", cumul_diff_adj: " << abei.cumulative_diff_adjusted << " (current mainchain cumul_diff_adj: " << m_db_blocks.back()->cumulative_diff_adjusted << ", ki lookup total: " << ki_lookup_total <<")" , LOG_LEVEL_0); - if (is_reorganize_required(*m_db_blocks.back(), abei, proof)) + if (is_reorganize_required(*m_db_blocks.back(), alt_chain, proof)) { auto a = epee::misc_utils::create_scope_leave_handler([&]() { m_is_reorganize_in_process = false; }); CHECK_AND_ASSERT_THROW_MES(!m_is_reorganize_in_process, "Detected recursive reorganzie"); @@ -1552,13 +1900,30 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: bvc.m_verification_failed = true; return r; } - bvc.added_to_altchain = true; + bvc.m_added_to_altchain = true; + + //protect ourself from altchains container flood + if (m_alternative_chains.size() > m_core_runtime_config.max_alt_blocks) + prune_aged_alt_blocks(); + return true; }else { //block orphaned bvc.m_marked_as_orphaned = true; - LOG_PRINT_RED_L0("Block recognized as orphaned and rejected, id = " << id << "," << ENDL << "parent id = " << b.prev_id << ENDL << "height = " << coinbase_height); + + if (m_invalid_blocks.count(id) != 0) + { + LOG_PRINT_RED_L0("Block recognized as blacklisted and rejected, id = " << id << "," << ENDL << "parent id = " << b.prev_id << ENDL << "height = " << coinbase_height); + } + else if (m_invalid_blocks.count(b.prev_id) != 0) + { + LOG_PRINT_RED_L0("Block recognized as orphaned (parent " << b.prev_id << " is in blacklist) and rejected, id = " << id << "," << ENDL << "parent id = " << b.prev_id << ENDL << "height = " << coinbase_height); + } + else + { + LOG_PRINT_RED_L0("Block recognized as orphaned and rejected, id = " << id << "," << ENDL << "parent id = " << b.prev_id << ENDL << "height = " << coinbase_height); + } } CHECK_AND_ASSERT_MES(validate_blockchain_prev_links(), false, "EPIC FAIL!"); @@ -1566,26 +1931,120 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: CATCH_ENTRY_CUSTOM("blockchain_storage::handle_alternative_block", bvc.m_verification_failed = true, false); } //------------------------------------------------------------------ -bool blockchain_storage::is_reorganize_required(const block_extended_info& main_chain_bei, const block_extended_info& alt_chain_bei, const crypto::hash& proof_alt) +wide_difficulty_type blockchain_storage::get_x_difficulty_after_height(uint64_t height, bool is_pos) { - if (main_chain_bei.cumulative_diff_adjusted < alt_chain_bei.cumulative_diff_adjusted) - return true; - else if (main_chain_bei.cumulative_diff_adjusted > alt_chain_bei.cumulative_diff_adjusted) - return false; - else // main_chain_bei.cumulative_diff_adjusted == alt_chain_bei.cumulative_diff_adjusted + CRITICAL_REGION_LOCAL(m_read_lock); + CHECK_AND_ASSERT_THROW_MES(height < m_db_blocks.size(), "Internal error: condition failed: height (" << height << ") < m_db_blocks.size() " << m_db_blocks.size()); + wide_difficulty_type diff = 0; + for (uint64_t i = height + 1; i != m_db_blocks.size(); i++) { - if (!is_pos_block(main_chain_bei.bl)) - return false; // do not reorganize on the same cummul diff if it's a PoW block + auto bei_ptr = m_db_blocks[i]; + if (is_pos_block(bei_ptr->bl) == is_pos) + { + diff = bei_ptr->difficulty; + break; + } + } + if (diff == 0) + { + //never met x type of block, that meanst that difficulty is current + diff = get_cached_next_difficulty(is_pos); + } + return diff; +} +//------------------------------------------------------------------ +bool blockchain_storage::is_reorganize_required(const block_extended_info& main_chain_bei, const alt_chain_type& alt_chain, const crypto::hash& proof_alt) +{ + //alt_chain - back is latest(top), first - connection with main chain + const block_extended_info& alt_chain_bei = alt_chain.back()->second; + const block_extended_info& connection_point = alt_chain.front()->second; - //in case of simultaneous PoS blocks are happened on the same height (quite common for PoS) - //we also try to weight them to guarantee consensus in network - if (std::memcmp(&main_chain_bei.stake_hash, &proof_alt, sizeof(main_chain_bei.stake_hash)) >= 0) + if (connection_point.height <= m_core_runtime_config.hard_fork_01_starts_after_height) + { + //use pre-hard fork, old-style comparing + if (main_chain_bei.cumulative_diff_adjusted < alt_chain_bei.cumulative_diff_adjusted) + return true; + else if (main_chain_bei.cumulative_diff_adjusted > alt_chain_bei.cumulative_diff_adjusted) return false; + else // main_chain_bei.cumulative_diff_adjusted == alt_chain_bei.cumulative_diff_adjusted + { + if (!is_pos_block(main_chain_bei.bl)) + return false; // do not reorganize on the same cummul diff if it's a PoW block + + //in case of simultaneous PoS blocks are happened on the same height (quite common for PoS) + //we also try to weight them to guarantee consensus in network + if (std::memcmp(&main_chain_bei.stake_hash, &proof_alt, sizeof(main_chain_bei.stake_hash)) >= 0) + return false; + + LOG_PRINT_L2("[is_reorganize_required]:TRUE, \"by order of memcmp\" main_stake_hash:" << &main_chain_bei.stake_hash << ", alt_stake_hash" << proof_alt); + return true; + } + } + else if (alt_chain_bei.height > m_core_runtime_config.hard_fork_01_starts_after_height) + { + //new rules, applied after HARD_FORK_1 + //to learn this algo please read https://github.com/hyle-team/docs/blob/master/zano/PoS_Analysis_and_improvements_proposal.pdf + + wide_difficulty_type difficulty_pos_at_split_point = get_x_difficulty_after_height(connection_point.height - 1, true); + wide_difficulty_type difficulty_pow_at_split_point = get_x_difficulty_after_height(connection_point.height - 1, false); + + difficulties main_cumul_diff = AUTO_VAL_INIT(main_cumul_diff); + difficulties alt_cumul_diff = AUTO_VAL_INIT(alt_cumul_diff); + //we use get_last_alt_x_block_cumulative_precise_adj_difficulty for getting both alt chain and main chain diff of given block types + + wide_difficulty_type alt_pos_diff_end = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain, alt_chain_bei.height, true); + wide_difficulty_type alt_pos_diff_begin = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), connection_point.height-1, true); + alt_cumul_diff.pos_diff = alt_pos_diff_end - alt_pos_diff_begin; - LOG_PRINT_L2("[is_reorganize_required]:TRUE, \"by order of memcmp\" main_stake_hash:" << &main_chain_bei.stake_hash << ", alt_stake_hash" << proof_alt); - return true; + wide_difficulty_type alt_pow_diff_end = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain, alt_chain_bei.height, false); + wide_difficulty_type alt_pow_diff_begin = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), connection_point.height - 1, false); + alt_cumul_diff.pow_diff = alt_pow_diff_end - alt_pow_diff_begin; + + wide_difficulty_type main_pos_diff_end = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), m_db_blocks.size()-1, true); + wide_difficulty_type main_pos_diff_begin = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), connection_point.height - 1, true); + main_cumul_diff.pos_diff = main_pos_diff_end - main_pos_diff_begin; + + wide_difficulty_type main_pow_diff_end = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), m_db_blocks.size() - 1, false); + wide_difficulty_type main_pow_diff_begin = get_last_alt_x_block_cumulative_precise_adj_difficulty(alt_chain_type(), connection_point.height - 1, false); + main_cumul_diff.pow_diff = main_pow_diff_end - main_pow_diff_begin; + + //TODO: measurement of precise cumulative difficult + boost::multiprecision::uint1024_t alt = get_a_to_b_relative_cumulative_difficulty(difficulty_pos_at_split_point, difficulty_pow_at_split_point, alt_cumul_diff, main_cumul_diff); + boost::multiprecision::uint1024_t main = get_a_to_b_relative_cumulative_difficulty(difficulty_pos_at_split_point, difficulty_pow_at_split_point, main_cumul_diff, alt_cumul_diff); + LOG_PRINT_L1("[FORK_CHOICE]: " << ENDL + << "difficulty_pow_at_split_point:" << difficulty_pow_at_split_point << ENDL + << "difficulty_pos_at_split_point:" << difficulty_pos_at_split_point << ENDL + << "alt_cumul_diff.pow_diff:" << alt_cumul_diff.pow_diff << ENDL + << "alt_cumul_diff.pos_diff:" << alt_cumul_diff.pos_diff << ENDL + << "main_cumul_diff.pow_diff:" << main_cumul_diff.pow_diff << ENDL + << "main_cumul_diff.pos_diff:" << main_cumul_diff.pos_diff << ENDL + << "alt:" << alt << ENDL + << "main:" << main << ENDL + ); + if (main < alt) + return true; + else if (main > alt) + return false; + else + { + if (!is_pos_block(main_chain_bei.bl)) + return false; // do not reorganize on the same cummul diff if it's a PoW block + + //in case of simultaneous PoS blocks are happened on the same height (quite common for PoS) + //we also try to weight them to guarantee consensus in network + if (std::memcmp(&main_chain_bei.stake_hash, &proof_alt, sizeof(main_chain_bei.stake_hash)) >= 0) + return false; + + LOG_PRINT_L1("[is_reorganize_required]:TRUE, \"by order of memcmp\" main_stake_hash:" << &main_chain_bei.stake_hash << ", alt_stake_hash" << proof_alt); + return true; + } + } + else + { + ASSERT_MES_AND_THROW("Unknown version of block"); } } + //------------------------------------------------------------------ bool blockchain_storage::pre_validate_relayed_block(block& bl, block_verification_context& bvc, const crypto::hash& id)const { @@ -2017,7 +2476,13 @@ bool blockchain_storage::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPU << out_ptr->out_no << " more than transaction outputs = " << tx_ptr->tx.vout.size() << ", for tx id = " << out_ptr->tx_id); const transaction& tx = tx_ptr->tx; + if (tx.vout[out_ptr->out_no].target.type() == typeid(txout_htlc)) + { + //silently return false, it's ok + return false; + } CHECK_AND_ASSERT_MES(tx.vout[out_ptr->out_no].target.type() == typeid(txout_to_key), false, "unknown tx out type"); + const txout_to_key& otk = boost::get(tx.vout[out_ptr->out_no].target); CHECK_AND_ASSERT_MES(tx_ptr->m_spent_flags.size() == tx.vout.size(), false, "internal error"); @@ -2025,12 +2490,16 @@ bool blockchain_storage::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPU if (tx_ptr->m_spent_flags[out_ptr->out_no]) return false; + // do not use burned coins + if (otk.key == null_pkey) + return false; + //check if transaction is unlocked - if (!is_tx_spendtime_unlocked(get_tx_unlock_time(tx))) + if (!is_tx_spendtime_unlocked(get_tx_unlock_time(tx, out_ptr->out_no))) return false; //use appropriate mix_attr out - uint8_t mix_attr = boost::get(tx.vout[out_ptr->out_no].target).mix_attr; + uint8_t mix_attr = otk.mix_attr; if(mix_attr == CURRENCY_TO_KEY_OUT_FORCED_NO_MIX) return false; //COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS call means that ring signature will have more than one entry. @@ -2042,7 +2511,7 @@ bool blockchain_storage::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPU COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); oen.global_amount_index = i; - oen.out_key = boost::get(tx.vout[out_ptr->out_no].target).key; + oen.out_key = otk.key; return true; } //------------------------------------------------------------------ @@ -2100,14 +2569,14 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO } if (result_outs.outs.size() < req.outs_count) { - LOG_PRINT_RED_L0("Not enough inputs for amount " << amount << ", needed " << req.outs_count << ", added " << result_outs.outs.size() << " good outs from " << up_index_limit << " unlocked of " << outs_container_size << " total"); + LOG_PRINT_RED_L0("Not enough inputs for amount " << print_money_brief(amount) << ", needed " << req.outs_count << ", added " << result_outs.outs.size() << " good outs from " << up_index_limit << " unlocked of " << outs_container_size << " total"); } }else { size_t added = 0; for (size_t i = 0; i != up_index_limit; i++) added += add_out_to_get_random_outs(result_outs, amount, i, req.outs_count, req.use_forced_mix_outs) ? 1 : 0; - LOG_PRINT_RED_L0("Not enough inputs for amount " << amount << ", needed " << req.outs_count << ", added " << added << " good outs from " << up_index_limit << " unlocked of " << outs_container_size << " total - respond with all good outs"); + LOG_PRINT_RED_L0("Not enough inputs for amount " << print_money_brief(amount) << ", needed " << req.outs_count << ", added " << added << " good outs from " << up_index_limit << " unlocked of " << outs_container_size << " total - respond with all good outs"); } } return true; @@ -2127,7 +2596,7 @@ bool blockchain_storage::is_pos_allowed() const return get_top_block_height() >= m_core_runtime_config.pos_minimum_heigh; } //------------------------------------------------------------------ -bool blockchain_storage::update_spent_tx_flags_for_input(uint64_t amount, const txout_v& o, bool spent) +bool blockchain_storage::update_spent_tx_flags_for_input(uint64_t amount, const txout_ref_v& o, bool spent) { if (o.type() == typeid(ref_by_id)) return update_spent_tx_flags_for_input(boost::get(o).tx_id, boost::get(o).n, spent); @@ -2205,31 +2674,6 @@ bool blockchain_storage::is_multisig_output_spent(const crypto::hash& multisig_i return source_tx_ptr->m_spent_flags[ms_out_index]; } //------------------------------------------------------------------ -// bool blockchain_storage::resync_spent_tx_flags() -// { -// LOG_PRINT_L0("Started re-building spent tx outputs data..."); -// CRITICAL_REGION_LOCAL(m_blockchain_lock); -// for(auto& tx: m_db_transactions) -// { -// if(is_coinbase(tx.second.tx)) -// continue; -// -// for(auto& in: tx.second.tx.vin) -// { -// CHECKED_GET_SPECIFIC_VARIANT(in, txin_to_key, in_to_key, false); -// if(in_to_key.key_offsets.size() != 1) -// continue; -// -// //direct spending -// if(!update_spent_tx_flags_for_input(in_to_key.amount, in_to_key.key_offsets[0], true)) -// return false; -// -// } -// } -// LOG_PRINT_L0("Finished re-building spent tx outputs data"); -// return true; -// } -//------------------------------------------------------------------ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset)const { CRITICAL_REGION_LOCAL(m_read_lock); @@ -2314,7 +2758,7 @@ bool blockchain_storage::forecast_difficulty(std::vector>& blocks) const { - LOG_ERROR("NOT IMPLEMENTED YET"); -// std::stringstream ss; -// CRITICAL_REGION_LOCAL(m_blockchain_lock); -// BOOST_FOREACH(const outputs_container::value_type& v, m_db_outputs) -// { -// const std::vector >& vals = v.second; -// if (vals.size()) -// { -// ss << "amount: " << print_money(v.first); -// uint64_t total_count = vals.size(); -// uint64_t unused_count = 0; -// for (size_t i = 0; i != vals.size(); i++) -// { -// bool used = false; -// auto it_tx = m_db_transactions.find(vals[i].first); -// if (it_tx == m_db_transactions.end()) -// { -// LOG_ERROR("Tx with id not found " << vals[i].first); -// } -// else -// { -// if (vals[i].second >= it_tx->second.m_spent_flags.size()) -// { -// LOG_ERROR("Tx with id " << vals[i].first << " in global index have wrong entry in global index, offset in tx = " << vals[i].second -// << ", it_tx->second.m_spent_flags.size()=" << it_tx->second.m_spent_flags.size() -// << ", it_tx->second.tx.vin.size()=" << it_tx->second.tx.vin.size()); -// } -// used = it_tx->second.m_spent_flags[vals[i].second]; -// -// } -// if (!used) -// ++unused_count; -// } -// ss << "\t total: " << total_count << "\t unused: " << unused_count << ENDL; -// } -// } -// LOG_PRINT_L0("OUTS: " << ENDL << ss.str()); + uint64_t count = 0; + for (uint64_t i = m_db_blocks.size() - 1; i != 0; --i) + { + auto block_ptr = m_db_blocks[i]; + if (is_pos_block(block_ptr->bl) == pos_blocks) + { + blocks.push_back(block_ptr); + ++count; + if (count >= n) + break; + } + } +} +//------------------------------------------------------------------ +void blockchain_storage::print_last_n_difficulty_numbers(uint64_t n) const +{ + + std::stringstream ss; + std::list> pos_blocks; + std::list> pow_blocks; + + get_last_n_x_blocks(n, true, pos_blocks); + get_last_n_x_blocks(n, false, pow_blocks); + ss << "PoS blocks difficulty:" << ENDL; + for (auto& bl_ptr : pos_blocks) + { + ss << bl_ptr->difficulty << ENDL; + } + + ss << "PoW blocks difficulty:" << ENDL; + for (auto& bl_ptr : pow_blocks) + { + ss << bl_ptr->difficulty << ENDL; + } + LOG_PRINT_L0("LAST BLOCKS:" << ss.str()); +} +//------------------------------------------------------------------ +void blockchain_storage::print_blockchain_outs_stats() const +{ + std::stringstream ss; + CRITICAL_REGION_LOCAL(m_read_lock); + + struct output_stat_t + { + uint64_t total = 0; + uint64_t unspent = 0; + uint64_t mixable = 0; + }; + + std::map outputs_stats; + + const uint64_t subitems_cnt = m_db_outputs.size(); + uint64_t progress = 0; + + auto lambda_handler = [&](uint64_t i, uint64_t amount, uint64_t index, const currency::global_output_entry& output_entry) -> bool + { + uint64_t progress_current = 20 * i / subitems_cnt; + if (progress_current != progress) + { + progress = progress_current; + LOG_PRINT_L0(progress * 5 << "%"); + } + + auto p_tx = m_db_transactions.find(output_entry.tx_id); + if (!p_tx) + { + LOG_ERROR("tx " << output_entry.tx_id << " not found"); + return true; // continue + } + if (output_entry.out_no >= p_tx->m_spent_flags.size()) + { + LOG_ERROR("tx with id " << output_entry.tx_id << " has wrong entry in global index, out_no = " << output_entry.out_no + << ", p_tx->m_spent_flags.size() = " << p_tx->m_spent_flags.size() + << ", p_tx->tx.vin.size() = " << p_tx->tx.vin.size()); + return true; // continue + } + if (p_tx->tx.vout.size() != p_tx->m_spent_flags.size()) + { + LOG_ERROR("Tx with id " << output_entry.tx_id << " has wrong entry in global index, out_no = " << output_entry.out_no + << ", p_tx->tx.vout.size() = " << p_tx->tx.vout.size() + << ", p_tx->m_spent_flags.size() = " << p_tx->m_spent_flags.size()); + return true; // continue + } + + auto& stat = outputs_stats[amount]; + ++stat.total; + + bool spent = p_tx->m_spent_flags[output_entry.out_no]; + if (!spent) + ++stat.unspent; + + if (!spent && p_tx->tx.vout[output_entry.out_no].target.type() == typeid(txout_to_key)) + { + if (boost::get(p_tx->tx.vout[output_entry.out_no].target).mix_attr != CURRENCY_TO_KEY_OUT_FORCED_NO_MIX) + ++stat.mixable; + } + return true; + }; + + m_db_outputs.enumerate_subitems(lambda_handler); + + ss << std::right << std::setw(15) << "amount" << std::setw(10) << "total" << std::setw(10) << "unspent" << std::setw(10) << "mixable" << ENDL; + for(auto it = outputs_stats.begin(); it != outputs_stats.end(); ++it) + ss << std::setw(15) << print_money_brief(it->first) << std::setw(10) << it->second.total << std::setw(10) << it->second.unspent << std::setw(10) << it->second.mixable << ENDL; + + LOG_PRINT_L0("OUTS: " << ENDL << ss.str()); } //------------------------------------------------------------------ void blockchain_storage::print_blockchain_outs(const std::string& file) const @@ -2586,11 +3100,126 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const +uint64_t blockchain_storage::get_blockchain_launch_timestamp()const +{ + if (m_blockchain_launch_timestamp) + return m_blockchain_launch_timestamp; + + if (m_db_blocks.size() > 2) + { + m_blockchain_launch_timestamp = m_db_blocks[1]->bl.timestamp; + } + return m_blockchain_launch_timestamp; +} +//------------------------------------------------------------------ +bool blockchain_storage::get_est_height_from_date(uint64_t date, uint64_t& res_h)const +{ + CRITICAL_REGION_LOCAL(m_read_lock); +#define GET_EST_HEIGHT_FROM_DATE_THRESHOLD 1440 + + res_h = 0; + + if (date < get_blockchain_launch_timestamp()) + { + return true; + } + + + uint64_t calculated_estimated_height = (date - get_blockchain_launch_timestamp()) / DIFFICULTY_TOTAL_TARGET; + + if (date > m_db_blocks[m_db_blocks.size() - 1]->bl.timestamp) + { + //that suspicious but also could be(in case someone just created wallet offline in + //console and then got it synchronyzing and last block had a little timestamp shift) + //let's just return 1 day behind for safety reasons. + if (m_db_blocks.size() > 1440) + { + res_h = m_db_blocks.size() - 1440; + } + else + { + //likely impossible, but just in case + res_h = 0; + } + return true; + } + if (calculated_estimated_height > m_db_blocks.size() - 1) + calculated_estimated_height = m_db_blocks.size() - 1; + + //goal is to get timestamp in window in between 1day+1hour and 1 hour before target(1 hour is just to be sure that + //we didn't miss actual wallet start because of timestamp and difficulty fluctuations) + uint64_t low_boundary = date - 90000; //1 day + 1 hour + uint64_t high_boundary = date - 3600; //1 hour + + //std::cout << "ENTRY: low_boundary(minutes):" << low_boundary/60 << " high_boundary(minutes): " << high_boundary / 60 << std::endl; + + uint64_t iteration_coun = 0; + uint64_t current_low_boundary = 0; + uint64_t current_height_boundary = m_db_blocks.size() - 1; + while (true) + { + iteration_coun++; + if (iteration_coun > 29) // Log2(CURRENCY_MAX_BLOCK_NUMBER) + { + LOG_ERROR("Internal error: too much iterations on get_est_height_from_date, date = " << date); + return true; + } + uint64_t ts = m_db_blocks[calculated_estimated_height]->bl.timestamp; + if (ts > high_boundary) + { + //we moved too much forward + + current_height_boundary = calculated_estimated_height; + CHECK_AND_ASSERT_MES(current_height_boundary > current_low_boundary, true, + "Internal error: current_hight_boundary(" << current_height_boundary << ") > current_low_boundary("<< current_low_boundary << ")"); + uint64_t offset = (current_height_boundary - current_low_boundary)/2; + if (offset <= 2) + { + //something really wrong with distribution of blocks, just use current_low_boundary to be sure that we didn't mess any transactions + res_h = current_low_boundary; + return true; + } + + //std::cout << "est_h:" << calculated_estimated_height << ", ts(min): " << ts / 60 << " distance to RIGHT minutes: " << int64_t((int64_t(ts) - int64_t(high_boundary))) / 60 << std::endl; + //std::cout << "OOFFSET: -" << offset << std::endl; + calculated_estimated_height -= offset; + } + else if (ts < low_boundary) + { + //we too much in past + current_low_boundary = calculated_estimated_height; + CHECK_AND_ASSERT_MES(current_height_boundary > current_low_boundary, true, + "Internal error: current_hight_boundary(" << current_height_boundary << ") > current_low_boundary(" << current_low_boundary << ")"); + uint64_t offset = (current_height_boundary - current_low_boundary) / 2; + if (offset <= 2) + { + //something really wrong with distribution of blocks, just use current_low_boundary to be sure that we didn't mess any transactions + res_h = current_low_boundary; + return true; + } + //CHECK_AND_ASSERT_MES(offset > 2, true, + // "offset is too low = " << offset); + + //std::cout << "est_h:" << calculated_estimated_height << ", ts(min): " << ts / 60 << " distance to LEFT minutes: " << int64_t((int64_t(low_boundary) - int64_t(ts))) / 60 << std::endl; + //std::cout << "OOFFSET: +" << offset << std::endl; + calculated_estimated_height += offset; + } + else + { + res_h = calculated_estimated_height; + break; + } + } + + LOG_PRINT_L0("[get_est_height_from_date] returned " << calculated_estimated_height << " with " << iteration_coun << " iterations"); + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height, bool need_global_indexes)const { CRITICAL_REGION_LOCAL(m_read_lock); blocks_direct_container blocks_direct; - if (!find_blockchain_supplement(qblock_ids, blocks_direct, total_height, start_height, max_count)) + if (!find_blockchain_supplement(qblock_ids, blocks_direct, total_height, start_height, max_count, minimum_height)) return false; for (auto& bd : blocks_direct) @@ -2605,11 +3234,13 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const +bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height, bool request_coinbase_info)const { CRITICAL_REGION_LOCAL(m_read_lock); if (!find_blockchain_supplement(qblock_ids, start_height)) return false; + if (minimum_height > start_height) + start_height = minimum_height; total_height = get_current_blockchain_size(); size_t count = 0; @@ -2619,7 +3250,9 @@ bool blockchain_storage::find_blockchain_supplement(const std::list mis; get_transactions_direct(m_db_blocks[i]->bl.tx_hashes, blocks.back().second, mis); - CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); + CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, block " << get_block_hash(m_db_blocks[i]->bl) << " [" << i << "] contains missing transactions: " << mis); + if(request_coinbase_info) + blocks.back().third = m_db_transactions.find(get_transaction_hash(m_db_blocks[i]->bl.miner_tx)); } return true; } @@ -2696,10 +3329,15 @@ bool blockchain_storage::push_transaction_to_global_outs_index(const transaction size_t i = 0; BOOST_FOREACH(const auto& ot, tx.vout) { - if (ot.target.type() == typeid(txout_to_key)) + if (ot.target.type() == typeid(txout_to_key) || ot.target.type() == typeid(txout_htlc)) { m_db_outputs.push_back_item(ot.amount, global_output_entry::construct(tx_id, i)); global_indexes.push_back(m_db_outputs.get_item_size(ot.amount) - 1); + if (ot.target.type() == typeid(txout_htlc) && !is_after_hardfork_3_zone()) + { + LOG_ERROR("Error: Transaction with txout_htlc before is_after_hardfork_3_zone(before height " << m_core_runtime_config.hard_fork_03_starts_after_height <<")"); + return false; + } } else if (ot.target.type() == typeid(txout_multisig)) { @@ -2737,8 +3375,14 @@ bool blockchain_storage::get_outs(uint64_t amount, std::list auto tx_ptr = m_db_transactions.find(out_entry_ptr->tx_id); CHECK_AND_ASSERT_MES(tx_ptr, false, "transactions outs global index consistency broken: can't find tx " << out_entry_ptr->tx_id << " in DB, for amount: " << amount << ", gindex: " << i); CHECK_AND_ASSERT_MES(tx_ptr->tx.vout.size() > out_entry_ptr->out_no, false, "transactions outs global index consistency broken: index in tx_outx == " << out_entry_ptr->out_no << " is greather than tx.vout size == " << tx_ptr->tx.vout.size() << ", for amount: " << amount << ", gindex: " << i); - CHECK_AND_ASSERT_MES(tx_ptr->tx.vout[out_entry_ptr->out_no].target.type() == typeid(txout_to_key), false, "transactions outs global index consistency broken: out #" << out_entry_ptr->out_no << " in tx " << out_entry_ptr->tx_id << " has wrong type, for amount: " << amount << ", gindex: " << i); - pkeys.push_back(boost::get(tx_ptr->tx.vout[out_entry_ptr->out_no].target).key); + //CHECK_AND_ASSERT_MES(tx_ptr->tx.vout[out_entry_ptr->out_no].target.type() == typeid(txout_to_key), false, "transactions outs global index consistency broken: out #" << out_entry_ptr->out_no << " in tx " << out_entry_ptr->tx_id << " has wrong type, for amount: " << amount << ", gindex: " << i); + if (tx_ptr->tx.vout[out_entry_ptr->out_no].target.type() == typeid(txout_to_key)) + { + pkeys.push_back(boost::get(tx_ptr->tx.vout[out_entry_ptr->out_no].target).key); + }else if(tx_ptr->tx.vout[out_entry_ptr->out_no].target.type() == typeid(txout_htlc)) + { + pkeys.push_back(boost::get(tx_ptr->tx.vout[out_entry_ptr->out_no].target).pkey_redeem); + } } return true; @@ -2750,7 +3394,7 @@ bool blockchain_storage::pop_transaction_from_global_index(const transaction& tx size_t i = tx.vout.size()-1; BOOST_REVERSE_FOREACH(const auto& ot, tx.vout) { - if (ot.target.type() == typeid(txout_to_key)) + if (ot.target.type() == typeid(txout_to_key) || ot.target.type() == typeid(txout_htlc)) { uint64_t sz= m_db_outputs.get_item_size(ot.amount); CHECK_AND_ASSERT_MES(sz, false, "transactions outs global index: empty index for amount: " << ot.amount); @@ -2870,7 +3514,7 @@ bool blockchain_storage::pop_alias_info(const extra_alias_entry& ai) m_db_addr_to_alias.set(local_alias_hist.back().m_address, local_copy); } - LOG_PRINT_MAGENTA("[ALIAS_UNREGISTERED]: " << ai.m_alias << ": " << get_account_address_as_str(ai.m_address) << " -> " << (!alias_history_ptr->empty() ? get_account_address_as_str(alias_history_ptr->back().m_address) : "(available)"), LOG_LEVEL_0); + LOG_PRINT_MAGENTA("[ALIAS_UNREGISTERED]: " << ai.m_alias << ": " << get_account_address_as_str(ai.m_address) << " -> " << (!local_alias_hist.empty() ? get_account_address_as_str(local_alias_hist.back().m_address) : "(available)"), LOG_LEVEL_1); return true; } //------------------------------------------------------------------ @@ -2910,11 +3554,11 @@ bool blockchain_storage::put_alias_info(const transaction & tx, extra_alias_entr //std::string signed_buff; //make_tx_extra_alias_entry(signed_buff, ai, true); std::string old_address = currency::get_account_address_as_str(local_alias_history.back().m_address); - bool r = crypto::check_signature(get_sign_buff_hash_for_alias_update(ai), local_alias_history.back().m_address.m_spend_public_key, ai.m_sign.back()); + bool r = crypto::check_signature(get_sign_buff_hash_for_alias_update(ai), local_alias_history.back().m_address.spend_public_key, ai.m_sign.back()); CHECK_AND_ASSERT_MES(r, false, "Failed to check signature, alias update failed." << ENDL << "alias: " << ai.m_alias << ENDL << "signed_buff_hash: " << get_sign_buff_hash_for_alias_update(ai) << ENDL - << "public key: " << local_alias_history.back().m_address.m_spend_public_key << ENDL + << "public key: " << local_alias_history.back().m_address.spend_public_key << ENDL << "new_address: " << get_account_address_as_str(ai.m_address) << ENDL << "signature: " << epee::string_tools::pod_to_hex(ai.m_sign) << ENDL << "alias_history.size() = " << local_alias_history.size()); @@ -2942,6 +3586,20 @@ bool blockchain_storage::put_alias_info(const transaction & tx, extra_alias_entr else { LOG_ERROR("Wrong m_addr_to_alias state: address not found " << get_account_address_as_str(local_alias_history.back().m_address)); + + std::stringstream ss; + ss << "History for alias " << ai.m_alias << ":" << ENDL; + size_t i = 0; + for (auto el : local_alias_history) + { + ss << std::setw(2) << i++ << " " + << get_account_address_as_str(el.m_address) << " " + << (el.m_sign.empty() ? " no sig " : " SIGNED ") << " " + << el.m_text_comment << ENDL; + } + + LOG_PRINT_L0(ss.str()); + } //update alias db @@ -2957,7 +3615,7 @@ bool blockchain_storage::put_alias_info(const transaction & tx, extra_alias_entr addr_to_alias_local2.insert(ai.m_alias); m_db_addr_to_alias.set(local_alias_history.back().m_address, addr_to_alias_local2); - LOG_PRINT_MAGENTA("[ALIAS_UPDATED]: " << ai.m_alias << ": from: " << old_address << " to " << get_account_address_as_str(ai.m_address), LOG_LEVEL_0); + LOG_PRINT_MAGENTA("[ALIAS_UPDATED]: " << ai.m_alias << ": from: " << old_address << " to " << get_account_address_as_str(ai.m_address), LOG_LEVEL_1); rise_core_event(CORE_EVENT_UPDATE_ALIAS, alias_info_to_rpc_update_alias_info(ai, old_address)); } @@ -3010,7 +3668,7 @@ uint64_t blockchain_storage::validate_alias_reward(const transaction& tx, const return true; } //------------------------------------------------------------------ -bool blockchain_storage::prevalidate_alias_info(const transaction& tx, extra_alias_entry& eae) +bool blockchain_storage::prevalidate_alias_info(const transaction& tx, const extra_alias_entry& eae) { bool r = validate_alias_name(eae.m_alias); @@ -3191,7 +3849,15 @@ namespace currency return true; } - + bool operator()(const txin_htlc& in) const + { + if (!m_bcs.is_after_hardfork_3_zone()) + { + LOG_ERROR("Error: Transaction with txin_htlc before is_after_hardfork_3_zone(before height " << m_bcs.get_core_runtime_config().hard_fork_03_starts_after_height << ")"); + return false; + } + return this->operator()(static_cast(in)); + } bool operator()(const txin_gen& in) const { return true; } bool operator()(const txin_multisig& in) const { @@ -3218,6 +3884,8 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const TIME_MEASURE_START_PD(tx_append_is_expired); CHECK_AND_ASSERT_MES(!is_tx_expired(tx), false, "Transaction can't be added to the blockchain since it's already expired. tx expiration time: " << get_tx_expiration_time(tx) << ", blockchain median time: " << get_tx_expiration_median()); TIME_MEASURE_FINISH_PD_COND(need_to_profile, tx_append_is_expired); + + CHECK_AND_ASSERT_MES(validate_tx_for_hardfork_specific_terms(tx, tx_id, bl_height), false, "tx " << tx_id << ": hardfork-specific validation failed"); TIME_MEASURE_START_PD(tx_process_extra); bool r = process_blockchain_tx_extra(tx); @@ -3230,7 +3898,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const TIME_MEASURE_START_PD(tx_process_inputs); - BOOST_FOREACH(const txin_v& in, tx.vin) + for(const txin_v& in : tx.vin) { if(!boost::apply_visitor(add_transaction_input_visitor(*this, m_db_spent_keys, tx_id, bl_id, bl_height), in)) { @@ -3310,7 +3978,7 @@ bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std:: bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) const { CRITICAL_REGION_LOCAL(m_read_lock); - bool res = check_tx_inputs(tx, tx_prefix_hash, &max_used_block_height); + bool res = check_tx_inputs(tx, tx_prefix_hash, max_used_block_height); if(!res) return false; CHECK_AND_ASSERT_MES(max_used_block_height < m_db_blocks.size(), false, "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_db_blocks.size()); get_block_hash(m_db_blocks[max_used_block_height]->bl, max_used_block_id); @@ -3411,8 +4079,7 @@ uint64_t blockchain_storage::tx_fee_median_for_height(uint64_t h)const //------------------------------------------------------------------ bool blockchain_storage::validate_all_aliases_for_new_median_mode() { - - LOG_PRINT_L0("Started reinitialization of median fee..."); + LOG_PRINT_L0("Started reinitialization of median fee..."); math_helper::once_a_time_seconds<10> log_idle; uint64_t sz = m_db_blocks.size(); for (uint64_t i = 0; i != sz; i++) @@ -3451,15 +4118,91 @@ bool blockchain_storage::validate_all_aliases_for_new_median_mode() return true; } //------------------------------------------------------------------ +bool blockchain_storage::print_tx_outputs_lookup(const crypto::hash& tx_id)const +{ + CRITICAL_REGION_LOCAL(m_read_lock); + auto tx_ptr = m_db_transactions.find(tx_id); + + if (!tx_ptr) + { + LOG_PRINT_RED_L0("Tx " << tx_id << " not found"); + return true; + } + + //amount -> index -> [{tx_id, rind_count}] + std::map > > > usage_stat; + std::stringstream strm_tx; + CHECK_AND_ASSERT_MES(tx_ptr->tx.vout.size() == tx_ptr->m_global_output_indexes.size(), false, "Internal error: output size missmatch"); + for (uint64_t i = 0; i!= tx_ptr->tx.vout.size();i++) + { + strm_tx << "[" << i << "]: " << print_money(tx_ptr->tx.vout[i].amount) << ENDL; + if (tx_ptr->tx.vout[i].target.type() != typeid(currency::txout_to_key)) + continue; + + usage_stat[tx_ptr->tx.vout[i].amount][tx_ptr->m_global_output_indexes[i]]; + + } + + LOG_PRINT_L0("Lookup in all transactions...."); + for (uint64_t i = 0; i != m_db_blocks.size(); i++) + { + auto block_ptr = m_db_blocks[i]; + for (auto block_tx_id : block_ptr->bl.tx_hashes) + { + auto block_tx_ptr = m_db_transactions.find(block_tx_id); + for (auto txi_in : block_tx_ptr->tx.vin) + { + if(txi_in.type() != typeid(currency::txin_to_key)) + continue; + currency::txin_to_key& txi_in_tokey = boost::get(txi_in); + uint64_t amount = txi_in_tokey.amount; + auto amount_it = usage_stat.find(amount); + if(amount_it == usage_stat.end()) + continue; + + for (txout_ref_v& off : txi_in_tokey.key_offsets) + { + if(off.type() != typeid(uint64_t)) + continue; + uint64_t index = boost::get(off); + auto index_it = amount_it->second.find(index); + if(index_it == amount_it->second.end()) + continue; + index_it->second.push_back(std::pair(block_tx_id, txi_in_tokey.key_offsets.size())); + } + } + } + } + + std::stringstream ss; + for (auto& amount : usage_stat) + { + for (auto& index : amount.second) + { + ss << "[" << print_money(amount.first) << ":" << index.first << "]" << ENDL ; + for (auto& list_entry : index.second) + { + ss << " " << list_entry.first << ": " << list_entry.second << ENDL; + } + } + } + + LOG_PRINT_L0("Results: " << ENDL << strm_tx.str() << ENDL << ss.str()); + + return true; +} +//------------------------------------------------------------------ bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) const { // check all tx's inputs for being already spent for (const txin_v& in : tx.vin) { - if (in.type() == typeid(txin_to_key)) + if (in.type() == typeid(txin_to_key) || in.type() == typeid(txin_htlc)) { - if (have_tx_keyimg_as_spent(boost::get(in).k_image)) + if (have_tx_keyimg_as_spent(get_to_key_input_from_txin_v(in).k_image)) + { return true; + } } else if (in.type() == typeid(txin_multisig)) { @@ -3478,25 +4221,22 @@ bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) const return false; } //------------------------------------------------------------------ -// bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) const -// { -// TIME_MEASURE_START_PD(tx_check_inputs_prefix_hash); -// crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); -// TIME_MEASURE_FINISH_PD(tx_check_inputs_prefix_hash); -// return check_tx_inputs(tx, tx_prefix_hash, pmax_used_block_height); -// } +bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash) const +{ + uint64_t stub = 0; + return check_tx_inputs(tx, tx_prefix_hash, stub); +} //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) const +bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& max_used_block_height) const { size_t sig_index = 0; - if(pmax_used_block_height) - *pmax_used_block_height = 0; + max_used_block_height = 0; std::vector sig_stub; const std::vector* psig = &sig_stub; TIME_MEASURE_START_PD(tx_check_inputs_loop); - BOOST_FOREACH(const auto& txin, tx.vin) + for(const auto& txin : tx.vin) { if (!m_is_in_checkpoint_zone) { @@ -3516,7 +4256,8 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::ha return false; } TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_kimage_check); - if (!check_tx_input(tx, sig_index, in_to_key, tx_prefix_hash, *psig, pmax_used_block_height)) + uint64_t max_unlock_time = 0; + if (!check_tx_input(tx, sig_index, in_to_key, tx_prefix_hash, *psig, max_used_block_height, max_unlock_time)) { LOG_ERROR("Failed to validate input #" << sig_index << " tx: " << tx_prefix_hash); return false; @@ -3525,12 +4266,34 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::ha else if (txin.type() == typeid(txin_multisig)) { const txin_multisig& in_ms = boost::get(txin); - if (!check_tx_input(tx, sig_index, in_ms, tx_prefix_hash, *psig, pmax_used_block_height)) + if (!check_tx_input(tx, sig_index, in_ms, tx_prefix_hash, *psig, max_used_block_height)) { LOG_ERROR("Failed to validate multisig input #" << sig_index << " (ms out id: " << in_ms.multisig_out_id << ") in tx: " << tx_prefix_hash); return false; } + } + else if (txin.type() == typeid(txin_htlc)) + { + if (!is_after_hardfork_3_zone()) + { + LOG_ERROR("Error: Transaction with txin_htlc before is_after_hardfork_3_zone(before height " << m_core_runtime_config.hard_fork_03_starts_after_height << ")"); + return false; + } + const txin_htlc& in_htlc = boost::get(txin); + CHECK_AND_ASSERT_MES(in_htlc.key_offsets.size(), false, "Empty in_to_key.key_offsets for input #" << sig_index << " tx: " << tx_prefix_hash); + TIME_MEASURE_START_PD(tx_check_inputs_loop_kimage_check); + if (have_tx_keyimg_as_spent(in_htlc.k_image)) + { + LOG_ERROR("Key image was already spent in blockchain: " << string_tools::pod_to_hex(in_htlc.k_image) << " for input #" << sig_index << " tx: " << tx_prefix_hash); + return false; + } + TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_kimage_check); + if (!check_tx_input(tx, sig_index, in_htlc, tx_prefix_hash, *psig, max_used_block_height)) + { + LOG_ERROR("Failed to validate multisig input #" << sig_index << " (ms out id: " << obj_to_json_str(in_htlc) << ") in tx: " << tx_prefix_hash); + return false; + } } sig_index++; } @@ -3554,16 +4317,16 @@ bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) const { return currency::is_tx_spendtime_unlocked(unlock_time, get_current_blockchain_size(), m_core_runtime_config.get_core_time()); } - //------------------------------------------------------------------ -bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) const +bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height, uint64_t& source_max_unlock_time_for_pos_coinbase) const { CRITICAL_REGION_LOCAL(m_read_lock); //TIME_MEASURE_START_PD(tx_check_inputs_loop_ch_in_get_keys_loop); std::vector output_keys; - if(!get_output_keys_for_input_with_checks(txin, output_keys, pmax_related_block_height)) + scan_for_keys_context scan_context = AUTO_VAL_INIT(scan_context); + if(!get_output_keys_for_input_with_checks(tx, txin, output_keys, max_related_block_height, source_max_unlock_time_for_pos_coinbase)) { LOG_PRINT_L0("Failed to get output keys for input #" << in_index << " (amount = " << print_money(txin.amount) << ", key_offset.size = " << txin.key_offsets.size() << ")"); return false; @@ -3575,61 +4338,116 @@ bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, for (auto& ptr : output_keys) output_keys_ptrs.push_back(&ptr); - return check_tokey_input(tx, in_index, txin, tx_prefix_hash, sig, output_keys_ptrs); + return check_input_signature(tx, in_index, txin, tx_prefix_hash, sig, output_keys_ptrs); } +//---------------------------------------------------------------- +struct outputs_visitor +{ + std::vector& m_results_collector; + blockchain_storage::scan_for_keys_context& m_scan_context; + const blockchain_storage& m_bch; + uint64_t& m_source_max_unlock_time_for_pos_coinbase; + outputs_visitor(std::vector& results_collector, + const blockchain_storage& bch, + uint64_t& source_max_unlock_time_for_pos_coinbase, + blockchain_storage::scan_for_keys_context& scan_context) + : m_results_collector(results_collector) + , m_bch(bch) + , m_source_max_unlock_time_for_pos_coinbase(source_max_unlock_time_for_pos_coinbase) + , m_scan_context(scan_context) + {} + bool handle_output(const transaction& source_tx, const transaction& validated_tx, const tx_out& out, uint64_t out_i) + { + //check tx unlock time + uint64_t source_out_unlock_time = get_tx_unlock_time(source_tx, out_i); + //let coinbase sources for PoS block to have locked inputs, the outputs supposed to be locked same way, except the reward + if (is_coinbase(validated_tx) && is_pos_block(validated_tx)) + { + CHECK_AND_ASSERT_MES(should_unlock_value_be_treated_as_block_height(source_out_unlock_time), false, "source output #" << out_i << " is locked by time, not by height, which is not allowed for PoS coinbase"); + if (source_out_unlock_time > m_source_max_unlock_time_for_pos_coinbase) + m_source_max_unlock_time_for_pos_coinbase = source_out_unlock_time; + } + else + { + if (!m_bch.is_tx_spendtime_unlocked(source_out_unlock_time)) + { + LOG_PRINT_L0("One of outputs for one of inputs have wrong tx.unlock_time = " << get_tx_unlock_time(source_tx, out_i)); + return false; + } + } + if (out.target.type() == typeid(txout_to_key)) + { + crypto::public_key pk = boost::get(out.target).key; + m_results_collector.push_back(pk); + } + else if (out.target.type() == typeid(txout_htlc)) + { + m_scan_context.htlc_outs.push_back(boost::get(out.target)); + crypto::public_key pk = null_pkey; + if (m_scan_context.htlc_is_expired) + { + pk = boost::get(out.target).pkey_refund; + } + else + { + pk = boost::get(out.target).pkey_redeem; + } + m_results_collector.push_back(pk); + }else + { + LOG_PRINT_L0("Output have wrong type id, which=" << out.target.which()); + return false; + } + + return true; + } +}; + //------------------------------------------------------------------ // Checks each referenced output for: // 1) source tx unlock time validity // 2) mixin restrictions // 3) general gindex/ref_by_id corectness -bool blockchain_storage::get_output_keys_for_input_with_checks(const txin_to_key& txin, std::vector& output_keys, uint64_t* pmax_related_block_height /* = NULL */) const +bool blockchain_storage::get_output_keys_for_input_with_checks(const transaction& tx, const txin_v& verifying_input, std::vector& output_keys, uint64_t& max_related_block_height, uint64_t& source_max_unlock_time_for_pos_coinbase, scan_for_keys_context& scan_context) const { CRITICAL_REGION_LOCAL(m_read_lock); - struct outputs_visitor - { - std::vector& m_results_collector; - const blockchain_storage& m_bch; - outputs_visitor(std::vector& results_collector, - const blockchain_storage& bch) :m_results_collector(results_collector), m_bch(bch) - {} - bool handle_output(const transaction& tx, const tx_out& out) - { - //check tx unlock time - if (!m_bch.is_tx_spendtime_unlocked(get_tx_unlock_time(tx))) - { - LOG_PRINT_L0("One of outputs for one of inputs have wrong tx.unlock_time = " << get_tx_unlock_time(tx)); - return false; - } - - if(out.target.type() != typeid(txout_to_key)) - { - LOG_PRINT_L0("Output have wrong type id, which=" << out.target.which()); - return false; - } - crypto::public_key pk = boost::get(out.target).key; - m_results_collector.push_back(pk); - return true; - } - }; - - outputs_visitor vi(output_keys, *this); - return scan_outputkeys_for_indexes(txin, vi, pmax_related_block_height); + outputs_visitor vi(output_keys, *this, source_max_unlock_time_for_pos_coinbase, scan_context); + return scan_outputkeys_for_indexes(tx, verifying_input, vi, max_related_block_height, scan_context); } +//------------------------------------------------------------------ +bool blockchain_storage::get_output_keys_for_input_with_checks(const transaction& tx, const txin_v& verifying_input, std::vector& output_keys, uint64_t& max_related_block_height, uint64_t& source_max_unlock_time_for_pos_coinbase) const +{ + scan_for_keys_context scan_context_dummy = AUTO_VAL_INIT(scan_context_dummy); + return get_output_keys_for_input_with_checks(tx, verifying_input, output_keys, max_related_block_height, source_max_unlock_time_for_pos_coinbase, scan_context_dummy); +} + //------------------------------------------------------------------ // Note: this function can be used for checking to_key inputs against either main chain or alt chain, that's why it has output_keys_ptrs parameter // Doesn't check spent flags, the caller must check it. -bool blockchain_storage::check_tokey_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, const std::vector& output_keys_ptrs) const +bool blockchain_storage::check_input_signature(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, const std::vector& output_keys_ptrs) const { - CRITICAL_REGION_LOCAL(m_read_lock); - - TIME_MEASURE_START_PD(tx_check_inputs_loop_ch_in_val_sig); - if (txin.key_offsets.size() != output_keys_ptrs.size()) { LOG_PRINT_L0("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys_ptrs.size()); return false; } + + return check_input_signature(tx, in_index, /*txin.key_offsets,*/ txin.amount, txin.k_image, txin.etc_details, tx_prefix_hash, sig, output_keys_ptrs); +} +//------------------------------------------------------------------ +bool blockchain_storage::check_input_signature(const transaction& tx, + size_t in_index, + uint64_t in_amount, + const crypto::key_image& in_k_image, + const std::vector& in_etc_details, + const crypto::hash& tx_prefix_hash, + const std::vector& sig, + const std::vector& output_keys_ptrs) const +{ + CRITICAL_REGION_LOCAL(m_read_lock); + + TIME_MEASURE_START_PD(tx_check_inputs_loop_ch_in_val_sig); if(m_is_in_checkpoint_zone) return true; @@ -3637,13 +4455,13 @@ bool blockchain_storage::check_tokey_input(const transaction& tx, size_t in_inde if (get_tx_flags(tx) & TX_FLAG_SIGNATURE_MODE_SEPARATE) { // check attachments, mentioned directly in this input - bool r = validate_attachment_info(txin.etc_details, tx.attachment, in_index != tx.vin.size() - 1); // attachment info can be omitted for all inputs, except the last one + bool r = validate_attachment_info(in_etc_details, tx.attachment, in_index != tx.vin.size() - 1); // attachment info can be omitted for all inputs, except the last one CHECK_AND_ASSERT_MES(r, false, "Failed to validate attachments in tx " << tx_prefix_hash << ": incorrect extra_attachment_info in etc_details in input #" << in_index); } else { // make sure normal tx does not have extra_attachment_info in etc_details - CHECK_AND_ASSERT_MES(!have_type_in_variant_container(txin.etc_details), false, "Incorrect using of extra_attachment_info in etc_details in input #" << in_index << " for tx " << tx_prefix_hash); + CHECK_AND_ASSERT_MES(!have_type_in_variant_container(in_etc_details), false, "Incorrect using of extra_attachment_info in etc_details in input #" << in_index << " for tx " << tx_prefix_hash); } // check signatures @@ -3661,14 +4479,14 @@ bool blockchain_storage::check_tokey_input(const transaction& tx, size_t in_inde LOG_PRINT_L4("CHECK RING SIGNATURE: tx_prefix_hash " << tx_prefix_hash << "tx_hash_for_signature" << tx_hash_for_signature - << "txin.k_image" << txin.k_image + << "in_k_image" << in_k_image << "key_ptr:" << *output_keys_ptrs[0] << "signature:" << sig[0]); - bool r = crypto::validate_key_image(txin.k_image); - CHECK_AND_ASSERT_MES(r, false, "key image for input #" << in_index << " is invalid: " << txin.k_image); + bool r = crypto::validate_key_image(in_k_image); + CHECK_AND_ASSERT_MES(r, false, "key image for input #" << in_index << " is invalid: " << in_k_image); - r = crypto::check_ring_signature(tx_hash_for_signature, txin.k_image, output_keys_ptrs, sig.data()); - CHECK_AND_ASSERT_MES(r, false, "failed to check ring signature for input #" << in_index << ENDL << dump_ring_sig_data(tx_hash_for_signature, txin.k_image, output_keys_ptrs, sig)); + r = crypto::check_ring_signature(tx_hash_for_signature, in_k_image, output_keys_ptrs, sig.data()); + CHECK_AND_ASSERT_MES(r, false, "failed to check ring signature for input #" << in_index << ENDL << dump_ring_sig_data(tx_hash_for_signature, in_k_image, output_keys_ptrs, sig)); if (need_to_check_extra_sign) { //here we check extra signature to validate that transaction was finalized by authorized subject @@ -3687,7 +4505,7 @@ bool blockchain_storage::check_ms_input(const transaction& tx, size_t in_index, #define LOC_CHK(cond, msg) CHECK_AND_ASSERT_MES(cond, false, "ms input check failed: ms_id: " << txin.multisig_out_id << ", input #" << in_index << " in tx " << tx_prefix_hash << ", refers to ms output #" << out_n << " in source tx " << get_transaction_hash(source_tx) << ENDL << msg) CRITICAL_REGION_LOCAL(m_read_lock); - uint64_t unlock_time = get_tx_unlock_time(source_tx); + uint64_t unlock_time = get_tx_unlock_time(source_tx, out_n); LOC_CHK(is_tx_spendtime_unlocked(unlock_time), "Source transaction is LOCKED! unlock_time: " << unlock_time << ", now is " << m_core_runtime_config.get_core_time() << ", blockchain size is " << get_current_blockchain_size()); LOC_CHK(source_tx.vout.size() > out_n, "internal error: out_n==" << out_n << " is out-of-bounds of source_tx.vout, size=" << source_tx.vout.size()); @@ -3767,7 +4585,7 @@ bool blockchain_storage::check_ms_input(const transaction& tx, size_t in_index, } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) const +bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height) const { CRITICAL_REGION_LOCAL(m_read_lock); @@ -3791,11 +4609,59 @@ bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, if (!check_ms_input(tx, in_index, txin, tx_prefix_hash, sig, source_tx_ptr->tx, n)) return false; - if (pmax_related_block_height != nullptr) - *pmax_related_block_height = source_tx_ptr->m_keeper_block_height; + max_related_block_height = source_tx_ptr->m_keeper_block_height; return true; #undef LOC_CHK +} +//------------------------------------------------------------------ +bool blockchain_storage::check_tx_input(const transaction& tx, size_t in_index, const txin_htlc& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height)const +{ + CRITICAL_REGION_LOCAL(m_read_lock); + + //TIME_MEASURE_START_PD(tx_check_inputs_loop_ch_in_get_keys_loop); + + std::vector output_keys; + scan_for_keys_context scan_contex = AUTO_VAL_INIT(scan_contex); + uint64_t source_max_unlock_time_for_pos_coinbase_dummy = AUTO_VAL_INIT(source_max_unlock_time_for_pos_coinbase_dummy); + if (!get_output_keys_for_input_with_checks(tx, txin, output_keys, max_related_block_height, source_max_unlock_time_for_pos_coinbase_dummy, scan_contex)) + { + LOG_PRINT_L0("Failed to get output keys for input #" << in_index << " (amount = " << print_money(txin.amount) << ", key_offset.size = " << txin.key_offsets.size() << ")"); + return false; + } + + CHECK_AND_ASSERT_THROW_MES(scan_contex.htlc_outs.size() == 1, "htlc output not found for input, tx: " << get_transaction_hash(tx)); + const txout_htlc& related_out = *scan_contex.htlc_outs.begin(); + bool use_sha256 = !(related_out.flags&CURRENCY_TXOUT_HTLC_FLAGS_HASH_TYPE_MASK); + if (use_sha256) + { + //doing sha256 hash + crypto::hash sha256 = crypto::sha256_hash(txin.hltc_origin.data(), txin.hltc_origin.size()); + CHECK_AND_ASSERT_THROW_MES(sha256 == related_out.htlc_hash, "htlc hash missmatched for tx: " << get_transaction_hash(tx) + << " calculated hash: " << sha256 << " expected hash(related_out.htlc_hash): " << related_out.htlc_hash); + } + else + { + //doing RIPEMD160 + crypto::hash160 ripemd160 = crypto::RIPEMD160_hash(txin.hltc_origin.data(), txin.hltc_origin.size()); + crypto::hash160 expected_ripemd160 = *(crypto::hash160*)&related_out.htlc_hash; + CHECK_AND_ASSERT_THROW_MES(ripemd160 == expected_ripemd160, "htlc hash missmatched for tx: " << get_transaction_hash(tx) + << " calculated hash: " << ripemd160 << " expected hash(related_out.htlc_hash): " << expected_ripemd160); + } + + + //TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_ch_in_get_keys_loop); + + + + std::vector output_keys_ptrs; + output_keys_ptrs.reserve(output_keys.size()); + for (auto& ptr : output_keys) + output_keys_ptrs.push_back(&ptr); + + CHECK_AND_ASSERT_THROW_MES(output_keys_ptrs.size() == 1, "Internal error: output_keys_ptrs.size() is not equal 1 for HTLC"); + + return check_input_signature(tx, in_index, txin.amount, txin.k_image, txin.etc_details, tx_prefix_hash, sig, output_keys_ptrs); } //------------------------------------------------------------------ uint64_t blockchain_storage::get_adjusted_time() const @@ -3921,9 +4787,9 @@ std::shared_ptr blockchain_storage::find_key_imag } for (auto& in : tx_chain_entry->tx.vin) { - if (in.type() == typeid(txin_to_key)) + if (in.type() == typeid(txin_to_key) || in.type() == typeid(txin_htlc)) { - if (boost::get(in).k_image == ki) + if (get_to_key_input_from_txin_v(in).k_image == ki) { id_result = get_transaction_hash(tx_chain_entry->tx); return tx_chain_entry; @@ -3940,6 +4806,12 @@ bool blockchain_storage::prune_aged_alt_blocks() CRITICAL_REGION_LOCAL1(m_alternative_chains_lock); uint64_t current_height = get_current_blockchain_size(); + size_t count_to_delete = 0; + if(m_alternative_chains.size() > m_core_runtime_config.max_alt_blocks) + count_to_delete = m_alternative_chains.size() - m_core_runtime_config.max_alt_blocks; + + std::map alts_to_delete; + for(auto it = m_alternative_chains.begin(); it != m_alternative_chains.end();) { if (current_height > it->second.height && current_height - it->second.height > CURRENCY_ALT_BLOCK_LIVETIME_COUNT) @@ -3948,9 +4820,28 @@ bool blockchain_storage::prune_aged_alt_blocks() } else { + if (count_to_delete) + { + if (!alts_to_delete.size()) + alts_to_delete[it->second.timestamp] = it; + else + { + if (it->second.timestamp >= alts_to_delete.rbegin()->first) + alts_to_delete[it->second.timestamp] = it; + + if (alts_to_delete.size() > count_to_delete) + alts_to_delete.erase(alts_to_delete.begin()); + } + } + ++it; } } + //now, if there was count_to_delete we should erase most oldest entries of altblocks + for (auto& itd : alts_to_delete) + { + m_alternative_chains.erase(itd.second); + } return true; } @@ -4030,6 +4921,108 @@ void blockchain_storage::get_pos_mining_estimate(uint64_t amount_coins, estimate_result = current_amount; } //------------------------------------------------------------------ +bool blockchain_storage::validate_tx_for_hardfork_specific_terms(const transaction& tx, const crypto::hash& tx_id) const +{ + uint64_t block_height = m_db_blocks.size(); + return validate_tx_for_hardfork_specific_terms(tx, tx_id, block_height); +} +//------------------------------------------------------------------ +bool blockchain_storage::validate_tx_for_hardfork_specific_terms(const transaction& tx, const crypto::hash& tx_id, uint64_t block_height) const +{ + auto is_allowed_before_hardfork2 = [&](const payload_items_v& el) -> bool + { + CHECK_AND_ASSERT_MES(el.type() != typeid(tx_payer), false, "tx " << tx_id << " contains tx_payer which is not allowed on height " << block_height); + CHECK_AND_ASSERT_MES(el.type() != typeid(tx_receiver), false, "tx " << tx_id << " contains tx_receiver which is not allowed on height " << block_height); + CHECK_AND_ASSERT_MES(el.type() != typeid(extra_alias_entry), false, "tx " << tx_id << " contains extra_alias_entry which is not allowed on height " << block_height); + return true; + }; + + auto is_allowed_before_hardfork1 = [&](const payload_items_v& el) -> bool + { + CHECK_AND_ASSERT_MES(el.type() != typeid(etc_tx_details_unlock_time2), false, "tx " << tx_id << " contains etc_tx_details_unlock_time2 which is not allowed on height " << block_height); + return true; + }; + + bool var_is_after_hardfork_1_zone = is_after_hardfork_1_zone(block_height); + bool var_is_after_hardfork_2_zone = is_after_hardfork_2_zone(block_height); + bool var_is_after_hardfork_3_zone = is_after_hardfork_3_zone(block_height); + + //inputs + for (const auto in : tx.vin) + { + if (in.type() == typeid(txin_htlc)) + { + if (!var_is_after_hardfork_3_zone) + return false; + } + } + //outputs + for (const auto out : tx.vout) + { + if (out.target.type() == typeid(txout_htlc)) + { + if (!var_is_after_hardfork_3_zone) + return false; + } + } + + //extra + for (const auto el : tx.extra) + { + if (!var_is_after_hardfork_1_zone && !is_allowed_before_hardfork1(el)) + return false; + if (!var_is_after_hardfork_2_zone && !is_allowed_before_hardfork2(el)) + return false; + } + + //attachments + for (const auto el : tx.attachment) + { + if (!var_is_after_hardfork_2_zone && !is_allowed_before_hardfork2(el)) + return false; + } + + + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::validate_pos_coinbase_outs_unlock_time(const transaction& miner_tx, uint64_t staked_amount, uint64_t source_max_unlock_time)const +{ + uint64_t major_unlock_time = get_tx_x_detail(miner_tx); + if (major_unlock_time) + { + //if there was etc_tx_details_unlock_time present in tx, then ignore etc_tx_details_unlock_time2 + if (major_unlock_time < source_max_unlock_time) + return false; + else + return true; + } + + CHECK_AND_ASSERT_MES(get_block_height(miner_tx) > m_core_runtime_config.hard_fork_01_starts_after_height, false, "error in block [" << get_block_height(miner_tx) << "] etc_tx_details_unlock_time2 can exist only after hard fork point : " << m_core_runtime_config.hard_fork_01_starts_after_height); + + //etc_tx_details_unlock_time2 can be kept only after hard_fork_1 point + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(miner_tx.extra, ut2); + CHECK_AND_ASSERT_MES(ut2.unlock_time_array.size() == miner_tx.vout.size(), false, "ut2.unlock_time_array.size()<" << ut2.unlock_time_array.size() + << "> != miner_tx.vout.size()<" << miner_tx.vout.size() << ">"); + + uint64_t amount_of_coins_in_unlock_in_range = 0; // amount of outputs locked for at least the same time + + for (uint64_t i = 0; i != miner_tx.vout.size(); i++) + { + uint64_t unlock_value = ut2.unlock_time_array[i]; + CHECK_AND_ASSERT_MES(should_unlock_value_be_treated_as_block_height(unlock_value), false, "output #" << i << " is locked by time, not buy height, which is not allowed for PoS coinbase"); + if (unlock_value >= source_max_unlock_time) + amount_of_coins_in_unlock_in_range += miner_tx.vout[i].amount; + } + + if (amount_of_coins_in_unlock_in_range >= staked_amount) + return true; + LOG_ERROR("amount_of_coins_in_unlock_in_range<" << amount_of_coins_in_unlock_in_range << "> is less then staked_amount<" << staked_amount); + return false; + +} +//------------------------------------------------------------------ bool blockchain_storage::validate_pos_block(const block& b, wide_difficulty_type basic_diff, uint64_t& amount, @@ -4059,11 +5052,11 @@ bool blockchain_storage::validate_pos_block(const block& b, //check actual time if it there uint64_t actual_ts = get_actual_timestamp(b); - if ((actual_ts > b.timestamp && actual_ts - b.timestamp > POS_MAC_ACTUAL_TIMESTAMP_TO_MINED) || - (actual_ts < b.timestamp && b.timestamp - actual_ts > POS_MAC_ACTUAL_TIMESTAMP_TO_MINED) + if ((actual_ts > b.timestamp && actual_ts - b.timestamp > POS_MAX_ACTUAL_TIMESTAMP_TO_MINED) || + (actual_ts < b.timestamp && b.timestamp - actual_ts > POS_MAX_ACTUAL_TIMESTAMP_TO_MINED) ) { - LOG_PRINT_L0("PoS block actual timestamp " << actual_ts << " differs from b.timestamp " << b.timestamp << " by " << ((int64_t)actual_ts - (int64_t)b.timestamp) << " s, it's more than allowed " << POS_MAC_ACTUAL_TIMESTAMP_TO_MINED << " s."); + LOG_PRINT_L0("PoS block actual timestamp " << actual_ts << " differs from b.timestamp " << b.timestamp << " by " << ((int64_t)actual_ts - (int64_t)b.timestamp) << " s, it's more than allowed " << POS_MAX_ACTUAL_TIMESTAMP_TO_MINED << " s."); return false; } @@ -4106,8 +5099,23 @@ bool blockchain_storage::validate_pos_block(const block& b, { // Do coinstake input validation for main chain only. // Txs in alternative PoS blocks (including miner_tx) are validated by validate_alt_block_txs() - r = check_tx_input(b.miner_tx, 1, coinstake_in, id, b.miner_tx.signatures[0], &max_related_block_height); + uint64_t source_max_unlock_time_for_pos_coinbase = 0; + r = check_tx_input(b.miner_tx, 1, coinstake_in, id, b.miner_tx.signatures[0], max_related_block_height, source_max_unlock_time_for_pos_coinbase); CHECK_AND_ASSERT_MES(r, false, "Failed to validate coinstake input in miner tx, block_id = " << get_block_hash(b)); + + if (get_block_height(b) > m_core_runtime_config.hard_fork_01_starts_after_height) + { + uint64_t last_pow_h = get_last_x_block_height(false); + CHECK_AND_ASSERT_MES(max_related_block_height <= last_pow_h, false, "Failed to validate coinbase in PoS block, condition failed: max_related_block_height(" << max_related_block_height << ") <= last_pow_h(" << last_pow_h << ")"); + //let's check that coinbase amount and unlock time + r = validate_pos_coinbase_outs_unlock_time(b.miner_tx, coinstake_in.amount, source_max_unlock_time_for_pos_coinbase); + CHECK_AND_ASSERT_MES(r, false, "Failed to validate_pos_coinbase_outs_unlock_time() in miner tx, block_id = " << get_block_hash(b) + << "source_max_unlock_time_for_pos_coinbase=" << source_max_unlock_time_for_pos_coinbase); + } + else + { + CHECK_AND_ASSERT_MES(is_tx_spendtime_unlocked(source_max_unlock_time_for_pos_coinbase), false, "Failed to validate coinbase in PoS block, condition failed: is_tx_spendtime_unlocked(source_max_unlock_time_for_pos_coinbase)(" << source_max_unlock_time_for_pos_coinbase << ")"); + } } uint64_t block_height = for_altchain ? split_height + alt_chain.size() : m_db_blocks.size(); @@ -4215,13 +5223,23 @@ uint64_t blockchain_storage::get_last_x_block_height(bool pos) const return 0; } //------------------------------------------------------------------ -wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise_difficulty(alt_chain_type& alt_chain, uint64_t block_height, bool pos) const +wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise_adj_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos) const { - uint64_t main_chain_first_block = block_height - 1; + wide_difficulty_type res = 0; + get_last_alt_x_block_cumulative_precise_difficulty(alt_chain, block_height, pos, res); + return res; +} +//------------------------------------------------------------------ +wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos, wide_difficulty_type& cumulative_diff_precise_adj) const +{ + uint64_t main_chain_first_block = block_height; for (auto it = alt_chain.rbegin(); it != alt_chain.rend(); it++) { if (is_pos_block((*it)->second.bl) == pos) + { + cumulative_diff_precise_adj = (*it)->second.cumulative_diff_precise_adjusted; return (*it)->second.cumulative_diff_precise; + } main_chain_first_block = (*it)->second.height - 1; } @@ -4232,11 +5250,27 @@ wide_difficulty_type blockchain_storage::get_last_alt_x_block_cumulative_precise for (uint64_t i = main_chain_first_block; i != 0; i--) { if (is_pos_block(m_db_blocks[i]->bl) == pos) + { + cumulative_diff_precise_adj = m_db_blocks[i]->cumulative_diff_precise_adjusted; return m_db_blocks[i]->cumulative_diff_precise; + } } + cumulative_diff_precise_adj = 0; return 0; } //------------------------------------------------------------------ +bool get_tx_from_cache(const crypto::hash& tx_id, transactions_map& tx_cache, transaction& tx, size_t& blob_size, uint64_t& fee) +{ + auto it = tx_cache.find(tx_id); + if (it == tx_cache.end()) + return false; + + tx = it->second; + blob_size = get_object_blobsize(tx); + fee = get_tx_fee(tx); + return true; +} +//------------------------------------------------------------------ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) { TIME_MEASURE_START_PD_MS(block_processing_time_0_ms); @@ -4250,8 +5284,6 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt return false; } - get_block_height(bl); - if(!check_block_timestamp_main(bl)) { LOG_PRINT_L0("Block with id: " << id << ENDL @@ -4291,7 +5323,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if (is_pos_bl) { bool r = validate_pos_block(bl, current_diffic, pos_coinstake_amount, this_coin_diff, proof_hash, id, false); - CHECK_AND_ASSERT_MES_CUSTOM(r, false, bvc.m_verification_failed = true, "validate_pos_block failed!!"); + CHECK_AND_ASSERT_MES_CUSTOM(r, false, bvc.m_verification_failed = true, "validate_pos_block failed!!"); } else { @@ -4352,13 +5384,18 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt size_t tx_processed_count = 0; uint64_t fee_summary = 0; + uint64_t burned_coins = 0; + std::list block_summary_kimages; for(const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; size_t blob_size = 0; uint64_t fee = 0; - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + + bool taken_from_cache = get_tx_from_cache(tx_id, bvc.m_onboard_transactions, tx, blob_size, fee); + bool taken_from_pool = m_tx_pool.take_tx(tx_id, tx, blob_size, fee); + if(!taken_from_cache && !taken_from_pool) { LOG_PRINT_L0("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); purge_block_data_from_blockchain(bl, tx_processed_count); @@ -4367,6 +5404,15 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt return false; } + if (!validate_tx_semantic(tx, blob_size)) + { + LOG_PRINT_L0("Block with id: " << id << " has at least one transaction with wrong semantic, tx_id: " << tx_id); + purge_block_data_from_blockchain(bl, tx_processed_count); + //add_block_as_invalid(bl, id); + bvc.m_verification_failed = true; + return false; + } + append_per_block_increments_for_tx(tx, gindices); //If we under checkpoints, ring signatures should be pruned @@ -4381,16 +5427,20 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt { LOG_PRINT_L0("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs."); currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); - m_tx_pool.add_transaction_to_black_list(tx); - CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + if (taken_from_pool) + { + bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); + m_tx_pool.add_transaction_to_black_list(tx); + CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + } purge_block_data_from_blockchain(bl, tx_processed_count); add_block_as_invalid(bl, id); - LOG_PRINT_L0("Block with id " << id << " added as invalid becouse of wrong inputs in transactions"); + LOG_PRINT_L0("Block with id " << id << " added as invalid because of wrong inputs in transactions"); bvc.m_verification_failed = true; return false; } TIME_MEASURE_FINISH_PD(tx_check_inputs_time); + burned_coins += get_burned_amount(tx); TIME_MEASURE_START_PD(tx_prapare_append); uint64_t current_bc_size = get_current_blockchain_size(); @@ -4399,11 +5449,14 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt TIME_MEASURE_START_PD(tx_append_time); if(!add_transaction_from_block(tx, tx_id, id, current_bc_size, actual_timestamp)) { - LOG_PRINT_L0("Block with id: " << id << " failed to add transaction to blockchain storage"); - currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); - m_tx_pool.add_transaction_to_black_list(tx); - CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + LOG_PRINT_L0("Block " << id << " contains tx " << tx_id << " that can't be added to the blockchain storage"); + if (taken_from_pool) + { + currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool add_res = m_tx_pool.add_tx(tx, tvc, true, true); + m_tx_pool.add_transaction_to_black_list(tx); + CHECK_AND_ASSERT_MES_NO_RET(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); + } purge_block_data_from_blockchain(bl, tx_processed_count); bvc.m_verification_failed = true; return false; @@ -4416,6 +5469,8 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt ++tx_processed_count; if (fee) block_fees.push_back(fee); + + read_keyimages_from_tx(tx, block_summary_kimages); } TIME_MEASURE_FINISH_PD(all_txs_insert_time_5); @@ -4450,6 +5505,10 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt if (is_pos_bl) bei.stake_hash = proof_hash; + ////////////////////////////////////////////////////////////////////////// + + //old style cumulative difficulty collecting + //precise difficulty - difficulty used to calculate next difficulty uint64_t last_x_h = get_last_x_block_height(is_pos_bl); if (!last_x_h) @@ -4473,11 +5532,33 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt size_t sequence_factor = get_current_sequence_factor(is_pos_bl); if (bei.height >= m_core_runtime_config.pos_minimum_heigh) cumulative_diff_delta = correct_difficulty_with_sequence_factor(sequence_factor, cumulative_diff_delta); - + + if (bei.height > BLOCKCHAIN_HEIGHT_FOR_POS_STRICT_SEQUENCE_LIMITATION && is_pos_bl && sequence_factor > BLOCK_POS_STRICT_SEQUENCE_LIMIT) + { + LOG_PRINT_RED_L0("Block " << id << " @ " << bei.height << " has too big sequence factor: " << sequence_factor << ", rejected"); + purge_block_data_from_blockchain(bl, tx_processed_count); + bvc.m_verification_failed = true; + return false; + } + bei.cumulative_diff_adjusted += cumulative_diff_delta; + ////////////////////////////////////////////////////////////////////////// + // rebuild cumulative_diff_precise_adjusted for whole period + wide_difficulty_type diff_precise_adj = correct_difficulty_with_sequence_factor(sequence_factor, current_diffic); + bei.cumulative_diff_precise_adjusted = last_x_h ? m_db_blocks[last_x_h]->cumulative_diff_precise_adjusted + diff_precise_adj : diff_precise_adj; + + ////////////////////////////////////////////////////////////////////////// + //etc - bei.already_generated_coins = already_generated_coins + base_reward; + if (already_generated_coins < burned_coins) + { + LOG_ERROR("Condition failed: already_generated_coins(" << already_generated_coins << ") >= burned_coins(" << burned_coins << ")"); + purge_block_data_from_blockchain(bl, tx_processed_count); + bvc.m_verification_failed = true; + return false; + } + bei.already_generated_coins = already_generated_coins - burned_coins + base_reward; auto blocks_index_ptr = m_db_blocks_index.get(id); if (blocks_index_ptr) @@ -4542,7 +5623,11 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt powpos_str_entry << "PoW:\t" << proof_hash; timestamp_str_entry << ", block ts: " << bei.bl.timestamp << " (diff: " << std::showpos << ts_diff << "s)"; } - LOG_PRINT_L1("+++++ BLOCK SUCCESSFULLY ADDED " << (is_pos_bl ? "[PoS]" : "[PoW]") << " Sq: " << sequence_factor + //explanation of this code will be provided later with public announce + set_lost_tx_unmixable_for_height(bei.height); + + + LOG_PRINT_L1("+++++ BLOCK SUCCESSFULLY ADDED " << (is_pos_bl ? "[PoS]" : "[PoW]") << "["<< static_cast(bei.bl.major_version) << "." << static_cast(bei.bl.minor_version) << "] "<< " Sq: " << sequence_factor << ENDL << "id:\t" << id << timestamp_str_entry.str() << ENDL << powpos_str_entry.str() << ENDL << "HEIGHT " << bei.height << ", difficulty: " << current_diffic << ", cumul_diff_precise: " << bei.cumulative_diff_precise << ", cumul_diff_adj: " << bei.cumulative_diff_adjusted << " (+" << cumulative_diff_delta << ")" @@ -4557,17 +5642,17 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt << "/" << etc_stuff_6 << "))"); - on_block_added(bei, id); + on_block_added(bei, id, block_summary_kimages); bvc.m_added_to_main_chain = true; return true; } //------------------------------------------------------------------ -void blockchain_storage::on_block_added(const block_extended_info& bei, const crypto::hash& id) +void blockchain_storage::on_block_added(const block_extended_info& bei, const crypto::hash& id, const std::list& bsk) { update_next_comulative_size_limit(); m_timestamps_median_cache.clear(); - m_tx_pool.on_blockchain_inc(bei.height, id); + m_tx_pool.on_blockchain_inc(bei.height, id, bsk); update_targetdata_cache_on_block_added(bei); @@ -4587,6 +5672,7 @@ void blockchain_storage::on_block_removed(const block_extended_info& bei) //------------------------------------------------------------------ void blockchain_storage::update_targetdata_cache_on_block_added(const block_extended_info& bei) { + CRITICAL_REGION_LOCAL(m_targetdata_cache_lock); if (bei.height == 0) return; //skip genesis std::list>& targetdata_cache = is_pos_block(bei.bl) ? m_pos_targetdata_cache : m_pow_targetdata_cache; @@ -4597,6 +5683,7 @@ void blockchain_storage::update_targetdata_cache_on_block_added(const block_exte //------------------------------------------------------------------ void blockchain_storage::update_targetdata_cache_on_block_removed(const block_extended_info& bei) { + CRITICAL_REGION_LOCAL(m_targetdata_cache_lock); std::list>& targetdata_cache = is_pos_block(bei.bl) ? m_pos_targetdata_cache : m_pow_targetdata_cache; if (targetdata_cache.size()) targetdata_cache.pop_back(); @@ -4606,6 +5693,7 @@ void blockchain_storage::update_targetdata_cache_on_block_removed(const block_ex //------------------------------------------------------------------ void blockchain_storage::load_targetdata_cache(bool is_pos)const { + CRITICAL_REGION_LOCAL(m_targetdata_cache_lock); std::list>& targetdata_cache = is_pos? m_pos_targetdata_cache: m_pow_targetdata_cache; targetdata_cache.clear(); uint64_t stop_ind = 0; @@ -4643,13 +5731,88 @@ bool blockchain_storage::update_next_comulative_size_limit() return true; } //------------------------------------------------------------------ -bool blockchain_storage::add_new_block(const block& bl_, block_verification_context& bvc) +bool blockchain_storage::is_after_hardfork_1_zone()const +{ + return is_after_hardfork_1_zone(m_db_blocks.size()); +} +//------------------------------------------------------------------ +bool blockchain_storage::is_after_hardfork_1_zone(uint64_t height)const +{ + if (height > m_core_runtime_config.hard_fork_01_starts_after_height) + return true; + return false; +} +//------------------------------------------------------------------ +bool blockchain_storage::is_after_hardfork_2_zone()const +{ + return is_after_hardfork_2_zone(m_db_blocks.size()); +} +//------------------------------------------------------------------ +bool blockchain_storage::is_after_hardfork_3_zone()const +{ + return is_after_hardfork_3_zone(m_db_blocks.size()); +} +//------------------------------------------------------------------ +bool blockchain_storage::is_after_hardfork_2_zone(uint64_t height)const +{ + if (height > m_core_runtime_config.hard_fork_02_starts_after_height) + return true; + return false; +} +//------------------------------------------------------------------ +bool blockchain_storage::is_after_hardfork_3_zone(uint64_t height)const +{ + if (height > m_core_runtime_config.hard_fork_03_starts_after_height) + return true; + return false; +} +//------------------------------------------------------------------ +bool blockchain_storage::prevalidate_block(const block& bl) +{ + //before hard_fork1 + if (bl.major_version == BLOCK_MAJOR_VERSION_INITIAL && get_block_height(bl) <= m_core_runtime_config.hard_fork_01_starts_after_height) + return true; + + + //after hard_fork1 and before hard_fork3 + if ( get_block_height(bl) > m_core_runtime_config.hard_fork_01_starts_after_height && + get_block_height(bl) <= m_core_runtime_config.hard_fork_03_starts_after_height + ) + { + if (bl.major_version <= HF1_BLOCK_MAJOR_VERSION ) + return true; + else + return false; + } + + //after hard_fork3 + if (bl.major_version > CURRENT_BLOCK_MAJOR_VERSION) + { + LOG_ERROR("prevalidation failed for block " << get_block_hash(bl) << ": major block version " << static_cast(bl.major_version) << " is incorrect, " << CURRENT_BLOCK_MAJOR_VERSION << " is expected" << ENDL + << obj_to_json_str(bl)); + return false; + } + + if (bl.minor_version > CURRENT_BLOCK_MINOR_VERSION) + { + //this means that binary block is compatible, but semantics got changed due to hardfork, daemon should be updated + LOG_PRINT_MAGENTA("Block's MINOR_VERSION is: " << bl.minor_version + << ", while current build supports not bigger then " << CURRENT_BLOCK_MINOR_VERSION + << ", please make sure you using latest version.", LOG_LEVEL_0 + ); + return false; + } + + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::add_new_block(const block& bl, block_verification_context& bvc) { try { m_db.begin_transaction(); - block bl = bl_; + //block bl = bl_; crypto::hash id = get_block_hash(bl); CRITICAL_REGION_LOCAL(m_tx_pool); //CRITICAL_REGION_LOCAL1(m_read_lock); @@ -4662,7 +5825,18 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont return false; } + if (!prevalidate_block(bl)) + { + LOG_PRINT_RED_L0("block with id = " << id << " failed to prevalidate"); + bvc.m_added_to_main_chain = false; + bvc.m_verification_failed = true; + m_db.commit_transaction(); + return false; + } + + //check that block refers to chain tail + if (!(bl.prev_id == get_top_block_id())) { @@ -4729,7 +5903,8 @@ bool blockchain_storage::truncate_blockchain(uint64_t to_height) uint64_t inital_height = get_current_blockchain_size(); while (get_current_blockchain_size() > to_height) { - pop_block_from_blockchain(); + transactions_map ot; + pop_block_from_blockchain(ot); } CRITICAL_REGION_LOCAL(m_alternative_chains_lock); m_alternative_chains.clear(); @@ -4826,7 +6001,7 @@ bool blockchain_storage::build_stake_modifier(stake_modifier_type& sm, const alt else { bool r = string_tools::parse_tpod_from_hex_string(POS_STARTER_KERNEL_HASH, sm.last_pos_kernel_id); - CHECK_AND_ASSERT_MES(r, false, "Failed to parse POS_STARTER_MODFIFIER"); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse POS_STARTER_KERNEL_HASH"); } sm.last_pow_id = get_block_hash(pbei_last_pow->bl); @@ -4882,12 +6057,12 @@ bool blockchain_storage::scan_pos(const COMMAND_RPC_SCAN_POS::request& sp, COMMA LOG_PRINT_GREEN("Found kernel: amount=" << print_money(sp.pos_entries[i].amount) << ", key_image" << sp.pos_entries[i].keyimage, LOG_LEVEL_0); rsp.index = i; rsp.block_timestamp = ts; - rsp.status = CORE_RPC_STATUS_OK; + rsp.status = API_RETURN_CODE_OK; return true; } } } - rsp.status = CORE_RPC_STATUS_NOT_FOUND; + rsp.status = API_RETURN_CODE_NOT_FOUND; return false; } //------------------------------------------------------------------ @@ -5042,8 +6217,17 @@ void blockchain_storage::calculate_local_gindex_lookup_table_for_height(uint64_t } } //------------------------------------------------------------------ -bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, std::set& collected_keyimages, const crypto::hash& bl_id, const crypto::hash& input_tx_hash, size_t input_index, - const std::vector& input_sigs, uint64_t split_height, const alt_chain_type& alt_chain, const std::set& alt_chain_block_ids, uint64_t& ki_lookuptime, +bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, + std::unordered_set& collected_keyimages, + const txs_by_id_and_height_altchain& alt_chain_tx_ids, + const crypto::hash& bl_id, + const crypto::hash& input_tx_hash, + size_t input_index, + const std::vector& input_sigs, + uint64_t split_height, + const alt_chain_type& alt_chain, + const std::unordered_set& alt_chain_block_ids, + uint64_t& ki_lookuptime, uint64_t* p_max_related_block_height /* = nullptr */) const { // Main and alt chain outline: @@ -5077,23 +6261,24 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s if (p_max_related_block_height != nullptr) *p_max_related_block_height = 0; - CHECK_AND_ASSERT_MES(input_index < input_tx.vin.size() && input_tx.vin[input_index].type() == typeid(txin_to_key), false, "invalid input index: " << input_index); - const txin_to_key& input = boost::get(input_tx.vin[input_index]); + CHECK_AND_ASSERT_MES(input_index < input_tx.vin.size(), false, "invalid input index: " << input_index); + const txin_v& input_v = input_tx.vin[input_index]; + const txin_to_key& input_to_key = get_to_key_input_from_txin_v(input_v); // check case b1: key_image spent status in main chain, should be either non-spent or has spent height >= split_height - auto p = m_db_spent_keys.get(input.k_image); - CHECK_AND_ASSERT_MES(p == nullptr || *p >= split_height, false, "key image " << input.k_image << " has been already spent in main chain at height " << *p << ", split height: " << split_height); + auto p = m_db_spent_keys.get(input_to_key.k_image); + CHECK_AND_ASSERT_MES(p == nullptr || *p >= split_height, false, "key image " << input_to_key.k_image << " has been already spent in main chain at height " << *p << ", split height: " << split_height); TIME_MEASURE_START(ki_lookup_time); //check key_image in altchain //check among this alt block already collected key images first - if (collected_keyimages.find(input.k_image) != collected_keyimages.end()) + if (collected_keyimages.find(input_to_key.k_image) != collected_keyimages.end()) { // cases b2, b3 - LOG_ERROR("key image " << input.k_image << " already spent in this alt block"); + LOG_ERROR("key image " << input_to_key.k_image << " already spent in this alt block"); return false; } - auto ki_it = m_altblocks_keyimages.find(input.k_image); + auto ki_it = m_altblocks_keyimages.find(input_to_key.k_image); if (ki_it != m_altblocks_keyimages.end()) { //have some entry for this key image. Check if this key image belongs to this alt chain @@ -5103,18 +6288,18 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s if (alt_chain_block_ids.find(h) != alt_chain_block_ids.end()) { // cases b2, b3 - LOG_ERROR("key image " << input.k_image << " already spent in altchain"); + LOG_ERROR("key image " << input_to_key.k_image << " already spent in altchain"); return false; } } } //update altchain with key image - collected_keyimages.insert(input.k_image); + collected_keyimages.insert(input_to_key.k_image); TIME_MEASURE_FINISH(ki_lookup_time); ki_lookuptime = ki_lookup_time; - std::vector abs_key_offsets = relative_output_offsets_to_absolute(input.key_offsets); - CHECK_AND_ASSERT_MES(abs_key_offsets.size() > 0 && abs_key_offsets.size() == input.key_offsets.size(), false, "internal error: abs_key_offsets.size()==" << abs_key_offsets.size() << ", input.key_offsets.size()==" << input.key_offsets.size()); + std::vector abs_key_offsets = relative_output_offsets_to_absolute(input_to_key.key_offsets); + CHECK_AND_ASSERT_MES(abs_key_offsets.size() > 0 && abs_key_offsets.size() == input_to_key.key_offsets.size(), false, "internal error: abs_key_offsets.size()==" << abs_key_offsets.size() << ", input_to_key.key_offsets.size()==" << input_to_key.key_offsets.size()); // eventually we should found all public keys for all outputs this input refers to, for checking ring signature std::vector pub_keys(abs_key_offsets.size(), null_pkey); @@ -5124,33 +6309,35 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s uint64_t global_outs_for_amount = 0; //figure out if this amount touched alt_chain amount's index and if it is, get bool amount_touched_altchain = false; - //auto abg_it = abei.gindex_lookup_table.find(input.amount); + //auto abg_it = abei.gindex_lookup_table.find(input_to_key.amount); //if (abg_it == abei.gindex_lookup_table.end()) if (!alt_chain.empty()) { - auto abg_it = alt_chain.back()->second.gindex_lookup_table.find(input.amount); + auto abg_it = alt_chain.back()->second.gindex_lookup_table.find(input_to_key.amount); if (abg_it != alt_chain.back()->second.gindex_lookup_table.end()) { amount_touched_altchain = true; - //Notice: since transactions is not allowed to refer to each other in one block, then we can consider that index in - //tx input would be always less then top for previous block, so just take it - global_outs_for_amount = abg_it->second; + // local gindex lookup table contains last used gindex, so we can't get total number of outs + // just skip setting global_outs_for_amount } else { //quite easy, - global_outs_for_amount = m_db_outputs.get_item_size(input.amount); + global_outs_for_amount = m_db_outputs.get_item_size(input_to_key.amount); } } else { //quite easy, - global_outs_for_amount = m_db_outputs.get_item_size(input.amount); + global_outs_for_amount = m_db_outputs.get_item_size(input_to_key.amount); } CHECK_AND_ASSERT_MES(pub_keys.size() == abs_key_offsets.size(), false, "pub_keys.size()==" << pub_keys.size() << " != abs_key_offsets.size()==" << abs_key_offsets.size()); // just a little bit of paranoia std::vector pub_key_pointers; + + uint64_t height_of_current_alt_block = alt_chain.size() ? alt_chain.back()->second.height + 1 : split_height + 1; + for (size_t pk_n = 0; pk_n < pub_keys.size(); ++pk_n) { crypto::public_key& pk = pub_keys[pk_n]; @@ -5160,8 +6347,8 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s if (off.type() == typeid(uint64_t)) { uint64_t offset_gindex = boost::get(off); - CHECK_AND_ASSERT_MES(offset_gindex < global_outs_for_amount, false, - "invalid global output index " << offset_gindex << " for amount=" << input.amount << + CHECK_AND_ASSERT_MES(amount_touched_altchain || (offset_gindex < global_outs_for_amount), false, + "invalid global output index " << offset_gindex << " for amount=" << input_to_key.amount << ", max is " << global_outs_for_amount << ", referred to by offset #" << pk_n << ", amount_touched_altchain = " << amount_touched_altchain); @@ -5170,7 +6357,7 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s bool found_the_key = false; for (auto alt_it = alt_chain.rbegin(); alt_it != alt_chain.rend(); alt_it++) { - auto it_aag = (*alt_it)->second.gindex_lookup_table.find(input.amount); + auto it_aag = (*alt_it)->second.gindex_lookup_table.find(input_to_key.amount); if (it_aag == (*alt_it)->second.gindex_lookup_table.end()) { CHECK_AND_ASSERT_MES(alt_it != alt_chain.rbegin(), false, "internal error: was marked as amount_touched_altchain but unable to find on first entry"); @@ -5179,23 +6366,51 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s } if (offset_gindex >= it_aag->second) { + //source tx found in altchain //GOT IT!! - //TODO: At the moment we ignore check of mix_attr again mixing to simplify alt chain check, but in future consider it for stronger validation + //TODO: At the moment we ignore check of mix_attr against mixing to simplify alt chain check, but in future consider it for stronger validation uint64_t local_offset = offset_gindex - it_aag->second; auto& alt_keys = (*alt_it)->second.outputs_pub_keys; - CHECK_AND_ASSERT_MES(local_offset < alt_keys[input.amount].size(), false, "Internal error: local_offset=" << local_offset << " while alt_keys[" << input.amount << " ].size()=" << alt_keys.size()); - pk = alt_keys[input.amount][local_offset]; + CHECK_AND_ASSERT_MES(local_offset < alt_keys[input_to_key.amount].size(), false, "Internal error: local_offset=" << local_offset << " while alt_keys[" << input_to_key.amount << " ].size()=" << alt_keys.size()); + const output_key_or_htlc_v& out_in_alt = alt_keys[input_to_key.amount][local_offset]; + + /* + here we do validation against compatibility of input and output type + + TxOutput | TxInput | Allowed + ---------------------------- + HTLC | HTLC | ONLY IF HTLC NOT EXPIRED + HTLC | TO_KEY | ONLY IF HTLC IS EXPIRED + TO_KEY | HTLC | NOT + TO_KEY | TO_KEY | YES + */ + uint64_t height_of_source_block = (*alt_it)->second.height; + CHECK_AND_ASSERT_MES(height_of_current_alt_block > height_of_source_block, false, "Intenral error: height_of_current_alt_block > height_of_source_block failed"); + bool r = is_output_allowed_for_input(out_in_alt, input_v, height_of_current_alt_block - height_of_source_block); + CHECK_AND_ASSERT_MES(r, false, "Input and output incompatible type"); + + if (out_in_alt.type() == typeid(crypto::public_key)) + { + pk = boost::get(out_in_alt); + } + else + { + const txout_htlc& out_htlc = boost::get(out_in_alt); + bool htlc_expired = out_htlc.expiration > (height_of_current_alt_block - height_of_source_block) ? false:true; + pk = htlc_expired ? out_htlc.pkey_refund : out_htlc.pkey_redeem; + //input_v + } pub_key_pointers.push_back(&pk); found_the_key = true; break; } } if (found_the_key) - break; + continue; //otherwise lookup in main chain index } - auto p = m_db_outputs.get_subitem(input.amount, offset_gindex); - CHECK_AND_ASSERT_MES(p != nullptr, false, "global output was not found, amount: " << input.amount << ", gindex: " << offset_gindex << ", referred to by offset #" << pk_n); + auto p = m_db_outputs.get_subitem(input_to_key.amount, offset_gindex); + CHECK_AND_ASSERT_MES(p != nullptr, false, "global output was not found, amount: " << input_to_key.amount << ", gindex: " << offset_gindex << ", referred to by offset #" << pk_n); tx_id = p->tx_id; out_n = p->out_no; } @@ -5204,16 +6419,90 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s auto &rbi = boost::get(off); tx_id = rbi.tx_id; out_n = rbi.n; + //look up in alt-chain transactions fist + auto it = alt_chain_tx_ids.find(tx_id); + if (it != alt_chain_tx_ids.end()) + { + uint64_t height_of_source_block = it->second.second; + CHECK_AND_ASSERT_MES(height_of_current_alt_block > height_of_source_block, false, "Intenral error: height_of_current_alt_block > height_of_source_block failed"); + + /* + here we do validation against compatibility of input and output type + + TxOutput | TxInput | Allowed + ---------------------------- + HTLC | HTLC | ONLY IF HTLC NOT EXPIRED + HTLC | TO_KEY | ONLY IF HTLC IS EXPIRED + TO_KEY | HTLC | NOT + TO_KEY | TO_KEY | YES + */ + + //source tx found in altchain + CHECK_AND_ASSERT_MES(it->second.first.vout.size() > out_n, false, "Internal error: out_n(" << out_n << ") >= it->second.vout.size()(" << it->second.first.vout.size() << ")"); + txout_target_v out_target_v = it->second.first.vout[out_n].target; + + bool r = is_output_allowed_for_input(out_target_v, input_v, height_of_current_alt_block - height_of_source_block); + CHECK_AND_ASSERT_MES(r, false, "Input and output incompatible type"); + + + if (out_target_v.type() == typeid(txout_htlc)) + { + //source is hltc out + const txout_htlc& htlc = boost::get(out_target_v); + bool htlc_expired = htlc.expiration > (height_of_current_alt_block - height_of_source_block) ? false : true; + pk = htlc_expired ? htlc.pkey_refund : htlc.pkey_redeem; + pub_key_pointers.push_back(&pk); + continue; + } + else if (out_target_v.type() == typeid(txout_to_key)) + { + //source is to_key out + pk = boost::get(out_target_v).key; + pub_key_pointers.push_back(&pk); + continue; + } + else + { + ASSERT_MES_AND_THROW("Unexpected out type for tx_in in altblock: " << out_target_v.type().name()); + } + } + } auto p = m_db_transactions.get(tx_id); CHECK_AND_ASSERT_MES(p != nullptr && out_n < p->tx.vout.size(), false, "can't find output #" << out_n << " for tx " << tx_id << " referred by offset #" << pk_n); auto &t = p->tx.vout[out_n].target; - CHECK_AND_ASSERT_MES(t.type() == typeid(txout_to_key), false, "txin_to_key input offset #" << pk_n << " refers to incorrect output type " << t.type().name()); - auto& out_tk = boost::get(t); - pk = out_tk.key; - bool mixattr_ok = is_mixattr_applicable_for_fake_outs_counter(out_tk.mix_attr, abs_key_offsets.size() - 1); - CHECK_AND_ASSERT_MES(mixattr_ok, false, "input offset #" << pk_n << " violates mixin restrictions: mix_attr = " << static_cast(out_tk.mix_attr) << ", input's key_offsets.size = " << abs_key_offsets.size()); + + /* + here we do validation against compatibility of input and output type + + TxOutput | TxInput | Allowed + ---------------------------- + HTLC | HTLC | ONLY IF HTLC NOT EXPIRED + HTLC | TO_KEY | ONLY IF HTLC IS EXPIRED + TO_KEY | HTLC | NOT + TO_KEY | TO_KEY | YES + */ + uint64_t height_of_source_block = p->m_keeper_block_height; + CHECK_AND_ASSERT_MES(height_of_current_alt_block > height_of_source_block, false, "Intenral error: height_of_current_alt_block > height_of_source_block failed"); + bool r = is_output_allowed_for_input(t, input_v, height_of_current_alt_block - height_of_source_block); + CHECK_AND_ASSERT_MES(r, false, "Input and output incompatible type"); + + if (t.type() == typeid(txout_to_key)) + { + const txout_to_key& out_tk = boost::get(t); + pk = out_tk.key; + + bool mixattr_ok = is_mixattr_applicable_for_fake_outs_counter(out_tk.mix_attr, abs_key_offsets.size() - 1); + CHECK_AND_ASSERT_MES(mixattr_ok, false, "input offset #" << pk_n << " violates mixin restrictions: mix_attr = " << static_cast(out_tk.mix_attr) << ", input's key_offsets.size = " << abs_key_offsets.size()); + + } + else if (t.type() == typeid(txout_htlc)) + { + const txout_htlc& htlc = boost::get(t); + bool htlc_expired = htlc.expiration > (height_of_current_alt_block - height_of_source_block) ? false : true; + pk = htlc_expired ? htlc.pkey_refund : htlc.pkey_redeem; + } // case b4 (make sure source tx in the main chain is preceding split point, otherwise this referece is invalid) CHECK_AND_ASSERT_MES(p->m_keeper_block_height < split_height, false, "input offset #" << pk_n << " refers to main chain tx " << tx_id << " at height " << p->m_keeper_block_height << " while split height is " << split_height); @@ -5223,12 +6512,15 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s // TODO: consider checking p->tx for unlock time validity as it's checked in get_output_keys_for_input_with_checks() // make sure it was actually found - CHECK_AND_ASSERT_MES(pk != null_pkey, false, "Can't determine output public key for offset " << pk_n); + + // let's disable this check due to missing equal check in main chain validation code + //TODO: implement more strict validation with next hard fork + //CHECK_AND_ASSERT_MES(pk != null_pkey, false, "Can't determine output public key for offset " << pk_n << " in related tx: " << tx_id << ", out_n = " << out_n); pub_key_pointers.push_back(&pk); } // do input checks (attachment_info, ring signature and extra signature, etc.) - r = check_tokey_input(input_tx, input_index, input, input_tx_hash, input_sigs, pub_key_pointers); + r = check_input_signature(input_tx, input_index, input_to_key, input_tx_hash, input_sigs, pub_key_pointers); CHECK_AND_ASSERT_MES(r, false, "to_key input validation failed"); // TODO: consider checking input_tx for valid extra attachment info as it's checked in check_tx_inputs() @@ -5236,6 +6528,73 @@ bool blockchain_storage::validate_alt_block_input(const transaction& input_tx, s return true; } //------------------------------------------------------------------ +bool blockchain_storage::is_output_allowed_for_input(const txout_target_v& out_v, const txin_v& in_v, uint64_t top_minus_source_height)const +{ + + /* + TxOutput | TxInput | Allowed + ---------------------------- + HTLC | HTLC | ONLY IF HTLC NOT EXPIRED + HTLC | TO_KEY | ONLY IF HTLC IS EXPIRED + TO_KEY | HTLC | NOT + TO_KEY | TO_KEY | YES + */ + + + if (out_v.type() == typeid(txout_to_key)) + { + return is_output_allowed_for_input(boost::get(out_v), in_v); + } + else if (out_v.type() == typeid(txout_htlc)) + { + return is_output_allowed_for_input(boost::get(out_v), in_v, top_minus_source_height); + } + else + { + LOG_ERROR("[scan_outputkeys_for_indexes]: Wrong output type in : " << out_v.type().name()); + return false; + } + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::is_output_allowed_for_input(const txout_htlc& out_v, const txin_v& in_v, uint64_t top_minus_source_height)const +{ + bool htlc_expired = out_v.expiration > (top_minus_source_height) ? false : true; + if (!htlc_expired) + { + //HTLC IS NOT expired, can be used ONLY by pkey_before_expiration and ONLY by HTLC input + CHECK_AND_ASSERT_MES(in_v.type() == typeid(txin_htlc), false, "[TXOUT_HTLC]: Unexpected output type of non-HTLC input"); + } + else + { + //HTLC IS expired, can be used ONLY by pkey_after_expiration and ONLY by to_key input + CHECK_AND_ASSERT_MES(in_v.type() == typeid(txin_to_key), false, "[TXOUT_HTLC]: Unexpected output type of HTLC input"); + } + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::is_output_allowed_for_input(const txout_to_key& out_v, const txin_v& in_v)const +{ + //HTLC input CAN'T refer to regular to_key output + CHECK_AND_ASSERT_MES(in_v.type() != typeid(txin_htlc), false, "[TXOUT_TO_KEY]: Unexpected output type of HTLC input"); + return true; +} +//------------------------------------------------------------------ +bool blockchain_storage::is_output_allowed_for_input(const output_key_or_htlc_v& out_v, const txin_v& in_v, uint64_t top_minus_source_height)const +{ + if (out_v.type() == typeid(crypto::public_key)) + { + return is_output_allowed_for_input(txout_to_key(), in_v); + } + else if (out_v.type() == typeid(txout_htlc)) + { + return is_output_allowed_for_input(boost::get(out_v), in_v, top_minus_source_height); + } + else { + ASSERT_MES_AND_THROW("Unexpected type in output_key_or_htlc_v: " << out_v.type().name()); + } +} +//------------------------------------------------------------------ bool blockchain_storage::validate_alt_block_ms_input(const transaction& input_tx, const crypto::hash& input_tx_hash, size_t input_index, const std::vector& input_sigs, uint64_t split_height, const alt_chain_type& alt_chain) const { // Main and alt chain outline: @@ -5360,7 +6719,11 @@ bool blockchain_storage::get_transaction_from_pool_or_db(const crypto::hash& tx_ if (!m_tx_pool.get_transaction(tx_id, *tx_ptr)) // first try to get from the pool { auto p = m_db_transactions.get(tx_id); // if not found in the pool -- get from the DB - CHECK_AND_ASSERT_MES(p != nullptr, false, "can't get tx " << tx_id << " neither from the pool, nor from db_transactions"); + if (p == nullptr) + { + return false; + } + //CHECK_AND_ASSERT_MES(p != nullptr, false, "can't get tx " << tx_id << " neither from the pool, nor from db_transactions"); CHECK_AND_ASSERT_MES(p->m_keeper_block_height >= min_allowed_block_height, false, "tx " << tx_id << " found in the main chain at height " << p->m_keeper_block_height << " while required min allowed height is " << min_allowed_block_height); *tx_ptr = p->tx; } @@ -5373,7 +6736,7 @@ bool blockchain_storage::update_alt_out_indexes_for_tx_in_block(const transactio //add tx outputs to gindex_lookup_table for (auto o : tx.vout) { - if (o.target.type() == typeid(txout_to_key)) + if (o.target.type() == typeid(txout_to_key) || o.target.type() == typeid(txout_htlc)) { //LOG_PRINT_MAGENTA("ALT_OUT KEY ON H[" << abei.height << "] AMOUNT: " << o.amount, LOG_LEVEL_0); // first, look at local gindexes tables @@ -5383,18 +6746,28 @@ bool blockchain_storage::update_alt_out_indexes_for_tx_in_block(const transactio abei.gindex_lookup_table[o.amount] = m_db_outputs.get_item_size(o.amount); //LOG_PRINT_MAGENTA("FIRST TOUCH: size=" << abei.gindex_lookup_table[o.amount], LOG_LEVEL_0); } - abei.outputs_pub_keys[o.amount].push_back(boost::get(o.target).key); + if (o.target.type() == typeid(txout_to_key)) + { + abei.outputs_pub_keys[o.amount].push_back(boost::get(o.target).key); + } + else + { + abei.outputs_pub_keys[o.amount].push_back(boost::get(o.target)); + } + //TODO: At the moment we ignore check of mix_attr again mixing to simplify alt chain check, but in future consider it for stronger validation } } return true; } -bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::hash& id, std::set& collected_keyimages, alt_block_extended_info& abei, const alt_chain_type& alt_chain, uint64_t split_height, uint64_t& ki_lookup_time_total) const +bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::hash& id, std::unordered_set& collected_keyimages, alt_block_extended_info& abei, const alt_chain_type& alt_chain, uint64_t split_height, uint64_t& ki_lookup_time_total) const { uint64_t height = abei.height; bool r = false; - std::set alt_chain_block_ids; + std::unordered_set alt_chain_block_ids; + txs_by_id_and_height_altchain alt_chain_tx_ids; + alt_chain_block_ids.insert(id); // prepare data structure for output global indexes tracking within current alt chain @@ -5416,8 +6789,13 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha for (auto& ch : alt_chain) { alt_chain_block_ids.insert(get_block_hash(ch->second.bl)); + for (auto & on_board_tx : ch->second.onboard_transactions) + { + alt_chain_tx_ids.insert(txs_by_id_and_height_altchain::value_type(on_board_tx.first, txs_by_id_and_height_altchain::value_type::second_type(on_board_tx.second, ch->second.height))); + } + //TODO: consider performance optimization (get_transaction_hash might slow down deep reorganizations ) + alt_chain_tx_ids.insert(txs_by_id_and_height_altchain::value_type(get_transaction_hash(ch->second.bl.miner_tx), txs_by_id_and_height_altchain::value_type::second_type(ch->second.bl.miner_tx, ch->second.height))); } - } else { @@ -5431,7 +6809,7 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha CHECK_AND_ASSERT_MES(b.miner_tx.signatures.size() == 1 && b.miner_tx.vin.size() == 2, false, "invalid PoS block's miner_tx, signatures size = " << b.miner_tx.signatures.size() << ", miner_tx.vin.size() = " << b.miner_tx.vin.size()); uint64_t max_related_block_height = 0; uint64_t ki_lookup = 0; - r = validate_alt_block_input(b.miner_tx, collected_keyimages, id, get_block_hash(b), 1, b.miner_tx.signatures[0], split_height, alt_chain, alt_chain_block_ids, ki_lookup, &max_related_block_height); + r = validate_alt_block_input(b.miner_tx, collected_keyimages, alt_chain_tx_ids, id, get_block_hash(b), 1, b.miner_tx.signatures[0], split_height, alt_chain, alt_chain_block_ids, ki_lookup, &max_related_block_height); CHECK_AND_ASSERT_MES(r, false, "miner tx " << get_transaction_hash(b.miner_tx) << ": validation failed"); ki_lookup_time_total += ki_lookup; // check stake age @@ -5441,18 +6819,24 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha } update_alt_out_indexes_for_tx_in_block(b.miner_tx, abei); + CHECK_AND_ASSERT_MES(validate_tx_for_hardfork_specific_terms(b.miner_tx, null_hash, height), false, "miner tx hardfork-specific validation failed"); + for (auto tx_id : b.tx_hashes) { std::shared_ptr tx_ptr; - CHECK_AND_ASSERT_MES(get_transaction_from_pool_or_db(tx_id, tx_ptr, split_height), false, "failed to get alt block tx " << tx_id << " with split_height == " << split_height); - transaction& tx = *tx_ptr; + auto it = abei.onboard_transactions.find(tx_id); + if (it == abei.onboard_transactions.end()) + { + CHECK_AND_ASSERT_MES(get_transaction_from_pool_or_db(tx_id, tx_ptr, split_height), false, "failed to get alt block tx " << tx_id << " with split_height == " << split_height); + } + const transaction& tx = it == abei.onboard_transactions.end() ? *tx_ptr : it->second; CHECK_AND_ASSERT_MES(tx.signatures.size() == tx.vin.size(), false, "invalid tx: tx.signatures.size() == " << tx.signatures.size() << ", tx.vin.size() == " << tx.vin.size()); for (size_t n = 0; n < tx.vin.size(); ++n) { - if (tx.vin[n].type() == typeid(txin_to_key)) + if (tx.vin[n].type() == typeid(txin_to_key) || tx.vin[n].type() == typeid(txin_htlc)) { uint64_t ki_lookup = 0; - r = validate_alt_block_input(tx, collected_keyimages, id, tx_id, n, tx.signatures[n], split_height, alt_chain, alt_chain_block_ids, ki_lookup); + r = validate_alt_block_input(tx, collected_keyimages, alt_chain_tx_ids, id, tx_id, n, tx.signatures[n], split_height, alt_chain, alt_chain_block_ids, ki_lookup); CHECK_AND_ASSERT_MES(r, false, "tx " << tx_id << ", input #" << n << ": validation failed"); ki_lookup_time_total += ki_lookup; } @@ -5471,6 +6855,8 @@ bool blockchain_storage::validate_alt_block_txs(const block& b, const crypto::ha CHECK_AND_ASSERT_MES(false, false, "input #" << n << " has unexpected type (" << tx.vin[n].type().name() << "), tx " << tx_id); } } + + CHECK_AND_ASSERT_MES(validate_tx_for_hardfork_specific_terms(tx, tx_id, height), false, "tx " << tx_id << ": hardfork-specific validation failed"); // Updating abei (and not updating alt_chain) during this cycle is safe because txs in the same block can't reference one another, // so only valid references are either to previous alt blocks (accessed via alt_chain) or to main chain blocks. diff --git a/src/currency_core/blockchain_storage.h b/src/currency_core/blockchain_storage.h index eaa1274..5c3b995 100644 --- a/src/currency_core/blockchain_storage.h +++ b/src/currency_core/blockchain_storage.h @@ -133,6 +133,12 @@ namespace currency } }; + struct scan_for_keys_context + { + bool htlc_is_expired; + std::list htlc_outs; + }; + // == Output indexes local lookup table conception == // Main chain gindex table (outputs_container) contains data which is valid only for the most recent block. // Thus it can't be used to get output's global index for any arbitrary height because there's no height data. @@ -144,7 +150,9 @@ namespace currency // retrieve gindex from local_gindex_lookup_table # there are outputs having given amount after the given height // else: // retrieve gindex from main chain gindex table # not outputs having given amount are present after the given height - // + // + + typedef boost::variant output_key_or_htlc_v; struct alt_block_extended_info: public block_extended_info { @@ -152,17 +160,22 @@ namespace currency std::map gindex_lookup_table; // {amount -> pub_keys} map of outputs' pub_keys appeared in this alt block ( index_in_vector == output_gindex - gindex_lookup_table[output_amount] ) - std::map > outputs_pub_keys; + std::map > outputs_pub_keys; + + //date added to alt chain storage + uint64_t timestamp; + + //transactions associated with the block + transactions_map onboard_transactions; }; typedef std::unordered_map alt_chain_container; - //typedef std::list alt_chain_type; - typedef std::vector alt_chain_type; + typedef std::vector alt_chain_type; // alternative subchain, front -> mainchain(split point), back -> alternative head typedef std::unordered_map blocks_ext_by_hash; typedef tools::db::basic_key_to_array_accessor outputs_container; // out_amount => ['global_output', ...] typedef tools::db::cached_key_value_accessor key_images_container; - typedef std::list, std::list > > > blocks_direct_container; + typedef std::list, std::list >, std::shared_ptr > > blocks_direct_container; friend struct add_transaction_input_visitor; //--------------------------------------------------------------------------------- @@ -184,6 +197,7 @@ namespace currency //------------- modifying members -------------- bool add_new_block(const block& bl_, block_verification_context& bvc); + bool prevalidate_block(const block& bl); bool clear(); bool reset_and_set_genesis_block(const block& b); //debug function @@ -221,8 +235,17 @@ namespace currency bool have_tx_keyimg_as_spent(const crypto::key_image &key_im, uint64_t before_height = UINT64_MAX) const; std::shared_ptr get_tx(const crypto::hash &id) const; + template - bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL) const ; + bool scan_outputkeys_for_indexes(const transaction &validated_tx, const txin_to_key& tx_in_to_key, visitor_t& vis) + { + scan_for_keys_context cntx_stub = AUTO_VAL_INIT(cntx_stub); + uint64_t stub = 0; + return scan_outputkeys_for_indexes(validated_tx, tx_in_to_key, vis, stub, cntx_stub); + } + template + bool scan_outputkeys_for_indexes(const transaction &validated_tx, const txin_v& verified_input, visitor_t& vis, uint64_t& max_related_block_height, scan_for_keys_context& /*scan_context*/) const; + uint64_t get_current_blockchain_size() const; uint64_t get_top_block_height() const; @@ -230,12 +253,13 @@ namespace currency crypto::hash get_top_block_id(uint64_t& height) const; bool get_top_block(block& b) const; wide_difficulty_type get_next_diff_conditional(bool pos) const; - wide_difficulty_type get_next_diff_conditional2(bool pos, const alt_chain_type& alt_chain, uint64_t split_height) const; + wide_difficulty_type get_next_diff_conditional2(bool pos, const alt_chain_type& alt_chain, uint64_t split_height, const alt_block_extended_info& abei) const; wide_difficulty_type get_cached_next_difficulty(bool pos) const; - typedef bool fill_block_template_func_t(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height); + bool create_block_template(block& b, const account_public_address& miner_address, const account_public_address& stakeholder_address, wide_difficulty_type& di, uint64_t& height, const blobdata& ex_nonce, bool pos, const pos_entry& pe, fill_block_template_func_t custom_fill_block_template_func = nullptr) const; bool create_block_template(block& b, const account_public_address& miner_address, wide_difficulty_type& di, uint64_t& height, const blobdata& ex_nonce) const; + bool create_block_template(const create_block_template_params& params, create_block_template_response& resp) const; bool have_block(const crypto::hash& id) const; size_t get_total_transactions()const; @@ -243,8 +267,8 @@ namespace currency bool get_short_chain_history(std::list& ids)const; bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)const; bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset)const; - bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; - bool find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; + bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0, bool need_global_indexes = false)const; + bool find_blockchain_supplement(const std::list& qblock_ids, blocks_direct_container& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count, uint64_t minimum_height = 0, bool request_coinbase_info = false)const; //bool find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count)const; bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp)const; bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res)const; @@ -260,14 +284,27 @@ namespace currency uint64_t get_aliases_count()const; uint64_t get_block_h_older_then(uint64_t timestamp) const; bool validate_tx_service_attachmens_in_services(const tx_service_attachment& a, size_t i, const transaction& tx)const; - bool check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL)const; - bool check_tx_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL)const; - bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL)const; - //bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL)const; - bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id)const; + bool check_tx_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height, uint64_t& source_max_unlock_time_for_pos_coinbase)const; + bool check_tx_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height)const; + bool check_tx_input(const transaction& tx, size_t in_index, const txin_htlc& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t& max_related_block_height)const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& max_used_block_height)const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash) const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t& max_used_block_height, crypto::hash& max_used_block_id)const; bool check_ms_input(const transaction& tx, size_t in_index, const txin_multisig& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, const transaction& source_tx, size_t out_n) const; - bool get_output_keys_for_input_with_checks(const txin_to_key& txin, std::vector& output_keys, uint64_t* pmax_related_block_height = NULL) const; - bool check_tokey_input(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, const std::vector& output_keys_ptrs) const; + bool validate_tx_for_hardfork_specific_terms(const transaction& tx, const crypto::hash& tx_id, uint64_t block_height) const; + bool validate_tx_for_hardfork_specific_terms(const transaction& tx, const crypto::hash& tx_id) const; + bool get_output_keys_for_input_with_checks(const transaction& tx, const txin_v& verified_input, std::vector& output_keys, uint64_t& max_related_block_height, uint64_t& source_max_unlock_time_for_pos_coinbase, scan_for_keys_context& scan_context) const; + bool get_output_keys_for_input_with_checks(const transaction& tx, const txin_v& verified_input, std::vector& output_keys, uint64_t& max_related_block_height, uint64_t& source_max_unlock_time_for_pos_coinbase) const; + bool check_input_signature(const transaction& tx, size_t in_index, const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, const std::vector& output_keys_ptrs) const; + bool check_input_signature(const transaction& tx, + size_t in_index, + uint64_t in_amount, + const crypto::key_image& k_image, + const std::vector& in_etc_details, + const crypto::hash& tx_prefix_hash, + const std::vector& sig, + const std::vector& output_keys_ptrs) const; + uint64_t get_current_comulative_blocksize_limit()const; uint64_t get_current_hashrate(size_t aprox_count)const; uint64_t get_seconds_between_last_n_block(size_t n)const; @@ -282,7 +319,7 @@ namespace currency i_core_event_handler* get_event_handler() const; uint64_t get_last_timestamps_check_window_median() const; uint64_t get_last_n_blocks_timestamps_median(size_t n) const; - bool prevalidate_alias_info(const transaction& tx, extra_alias_entry& eae); + bool prevalidate_alias_info(const transaction& tx, const extra_alias_entry& eae); bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, const boost::multiprecision::uint128_t& already_generated_coins) const; performnce_data& get_performnce_data()const; bool validate_instance(const std::string& path); @@ -304,6 +341,7 @@ namespace currency bool build_stake_modifier(stake_modifier_type& sm, const alt_chain_type& alt_chain = alt_chain_type(), uint64_t split_height = 0, crypto::hash *p_last_block_hash = nullptr) const; bool scan_pos(const COMMAND_RPC_SCAN_POS::request& sp, COMMAND_RPC_SCAN_POS::response& rsp)const; + bool validate_pos_coinbase_outs_unlock_time(const transaction& miner_tx, uint64_t staked_amount, uint64_t source_max_unlock_time)const; bool validate_pos_block(const block& b, const crypto::hash& id, bool for_altchain)const; bool validate_pos_block(const block& b, wide_difficulty_type basic_diff, const crypto::hash& id, bool for_altchain)const; bool validate_pos_block(const block& b, @@ -408,7 +446,7 @@ namespace currency template void serialize(archive_t & ar, const unsigned int version); - + bool get_est_height_from_date(uint64_t date, uint64_t& res_h)const; //debug functions @@ -419,8 +457,9 @@ namespace currency void print_blockchain_with_tx(uint64_t start_index, uint64_t end_index) const; void print_blockchain_index() const; void print_blockchain_outs(const std::string& file) const; - void print_blockchain_outs_stat() const; + void print_blockchain_outs_stats() const; void print_db_cache_perfeormance_data() const; + void print_last_n_difficulty_numbers(uint64_t n) const; bool calc_tx_cummulative_blob(const block& bl)const; bool get_outs_index_stat(outs_index_stat& outs_stat)const; bool print_lookup_key_image(const crypto::key_image& ki) const; @@ -429,6 +468,9 @@ namespace currency void inspect_blocks_index() const; bool rebuild_tx_fee_medians(); bool validate_all_aliases_for_new_median_mode(); + bool print_tx_outputs_lookup(const crypto::hash& tx_id) const; + uint64_t get_last_x_block_height(bool pos)const; + bool is_tx_spendtime_unlocked(uint64_t unlock_time)const; private: //-------------- DB containers -------------- @@ -444,6 +486,8 @@ namespace currency typedef tools::db::basic_key_value_accessor per_block_gindex_increments_container; // height => [(amount, gindex_increment), ...] //----------------------------------------- + + typedef std::unordered_map > txs_by_id_and_height_altchain; tx_memory_pool& m_tx_pool; mutable bc_attachment_services_manager m_services_mgr; @@ -511,6 +555,7 @@ namespace currency mutable wide_difficulty_type m_cached_next_pow_difficulty; mutable wide_difficulty_type m_cached_next_pos_difficulty; + mutable critical_section m_targetdata_cache_lock; mutable std::list > m_pos_targetdata_cache; mutable std::list > m_pow_targetdata_cache; //work like a cache to avoid recalculation on read operations @@ -518,42 +563,55 @@ namespace currency mutable uint64_t m_current_fee_median_effective_index; bool m_is_reorganize_in_process; mutable std::atomic m_deinit_is_done; - + mutable uint64_t m_blockchain_launch_timestamp; bool init_tx_fee_median(); bool update_tx_fee_median(); - void initialize_db_solo_options_values(); + void store_db_solo_options_values(); + bool set_lost_tx_unmixable(); + bool set_lost_tx_unmixable_for_height(uint64_t height); + void patch_out_if_needed(txout_to_key& out, const crypto::hash& tx_id, uint64_t n)const ; bool switch_to_alternative_blockchain(alt_chain_type& alt_chain); void purge_alt_block_txs_hashs(const block& b); void add_alt_block_txs_hashs(const block& b); - bool pop_block_from_blockchain(); + bool pop_block_from_blockchain(transactions_map& onboard_transactions); bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count); - bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count, uint64_t& fee); - bool purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee); + bool purge_block_data_from_blockchain(const block& b, size_t processed_tx_count, uint64_t& fee, transactions_map& onboard_transactions); + bool purge_transaction_from_blockchain(const crypto::hash& tx_id, uint64_t& fee, transaction& tx); bool purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check); wide_difficulty_type get_next_difficulty_for_alternative_chain(const alt_chain_type& alt_chain, block_extended_info& bei, bool pos) const; bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); std::string print_alt_chain(alt_chain_type alt_chain); bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); - bool is_reorganize_required(const block_extended_info& main_chain_bei, const block_extended_info& alt_chain_bei, const crypto::hash& proof_alt); + bool is_reorganize_required(const block_extended_info& main_chain_bei, const alt_chain_type& alt_chain, const crypto::hash& proof_alt); + wide_difficulty_type get_x_difficulty_after_height(uint64_t height, bool is_pos); bool purge_keyimage_from_big_heap(const crypto::key_image& ki, const crypto::hash& id); bool purge_altblock_keyimages_from_big_heap(const block& b, const crypto::hash& id); - bool append_altblock_keyimages_to_big_heap(const crypto::hash& block_id, const std::set& alt_block_keyimages); - bool validate_alt_block_input(const transaction& input_tx, std::set& collected_keyimages, const crypto::hash& bl_id, const crypto::hash& input_tx_hash, size_t input_index, const std::vector& input_sigs, uint64_t split_height, const alt_chain_type& alt_chain, const std::set& alt_chain_block_ids, uint64_t& ki_lookuptime, uint64_t* p_max_related_block_height = nullptr) const; + bool append_altblock_keyimages_to_big_heap(const crypto::hash& block_id, const std::unordered_set& alt_block_keyimages); + bool validate_alt_block_input(const transaction& input_tx, + std::unordered_set& collected_keyimages, + const txs_by_id_and_height_altchain& alt_chain_tx_ids, + const crypto::hash& bl_id, + const crypto::hash& input_tx_hash, + size_t input_index, + const std::vector& input_sigs, + uint64_t split_height, + const alt_chain_type& alt_chain, + const std::unordered_set& alt_chain_block_ids, + uint64_t& ki_lookuptime, + uint64_t* p_max_related_block_height = nullptr) const; bool validate_alt_block_ms_input(const transaction& input_tx, const crypto::hash& input_tx_hash, size_t input_index, const std::vector& input_sigs, uint64_t split_height, const alt_chain_type& alt_chain) const; - bool validate_alt_block_txs(const block& b, const crypto::hash& id, std::set& collected_keyimages, alt_block_extended_info& abei, const alt_chain_type& alt_chain, uint64_t split_height, uint64_t& ki_lookup_time_total) const; + bool validate_alt_block_txs(const block& b, const crypto::hash& id, std::unordered_set& collected_keyimages, alt_block_extended_info& abei, const alt_chain_type& alt_chain, uint64_t split_height, uint64_t& ki_lookup_time_total) const; bool update_alt_out_indexes_for_tx_in_block(const transaction& tx, alt_block_extended_info& abei)const; bool get_transaction_from_pool_or_db(const crypto::hash& tx_id, std::shared_ptr& tx_ptr, uint64_t min_allowed_block_height = 0) const; - + void get_last_n_x_blocks(uint64_t n, bool pos_blocks, std::list>& blocks) const; bool prevalidate_miner_transaction(const block& b, uint64_t height, bool pos)const; - bool validate_transaction(const block& b, uint64_t height, const transaction& tx)const; - bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); + bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, uint64_t timestamp); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); bool add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i, uint64_t mix_count, bool use_only_forced_to_mix = false) const; - bool is_tx_spendtime_unlocked(uint64_t unlock_time)const; bool add_block_as_invalid(const block& bl, const crypto::hash& h); bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); size_t find_end_of_allowed_index(uint64_t amount)const; @@ -561,7 +619,7 @@ namespace currency bool check_block_timestamp(std::vector timestamps, const block& b)const; std::vector get_last_n_blocks_timestamps(size_t n)const; const std::vector& get_txin_etc_options(const txin_v& in)const; - void on_block_added(const block_extended_info& bei, const crypto::hash& id); + void on_block_added(const block_extended_info& bei, const crypto::hash& id, const std::list& bsk); void on_block_removed(const block_extended_info& bei); void update_targetdata_cache_on_block_added(const block_extended_info& bei); void update_targetdata_cache_on_block_removed(const block_extended_info& bei); @@ -587,7 +645,7 @@ namespace currency // bool build_stake_modifier_for_alt(const alt_chain_type& alt_chain, stake_modifier_type& sm); template bool enum_blockchain(visitor_t& v, const alt_chain_type& alt_chain = alt_chain_type(), uint64_t split_height = 0) const; - bool update_spent_tx_flags_for_input(uint64_t amount, const txout_v& o, bool spent); + bool update_spent_tx_flags_for_input(uint64_t amount, const txout_ref_v& o, bool spent); bool update_spent_tx_flags_for_input(uint64_t amount, uint64_t global_index, bool spent); bool update_spent_tx_flags_for_input(const crypto::hash& multisig_id, uint64_t spent_height); bool update_spent_tx_flags_for_input(const crypto::hash& tx_id, size_t n, bool spent); @@ -596,6 +654,17 @@ namespace currency void pop_block_from_per_block_increments(uint64_t height_); void calculate_local_gindex_lookup_table_for_height(uint64_t split_height, std::map& increments) const; void do_erase_altblock(alt_chain_container::iterator it); + uint64_t get_blockchain_launch_timestamp()const; + bool is_output_allowed_for_input(const txout_target_v& out_v, const txin_v& in_v, uint64_t top_minus_source_height)const; + bool is_output_allowed_for_input(const output_key_or_htlc_v& out_v, const txin_v& in_v, uint64_t top_minus_source_height)const; + bool is_output_allowed_for_input(const txout_to_key& out_v, const txin_v& in_v)const; + bool is_output_allowed_for_input(const txout_htlc& out_v, const txin_v& in_v, uint64_t top_minus_source_height)const; + bool is_after_hardfork_1_zone()const; + bool is_after_hardfork_1_zone(uint64_t height)const; + bool is_after_hardfork_2_zone()const; + bool is_after_hardfork_2_zone(uint64_t height)const; + bool is_after_hardfork_3_zone()const; + bool is_after_hardfork_3_zone(uint64_t height)const; @@ -603,8 +672,8 @@ namespace currency //POS wide_difficulty_type get_adjusted_cumulative_difficulty_for_next_pos(wide_difficulty_type next_diff)const; wide_difficulty_type get_adjusted_cumulative_difficulty_for_next_alt_pos(alt_chain_type& alt_chain, uint64_t block_height, wide_difficulty_type next_diff, uint64_t connection_height)const; - uint64_t get_last_x_block_height(bool pos)const; - wide_difficulty_type get_last_alt_x_block_cumulative_precise_difficulty(alt_chain_type& alt_chain, uint64_t block_height, bool pos)const; + wide_difficulty_type get_last_alt_x_block_cumulative_precise_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos, wide_difficulty_type& cumulative_diff_precise_adj)const; + wide_difficulty_type get_last_alt_x_block_cumulative_precise_adj_difficulty(const alt_chain_type& alt_chain, uint64_t block_height, bool pos) const; size_t get_current_sequence_factor_for_alt(alt_chain_type& alt_chain, bool pos, uint64_t connection_height)const; }; @@ -639,25 +708,28 @@ namespace currency return !keep_going; } - - //------------------------------------------------------------------ //------------------------------------------------------------------ template - bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) const + bool blockchain_storage::scan_outputkeys_for_indexes(const transaction &validated_tx, const txin_v& verified_input, visitor_t& vis, uint64_t& max_related_block_height, scan_for_keys_context& scan_context) const { + const txin_to_key& input_to_key = get_to_key_input_from_txin_v(verified_input); + + uint64_t amount = input_to_key.amount; + const std::vector& key_offsets = input_to_key.key_offsets; + CRITICAL_REGION_LOCAL(m_read_lock); TIME_MEASURE_START_PD(tx_check_inputs_loop_scan_outputkeys_get_item_size); - uint64_t outs_count_for_amount = m_db_outputs.get_item_size(tx_in_to_key.amount); + uint64_t outs_count_for_amount = m_db_outputs.get_item_size(amount); TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_scan_outputkeys_get_item_size); if (!outs_count_for_amount) return false; TIME_MEASURE_START_PD(tx_check_inputs_loop_scan_outputkeys_relative_to_absolute); - std::vector absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); + std::vector absolute_offsets = relative_output_offsets_to_absolute(key_offsets); TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_scan_outputkeys_relative_to_absolute); TIME_MEASURE_START_PD(tx_check_inputs_loop_scan_outputkeys_loop); size_t output_index = 0; - for(const txout_v& o : absolute_offsets) + for(const txout_ref_v& o : absolute_offsets) { crypto::hash tx_id = null_hash; size_t n = 0; @@ -675,7 +747,7 @@ namespace currency LOG_ERROR("Wrong index in transaction inputs: " << i << ", expected maximum " << outs_count_for_amount - 1); return false; } - auto out_ptr = m_db_outputs.get_subitem(tx_in_to_key.amount, i); + auto out_ptr = m_db_outputs.get_subitem(amount, i); tx_id = out_ptr->tx_id; n = out_ptr->out_no; TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_scan_outputkeys_loop_get_subitem); @@ -688,25 +760,66 @@ namespace currency //check mix_attr TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_scan_outputkeys_loop_find_tx); - CHECKED_GET_SPECIFIC_VARIANT(tx_ptr->tx.vout[n].target, const txout_to_key, outtk, false); - - CHECK_AND_ASSERT_MES(tx_in_to_key.key_offsets.size() >= 1, false, "internal error: tx input has empty key_offsets"); // should never happen as input correctness must be handled by the caller - bool mixattr_ok = is_mixattr_applicable_for_fake_outs_counter(outtk.mix_attr, tx_in_to_key.key_offsets.size() - 1); - CHECK_AND_ASSERT_MES(mixattr_ok, false, "tx output #" << output_index << " violates mixin restrictions: mix_attr = " << static_cast(outtk.mix_attr) << ", key_offsets.size = " << tx_in_to_key.key_offsets.size()); + //CHECKED_GET_SPECIFIC_VARIANT(tx_ptr->tx.vout[n].target, const txout_to_key, outtk, false); + CHECK_AND_ASSERT_MES(key_offsets.size() >= 1, false, "internal error: tx input has empty key_offsets"); // should never happen as input correctness must be handled by the caller + + /* + TxOutput | TxInput | Allowed + ---------------------------- + HTLC | HTLC | ONLY IF HTLC NOT EXPIRED + HTLC | TO_KEY | ONLY IF HTLC IS EXPIRED + TO_KEY | HTLC | NOT + TO_KEY | TO_KEY | YES + */ + + bool r = is_output_allowed_for_input(tx_ptr->tx.vout[n].target, verified_input, get_current_blockchain_size() - tx_ptr->m_keeper_block_height); + CHECK_AND_ASSERT_MES(r, false, "Input and output incompatible type"); + + if (tx_ptr->tx.vout[n].target.type() == typeid(txout_to_key)) + { + CHECKED_GET_SPECIFIC_VARIANT(tx_ptr->tx.vout[n].target, const txout_to_key, outtk, false); + //fix for burned money + patch_out_if_needed(const_cast(outtk), tx_id, n); + + bool mixattr_ok = is_mixattr_applicable_for_fake_outs_counter(outtk.mix_attr, key_offsets.size() - 1); + CHECK_AND_ASSERT_MES(mixattr_ok, false, "tx output #" << output_index << " violates mixin restrictions: mix_attr = " << static_cast(outtk.mix_attr) << ", key_offsets.size = " << key_offsets.size()); + } + else if (tx_ptr->tx.vout[n].target.type() == typeid(txout_htlc)) + { + //check for spend flags + CHECK_AND_ASSERT_MES(tx_ptr->m_spent_flags.size() > n, false, + "Internal error: tx_ptr->m_spent_flags.size(){" << tx_ptr->m_spent_flags.size() << "} > n{" << n << "}"); + CHECK_AND_ASSERT_MES(tx_ptr->m_spent_flags[n] == false, false, "HTLC out already spent, double spent attempt detected"); + + const txout_htlc& htlc_out = boost::get(tx_ptr->tx.vout[n].target); + if (htlc_out.expiration > get_current_blockchain_size() - tx_ptr->m_keeper_block_height) + { + //HTLC IS NOT expired, can be used ONLY by pkey_before_expiration and ONLY by HTLC input + scan_context.htlc_is_expired = false; + } + else + { + //HTLC IS expired, can be used ONLY by pkey_after_expiration and ONLY by to_key input + scan_context.htlc_is_expired = true; + } + }else + { + LOG_ERROR("[scan_outputkeys_for_indexes]: Wrong output type in : " << tx_ptr->tx.vout[n].target.type().name()); + return false; + } + TIME_MEASURE_START_PD(tx_check_inputs_loop_scan_outputkeys_loop_handle_output); - if (!vis.handle_output(tx_ptr->tx, tx_ptr->tx.vout[n])) + if (!vis.handle_output(tx_ptr->tx, validated_tx, tx_ptr->tx.vout[n], n)) { LOG_PRINT_L0("Failed to handle_output for output id = " << tx_id << ", no " << n); return false; } TIME_MEASURE_FINISH_PD(tx_check_inputs_loop_scan_outputkeys_loop_handle_output); - if (pmax_related_block_height) - { - if (*pmax_related_block_height < tx_ptr->m_keeper_block_height) - *pmax_related_block_height = tx_ptr->m_keeper_block_height; - } + if (max_related_block_height < tx_ptr->m_keeper_block_height) + max_related_block_height = tx_ptr->m_keeper_block_height; + ++output_index; } diff --git a/src/currency_core/blockchain_storage_basic.h b/src/currency_core/blockchain_storage_basic.h index 05709ae..8a48700 100644 --- a/src/currency_core/blockchain_storage_basic.h +++ b/src/currency_core/blockchain_storage_basic.h @@ -6,6 +6,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#include #include #include @@ -17,7 +18,7 @@ #include "currency_basic.h" #include "difficulty.h" - +#include "currency_protocol/blobdatatype.h" namespace currency { @@ -47,6 +48,7 @@ namespace currency uint64_t block_cumulative_size; wide_difficulty_type cumulative_diff_adjusted; wide_difficulty_type cumulative_diff_precise; + wide_difficulty_type cumulative_diff_precise_adjusted; wide_difficulty_type difficulty; boost::multiprecision::uint128_t already_generated_coins; crypto::hash stake_hash; //TODO: unused field for PoW blocks, subject for refactoring @@ -62,6 +64,7 @@ namespace currency FIELD(block_cumulative_size) FIELD(cumulative_diff_adjusted) FIELD(cumulative_diff_precise) + FIELD(cumulative_diff_precise_adjusted) FIELD(difficulty) FIELD(already_generated_coins) FIELD(stake_hash) @@ -123,6 +126,34 @@ namespace currency } }; + typedef bool fill_block_template_func_t(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height); + + struct create_block_template_params + { + account_public_address miner_address; + account_public_address stakeholder_address; + blobdata ex_nonce; + bool pos = false; + pos_entry pe; + std::list explicit_txs; + fill_block_template_func_t *pcustom_fill_block_template_func; + }; + + struct create_block_template_response + { + block b; + wide_difficulty_type diffic; + uint64_t height; + }; + + typedef std::unordered_map transactions_map; + + struct block_ws_txs + { + block b; + transactions_map onboard_transactions; + }; + } \ No newline at end of file diff --git a/src/currency_core/blockchain_storage_boost_serialization.h b/src/currency_core/blockchain_storage_boost_serialization.h index a16a36d..ce1babf 100644 --- a/src/currency_core/blockchain_storage_boost_serialization.h +++ b/src/currency_core/blockchain_storage_boost_serialization.h @@ -6,6 +6,8 @@ #pragma once +#include "currency_boost_serialization.h" + namespace boost { namespace serialization @@ -21,6 +23,8 @@ namespace boost ar & te.m_spent_flags; } + // The following method is used in tests only atm + // TODO: Consider to remove completely template void serialize(archive_t & ar, currency::block_extended_info& ei, const unsigned int version) { @@ -32,17 +36,12 @@ namespace boost ar & ei.block_cumulative_size; ar & ei.already_generated_coins; ar & ei.stake_hash; + + ar & ei.cumulative_diff_precise_adjusted; + //ar & ei.version; + ar & ei.this_block_tx_fee_median; + ar & ei.effective_tx_fee_median; } - template - void serialize(archive_t & ar, currency::extra_alias_entry_base& ai, const unsigned int version) - { - ar & ai.m_address.m_spend_public_key; - ar & ai.m_address.m_view_public_key; - ar & ai.m_view_key; - ar & ai.m_sign; - ar & ai.m_text_comment; - //ar & ai.; - } } } diff --git a/src/currency_core/checkpoints_create.h b/src/currency_core/checkpoints_create.h index e404c65..ea87d78 100644 --- a/src/currency_core/checkpoints_create.h +++ b/src/currency_core/checkpoints_create.h @@ -9,12 +9,24 @@ #include "checkpoints.h" #include "misc_log_ex.h" -#define ADD_CHECKPOINT(h, hash) CHECK_AND_ASSERT(checkpoints.add_checkpoint(h, hash), false); +#define ADD_CHECKPOINT(h, hash) CHECK_AND_ASSERT(checkpoints.add_checkpoint(h, hash), false) + +namespace currency +{ -namespace currency { inline bool create_checkpoints(currency::checkpoints& checkpoints) { +#ifdef TESTNET + //ADD_CHECKPOINT(50000, "492ef71f5d722a8a182d65eb0ff731b740e023a2d64881f43db9af7b39ba7988"); +#else + // MAINNET + ADD_CHECKPOINT(425000, "46a6c36d5dec2d484d5e4845a8525ca322aafc06915ed9c8da2a241b51b7d1e8"); + ADD_CHECKPOINT(525000, "8c1ac57e67448130207a224b2d6e33ccdc64d6dd1c59dbcf9ad2361dc0d07d51"); + ADD_CHECKPOINT(600000, "d9fe316086e1aaea07d94082973ec764eff5fc5a05ed6e1eca273cee59daeeb4"); + ADD_CHECKPOINT(900000, "2205b73cd79d4937b087b02a8b001171b73c34464bc4a952834eaf7c2bd63e86"); +#endif return true; } -} + +} // namespace currency diff --git a/src/currency_core/core_runtime_config.h b/src/currency_core/core_runtime_config.h index f046ce0..a2bbe84 100644 --- a/src/currency_core/core_runtime_config.h +++ b/src/currency_core/core_runtime_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2020 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Boolberry developers // Distributed under the MIT/X11 software license, see the accompanying @@ -18,9 +18,26 @@ namespace currency uint64_t pos_minimum_heigh; //height uint64_t tx_pool_min_fee; uint64_t tx_default_fee; + uint64_t max_alt_blocks; crypto::public_key alias_validation_pubkey; core_time_func_t get_core_time; - + + uint64_t hard_fork_01_starts_after_height; + uint64_t hard_fork_02_starts_after_height; + uint64_t hard_fork_03_starts_after_height; + + bool is_hardfork_active_for_height(size_t hardfork_id, uint64_t height) const + { + switch (hardfork_id) + { + case 0: return true; + case 1: return height > hard_fork_01_starts_after_height; + case 2: return height > hard_fork_02_starts_after_height; + case 3: return height > hard_fork_03_starts_after_height; + default: return false; + } + } + static uint64_t _default_core_time_function() { return time(NULL); @@ -34,6 +51,12 @@ namespace currency pc.pos_minimum_heigh = POS_START_HEIGHT; pc.tx_pool_min_fee = TX_MINIMUM_FEE; pc.tx_default_fee = TX_DEFAULT_FEE; + pc.max_alt_blocks = CURRENCY_ALT_BLOCK_MAX_COUNT; + + pc.hard_fork_01_starts_after_height = ZANO_HARDFORK_01_AFTER_HEIGHT; + pc.hard_fork_02_starts_after_height = ZANO_HARDFORK_02_AFTER_HEIGHT; + pc.hard_fork_03_starts_after_height = ZANO_HARDFORK_03_AFTER_HEIGHT; + pc.get_core_time = &core_runtime_config::_default_core_time_function; bool r = epee::string_tools::hex_to_pod(ALIAS_SHORT_NAMES_VALIDATION_PUB_KEY, pc.alias_validation_pubkey); CHECK_AND_ASSERT_THROW_MES(r, "failed to parse alias_validation_pub_key"); diff --git a/src/currency_core/currency_basic.h b/src/currency_core/currency_basic.h index 81d5bac..274a2f2 100644 --- a/src/currency_core/currency_basic.h +++ b/src/currency_core/currency_basic.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Copyright (c) 2014-2015 The Boolberry developers @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include @@ -51,32 +51,84 @@ namespace currency const static crypto::signature null_sig = AUTO_VAL_INIT(null_sig); const static crypto::key_derivation null_derivation = AUTO_VAL_INIT(null_derivation); + const static crypto::hash gdefault_genesis = epee::string_tools::hex_to_pod("CC608F59F8080E2FBFE3C8C80EB6E6A953D47CF2D6AEBD345BADA3A1CAB99852"); + typedef std::string payment_id_t; /************************************************************************/ /* */ /************************************************************************/ - //since structure used in blockchain as a key accessor, then be sure that there is no padding inside +//since structure used in blockchain as a key accessor, then be sure that there is no padding inside +#pragma pack(push, 1) + struct account_public_address_old + { + crypto::public_key spend_public_key; + crypto::public_key view_public_key; + + BEGIN_SERIALIZE_OBJECT() + FIELD(spend_public_key) + FIELD(view_public_key) + END_SERIALIZE() + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(spend_public_key, "m_spend_public_key") + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(view_public_key, "m_view_public_key") + END_KV_SERIALIZE_MAP() + }; +#pragma pack(pop) + + +#define ACCOUNT_PUBLIC_ADDRESS_SERIZALIZATION_VER 1 + +#define ACCOUNT_PUBLIC_ADDRESS_FLAG_AUDITABLE 0x01 // auditable address + +//since structure used in blockchain as a key accessor, then be sure that there is no padding inside #pragma pack(push, 1) struct account_public_address { - crypto::public_key m_spend_public_key; - crypto::public_key m_view_public_key; + crypto::public_key spend_public_key; + crypto::public_key view_public_key; + uint8_t flags; + DEFINE_SERIALIZATION_VERSION(ACCOUNT_PUBLIC_ADDRESS_SERIZALIZATION_VER) BEGIN_SERIALIZE_OBJECT() - FIELD(m_spend_public_key) - FIELD(m_view_public_key) + FIELD(spend_public_key) + FIELD(view_public_key) + FIELD(flags) END_SERIALIZE() - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_spend_public_key) - KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_view_public_key) - END_KV_SERIALIZE_MAP() + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(spend_public_key, "m_spend_public_key") + KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(view_public_key, "m_view_public_key") + KV_SERIALIZE(flags) + END_KV_SERIALIZE_MAP() + + bool is_auditable() const + { + return (flags & ACCOUNT_PUBLIC_ADDRESS_FLAG_AUDITABLE) != 0; + } + + static account_public_address from_old(const account_public_address_old& rhs) + { + account_public_address result = AUTO_VAL_INIT(result); + result.spend_public_key = rhs.spend_public_key; + result.view_public_key = rhs.view_public_key; + return result; + } + + account_public_address_old to_old() const + { + account_public_address_old result = AUTO_VAL_INIT(result); + result.spend_public_key = spend_public_key; + result.view_public_key = view_public_key; + return result; + } }; #pragma pack(pop) + const static account_public_address null_pub_addr = AUTO_VAL_INIT(null_pub_addr); typedef std::vector ring_signature; @@ -139,7 +191,7 @@ namespace currency END_SERIALIZE() }; - typedef boost::variant txout_v; + typedef boost::variant txout_ref_v; struct signed_parts @@ -159,7 +211,7 @@ namespace currency struct txin_to_key { uint64_t amount; - std::vector key_offsets; + std::vector key_offsets; crypto::key_image k_image; // double spending protection std::vector etc_details; //this flag used when TX_FLAG_SIGNATURE_MODE_SEPARATE flag is set, point to which amount of outputs(starting from zero) used in signature @@ -171,6 +223,15 @@ namespace currency END_SERIALIZE() }; + struct txin_htlc: public txin_to_key + { + std::string hltc_origin; + BEGIN_SERIALIZE_OBJECT() + FIELD(hltc_origin) + FIELDS(*static_cast(this)) + END_SERIALIZE() + }; + struct txin_multisig { uint64_t amount; @@ -197,9 +258,28 @@ namespace currency END_SERIALIZE() }; - typedef boost::variant txin_v; +#define CURRENCY_TXOUT_HTLC_FLAGS_HASH_TYPE_MASK 0x01 // 0 - SHA256, 1 - RIPEMD160 - typedef boost::variant txout_target_v; + struct txout_htlc + { + crypto::hash htlc_hash; + uint8_t flags; //select type of the hash, may be some extra info in future + uint64_t expiration; + crypto::public_key pkey_redeem; //works before expiration + crypto::public_key pkey_refund; //works after expiration + + BEGIN_SERIALIZE_OBJECT() + FIELD(htlc_hash) + FIELD(flags) + VARINT_FIELD(expiration) + FIELD(pkey_redeem) + FIELD(pkey_refund) + END_SERIALIZE() + }; + + typedef boost::variant txin_v; + + typedef boost::variant txout_target_v; //typedef std::pair out_t; struct tx_out @@ -224,9 +304,30 @@ namespace currency END_SERIALIZE() }; + struct tx_payer_old + { + account_public_address_old acc_addr; + + BEGIN_SERIALIZE() + FIELD(acc_addr) + END_SERIALIZE() + }; + struct tx_payer { - account_public_address acc_addr; + tx_payer() = default; + tx_payer(const tx_payer_old& old) : acc_addr(account_public_address::from_old(old.acc_addr)) {} + + account_public_address acc_addr{}; + + BEGIN_SERIALIZE() + FIELD(acc_addr) + END_SERIALIZE() + }; + + struct tx_receiver_old + { + account_public_address_old acc_addr; BEGIN_SERIALIZE() FIELD(acc_addr) @@ -235,7 +336,10 @@ namespace currency struct tx_receiver { - account_public_address acc_addr; + tx_receiver() = default; + tx_receiver(const tx_receiver_old& old) : acc_addr(account_public_address::from_old(old.acc_addr)) {} + + account_public_address acc_addr{}; BEGIN_SERIALIZE() FIELD(acc_addr) @@ -255,13 +359,13 @@ namespace currency END_SERIALIZE() }; - struct tx_message + struct tx_derivation_hint { std::string msg; BEGIN_SERIALIZE() FIELD(msg) - END_SERIALIZE() + END_SERIALIZE() }; struct tx_service_attachment @@ -297,33 +401,83 @@ namespace currency }; - struct extra_alias_entry_base + struct extra_alias_entry_base_old { - account_public_address m_address; + account_public_address_old m_address; std::string m_text_comment; std::vector m_view_key; // only one or zero elments expected (std::vector is using as memory efficient container for such a case) std::vector m_sign; // only one or zero elments expected (std::vector is using as memory efficient container for such a case) - //uint8_t flags; BEGIN_SERIALIZE() FIELD(m_address) FIELD(m_text_comment) FIELD(m_view_key) FIELD(m_sign) - //FIELD(flags) END_SERIALIZE() }; - struct extra_alias_entry: public extra_alias_entry_base + struct extra_alias_entry_old : public extra_alias_entry_base_old { std::string m_alias; BEGIN_SERIALIZE() FIELD(m_alias) - FIELDS(*static_cast(this)) + FIELDS(*static_cast(this)) END_SERIALIZE() }; + struct extra_alias_entry_base + { + extra_alias_entry_base() = default; + extra_alias_entry_base(const extra_alias_entry_base_old& old) + : m_address(account_public_address::from_old(old.m_address)) + , m_text_comment(old.m_text_comment) + , m_view_key(old.m_view_key) + , m_sign(old.m_sign) + { + } + + account_public_address m_address; + std::string m_text_comment; + std::vector m_view_key; // only one or zero elments expected (std::vector is using as memory efficient container for such a case) + std::vector m_sign; // only one or zero elments expected (std::vector is using as memory efficient container for such a case) + + BEGIN_SERIALIZE() + FIELD(m_address) + FIELD(m_text_comment) + FIELD(m_view_key) + FIELD(m_sign) + END_SERIALIZE() + }; + + struct extra_alias_entry : public extra_alias_entry_base + { + extra_alias_entry() = default; + extra_alias_entry(const extra_alias_entry_old& old) + : extra_alias_entry_base(old) + , m_alias(old.m_alias) + { + } + + std::string m_alias; + + BEGIN_SERIALIZE() + FIELD(m_alias) + FIELDS(*static_cast(this)) + END_SERIALIZE() + + extra_alias_entry_old to_old() const + { + extra_alias_entry_old result = AUTO_VAL_INIT(result); + result.m_address = m_address.to_old(); + result.m_text_comment = m_text_comment; + result.m_view_key = m_view_key; + result.m_sign = m_sign; + result.m_alias = m_alias; + return result; + } + }; + struct extra_padding @@ -336,7 +490,7 @@ namespace currency }; - //number of block (or time), used as a limitation: spend this tx not early then block/time + //number of block (or timestamp if v bigger then CURRENCY_MAX_BLOCK_NUMBER), used as a limitation: spend this tx not early then block/time struct etc_tx_details_unlock_time { uint64_t v; @@ -345,6 +499,16 @@ namespace currency END_SERIALIZE() }; + //number of block (or timestamp if unlock_time_array[i] bigger then CURRENCY_MAX_BLOCK_NUMBER), used as a limitation: spend this tx not early then block/time + //unlock_time_array[i], i - index of output, unlock_time_array.size() == vout.size() + struct etc_tx_details_unlock_time2 + { + std::vector unlock_time_array; + BEGIN_SERIALIZE() + FIELD(unlock_time_array) + END_SERIALIZE() + }; + struct etc_tx_details_expiration_time { uint64_t v; @@ -361,7 +525,7 @@ namespace currency uint64_t v; BEGIN_SERIALIZE() VARINT_FIELD(v) - END_SERIALIZE() + END_SERIALIZE() }; struct etc_tx_details_flags @@ -372,7 +536,7 @@ namespace currency END_SERIALIZE() }; - struct etc_tx_derivation_hint + struct etc_tx_flags16_t { uint16_t v; BEGIN_SERIALIZE() @@ -380,16 +544,22 @@ namespace currency END_SERIALIZE() }; - typedef boost::mpl::vector all_payload_types; - typedef boost::make_variant_over::type attachment_v; - typedef boost::make_variant_over::type extra_v; + typedef boost::mpl::vector21< + tx_service_attachment, tx_comment, tx_payer_old, tx_receiver_old, tx_derivation_hint, std::string, tx_crypto_checksum, etc_tx_time, etc_tx_details_unlock_time, etc_tx_details_expiration_time, + etc_tx_details_flags, crypto::public_key, extra_attachment_info, extra_alias_entry_old, extra_user_data, extra_padding, etc_tx_flags16_t, etc_tx_details_unlock_time2, + tx_payer, tx_receiver, extra_alias_entry + > all_payload_types; + typedef boost::make_variant_over::type payload_items_v; + typedef payload_items_v extra_v; + typedef payload_items_v attachment_v; + class transaction_prefix { public: // tx version information - size_t version; + uint64_t version{}; //extra std::vector extra; std::vector vin; @@ -553,19 +723,23 @@ namespace currency uint64_t index; crypto::key_image keyimage; uint64_t block_timestamp; + uint64_t stake_unlock_time; //not for serialization uint64_t wallet_index; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) KV_SERIALIZE(index) + KV_SERIALIZE(stake_unlock_time) KV_SERIALIZE(block_timestamp) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(keyimage) END_KV_SERIALIZE_MAP() }; -} + +} // namespace currency POD_MAKE_HASHABLE(currency, account_public_address); +POD_MAKE_HASHABLE(currency, account_public_address_old); BLOB_SERIALIZER(currency::txout_to_key); @@ -586,10 +760,10 @@ SET_VARIANT_TAGS(currency::transaction, 5, "tx"); SET_VARIANT_TAGS(currency::block, 6, "block"); //attachment_v definitions SET_VARIANT_TAGS(currency::tx_comment, 7, "comment"); -SET_VARIANT_TAGS(currency::tx_payer, 8, "payer"); +SET_VARIANT_TAGS(currency::tx_payer_old, 8, "payer"); SET_VARIANT_TAGS(std::string, 9, "string"); SET_VARIANT_TAGS(currency::tx_crypto_checksum, 10, "checksum"); -SET_VARIANT_TAGS(currency::tx_message, 11, "message"); +SET_VARIANT_TAGS(currency::tx_derivation_hint, 11, "derivation_hint"); SET_VARIANT_TAGS(currency::tx_service_attachment, 12, "attachment"); //SET_VARIANT_TAGS(currency::tx_onetime_secret_key, 13, "sec_key"); SET_VARIANT_TAGS(currency::etc_tx_details_unlock_time, 14, "unlock_time"); @@ -600,10 +774,10 @@ SET_VARIANT_TAGS(currency::signed_parts, 17, "signed_outs"); //extra_v definitions SET_VARIANT_TAGS(currency::extra_attachment_info, 18, "extra_attach_info"); SET_VARIANT_TAGS(currency::extra_user_data, 19, "user_data"); -SET_VARIANT_TAGS(currency::extra_alias_entry, 20, "alias_entry"); +SET_VARIANT_TAGS(currency::extra_alias_entry_old, 20, "alias_entry"); SET_VARIANT_TAGS(currency::extra_padding, 21, "extra_padding"); SET_VARIANT_TAGS(crypto::public_key, 22, "pub_key"); -SET_VARIANT_TAGS(currency::etc_tx_derivation_hint, 23, "derive_hint"); +SET_VARIANT_TAGS(currency::etc_tx_flags16_t, 23, "etc_tx_flags16"); SET_VARIANT_TAGS(uint16_t, 24, "derive_xor"); //txout_v SET_VARIANT_TAGS(currency::ref_by_id, 25, "ref_by_id"); @@ -611,6 +785,20 @@ SET_VARIANT_TAGS(uint64_t, 26, "uint64_t"); //etc SET_VARIANT_TAGS(currency::etc_tx_time, 27, "etc_tx_time"); SET_VARIANT_TAGS(uint32_t, 28, "uint32_t"); -SET_VARIANT_TAGS(currency::tx_receiver, 29, "payer"); +SET_VARIANT_TAGS(currency::tx_receiver_old, 29, "payer"); // -- original +//SET_VARIANT_TAGS(currency::tx_receiver_old, 29, "receiver"); +SET_VARIANT_TAGS(currency::etc_tx_details_unlock_time2, 30, "unlock_time2"); + +SET_VARIANT_TAGS(currency::tx_payer, 31, "payer2"); +SET_VARIANT_TAGS(currency::tx_receiver, 32, "receiver2"); + +// @#@ TODO @#@ +SET_VARIANT_TAGS(currency::extra_alias_entry, 33, "alias_entry2"); + +//htlc +SET_VARIANT_TAGS(currency::txin_htlc, 34, "txin_htlc"); +SET_VARIANT_TAGS(currency::txout_htlc, 35, "txout_htlc"); + + #undef SET_VARIANT_TAGS diff --git a/src/currency_core/currency_boost_serialization.h b/src/currency_core/currency_boost_serialization.h index 6802afd..96f95bd 100644 --- a/src/currency_core/currency_boost_serialization.h +++ b/src/currency_core/currency_boost_serialization.h @@ -29,10 +29,18 @@ namespace boost template inline void serialize(Archive &a, currency::account_public_address &x, const boost::serialization::version_type ver) { - a & x.m_spend_public_key; - a & x.m_view_public_key; + //a & x.version; + a & x.flags; + a & x.spend_public_key; + a & x.view_public_key; } + template + inline void serialize(Archive &a, currency::account_public_address_old &x, const boost::serialization::version_type ver) + { + a & x.spend_public_key; + a & x.view_public_key; + } template inline void serialize(Archive &a, currency::txout_to_key &x, const boost::serialization::version_type ver) @@ -48,6 +56,16 @@ namespace boost a & x.keys; } + template + inline void serialize(Archive &a, currency::txout_htlc &x, const boost::serialization::version_type ver) + { + a & x.expiration; + a & x.flags; + a & x.htlc_hash; + a & x.pkey_redeem; + a & x.pkey_refund; + } + template inline void serialize(Archive &a, currency::txin_gen &x, const boost::serialization::version_type ver) { @@ -75,6 +93,16 @@ namespace boost a & x.etc_details; } + template + inline void serialize(Archive &a, currency::txin_htlc &x, const boost::serialization::version_type ver) + { + a & x.amount; + a & x.etc_details; + a & x.hltc_origin; + a & x.k_image; + a & x.key_offsets; + } + template inline void serialize(Archive &a, currency::tx_out &x, const boost::serialization::version_type ver) { @@ -90,16 +118,30 @@ namespace boost a & x.comment; } + template + inline void serialize(Archive &a, currency::tx_payer_old &x, const boost::serialization::version_type ver) + { + a & x.acc_addr; + } + template inline void serialize(Archive &a, currency::tx_payer &x, const boost::serialization::version_type ver) { a & x.acc_addr; } + + template + inline void serialize(Archive &a, currency::tx_receiver_old &x, const boost::serialization::version_type ver) + { + a & x.acc_addr; + } + template inline void serialize(Archive &a, currency::tx_receiver &x, const boost::serialization::version_type ver) { a & x.acc_addr; } + template inline void serialize(Archive &a, currency::tx_crypto_checksum &x, const boost::serialization::version_type ver) { @@ -107,7 +149,7 @@ namespace boost a & x.derivation_hash; } template - inline void serialize(Archive &a, currency::tx_message &x, const boost::serialization::version_type ver) + inline void serialize(Archive &a, currency::tx_derivation_hint &x, const boost::serialization::version_type ver) { a & x.msg; } @@ -135,14 +177,6 @@ namespace boost a & x.m_sign; } - - template - inline void serialize(Archive &a, currency::signed_parts &x, const boost::serialization::version_type ver) - { - a & x.n_outs; - a & x.n_extras; - } - template inline void serialize(Archive &a, currency::extra_alias_entry &x, const boost::serialization::version_type ver) { @@ -150,6 +184,29 @@ namespace boost a & static_cast(x); } + template + inline void serialize(Archive &a, currency::extra_alias_entry_base_old &x, const boost::serialization::version_type ver) + { + a & x.m_address; + a & x.m_text_comment; + a & x.m_view_key; + a & x.m_sign; + } + + template + inline void serialize(Archive &a, currency::extra_alias_entry_old &x, const boost::serialization::version_type ver) + { + a & x.m_alias; + a & static_cast(x); + } + + template + inline void serialize(Archive &a, currency::signed_parts &x, const boost::serialization::version_type ver) + { + a & x.n_outs; + a & x.n_extras; + } + template inline void serialize(Archive &a, currency::extra_padding &x, const boost::serialization::version_type ver) { @@ -171,7 +228,7 @@ namespace boost inline void serialize(Archive &a, currency::keypair &kp, const boost::serialization::version_type ver) { a & kp.pub; - a & kp.sec; + a & kp.sec; } template @@ -189,16 +246,25 @@ namespace boost { a & at.v; } + + template + inline void serialize(Archive &a, currency::etc_tx_details_unlock_time2 &at, const boost::serialization::version_type ver) + { + a & at.unlock_time_array; + } + template inline void serialize(Archive &a, currency::etc_tx_details_expiration_time &at, const boost::serialization::version_type ver) { a & at.v; } + template inline void serialize(Archive &a, currency::etc_tx_details_flags &at, const boost::serialization::version_type ver) { a & at.v; } + template inline void serialize(Archive &a, currency::etc_tx_time &at, const boost::serialization::version_type ver) { @@ -206,7 +272,7 @@ namespace boost } template - inline void serialize(Archive &a, currency::etc_tx_derivation_hint &at, const boost::serialization::version_type ver) + inline void serialize(Archive &a, currency::etc_tx_flags16_t&at, const boost::serialization::version_type ver) { a & at.v; } diff --git a/src/currency_core/currency_config.h b/src/currency_core/currency_config.h index 2a30e8a..1b658a6 100644 --- a/src/currency_core/currency_config.h +++ b/src/currency_core/currency_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying @@ -7,30 +7,34 @@ #pragma once -#define CURRENCY_FORMATION_VERSION 80 - +#ifndef TESTNET +#define CURRENCY_FORMATION_VERSION 84 +#else +#define CURRENCY_FORMATION_VERSION 88 +#endif + +#define CURRENCY_GENESIS_NONCE (CURRENCY_FORMATION_VERSION + 101011010121) //bender's nightmare + + #define CURRENCY_MAX_BLOCK_NUMBER 500000000 #define CURRENCY_MAX_BLOCK_SIZE 500000000 // block header blob limit, never used! #define CURRENCY_TX_MAX_ALLOWED_OUTS 2000 -#define CURRENCY_PUBLIC_ADDRESS_TEXTBLOB_VER 0 -#define CURRENCY_PUBLIC_ADDRESS_BASE58_PREFIX 197 // addresses start with 'Z' +#define CURRENCY_PUBLIC_ADDRESS_BASE58_PREFIX 0xc5 // addresses start with 'Zx' #define CURRENCY_PUBLIC_INTEG_ADDRESS_BASE58_PREFIX 0x3678 // integrated addresses start with 'iZ' +#define CURRENCY_PUBLIC_INTEG_ADDRESS_V2_BASE58_PREFIX 0x36f8 // integrated addresses start with 'iZ' (new format) +#define CURRENCY_PUBLIC_AUDITABLE_ADDRESS_BASE58_PREFIX 0x98c8 // auditable addresses start with 'aZx' +#define CURRENCY_PUBLIC_AUDITABLE_INTEG_ADDRESS_BASE58_PREFIX 0x8a49 // auditable integrated addresses start with 'aiZX' #define CURRENCY_MINED_MONEY_UNLOCK_WINDOW 10 #define CURRENT_TRANSACTION_VERSION 1 -#define CURRENT_BLOCK_MAJOR_VERSION 1 +#define HF1_BLOCK_MAJOR_VERSION 1 +#define CURRENT_BLOCK_MAJOR_VERSION 2 + #define CURRENT_BLOCK_MINOR_VERSION 0 #define CURRENCY_BLOCK_FUTURE_TIME_LIMIT 60*60*2 #define CURRENCY_POS_BLOCK_FUTURE_TIME_LIMIT 60*20 #define BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW 60 - - -// TOTAL_MONEY_SUPPLY - total number coins to be generated -#define TOTAL_MONEY_SUPPLY ((uint64_t)(-1)) - -#define EMISSION_POS_REWARD_DEVIDER 5425518 // originally based on formula "1/((sqrt(1.05, (740*366)) - 1))" which is avr 5% per year, and then adjusted -#define EMISSION_POW_REWARD_DEVIDER 27098500 // originally based on formula "1/((sqrt(1.01, (740*366)) - 1))" which is avr 1% per year, and then adjusted #define POS_START_HEIGHT 0 @@ -44,23 +48,17 @@ // COIN - number of smallest units in one coin #define COIN ((uint64_t)1000000000000) // pow(10, CURRENCY_DISPLAY_DECIMAL_POINT) #define BASE_REWARD_DUST_THRESHOLD ((uint64_t)1000000) // pow(10, 6) - change this will cause hard-fork! -#define DEFAULT_DUST_THRESHOLD ((uint64_t)0)//((uint64_t)100000) // pow(10, 5) +#define DEFAULT_DUST_THRESHOLD ((uint64_t)0) -#define TX_DEFAULT_FEE ((uint64_t)100000) // pow(10, 5) -#define TX_MINIMUM_FEE ((uint64_t)100000) // pow(10, 5) +#define TX_DEFAULT_FEE ((uint64_t)10000000000) // .01 +#define TX_MINIMUM_FEE ((uint64_t)10000000000) // .01 -// #define CURRENCY_FIXED_REWARD_ZONE_HEIGHT 300 // blocks will have fixed reward up to this height (including) -// #define CURRENCY_FIXED_REWARD_ZONE_REWARD_AMOUNT ((uint64_t)100000000) // should be TX_MINIMUM_FEE * CURRENCY_FIXED_REWARD_ZONE_FEE_MULTIPLIER -// #define CURRENCY_FIXED_REWARD_ZONE_FEE_MULTIPLIER 1000 // reward in minimum fees for a block in the zone - -#define CURRENCY_BLOCK_REWARD 1000000000000 // 1.0 coin +#define CURRENCY_BLOCK_REWARD 1000000000000 // 1.0 coin == pow(10, CURRENCY_DISPLAY_DECIMAL_POINT) #define WALLET_MAX_ALLOWED_OUTPUT_AMOUNT ((uint64_t)0xffffffffffffffffLL) #define CURRENCY_MINER_TX_MAX_OUTS CURRENCY_TX_MAX_ALLOWED_OUTS -#define ORPHANED_BLOCKS_MAX_COUNT 100 - #define DIFFICULTY_STARTER 1 #define DIFFICULTY_POS_TARGET 120 // seconds #define DIFFICULTY_POW_TARGET 120 // seconds @@ -96,19 +94,23 @@ #define CURRENCY_ALT_BLOCK_LIVETIME_COUNT (CURRENCY_BLOCKS_PER_DAY*7)//one week +#define CURRENCY_ALT_BLOCK_MAX_COUNT 43200 //30 days #define CURRENCY_MEMPOOL_TX_LIVETIME 345600 //seconds, 4 days + #ifndef TESTNET -#define P2P_DEFAULT_PORT (CURRENCY_FORMATION_VERSION+11121) +#define P2P_DEFAULT_PORT 11121 #define RPC_DEFAULT_PORT 11211 #define STRATUM_DEFAULT_PORT 11777 #define P2P_NETWORK_ID_TESTNET_FLAG 0 +#define P2P_MAINTAINERS_PUB_KEY "8f138bb73f6d663a3746a542770781a09579a7b84cb4125249e95530824ee607" #else -#define P2P_DEFAULT_PORT (CURRENCY_FORMATION_VERSION+11112) +#define P2P_DEFAULT_PORT (11112 + CURRENCY_FORMATION_VERSION) #define RPC_DEFAULT_PORT 12111 #define STRATUM_DEFAULT_PORT 11888 #define STRARUM_DEFAULT_PORT 51113 #define P2P_NETWORK_ID_TESTNET_FLAG 1 +#define P2P_MAINTAINERS_PUB_KEY "aaa2d7aabc8d383fd53a3ae898697b28f236ceade6bafc1eecff413a6a02272a" #endif #define P2P_NETWORK_ID_VER (CURRENCY_FORMATION_VERSION+0) @@ -119,14 +121,13 @@ #define P2P_LOCAL_GRAY_PEERLIST_LIMIT 5000 #define P2P_DEFAULT_CONNECTIONS_COUNT 8 -#define P2P_DEFAULT_HANDSHAKE_INTERVAL 60 //secondes +#define P2P_DEFAULT_HANDSHAKE_INTERVAL 60 //seconds #define P2P_DEFAULT_PACKET_MAX_SIZE 50000000 //50000000 bytes maximum packet size #define P2P_DEFAULT_PEERS_IN_HANDSHAKE 250 #define P2P_DEFAULT_CONNECTION_TIMEOUT 5000 //5 seconds #define P2P_DEFAULT_PING_CONNECTION_TIMEOUT 2000 //2 seconds #define P2P_DEFAULT_INVOKE_TIMEOUT 60*2*1000 //2 minutes #define P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT 10000 //10 seconds -#define P2P_MAINTAINERS_PUB_KEY "888db12b7e0cd325880c815ea13d7062f4c4a89dafc355f4d7b93a7f18342df3" #define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT 70 #define P2P_FAILED_ADDR_FORGET_SECONDS (60*5) //5 minutes @@ -137,7 +138,7 @@ //PoS definitions #define POS_SCAN_WINDOW 60*10 //seconds // 10 minutes #define POS_SCAN_STEP 15 //seconds -#define POS_MAC_ACTUAL_TIMESTAMP_TO_MINED (POS_SCAN_WINDOW+100) +#define POS_MAX_ACTUAL_TIMESTAMP_TO_MINED (POS_SCAN_WINDOW+100) #define POS_STARTER_KERNEL_HASH "00000000000000000006382a8d8f94588ce93a1351924f6ccb9e07dd287c6e4b" #define POS_MODFIFIER_INTERVAL 10 @@ -145,13 +146,16 @@ #define POS_MINIMUM_COINSTAKE_AGE 10 // blocks count -#define WALLET_FILE_SIGNATURE 0x1111012101101011LL //Bender's nightmare -#define WALLET_FILE_MAX_BODY_SIZE 0x88888888L //2GB +#define WALLET_FILE_SIGNATURE_OLD 0x1111012101101011LL // Bender's nightmare +#define WALLET_FILE_SIGNATURE_V2 0x1111011201101011LL // another Bender's nightmare +#define WALLET_FILE_BINARY_HEADER_VERSION 1001 + #define WALLET_FILE_MAX_KEYS_SIZE 10000 // #define WALLET_BRAIN_DATE_OFFSET 1543622400 -#define WALLET_BRAIN_DATE_QUANTUM 604800 //by last word we encode a number of week since launch of the project, +#define WALLET_BRAIN_DATE_QUANTUM 604800 //by last word we encode a number of week since launch of the project AND password flag, //which let us to address tools::mnemonic_encoding::NUMWORDS weeks after project launch - //which is about 30 years + //which is about 15 years +#define WALLET_BRAIN_DATE_MAX_WEEKS_COUNT 800 #define OFFER_MAXIMUM_LIFE_TIME (60*60*24*30) // 30 days @@ -162,7 +166,7 @@ -#define CURRENCY_NAME_ABR "ZAN" +#define CURRENCY_NAME_ABR "ZANO" #define CURRENCY_NAME_BASE "Zano" #define CURRENCY_NAME_SHORT_BASE "Zano" #ifndef TESTNET @@ -182,18 +186,25 @@ #define ALIAS_REWARDS_ACCOUNT_VIEW_SEC_KEY "0000000000000000000000000000000000000000000000000000000000000000" //burn alias money #define ALIAS_MINIMUM_PUBLIC_SHORT_NAME_ALLOWED 6 -#define ALIAS_SHORT_NAMES_VALIDATION_PUB_KEY "3b63cb2f3d425053f4120b10bced73d87e98c27b6e4bcfd123a5cfac688c551f" +#define ALIAS_SHORT_NAMES_VALIDATION_PUB_KEY "37947f7b6a5268c5d0a48bde73d7a426f0b5f24648f74024279540207dc70031" #define ALIAS_NAME_MAX_LEN 255 #define ALIAS_VALID_CHARS "0123456789abcdefghijklmnopqrstuvwxyz-." -#define ALIAS_COMMENT_MAX_SIZE_BYTES 400 +#define ALIAS_COMMENT_MAX_SIZE_BYTES 400 #define CURRENCY_CORE_INSTANCE_LOCK_FILE "lock.lck" -#define CURRENCY_POOLDATA_FOLDERNAME "poolstate" -#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME "blockchain" +#define CURRENCY_POOLDATA_FOLDERNAME_OLD "poolstate" +#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_OLD "blockchain" + + +#define CURRENCY_POOLDATA_FOLDERNAME_PREFIX "poolstate_" +#define CURRENCY_POOLDATA_FOLDERNAME_SUFFIX "_v1" +#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_PREFIX "blockchain_" +#define CURRENCY_BLOCKCHAINDATA_FOLDERNAME_SUFFIX "_v1" + #define P2P_NET_DATA_FILENAME "p2pstate.bin" #define MINER_CONFIG_FILENAME "miner_conf.json" #define GUI_SECURE_CONFIG_FILENAME "gui_secure_conf.bin" @@ -201,10 +212,11 @@ #define GUI_INTERNAL_CONFIG "gui_internal_config.bin" + #define CURRENT_TRANSACTION_CHAIN_ENTRY_ARCHIVE_VER 3 #define CURRENT_BLOCK_EXTENDED_INFO_ARCHIVE_VER 1 -#define BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION CURRENCY_FORMATION_VERSION + 4 +#define BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION CURRENCY_FORMATION_VERSION + 11 #define BLOCKCHAIN_STORAGE_MINOR_COMPATIBILITY_VERSION 1 @@ -212,13 +224,27 @@ #define BC_OFFERS_CURRENCY_MARKET_FILENAME "market.bin" -#define WALLET_FILE_SERIALIZATION_VERSION (CURRENCY_FORMATION_VERSION+62) +#define WALLET_FILE_SERIALIZATION_VERSION (CURRENCY_FORMATION_VERSION+69) + #define CURRENT_MEMPOOL_ARCHIVE_VER (CURRENCY_FORMATION_VERSION+31) +//hard forks section +#define BLOCK_MAJOR_VERSION_GENESIS 1 +#define BLOCK_MINOR_VERSION_GENESIS 0 +#define BLOCK_MAJOR_VERSION_INITIAL 0 +#ifndef TESTNET +#define ZANO_HARDFORK_01_AFTER_HEIGHT 194624 +#define ZANO_HARDFORK_02_AFTER_HEIGHT 999999 +#define ZANO_HARDFORK_03_AFTER_HEIGHT 1082577 +#else +#define ZANO_HARDFORK_01_AFTER_HEIGHT 1440 +#define ZANO_HARDFORK_02_AFTER_HEIGHT 1800 +#define ZANO_HARDFORK_03_AFTER_HEIGHT 1801 +#endif static_assert(CURRENCY_MINER_TX_MAX_OUTS <= CURRENCY_TX_MAX_ALLOWED_OUTS, "Miner tx must obey normal tx max outs limit"); static_assert(PREMINE_AMOUNT / WALLET_MAX_ALLOWED_OUTPUT_AMOUNT < CURRENCY_MINER_TX_MAX_OUTS, "Premine can't be divided into reasonable number of outs"); -//static_assert(CURRENCY_FIXED_REWARD_ZONE_REWARD_AMOUNT == TX_MINIMUM_FEE * CURRENCY_FIXED_REWARD_ZONE_FEE_MULTIPLIER, "CURRENCY_FIXED_REWARD_ZONE_REWARD_AMOUNT is incorrect with regard to TX_MINIMUM_FEE"); + diff --git a/src/currency_core/currency_core.cpp b/src/currency_core/currency_core.cpp index 0e4e66c..cd0b5d8 100644 --- a/src/currency_core/currency_core.cpp +++ b/src/currency_core/currency_core.cpp @@ -18,6 +18,10 @@ using namespace epee; #include "currency_core/currency_config.h" #include "currency_format_utils.h" #include "misc_language.h" +#include "string_coding.h" +#include "tx_semantic_validation.h" + +#define MINIMUM_REQUIRED_FREE_SPACE_BYTES (1024 * 1024 * 100) DISABLE_VS_WARNINGS(4355) #undef LOG_DEFAULT_CHANNEL @@ -27,15 +31,18 @@ namespace currency { //----------------------------------------------------------------------------------------------- - core::core(i_currency_protocol* pprotocol): - m_mempool(m_blockchain_storage, pprotocol), - m_blockchain_storage(m_mempool), - m_miner(this, m_blockchain_storage), - m_miner_address(boost::value_initialized()), - m_starter_message_showed(false) + core::core(i_currency_protocol* pprotocol) + : m_mempool(m_blockchain_storage, pprotocol) + , m_blockchain_storage(m_mempool) + , m_miner(this, m_blockchain_storage) + , m_miner_address(boost::value_initialized()) + , m_starter_message_showed(false) + , m_critical_error_handler(nullptr) + , m_stop_after_height(0) { set_currency_protocol(pprotocol); } + //----------------------------------------------------------------------------------- void core::set_currency_protocol(i_currency_protocol* pprotocol) { if(pprotocol) @@ -46,6 +53,11 @@ namespace currency m_mempool.set_protocol(m_pprotocol); } //----------------------------------------------------------------------------------- + void core::set_critical_error_handler(i_critical_error_handler* handler) + { + m_critical_error_handler = handler; + } + //----------------------------------------------------------------------------------- bool core::set_checkpoints(checkpoints&& chk_pts) { return m_blockchain_storage.set_checkpoints(std::move(chk_pts)); @@ -64,6 +76,11 @@ namespace currency bool core::handle_command_line(const boost::program_options::variables_map& vm) { m_config_folder = command_line::get_arg(vm, command_line::arg_data_dir); + m_stop_after_height = static_cast(command_line::get_arg(vm, command_line::arg_stop_after_height)); + if (m_stop_after_height != 0) + { + LOG_PRINT_YELLOW("Daemon will STOP after block " << m_stop_after_height, LOG_LEVEL_0); + } return true; } //----------------------------------------------------------------------------------------------- @@ -129,7 +146,10 @@ namespace currency { bool r = handle_command_line(vm); - r = m_mempool.init(m_config_folder); + uint64_t available_space = 0; + CHECK_AND_ASSERT_MES(!check_if_free_space_critically_low(&available_space), false, "free space in data folder is critically low: " << std::fixed << available_space / (1024 * 1024) << " MB"); + + r = m_mempool.init(m_config_folder, vm); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); r = m_blockchain_storage.init(m_config_folder, vm); @@ -170,15 +190,45 @@ namespace currency return true; } //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block, const crypto::hash& tx_hash_ /* = null_hash */) + { + TIME_MEASURE_START_MS(wait_lock_time); + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); + TIME_MEASURE_FINISH_MS(wait_lock_time); + + crypto::hash tx_hash = tx_hash_; + if (tx_hash == null_hash) + tx_hash = get_transaction_hash(tx); + + TIME_MEASURE_START_MS(add_new_tx_time); + bool r = add_new_tx(tx, tx_hash, get_object_blobsize(tx), tvc, kept_by_block); + TIME_MEASURE_FINISH_MS(add_new_tx_time); + + if(tvc.m_verification_failed) + {LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);} + else if(tvc.m_verification_impossible) + {LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);} + + if (tvc.m_added_to_pool) + { + LOG_PRINT_L2("incoming tx " << tx_hash << " was added to the pool"); + } + LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX1]: timing " << wait_lock_time + << "/" << add_new_tx_time); + return r; + } + //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block) { + CHECK_AND_ASSERT_MES(!kept_by_block, false, "Transaction associated with block came throw handle_incoming_tx!(not allowed anymore)"); + tvc = boost::value_initialized(); //want to process all transactions sequentially TIME_MEASURE_START_MS(wait_lock_time); CRITICAL_REGION_LOCAL(m_incoming_tx_lock); TIME_MEASURE_FINISH_MS(wait_lock_time); - if(tx_blob.size() > get_max_tx_size()) + if(tx_blob.size() > CURRENCY_MAX_TRANSACTION_BLOB_SIZE) { LOG_PRINT_L0("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected"); tvc.m_verification_failed = true; @@ -196,43 +246,19 @@ namespace currency } TIME_MEASURE_FINISH_MS(parse_tx_time); - - TIME_MEASURE_START_MS(check_tx_syntax_time); - if(!check_tx_syntax(tx)) - { - LOG_PRINT_L0("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " syntax, rejected"); - tvc.m_verification_failed = true; - return false; - } - TIME_MEASURE_FINISH_MS(check_tx_syntax_time); - TIME_MEASURE_START_MS(check_tx_semantic_time); - if(!check_tx_semantic(tx, kept_by_block)) + if(!validate_tx_semantic(tx, tx_blob.size())) { - LOG_PRINT_L0("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); + LOG_PRINT_L0("WRONG TRANSACTION SEMANTICS, Failed to check tx " << tx_hash << " semantic, rejected"); tvc.m_verification_failed = true; return false; } TIME_MEASURE_FINISH_MS(check_tx_semantic_time); - TIME_MEASURE_START_MS(add_new_tx_time); - bool r = add_new_tx(tx, tx_hash, get_object_blobsize(tx), tvc, kept_by_block); - TIME_MEASURE_FINISH_MS(add_new_tx_time); - - if(tvc.m_verification_failed) - {LOG_PRINT_RED_L0("Transaction verification failed: " << tx_hash);} - else if(tvc.m_verification_impossible) - {LOG_PRINT_RED_L0("Transaction verification impossible: " << tx_hash);} - - if (tvc.m_added_to_pool) - { - LOG_PRINT_L2("incoming tx " << tx_hash << " was added to the pool"); - } - LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX]: timing " << wait_lock_time + bool r = handle_incoming_tx(tx, tvc, kept_by_block, tx_hash); + LOG_PRINT_L2("[CORE HANDLE_INCOMING_TX2]: timing " << wait_lock_time << "/" << parse_tx_time - << "/" << check_tx_syntax_time - << "/" << check_tx_semantic_time - << "/" << add_new_tx_time); + << "/" << check_tx_semantic_time); return r; } //----------------------------------------------------------------------------------------------- @@ -283,88 +309,9 @@ namespace currency return true; } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_semantic(const transaction& tx, bool kept_by_block) - { - if(!tx.vin.size()) - { - LOG_PRINT_RED_L0("tx with empty inputs, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_inputs_types_supported(tx)) - { - LOG_PRINT_RED_L0("unsupported input types for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_outs_valid(tx)) - { - LOG_PRINT_RED_L0("tx with invalid outputs, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - if(!check_money_overflow(tx)) - { - LOG_PRINT_RED_L0("tx has money overflow, rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - - uint64_t amount_in = 0; - get_inputs_money_amount(tx, amount_in); - uint64_t amount_out = get_outs_money_amount(tx); - - if(amount_in < amount_out) - { - LOG_PRINT_RED_L0("tx with wrong amounts: ins " << amount_in << ", outs " << amount_out << ", rejected for tx id= " << get_transaction_hash(tx)); - return false; - } - - if(!kept_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_comulative_blocksize_limit() - CURRENCY_COINBASE_BLOB_RESERVED_SIZE) - { - LOG_PRINT_RED_L0("tx has too big size " << get_object_blobsize(tx) << ", expected no bigger than " << m_blockchain_storage.get_current_comulative_blocksize_limit() - CURRENCY_COINBASE_BLOB_RESERVED_SIZE); - return false; - } - - //check if tx use different key images - if(!check_tx_inputs_keyimages_diff(tx)) - { - LOG_PRINT_RED_L0("tx inputs have the same key images"); - return false; - } - - if(!check_tx_extra(tx)) - { - LOG_PRINT_RED_L0("tx has wrong extra, rejected"); - return false; - } - - return true; - } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_extra(const transaction& tx) - { - tx_extra_info ei = AUTO_VAL_INIT(ei); - bool r = parse_and_validate_tx_extra(tx, ei); - if(!r) - return false; - return true; - } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_inputs_keyimages_diff(const transaction& tx) - { - std::unordered_set ki; - BOOST_FOREACH(const auto& in, tx.vin) - { - if (in.type() == typeid(txin_to_key)) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); - if (!ki.insert(tokey_in.k_image).second) - return false; - } - } - return true; - } //----------------------------------------------------------------------------------------------- bool core::add_new_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block) { @@ -406,6 +353,11 @@ namespace currency return m_blockchain_storage.create_block_template(b, adr, stakeholder_address, diffic, height, ex_nonce, pos, pe); } //----------------------------------------------------------------------------------------------- + bool core::get_block_template(const create_block_template_params& params, create_block_template_response& resp) + { + return m_blockchain_storage.create_block_template(params, resp); + } + //----------------------------------------------------------------------------------------------- bool core::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); @@ -547,7 +499,25 @@ namespace currency //----------------------------------------------------------------------------------------------- bool core::add_new_block(const block& b, block_verification_context& bvc) { - return m_blockchain_storage.add_new_block(b, bvc); + bool r = m_blockchain_storage.add_new_block(b, bvc); + if (r && bvc.m_added_to_main_chain) + { + uint64_t h = get_block_height(b); + auto& crc = m_blockchain_storage.get_core_runtime_config(); + if (h == crc.hard_fork_01_starts_after_height + 1) + { LOG_PRINT_GREEN("Hardfork 1 activated at height " << h, LOG_LEVEL_0); } + else if (h == crc.hard_fork_02_starts_after_height + 1) + { LOG_PRINT_GREEN("Hardfork 2 activated at height " << h, LOG_LEVEL_0); } + + if (h == m_stop_after_height) + { + LOG_PRINT_YELLOW("Reached block " << h << ", the daemon will now stop as requested", LOG_LEVEL_0); + if (m_critical_error_handler) + return m_critical_error_handler->on_immediate_stop_requested(); + return false; + } + } + return r; } //----------------------------------------------------------------------------------------------- bool core::parse_block(const blobdata& block_blob, block& b, block_verification_context& bvc) @@ -576,7 +546,6 @@ namespace currency //----------------------------------------------------------------------------------------------- bool core::handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate) { - bvc = AUTO_VAL_INIT_T(block_verification_context); block b = AUTO_VAL_INIT(b); if (!parse_block(block_blob, b, bvc)) { @@ -614,11 +583,6 @@ namespace currency { return parse_and_validate_tx_from_blob(blob, tx, tx_hash); } - //----------------------------------------------------------------------------------------------- - bool core::check_tx_syntax(const transaction& tx) - { - return true; - } //----------------------------------------------------------------------------------------------- bool core::get_pool_transactions(std::list& txs) { @@ -678,6 +642,7 @@ namespace currency } m_prune_alt_blocks_interval.do_call([this](){return m_blockchain_storage.prune_aged_alt_blocks();}); + m_check_free_space_interval.do_call([this](){ check_free_space(); return true; }); m_miner.on_idle(); m_mempool.on_idle(); return true; @@ -702,7 +667,40 @@ namespace currency l->on_blockchain_update(); } //----------------------------------------------------------------------------------------------- + bool core::check_if_free_space_critically_low(uint64_t* p_available_space /* = nullptr */) + { + namespace fs = boost::filesystem; + + try + { + CHECK_AND_ASSERT_MES(tools::create_directories_if_necessary(m_config_folder), false, "create_directories_if_necessary failed: " << m_config_folder); + std::wstring config_folder_w = epee::string_encoding::utf8_to_wstring(m_config_folder); + fs::space_info si = fs::space(config_folder_w); + if (p_available_space != nullptr) + *p_available_space = si.available; + return si.available < MINIMUM_REQUIRED_FREE_SPACE_BYTES; + } + catch (std::exception& e) + { + LOG_ERROR("failed to determine free space: " << e.what()); + return false; + } + catch (...) + { + LOG_ERROR("failed to determine free space: unknown exception"); + return false; + } + } + + void core::check_free_space() + { + if (!m_critical_error_handler) + return; + + uint64_t available_space = 0; + if (check_if_free_space_critically_low(&available_space)) + m_critical_error_handler->on_critical_low_free_space(available_space, MINIMUM_REQUIRED_FREE_SPACE_BYTES); + } + //----------------------------------------------------------------------------------------------- } -#undef LOG_DEFAULT_CHANNEL -#define LOG_DEFAULT_CHANNEL NULL \ No newline at end of file diff --git a/src/currency_core/currency_core.h b/src/currency_core/currency_core.h index 158c630..354fd77 100644 --- a/src/currency_core/currency_core.h +++ b/src/currency_core/currency_core.h @@ -21,7 +21,7 @@ #include "warnings.h" #include "crypto/hash.h" -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4355) namespace currency @@ -41,6 +41,7 @@ namespace currency core(i_currency_protocol* pprotocol); bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, currency_connection_context& context)const ; bool on_idle(); + bool handle_incoming_tx(const transaction& tx, tx_verification_context& tvc, bool kept_by_block, const crypto::hash& tx_hash_ = null_hash); bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool kept_by_block); bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true); bool handle_incoming_block(const block& b, block_verification_context& bvc, bool update_miner_blocktemplate = true); @@ -53,7 +54,8 @@ namespace currency //-------------------- i_miner_handler ----------------------- virtual bool handle_block_found(const block& b, block_verification_context* p_verification_result = nullptr); - virtual bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()); + virtual bool get_block_template(const create_block_template_params& params, create_block_template_response& resp); + bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()); miner& get_miner(){ return m_miner; } static void init_options(boost::program_options::options_description& desc); @@ -81,6 +83,8 @@ namespace currency size_t get_alternative_blocks_count(); void set_currency_protocol(i_currency_protocol* pprotocol); + void set_critical_error_handler(i_critical_error_handler *handler); + i_critical_error_handler* get_critical_error_handler() const { return m_critical_error_handler; } bool set_checkpoints(checkpoints&& chk_pts); bool get_pool_transactions(std::list& txs); @@ -116,12 +120,6 @@ namespace currency bool add_new_block(const block& b, block_verification_context& bvc); bool load_state_data(); bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob); - bool check_tx_extra(const transaction& tx); - - bool check_tx_syntax(const transaction& tx); - //check correct values, amounts and all lightweight checks not related with database - bool check_tx_semantic(const transaction& tx, bool kept_by_block); - //check if tx already in memory pool or in main blockchain bool is_key_image_spent(const crypto::key_image& key_im); @@ -130,20 +128,25 @@ namespace currency bool update_miner_block_template(); bool handle_command_line(const boost::program_options::variables_map& vm); bool on_update_blocktemplate_interval(); - bool check_tx_inputs_keyimages_diff(const transaction& tx); void notify_blockchain_update_listeners(); + bool check_if_free_space_critically_low(uint64_t* p_available_space = nullptr); + void check_free_space(); + blockchain_storage m_blockchain_storage; tx_memory_pool m_mempool; i_currency_protocol* m_pprotocol; + i_critical_error_handler* m_critical_error_handler; critical_section m_incoming_tx_lock; miner m_miner; account_public_address m_miner_address; std::string m_config_folder; + uint64_t m_stop_after_height; currency_protocol_stub m_protocol_stub; math_helper::once_a_time_seconds<60*60*12, false> m_prune_alt_blocks_interval; + math_helper::once_a_time_seconds<60, true> m_check_free_space_interval; friend class tx_validate_inputs; std::atomic m_starter_message_showed; @@ -152,4 +155,4 @@ namespace currency }; } -POP_WARNINGS +POP_VS_WARNINGS diff --git a/src/currency_core/currency_format_utils.cpp b/src/currency_core/currency_format_utils.cpp index cc73db4..7c014f2 100644 --- a/src/currency_core/currency_format_utils.cpp +++ b/src/currency_core/currency_format_utils.cpp @@ -7,7 +7,9 @@ #include "include_base_utils.h" #include -#include +#ifndef ANDROID_BUILD + #include +#endif using namespace epee; #include "print_fixed_point_helper.h" @@ -27,6 +29,7 @@ using namespace epee; #include "genesis.h" #include "genesis_acc.h" #include "common/mnemonic-encoding.h" +#include "crypto/bitcoin/sha256_helper.h" namespace currency { @@ -87,7 +90,7 @@ namespace currency bool pos, const pos_entry& pe) { - uint64_t block_reward; + uint64_t block_reward = 0; if (!get_block_reward(pos, median_size, current_block_size, already_generated_coins, block_reward, height)) { LOG_ERROR("Block is too big"); @@ -107,21 +110,54 @@ namespace currency out_amounts.resize(out_amounts.size() - 1); } + std::vector destinations; for (auto a : out_amounts) { - tx_destination_entry de; + tx_destination_entry de = AUTO_VAL_INIT(de); de.addr.push_back(miner_address); de.amount = a; + if (pe.stake_unlock_time && pe.stake_unlock_time > height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW) + { + //this means that block is creating after hardfork_1 and unlock_time is needed to set for every destination separately + de.unlock_time = height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW; + } destinations.push_back(de); } if (pos) - destinations.push_back(tx_destination_entry(pe.amount, stakeholder_address)); + { + uint64_t stake_lock_time = 0; + if (pe.stake_unlock_time && pe.stake_unlock_time > height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW) + stake_lock_time = pe.stake_unlock_time; + destinations.push_back(tx_destination_entry(pe.amount, stakeholder_address, stake_lock_time)); + } + return construct_miner_tx(height, median_size, already_generated_coins, current_block_size, fee, destinations, tx, extra_nonce, max_outs, pos, pe); } //------------------------------------------------------------------ + bool apply_unlock_time(const std::vector& destinations, transaction& tx) + { + currency::etc_tx_details_unlock_time2 unlock_time2 = AUTO_VAL_INIT(unlock_time2); + unlock_time2.unlock_time_array.resize(destinations.size()); + bool found_unlock_time = false; + for (size_t i = 0; i != unlock_time2.unlock_time_array.size(); i++) + { + if (destinations[i].unlock_time) + { + found_unlock_time = true; + unlock_time2.unlock_time_array[i] = destinations[i].unlock_time; + } + } + if (found_unlock_time) + { + tx.extra.push_back(unlock_time2); + } + + return true; + } + //------------------------------------------------------------------ bool construct_miner_tx(size_t height, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t current_block_size, uint64_t fee, @@ -144,9 +180,12 @@ namespace currency if (!add_tx_extra_userdata(tx, extra_nonce)) return false; + //at this moment we do apply_unlock_time only for coin_base transactions + apply_unlock_time(destinations, tx); //we always add extra_padding with 2 bytes length to make possible for get_block_template to adjust cumulative size tx.extra.push_back(extra_padding()); + txin_gen in; in.height = height; tx.vin.push_back(in); @@ -167,27 +206,32 @@ namespace currency std::set deriv_cache; for (auto& d : destinations) { - bool r = construct_tx_out(d, txkey.sec, no, tx, deriv_cache); + bool r = construct_tx_out(d, txkey.sec, no, tx, deriv_cache, account_keys()); CHECK_AND_ASSERT_MES(r, false, "Failed to contruct miner tx out"); no++; } - + tx.version = CURRENT_TRANSACTION_VERSION; - set_tx_unlock_time(tx, height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + if (!have_type_in_variant_container(tx.extra)) + { + //if stake unlock time was not set, then we can use simple "whole transaction" lock scheme + set_tx_unlock_time(tx, height + CURRENCY_MINED_MONEY_UNLOCK_WINDOW); + } + return true; } //--------------------------------------------------------------- bool derive_ephemeral_key_helper(const account_keys& ack, const crypto::public_key& tx_public_key, size_t real_output_index, keypair& in_ephemeral) { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); - bool r = crypto::generate_key_derivation(tx_public_key, ack.m_view_secret_key, recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + bool r = crypto::generate_key_derivation(tx_public_key, ack.view_secret_key, recv_derivation); + CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.view_secret_key << ")"); - r = crypto::derive_public_key(recv_derivation, real_output_index, ack.m_account_address.m_spend_public_key, in_ephemeral.pub); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to derive_public_key(" << recv_derivation << ", " << real_output_index << ", " << ack.m_account_address.m_spend_public_key << ")"); + r = crypto::derive_public_key(recv_derivation, real_output_index, ack.account_address.spend_public_key, in_ephemeral.pub); + CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to derive_public_key(" << recv_derivation << ", " << real_output_index << ", " << ack.account_address.spend_public_key << ")"); - crypto::derive_secret_key(recv_derivation, real_output_index, ack.m_spend_secret_key, in_ephemeral.sec); + crypto::derive_secret_key(recv_derivation, real_output_index, ack.spend_secret_key, in_ephemeral.sec); return true; } //--------------------------------------------------------------- @@ -270,6 +314,46 @@ namespace currency return string_tools::get_xtype_from_string(amount, str_amount); } //-------------------------------------------------------------------------------- + bool parse_tracking_seed(const std::string& tracking_seed, account_public_address& address, crypto::secret_key& view_sec_key, uint64_t& creation_timestamp) + { + std::vector parts; + boost::split(parts, tracking_seed, [](char x){ return x == ':'; } ); + if (parts.size() != 2 && parts.size() != 3) + return false; + + if (!get_account_address_from_str(address, parts[0])) + return false; + + if (!address.is_auditable()) + return false; + + if (!epee::string_tools::parse_tpod_from_hex_string(parts[1], view_sec_key)) + return false; + + crypto::public_key view_pub_key = AUTO_VAL_INIT(view_pub_key); + if (!crypto::secret_key_to_public_key(view_sec_key, view_pub_key)) + return false; + + if (view_pub_key != address.view_public_key) + return false; + + creation_timestamp = 0; + if (parts.size() == 3) + { + // parse timestamp + int64_t ts = 0; + if (!epee::string_tools::string_to_num_fast(parts[2], ts)) + return false; + + if (ts < WALLET_BRAIN_DATE_OFFSET) + return false; + + creation_timestamp = ts; + } + + return true; + } + //-------------------------------------------------------------------------------- std::string print_stake_kernel_info(const stake_kernel& sk) { std::stringstream ss; @@ -336,15 +420,10 @@ namespace currency //--------------------------------------------------------------- crypto::hash get_sign_buff_hash_for_alias_update(const extra_alias_entry& ai) { - const extra_alias_entry* pale = &ai; - extra_alias_entry eae_local = AUTO_VAL_INIT(eae_local); - if (ai.m_sign.size()) - { - eae_local = ai; - eae_local.m_sign.clear(); - pale = &eae_local; - } - return get_object_hash(*pale); + extra_alias_entry_old ai_old = ai.to_old(); + if (ai_old.m_sign.size()) + ai_old.m_sign.clear(); + return get_object_hash(ai_old); } //--------------------------------------------------------------- struct tx_extra_handler : public boost::static_visitor @@ -378,13 +457,16 @@ namespace currency rei.m_attachment_info = ai; return true; } - bool operator()(const extra_alias_entry& ae) const { ENSURE_ONETIME(was_alias, "alias"); rei.m_alias = ae; return true; } + bool operator()(const extra_alias_entry_old& ae) const + { + return operator()(static_cast(ae)); + } bool operator()(const extra_user_data& ud) const { ENSURE_ONETIME(was_userdata, "userdata"); @@ -467,14 +549,25 @@ namespace currency //--------------------------------------------------------------- bool derive_public_key_from_target_address(const account_public_address& destination_addr, const crypto::secret_key& tx_sec_key, size_t index, crypto::public_key& out_eph_public_key, crypto::key_derivation& derivation) { - bool r = crypto::generate_key_derivation(destination_addr.m_view_public_key, tx_sec_key, derivation); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << destination_addr.m_view_public_key << ", " << tx_sec_key << ")"); + bool r = crypto::generate_key_derivation(destination_addr.view_public_key, tx_sec_key, derivation); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to generate_key_derivation(" << destination_addr.view_public_key << ", " << tx_sec_key << ")"); - r = crypto::derive_public_key(derivation, index, destination_addr.m_spend_public_key, out_eph_public_key); - CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to derive_public_key(" << derivation << ", " << index << ", " << destination_addr.m_view_public_key << ")"); + r = crypto::derive_public_key(derivation, index, destination_addr.spend_public_key, out_eph_public_key); + CHECK_AND_ASSERT_MES(r, false, "at creation outs: failed to derive_public_key(" << derivation << ", " << index << ", " << destination_addr.view_public_key << ")"); return r; } //--------------------------------------------------------------- + uint16_t get_derivation_hint(const crypto::key_derivation& derivation) + { + crypto::hash h = blake2_hash(&derivation, sizeof(derivation)); + + uint16_t* pderiv_hash_as_array = (uint16_t*)&h; + uint16_t res = pderiv_hash_as_array[0]; + for (size_t i = 1; i != sizeof(h) / sizeof(uint16_t); i++) + res ^= pderiv_hash_as_array[i]; + return res; + } + //--------------------------------------------------------------- uint64_t get_string_uint64_hash(const std::string& str) { crypto::hash h = crypto::cn_fast_hash(str.data(), str.size()); @@ -482,16 +575,41 @@ namespace currency return phash_as_array[0] ^ phash_as_array[1] ^ phash_as_array[2] ^ phash_as_array[3]; } //--------------------------------------------------------------- - uint16_t get_derivation_xor(const crypto::key_derivation& derivation) + tx_derivation_hint make_tx_derivation_hint_from_uint16(uint16_t hint) { - uint16_t* pderiv_as_array = (uint16_t*)&derivation; - uint16_t res = pderiv_as_array[0]; - for (size_t i = 1; i != sizeof(derivation) / sizeof(uint16_t); i++) - res ^= pderiv_as_array[i]; - return res; + tx_derivation_hint dh = AUTO_VAL_INIT(dh); + dh.msg.assign((const char*)&hint, sizeof(hint)); + return dh; } //--------------------------------------------------------------- - bool construct_tx_out(const tx_destination_entry& de, const crypto::secret_key& tx_sec_key, size_t output_index, transaction& tx, std::set& deriv_cache, uint8_t tx_outs_attr) +// bool get_uint16_from_tx_derivation_hint(const tx_derivation_hint& dh, uint16_t& hint) +// { +// tx_derivation_hint dh; +// if (dh.msg.size() != sizeof(hint)) +// return false; +// hint = *((uint16_t*)dh.msg.data()); +// return true; +// } + //--------------------------------------------------------------- + std::string generate_origin_for_htlc(const txout_htlc& htlc, const account_keys& acc_keys) + { + std::string blob; + string_tools::apped_pod_to_strbuff(blob, htlc.pkey_redeem); + string_tools::apped_pod_to_strbuff(blob, htlc.pkey_refund); + string_tools::apped_pod_to_strbuff(blob, acc_keys.spend_secret_key); + crypto::hash origin_hs = crypto::cn_fast_hash(blob.data(), blob.size()); + std::string origin_blob; + string_tools::apped_pod_to_strbuff(origin_blob, origin_hs); + return origin_blob; + } + //--------------------------------------------------------------- + bool construct_tx_out(const tx_destination_entry& de, const crypto::secret_key& tx_sec_key, size_t output_index, transaction& tx, std::set& deriv_cache, const account_keys& self, uint8_t tx_outs_attr) + { + finalized_tx result = AUTO_VAL_INIT(result); + return construct_tx_out(de, tx_sec_key, output_index, tx, deriv_cache, self, result, tx_outs_attr); + } + //--------------------------------------------------------------- + bool construct_tx_out(const tx_destination_entry& de, const crypto::secret_key& tx_sec_key, size_t output_index, transaction& tx, std::set& deriv_cache, const account_keys& self, finalized_tx& result, uint8_t tx_outs_attr) { CHECK_AND_ASSERT_MES(de.addr.size() == 1 || (de.addr.size() > 1 && de.minimum_sigs <= de.addr.size()), false, "Invalid destination entry: amount: " << de.amount << " minimum_sigs: " << de.minimum_sigs << " addr.size(): " << de.addr.size()); @@ -500,7 +618,7 @@ namespace currency for (auto& apa : de.addr) { crypto::public_key out_eph_public_key = AUTO_VAL_INIT(out_eph_public_key); - if (apa.m_spend_public_key == null_pkey && apa.m_view_public_key == null_pkey) + if (apa.spend_public_key == null_pkey && apa.view_public_key == null_pkey) { //burning money(for example alias reward) out_eph_public_key = null_pkey; @@ -510,12 +628,12 @@ namespace currency crypto::key_derivation derivation = AUTO_VAL_INIT(derivation); bool r = derive_public_key_from_target_address(apa, tx_sec_key, output_index, out_eph_public_key, derivation); CHECK_AND_ASSERT_MES(r, false, "failed to derive_public_key_from_target_address"); - etc_tx_derivation_hint dh = AUTO_VAL_INIT(dh); - dh.v = get_derivation_xor(derivation); - if (deriv_cache.count(dh.v) == 0) + + uint16_t hint = get_derivation_hint(derivation); + if (deriv_cache.count(hint) == 0) { - tx.extra.push_back(dh); - deriv_cache.insert(dh.v); + tx.extra.push_back(make_tx_derivation_hint_from_uint16(hint)); + deriv_cache.insert(hint); } } target_keys.push_back(out_eph_public_key); @@ -523,12 +641,65 @@ namespace currency tx_out out; out.amount = de.amount; - if (target_keys.size() == 1) + if (de.htlc_options.expiration != 0) + { + const destination_option_htlc_out& htlc_dest = de.htlc_options; + //out htlc + CHECK_AND_ASSERT_MES(target_keys.size() == 1, false, "Unexpected htl keys count = " << target_keys.size() << ", expected ==1"); + txout_htlc htlc = AUTO_VAL_INIT(htlc); + htlc.expiration = htlc_dest.expiration; + htlc.flags = 0; //0 - SHA256, 1 - RIPEMD160, by default leave SHA256 + //receiver key + htlc.pkey_redeem = *target_keys.begin(); + //generate refund key + crypto::key_derivation derivation = AUTO_VAL_INIT(derivation); + crypto::public_key out_eph_public_key = AUTO_VAL_INIT(out_eph_public_key); + bool r = derive_public_key_from_target_address(self.account_address, tx_sec_key, output_index, out_eph_public_key, derivation); + CHECK_AND_ASSERT_MES(r, false, "failed to derive_public_key_from_target_address"); + htlc.pkey_refund = out_eph_public_key; + //add derivation hint for refund address + uint16_t hint = get_derivation_hint(derivation); + if (deriv_cache.count(hint) == 0) + { + tx.extra.push_back(make_tx_derivation_hint_from_uint16(hint)); + deriv_cache.insert(hint); + } + + + if (htlc_dest.htlc_hash == null_hash) + { + //we use deterministic origin, to make possible access origin on different wallets copies + + result.htlc_origin = generate_origin_for_htlc(htlc, self); + + //calculate hash + if (!htlc.flags&CURRENCY_TXOUT_HTLC_FLAGS_HASH_TYPE_MASK) + { + htlc.htlc_hash = crypto::sha256_hash(result.htlc_origin.data(), result.htlc_origin.size()); + } + else + { + crypto::hash160 h160 = crypto::RIPEMD160_hash(result.htlc_origin.data(), result.htlc_origin.size()); + std::memcpy(&htlc.htlc_hash, &h160, sizeof(h160)); + } + } + else + { + htlc.htlc_hash = htlc_dest.htlc_hash; + } + out.target = htlc; + } + else if (target_keys.size() == 1) { //out to key - txout_to_key tk; + txout_to_key tk = AUTO_VAL_INIT(tk); tk.key = target_keys.back(); - tk.mix_attr = tx_outs_attr; + + if (de.addr.front().is_auditable()) // check only the first address because there's only one in this branch + tk.mix_attr = CURRENCY_TO_KEY_OUT_FORCED_NO_MIX; // override mix_attr to 1 for auditable target addresses + else + tk.mix_attr = tx_outs_attr; + out.target = tk; } else @@ -566,7 +737,8 @@ namespace currency //put hash into extra std::stringstream ss; binary_archive oar(ss); - ::do_serialize(oar, const_cast&>(attachment)); + if (!::do_serialize(oar, const_cast&>(attachment))) + return; std::string buff = ss.str(); eai.sz = buff.size(); eai.hsh = get_blob_hash(buff); @@ -605,9 +777,14 @@ namespace currency crypto::chacha_crypt(m.acc_addr, m_key); m_was_crypted_entries = true; } - void operator()(tx_message& m) + void operator()(tx_payer_old& pr) { - crypto::chacha_crypt(m.msg, m_key); + crypto::chacha_crypt(pr.acc_addr, m_key); + m_was_crypted_entries = true; + } + void operator()(tx_receiver_old& m) + { + crypto::chacha_crypt(m.acc_addr, m_key); m_was_crypted_entries = true; } void operator()(tx_service_attachment& sa) @@ -645,13 +822,6 @@ namespace currency rdecrypted_att.push_back(local_comment); } - void operator()(const tx_message& m) - { - tx_message local_msg = m; - crypto::chacha_crypt(local_msg.msg, rkey); - rdecrypted_att.push_back(local_msg); - } - void operator()(const tx_service_attachment& sa) { tx_service_attachment local_sa = sa; @@ -679,7 +849,18 @@ namespace currency crypto::chacha_crypt(receiver_local.acc_addr, rkey); rdecrypted_att.push_back(receiver_local); } - + void operator()(const tx_payer_old& pr) + { + tx_payer_old payer_local = pr; + crypto::chacha_crypt(payer_local.acc_addr, rkey); + rdecrypted_att.push_back(payer_local); + } + void operator()(const tx_receiver_old& pr) + { + tx_receiver_old receiver_local = pr; + crypto::chacha_crypt(receiver_local.acc_addr, rkey); + rdecrypted_att.push_back(receiver_local); + } template void operator()(const attachment_t& att) { @@ -728,15 +909,15 @@ namespace currency { crypto::public_key tx_pub_key = currency::get_tx_pub_key_from_extra(tx); - bool r = crypto::generate_key_derivation(tx_pub_key, acc_keys.m_view_secret_key, derivation); + bool r = crypto::generate_key_derivation(tx_pub_key, acc_keys.view_secret_key, derivation); CHECK_AND_ASSERT_MES(r, null_derivation, "failed to generate_key_derivation"); - LOG_PRINT_GREEN("DECRYPTING ON KEY: " << epee::string_tools::pod_to_hex(derivation) << ", key derived from destination addr: " << currency::get_account_address_as_str(acc_keys.m_account_address), LOG_LEVEL_0); + LOG_PRINT_GREEN("DECRYPTING ON KEY: " << epee::string_tools::pod_to_hex(derivation) << ", key derived from destination addr: " << currency::get_account_address_as_str(acc_keys.account_address), LOG_LEVEL_0); } else { derivation = crypto_info.encrypted_key_derivation; - crypto::chacha_crypt(derivation, acc_keys.m_spend_secret_key); - LOG_PRINT_GREEN("DECRYPTING ON KEY: " << epee::string_tools::pod_to_hex(derivation) << ", key decrypted from sender address: " << currency::get_account_address_as_str(acc_keys.m_account_address), LOG_LEVEL_0); + crypto::chacha_crypt(derivation, acc_keys.spend_secret_key); + LOG_PRINT_GREEN("DECRYPTING ON KEY: " << epee::string_tools::pod_to_hex(derivation) << ", key decrypted from sender address: " << currency::get_account_address_as_str(acc_keys.account_address), LOG_LEVEL_0); } //validate derivation we here. Yoda style @@ -783,7 +964,7 @@ namespace currency void encrypt_attachments(transaction& tx, const account_keys& sender_keys, const account_public_address& destination_addr, const keypair& tx_random_key) { crypto::key_derivation derivation = AUTO_VAL_INIT(derivation); - bool r = crypto::generate_key_derivation(destination_addr.m_view_public_key, tx_random_key.sec, derivation); + bool r = crypto::generate_key_derivation(destination_addr.view_public_key, tx_random_key.sec, derivation); CHECK_AND_ASSERT_MES(r, void(), "failed to generate_key_derivation"); bool was_attachment_crypted_entries = false; bool was_extra_crypted_entries = false; @@ -805,23 +986,37 @@ namespace currency chs.derivation_hash = *(uint32_t*)&hash_for_check_sum; //put encrypted derivation to let sender decrypt all this data from attachment/extra chs.encrypted_key_derivation = derivation; - crypto::chacha_crypt(chs.encrypted_key_derivation, sender_keys.m_spend_secret_key); + crypto::chacha_crypt(chs.encrypted_key_derivation, sender_keys.spend_secret_key); if (was_extra_crypted_entries) tx.extra.push_back(chs); else tx.attachment.push_back(chs); - - LOG_PRINT_GREEN("ENCRYPTING ATTACHMENTS ON KEY: " << epee::string_tools::pod_to_hex(derivation) - << " destination addr: " << currency::get_account_address_as_str(destination_addr) - << " tx_random_key.sec" << tx_random_key.sec - << " tx_random_key.pub" << tx_random_key.pub - << " sender address: " << currency::get_account_address_as_str(sender_keys.m_account_address) - , LOG_LEVEL_0); - } } //--------------------------------------------------------------- - uint64_t get_tx_type(const transaction& tx) + void load_wallet_transfer_info_flags(tools::wallet_public::wallet_transfer_info& x) + { + x.is_service = currency::is_service_tx(x.tx); + x.is_mixing = currency::does_tx_have_only_mixin_inputs(x.tx); + x.is_mining = currency::is_coinbase(x.tx); + if (!x.is_mining) + x.fee = currency::get_tx_fee(x.tx); + else + x.fee = 0; + x.show_sender = currency::is_showing_sender_addres(x.tx); + tx_out htlc_out = AUTO_VAL_INIT(htlc_out); + txin_htlc htlc_in = AUTO_VAL_INIT(htlc_in); + + x.tx_type = get_tx_type_ex(x.tx, htlc_out, htlc_in); + if(x.tx_type == GUI_TX_TYPE_HTLC_DEPOSIT && x.is_income == true) + { + //need to override amount + x.amount = htlc_out.amount; + } + } + + //--------------------------------------------------------------- + uint64_t get_tx_type_ex(const transaction& tx, tx_out& htlc_out, txin_htlc& htlc_in) { if (is_coinbase(tx)) return GUI_TX_TYPE_COIN_BASE; @@ -836,7 +1031,7 @@ namespace currency else return GUI_TX_TYPE_NEW_ALIAS; } - + // offers tx_service_attachment a = AUTO_VAL_INIT(a); if (get_type_in_variant_container(tx.attachment, a)) @@ -851,7 +1046,7 @@ namespace currency return GUI_TX_TYPE_CANCEL_OFFER; } } - + // escrow tx_service_attachment tsa = AUTO_VAL_INIT(tsa); if (bc_services::get_first_service_attachment_by_id(tx, BC_ESCROW_SERVICE_ID, BC_ESCROW_SERVICE_INSTRUCTION_RELEASE_TEMPLATES, tsa)) @@ -867,9 +1062,31 @@ namespace currency if (bc_services::get_first_service_attachment_by_id(tx, BC_ESCROW_SERVICE_ID, BC_ESCROW_SERVICE_INSTRUCTION_CANCEL_PROPOSAL, tsa)) return GUI_TX_TYPE_ESCROW_CANCEL_PROPOSAL; + for (auto o : tx.vout) + { + if (o.target.type() == typeid(txout_htlc)) + { + htlc_out = o; + return GUI_TX_TYPE_HTLC_DEPOSIT; + } + } + + if (get_type_in_variant_container(tx.vin, htlc_in)) + { + return GUI_TX_TYPE_HTLC_REDEEM; + } + + return GUI_TX_TYPE_NORMAL; } //--------------------------------------------------------------- + uint64_t get_tx_type(const transaction& tx) + { + tx_out htlc_out = AUTO_VAL_INIT(htlc_out); + txin_htlc htlc_in = AUTO_VAL_INIT(htlc_in); + return get_tx_type_ex(tx, htlc_out, htlc_in); + } + //--------------------------------------------------------------- size_t get_multisig_out_index(const std::vector& outs) { size_t n = 0; @@ -891,16 +1108,7 @@ namespace currency } return n; } - //--------------------------------------------------------------- - account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations) - { - for (const auto& de : destinations) - { - if (de.addr.size() == 1 && sender_account_keys.m_account_address != de.addr.back()) - return de.addr.back(); // return the first destination address that is non-multisig and not equal to the sender's address - } - return sender_account_keys.m_account_address; // otherwise, fallback to sender's address - } + //--------------------------------------------------------------- bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, @@ -927,7 +1135,7 @@ namespace currency shuffle, flags); } - + //--------------------------------------------------------------- bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, const std::vector& destinations, const std::vector& extra, @@ -941,8 +1149,49 @@ namespace currency bool shuffle, uint64_t flags) { + //extra copy operation, but creating transaction is not sensitive to this + finalize_tx_param ftp = AUTO_VAL_INIT(ftp); + ftp.sources = sources; + ftp.prepared_destinations = destinations; + ftp.extra = extra; + ftp.attachments = attachments; + ftp.unlock_time = unlock_time; + ftp.crypt_address = crypt_destination_addr; + ftp.expiration_time = expiration_time; + ftp.tx_outs_attr = tx_outs_attr; + ftp.shuffle = shuffle; + ftp.flags = flags; + + finalized_tx ft = AUTO_VAL_INIT(ft); + ft.tx = tx; + ft.one_time_key = one_time_secret_key; + bool r = construct_tx(sender_account_keys, ftp, ft); + tx = ft.tx; + one_time_secret_key = ft.one_time_key; + return r; + } + //--------------------------------------------------------------- + bool construct_tx(const account_keys& sender_account_keys, const finalize_tx_param& ftp, finalized_tx& result) + { + const std::vector& sources = ftp.sources; + const std::vector& destinations = ftp.prepared_destinations; + const std::vector& extra = ftp.extra; + const std::vector& attachments = ftp.attachments; + const uint64_t& unlock_time = ftp.unlock_time; + const account_public_address& crypt_destination_addr = ftp.crypt_address; + const uint64_t& expiration_time = ftp.expiration_time; + const uint8_t& tx_outs_attr = ftp.tx_outs_attr; + const bool& shuffle = ftp.shuffle; + const uint64_t& flags = ftp.flags; + + transaction& tx = result.tx; + crypto::secret_key& one_time_secret_key = result.one_time_key; + + result.ftp = ftp; CHECK_AND_ASSERT_MES(destinations.size() <= CURRENCY_TX_MAX_ALLOWED_OUTS, false, "Too many outs (" << destinations.size() << ")! Tx can't be constructed."); + bool watch_only_mode = sender_account_keys.spend_secret_key == null_skey; + bool append_mode = false; if (flags&TX_FLAG_SIGNATURE_MODE_SEPARATE && tx.vin.size()) append_mode = true; @@ -971,6 +1220,11 @@ namespace currency add_tx_pub_key_to_extra(tx, txkey.pub); one_time_secret_key = txkey.sec; + //add flags + etc_tx_flags16_t e = AUTO_VAL_INIT(e); + //todo: add some flags here + update_or_add_field_to_extra(tx.extra, e); + //include offers if need tx.attachment = attachments; encrypt_attachments(tx, sender_account_keys, crypt_destination_addr, txkey); @@ -1015,8 +1269,56 @@ namespace currency for (const tx_source_entry& src_entr : sources) { in_contexts.push_back(input_generation_context_data()); - if (!src_entr.is_multisig()) + if(src_entr.is_multisig()) + {//multisig input + txin_multisig input_multisig = AUTO_VAL_INIT(input_multisig); + summary_inputs_money += input_multisig.amount = src_entr.amount; + input_multisig.multisig_out_id = src_entr.multisig_id; + input_multisig.sigs_count = src_entr.ms_sigs_count; + tx.vin.push_back(input_multisig); + } + else if (src_entr.htlc_origin.size()) { + //htlc redeem + keypair& in_ephemeral = in_contexts.back().in_ephemeral; + //txin_to_key + if(src_entr.outputs.size() != 1) + { + LOG_ERROR("htlc in: wrong output src_entr.outputs.size() = " << src_entr.outputs.size()); + return false; + } + summary_inputs_money += src_entr.amount; + + //key_derivation recv_derivation; + crypto::key_image img; + if (!generate_key_image_helper(sender_account_keys, src_entr.real_out_tx_key, src_entr.real_output_in_tx_index, in_ephemeral, img)) + return false; + + //check that derivated key is equal with real output key + if (!(in_ephemeral.pub == src_entr.outputs[src_entr.real_output].second)) + { + LOG_ERROR("derived public key missmatch with output public key! " << ENDL << "derived_key:" + << string_tools::pod_to_hex(in_ephemeral.pub) << ENDL << "real output_public_key:" + << string_tools::pod_to_hex(src_entr.outputs[src_entr.real_output].second)); + return false; + } + + //put key image into tx input + txin_htlc input_to_key; + input_to_key.amount = src_entr.amount; + input_to_key.k_image = img; + input_to_key.hltc_origin = src_entr.htlc_origin; + + //fill outputs array and use relative offsets + BOOST_FOREACH(const tx_source_entry::output_entry& out_entry, src_entr.outputs) + input_to_key.key_offsets.push_back(out_entry.first); + + input_to_key.key_offsets = absolute_output_offsets_to_relative(input_to_key.key_offsets); + tx.vin.push_back(input_to_key); + } + else + { + //regular to key out keypair& in_ephemeral = in_contexts.back().in_ephemeral; //txin_to_key if (src_entr.real_output >= src_entr.outputs.size()) @@ -1041,7 +1343,26 @@ namespace currency } //put key image into tx input - txin_to_key input_to_key; + txin_v in_v; + txin_to_key* ptokey = nullptr; + if (src_entr.htlc_origin.size()) + { + //add txin_htlc + txin_htlc in_htlc = AUTO_VAL_INIT(in_htlc); + in_htlc.hltc_origin = src_entr.htlc_origin; + in_v = in_htlc; + txin_htlc& in_v_ref = boost::get(in_v); + ptokey = static_cast(&in_v_ref); + } + else + { + in_v = txin_to_key(); + txin_to_key& in_v_ref = boost::get(in_v); + ptokey = &in_v_ref; + } + txin_to_key& input_to_key = *ptokey; + + input_to_key.amount = src_entr.amount; input_to_key.k_image = img; @@ -1050,16 +1371,9 @@ namespace currency input_to_key.key_offsets.push_back(out_entry.first); input_to_key.key_offsets = absolute_output_offsets_to_relative(input_to_key.key_offsets); - tx.vin.push_back(input_to_key); - } - else - {//multisig input - txin_multisig input_multisig = AUTO_VAL_INIT(input_multisig); - summary_inputs_money += input_multisig.amount = src_entr.amount; - input_multisig.multisig_out_id = src_entr.multisig_id; - input_multisig.sigs_count = src_entr.ms_sigs_count; - tx.vin.push_back(input_multisig); + tx.vin.push_back(in_v); } + } // "Shuffle" outs @@ -1071,10 +1385,10 @@ namespace currency //fill outputs size_t output_index = tx.vout.size(); // in case of append mode we need to start output indexing from the last one + 1 std::set deriv_cache; - BOOST_FOREACH(const tx_destination_entry& dst_entr, shuffled_dsts) + for(const tx_destination_entry& dst_entr : shuffled_dsts) { CHECK_AND_ASSERT_MES(dst_entr.amount > 0, false, "Destination with wrong amount: " << dst_entr.amount); - bool r = construct_tx_out(dst_entr, txkey.sec, output_index, tx, deriv_cache, tx_outs_attr); + bool r = construct_tx_out(dst_entr, txkey.sec, output_index, tx, deriv_cache, sender_account_keys, result, tx_outs_attr); CHECK_AND_ASSERT_MES(r, false, "Failed to construc tx out"); output_index++; summary_outs_money += dst_entr.amount; @@ -1103,7 +1417,7 @@ namespace currency { CHECK_AND_ASSERT_MES(tsa.security.size() == 1, false, "Wrong tsa.security.size() = " << tsa.security.size()); - bool r = derive_public_key_from_target_address(sender_account_keys.m_account_address, one_time_secret_key, att_count, tsa.security.back()); + bool r = derive_public_key_from_target_address(sender_account_keys.account_address, one_time_secret_key, att_count, tsa.security.back()); CHECK_AND_ASSERT_MES(r, false, "Failed to derive_public_key_from_target_address"); } att_count++; @@ -1146,9 +1460,14 @@ namespace currency tx.signatures.push_back(std::vector()); std::vector& sigs = tx.signatures.back(); - if (!src_entr.is_multisig()) + if(src_entr.is_multisig()) { - // txin_to_key + // txin_multisig -- don't sign anything here (see also sign_multisig_input_in_tx()) + sigs.resize(src_entr.ms_keys_count, null_sig); // just reserve keys.size() null signatures (NOTE: not minimum_sigs!) + } + else + { + // regular txin_to_key or htlc ss_ring_s << "input #" << input_index << ", pub_keys:" << ENDL; std::vector keys_ptrs; BOOST_FOREACH(const tx_source_entry::output_entry& o, src_entr.outputs) @@ -1158,15 +1477,12 @@ namespace currency } sigs.resize(src_entr.outputs.size()); - crypto::generate_ring_signature(tx_hash_for_signature, boost::get(tx.vin[input_index]).k_image, keys_ptrs, in_contexts[in_context_index].in_ephemeral.sec, src_entr.real_output, sigs.data()); + if (!watch_only_mode) + crypto::generate_ring_signature(tx_hash_for_signature, get_to_key_input_from_txin_v(tx.vin[input_index]).k_image, keys_ptrs, in_contexts[in_context_index].in_ephemeral.sec, src_entr.real_output, sigs.data()); + ss_ring_s << "signatures:" << ENDL; - std::for_each(sigs.begin(), sigs.end(), [&](const crypto::signature& s){ss_ring_s << s << ENDL; }); - ss_ring_s << "prefix_hash:" << tx_prefix_hash << ENDL << "in_ephemeral_key: " << in_contexts[in_context_index].in_ephemeral.sec << ENDL << "real_output: " << src_entr.real_output << ENDL; - } - else - { - // txin_multisig -- don't sign anything here (see also sign_multisig_input_in_tx()) - sigs.resize(src_entr.ms_keys_count, null_sig); // just reserve keys.size() null signatures (NOTE: not minimum_sigs!) + std::for_each(sigs.begin(), sigs.end(), [&ss_ring_s](const crypto::signature& s) { ss_ring_s << s << ENDL; }); + ss_ring_s << "prefix_hash: " << tx_prefix_hash << ENDL << "in_ephemeral_key: " << in_contexts[in_context_index].in_ephemeral.sec << ENDL << "real_output: " << src_entr.real_output << ENDL; } if (src_entr.separately_signed_tx_complete) { @@ -1210,21 +1526,34 @@ namespace currency return reward; } //--------------------------------------------------------------- - std::string get_word_from_timstamp(uint64_t timestamp) + std::string get_word_from_timstamp(uint64_t timestamp, bool use_password) { uint64_t date_offset = timestamp > WALLET_BRAIN_DATE_OFFSET ? timestamp - WALLET_BRAIN_DATE_OFFSET : 0; uint64_t weeks_count = date_offset / WALLET_BRAIN_DATE_QUANTUM; - CHECK_AND_ASSERT_THROW_MES(weeks_count < std::numeric_limits::max(), "internal error: unable to converto to uint32, val = " << weeks_count); + CHECK_AND_ASSERT_THROW_MES(weeks_count < WALLET_BRAIN_DATE_MAX_WEEKS_COUNT, "SEED PHRASE need to be extended or refactored"); + + if (use_password) + weeks_count += WALLET_BRAIN_DATE_MAX_WEEKS_COUNT; + + CHECK_AND_ASSERT_THROW_MES(weeks_count < std::numeric_limits::max(), "internal error: unable to convert to uint32, val = " << weeks_count); uint32_t weeks_count_32 = static_cast(weeks_count); return tools::mnemonic_encoding::word_by_num(weeks_count_32); } //--------------------------------------------------------------- - uint64_t get_timstamp_from_word(std::string word) + uint64_t get_timstamp_from_word(std::string word, bool& password_used) { uint64_t count_of_weeks = tools::mnemonic_encoding::num_by_word(word); + if (count_of_weeks >= WALLET_BRAIN_DATE_MAX_WEEKS_COUNT) + { + count_of_weeks -= WALLET_BRAIN_DATE_MAX_WEEKS_COUNT; + password_used = true; + } + else { + password_used = false; + } uint64_t timestamp = count_of_weeks * WALLET_BRAIN_DATE_QUANTUM + WALLET_BRAIN_DATE_OFFSET; - + return timestamp; } //--------------------------------------------------------------- @@ -1304,7 +1633,7 @@ namespace currency bool get_inputs_money_amount(const transaction& tx, uint64_t& money) { money = 0; - BOOST_FOREACH(const auto& in, tx.vin) + for(const auto& in : tx.vin) { uint64_t this_amount = get_amount_from_variant(in); if (!this_amount) @@ -1328,12 +1657,11 @@ namespace currency //--------------------------------------------------------------- bool check_inputs_types_supported(const transaction& tx) { - BOOST_FOREACH(const auto& in, tx.vin) + for(const auto& in : tx.vin) { - CHECK_AND_ASSERT_MES(in.type() == typeid(txin_to_key) || in.type() == typeid(txin_multisig), false, "wrong variant type: " - << in.type().name() << ", expected " << typeid(txin_to_key).name() + CHECK_AND_ASSERT_MES(in.type() == typeid(txin_to_key) || in.type() == typeid(txin_multisig) || in.type() == typeid(txin_htlc), false, "wrong variant type: " + << in.type().name() << ", in transaction id=" << get_transaction_hash(tx)); - } return true; } @@ -1396,14 +1724,22 @@ namespace currency //----------------------------------------------------------------------------------------------- bool check_outs_valid(const transaction& tx) { - BOOST_FOREACH(const tx_out& out, tx.vout) + for(const tx_out& out : tx.vout) { - CHECK_AND_NO_ASSERT_MES(0 < out.amount, false, "zero amount ouput in transaction id=" << get_transaction_hash(tx)); + CHECK_AND_NO_ASSERT_MES(0 < out.amount, false, "zero amount output in transaction id=" << get_transaction_hash(tx)); if (out.target.type() == typeid(txout_to_key)) { if (!check_key(boost::get(out.target).key)) return false; } + else if (out.target.type() == typeid(txout_htlc)) + { + const txout_htlc& htlc = boost::get(out.target); + if (!check_key(htlc.pkey_redeem)) + return false; + if (!check_key(htlc.pkey_refund)) + return false; + } else if (out.target.type() == typeid(txout_multisig)) { const txout_multisig& ms = boost::get(out.target); @@ -1440,8 +1776,13 @@ namespace currency } else if (in.type() == typeid(txin_multisig)) { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_multisig, tokey_in, false); - this_amount = tokey_in.amount; + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_multisig, ms_in, false); + this_amount = ms_in.amount; + } + else if (in.type() == typeid(txin_htlc)) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_htlc, htlc_in, false); + this_amount = htlc_in.amount; } else { @@ -1487,14 +1828,16 @@ namespace currency bool is_out_to_acc(const account_keys& acc, const txout_to_key& out_key, const crypto::key_derivation& derivation, size_t output_index) { crypto::public_key pk; - derive_public_key(derivation, output_index, acc.m_account_address.m_spend_public_key, pk); + if (!derive_public_key(derivation, output_index, acc.account_address.spend_public_key, pk)) + return false; return pk == out_key.key; } //--------------------------------------------------------------- bool is_out_to_acc(const account_keys& acc, const txout_multisig& out_multisig, const crypto::key_derivation& derivation, size_t output_index) { crypto::public_key pk; - derive_public_key(derivation, output_index, acc.m_account_address.m_spend_public_key, pk); + if (!derive_public_key(derivation, output_index, acc.account_address.spend_public_key, pk)) + return false; auto it = std::find(out_multisig.keys.begin(), out_multisig.keys.end(), pk); if (out_multisig.keys.end() == it) return false; @@ -1512,14 +1855,19 @@ namespace currency bool check_tx_derivation_hint(const transaction& tx, const crypto::key_derivation& derivation) { bool found_der_xor = false; - uint16_t my_derive_xor = get_derivation_xor(derivation); + uint16_t hint = get_derivation_hint(derivation); + tx_derivation_hint dh = make_tx_derivation_hint_from_uint16(hint); for (auto& e : tx.extra) { - if (e.type() == typeid(etc_tx_derivation_hint)) + if (e.type() == typeid(tx_derivation_hint)) { - found_der_xor = true; - if (my_derive_xor == boost::get(e).v) - return true; + const tx_derivation_hint& tdh = boost::get(e); + if (tdh.msg.size() == sizeof(uint16_t)) + { + found_der_xor = true; + if (dh.msg == tdh.msg) + return true; + } } } //if tx doesn't have any hints - feature is not supported, use full scan @@ -1532,7 +1880,7 @@ namespace currency bool lookup_acc_outs_genesis(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered, crypto::key_derivation& derivation) { uint64_t offset = 0; - bool r = get_account_genesis_offset_by_address(get_account_address_as_str(acc.m_account_address), offset); + bool r = get_account_genesis_offset_by_address(get_account_address_as_str(acc.account_address), offset); if (!r) return true; @@ -1548,9 +1896,16 @@ namespace currency } //--------------------------------------------------------------- bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered, crypto::key_derivation& derivation) + { + std::list htlc_info_list; + return lookup_acc_outs(acc, tx, tx_pub_key, outs, money_transfered, derivation, htlc_info_list); + } + //--------------------------------------------------------------- + bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered, crypto::key_derivation& derivation, std::list& htlc_info_list) { money_transfered = 0; - generate_key_derivation(tx_pub_key, acc.m_view_secret_key, derivation); + bool r = generate_key_derivation(tx_pub_key, acc.view_secret_key, derivation); + CHECK_AND_ASSERT_MES(r, false, "unable to generate derivation from tx_pub = " << tx_pub_key << " * view_sec, invalid tx_pub?"); if (is_coinbase(tx) && get_block_height(tx) == 0 && tx_pub_key == ggenesis_tx_pub_key) { @@ -1558,12 +1913,11 @@ namespace currency return lookup_acc_outs_genesis(acc, tx, tx_pub_key, outs, money_transfered, derivation); } - if (!check_tx_derivation_hint(tx, derivation)) return true; size_t i = 0; - BOOST_FOREACH(const tx_out& o, tx.vout) + for(const tx_out& o : tx.vout) { if (o.target.type() == typeid(txout_to_key)) { @@ -1581,6 +1935,23 @@ namespace currency //don't count this money } } + else if (o.target.type() == typeid(txout_htlc)) + { + htlc_info hi = AUTO_VAL_INIT(hi); + const txout_htlc& htlc = boost::get(o.target); + if (is_out_to_acc(acc, htlc.pkey_redeem, derivation, i)) + { + hi.hltc_our_out_is_before_expiration = true; + htlc_info_list.push_back(hi); + outs.push_back(i); + } + else if (is_out_to_acc(acc, htlc.pkey_refund, derivation, i)) + { + hi.hltc_our_out_is_before_expiration = false; + htlc_info_list.push_back(hi); + outs.push_back(i); + } + } else { LOG_ERROR("Wrong type at lookup_acc_outs, unexpected type is: " << o.target.type().name()); @@ -1646,12 +2017,29 @@ namespace currency bool r = currency::parse_and_validate_block_from_blob(bl_entry.block, blextin_ptr->bl); bdde.block_ptr = blextin_ptr; CHECK_AND_ASSERT_MES(r, false, "failed to parse block from blob: " << string_tools::buff_to_hex_nodelimer(bl_entry.block)); + size_t i = 0; + if (bl_entry.tx_global_outs.size()) + { + CHECK_AND_ASSERT_MES(bl_entry.tx_global_outs.size() == bl_entry.txs.size(), false, "tx_global_outs count " << bl_entry.tx_global_outs.size() << " count missmatch with bl_entry.txs count " << bl_entry.txs.size()); + } + if (bl_entry.coinbase_global_outs.size()) + { + std::shared_ptr tche_ptr(new currency::transaction_chain_entry()); + tche_ptr->m_global_output_indexes = bl_entry.coinbase_global_outs; + bdde.coinbase_ptr = tche_ptr; + } for (const auto& tx_blob : bl_entry.txs) { std::shared_ptr tche_ptr(new currency::transaction_chain_entry()); r = parse_and_validate_tx_from_blob(tx_blob, tche_ptr->tx); CHECK_AND_ASSERT_MES(r, false, "failed to parse tx from blob: " << string_tools::buff_to_hex_nodelimer(tx_blob)); bdde.txs_ptr.push_back(tche_ptr); + if (bl_entry.tx_global_outs.size()) + { + CHECK_AND_ASSERT_MES(bl_entry.tx_global_outs[i].v.size() == tche_ptr->tx.vout.size(), false, "tx_global_outs for tx" << bl_entry.tx_global_outs[i].v.size() << " count missmatch with tche_ptr->tx.vout.size() count " << tche_ptr->tx.vout.size()); + tche_ptr->m_global_output_indexes = bl_entry.tx_global_outs[i].v; + } + i++; } } return true; @@ -1694,7 +2082,13 @@ namespace currency } return true; } - + //------------------------------------------------------------------ + bool validate_password(const std::string& password) + { + static const std::string allowed_password_symbols = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz~!?@#$%^&*_+|{}[]()<>:;\"'-=\\/.,"; + size_t n = password.find_first_not_of(allowed_password_symbols, 0); + return n == std::string::npos; + } //------------------------------------------------------------------ #define ANTI_OVERFLOW_AMOUNT 1000000 @@ -1828,9 +2222,7 @@ namespace currency //for future forks std::cout << "Currency name: \t\t" << CURRENCY_NAME << "(" << CURRENCY_NAME_SHORT << ")" << std::endl; - std::cout << "Money supply: \t\t" << print_money(TOTAL_MONEY_SUPPLY) << " coins" - << "(" << print_money(TOTAL_MONEY_SUPPLY) << "), dev bounties is ???" << std::endl; - + std::cout << "Money supply: \t\t " << CURRENCY_BLOCK_REWARD * CURRENCY_BLOCKS_PER_DAY * 365 << " coins per year" << std::endl; std::cout << "PoS block interval: \t" << DIFFICULTY_POS_TARGET << " seconds" << std::endl; std::cout << "PoW block interval: \t" << DIFFICULTY_POW_TARGET << " seconds" << std::endl; std::cout << "Total blocks per day: \t" << CURRENCY_BLOCKS_PER_DAY << " seconds" << std::endl; @@ -1865,19 +2257,22 @@ namespace currency std::string genesis_coinbase_tx_hex = ""; #endif + //genesis proof phrase: "Liverpool beat Barcelona: Greatest Champions League comebacks of all time" + //taken from: https://www.bbc.com/sport/football/48163330 + //sha3-256 from proof phrase: a074236b1354901d5dbc029c0ac4c05c948182c34f3030f32b0c93aee7ba275c (included in genesis block) + + blobdata tx_bl((const char*)&ggenesis_tx_raw, sizeof(ggenesis_tx_raw)); //string_tools::parse_hexstr_to_binbuff(genesis_coinbase_tx_hex, tx_bl); bool r = parse_and_validate_tx_from_blob(tx_bl, bl.miner_tx); CHECK_AND_ASSERT_MES(r, false, "failed to parse coinbase tx from hard coded blob"); - bl.major_version = CURRENT_BLOCK_MAJOR_VERSION; - bl.minor_version = CURRENT_BLOCK_MINOR_VERSION; + bl.major_version = BLOCK_MAJOR_VERSION_GENESIS; + bl.minor_version = BLOCK_MINOR_VERSION_GENESIS; bl.timestamp = 0; - bl.nonce = 101010121 + CURRENCY_FORMATION_VERSION; //bender's nightmare - //miner::find_nonce_for_given_block(bl, 1, 0,); + bl.nonce = CURRENCY_GENESIS_NONCE; LOG_PRINT_GREEN("Generated genesis: " << get_block_hash(bl), LOG_LEVEL_0); return true; } - //--------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- const crypto::hash& get_genesis_hash(bool need_to_set, const crypto::hash& h) { @@ -1898,11 +2293,11 @@ namespace currency return genesis_id; } //--------------------------------------------------------------- - std::vector relative_output_offsets_to_absolute(const std::vector& off) + std::vector relative_output_offsets_to_absolute(const std::vector& off) { //if array has both types of outs, then global index (uint64_t) should be first, and then the rest could be out_by_id - std::vector res = off; + std::vector res = off; for (size_t i = 1; i < res.size(); i++) { if (res[i].type() == typeid(ref_by_id)) @@ -1913,13 +2308,13 @@ namespace currency return res; } //--------------------------------------------------------------- - std::vector absolute_output_offsets_to_relative(const std::vector& off) + std::vector absolute_output_offsets_to_relative(const std::vector& off) { - std::vector res = off; + std::vector res = off; if (off.size() < 2) return res; - std::sort(res.begin(), res.end(), [](const txout_v& lft, const txout_v& rght) + std::sort(res.begin(), res.end(), [](const txout_ref_v& lft, const txout_ref_v& rght) { if (lft.type() == typeid(uint64_t)) { @@ -1974,14 +2369,14 @@ namespace currency //--------------------------------------------------------------- bool is_showing_sender_addres(const transaction& tx) { - return have_type_in_variant_container(tx.attachment); + return have_type_in_variant_container(tx.attachment) || have_type_in_variant_container(tx.attachment); } //--------------------------------------------------------------- - bool is_mixin_tx(const transaction& tx) + bool does_tx_have_only_mixin_inputs(const transaction& tx) { for (const auto& e : tx.vin) { - if (e.type() != typeid(txin_to_key)) + if (e.type() != typeid(txin_to_key) || e.type() != typeid(txin_multisig) || e.type() != typeid(txin_htlc)) return false; if (boost::get(e).key_offsets.size() < 2) return false; @@ -2008,8 +2403,8 @@ namespace currency //--------------------------------------------------------------- bool get_aliases_reward_account(account_public_address& acc) { - bool r = string_tools::parse_tpod_from_hex_string(ALIAS_REWARDS_ACCOUNT_SPEND_PUB_KEY, acc.m_spend_public_key); - r &= string_tools::parse_tpod_from_hex_string(ALIAS_REWARDS_ACCOUNT_VIEW_PUB_KEY, acc.m_view_public_key); + bool r = string_tools::parse_tpod_from_hex_string(ALIAS_REWARDS_ACCOUNT_SPEND_PUB_KEY, acc.spend_public_key); + r &= string_tools::parse_tpod_from_hex_string(ALIAS_REWARDS_ACCOUNT_VIEW_PUB_KEY, acc.view_public_key); return r; } //------------------------------------------------------------------ @@ -2078,6 +2473,20 @@ namespace currency return true; } + bool operator()(const etc_tx_details_unlock_time2& ee) + { + tv.type = "unlock_time"; + std::stringstream ss; + ss << "["; + for (auto v : ee.unlock_time_array) + { + ss << " " << v; + } + ss << "]"; + tv.short_view = ss.str(); + + return true; + } bool operator()(const etc_tx_details_expiration_time& ee) { tv.type = "expiration_time"; @@ -2115,6 +2524,10 @@ namespace currency return true; } + bool operator()(const extra_alias_entry_old& ee) + { + return operator()(static_cast(ee)); + } bool operator()(const extra_user_data& ee) { tv.type = "user_data"; @@ -2148,7 +2561,13 @@ namespace currency return true; } + bool operator()(const tx_payer_old&) + { + tv.type = "payer_old"; + tv.short_view = "(encrypted)"; + return true; + } bool operator()(const tx_receiver& ee) { //const tx_payer& ee = boost::get(extra); @@ -2157,9 +2576,16 @@ namespace currency return true; } - bool operator()(const tx_message& ee) + bool operator()(const tx_receiver_old& ee) { - tv.type = "message"; + tv.type = "receiver_old"; + tv.short_view = "(encrypted)"; + + return true; + } + bool operator()(const tx_derivation_hint& ee) + { + tv.type = "derivation_hint"; tv.short_view = std::to_string(ee.msg.size()) + " bytes"; tv.datails_view = epee::string_tools::buff_to_hex_nodelimer(ee.msg); @@ -2173,18 +2599,15 @@ namespace currency return true; } - bool operator()(const etc_tx_derivation_hint& dh) + bool operator()(const etc_tx_flags16_t& dh) { - tv.type = "XOR"; + tv.type = "FLAGS16"; tv.short_view = epee::string_tools::pod_to_hex(dh); tv.datails_view = epee::string_tools::pod_to_hex(dh); return true; } - }; - - //------------------------------------------------------------------ template bool fill_tx_rpc_payload_items(std::vector& target_vector, const t_container& tc) @@ -2229,6 +2652,12 @@ namespace currency } tei.outs.back().minimum_sigs = otm.minimum_sigs; } + else if (out.target.type() == typeid(txout_htlc)) + { + const txout_htlc& otk = boost::get(out.target); + tei.outs.back().pub_keys.push_back(epee::string_tools::pod_to_hex(otk.pkey_redeem) + "(htlc_pkey_redeem)"); + tei.outs.back().pub_keys.push_back(epee::string_tools::pod_to_hex(otk.pkey_refund) + "(htlc_pkey_refund)"); + } ++i; } @@ -2245,12 +2674,13 @@ namespace currency { tei.ins.back().amount = 0; } - else if (in.type() == typeid(txin_to_key)) + else if (in.type() == typeid(txin_to_key) || in.type() == typeid(txin_htlc)) { - txin_to_key& tk = boost::get(in); + //TODO: add htlc info + const txin_to_key& tk = get_to_key_input_from_txin_v(in); tei.ins.back().amount = tk.amount; tei.ins.back().kimage_or_ms_id = epee::string_tools::pod_to_hex(tk.k_image); - std::vector absolute_offsets = relative_output_offsets_to_absolute(tk.key_offsets); + std::vector absolute_offsets = relative_output_offsets_to_absolute(tk.key_offsets); for (auto& ao : absolute_offsets) { tei.ins.back().global_indexes.push_back(0); @@ -2264,6 +2694,10 @@ namespace currency tei.ins.back().global_indexes.back() = std::numeric_limits::max(); } } + if (in.type() == typeid(txin_htlc)) + { + tei.ins.back().htlc_origin = epee::string_tools::buff_to_hex_nodelimer(boost::get(in).hltc_origin); + } //tk.etc_details -> visualize it may be later } else if (in.type() == typeid(txin_multisig)) @@ -2332,7 +2766,7 @@ namespace currency { for (size_t n = 0; n < tx.vout.size(); ++n) { - if (tx.vout[n].target.type() == typeid(txout_to_key)) + if (tx.vout[n].target.type() == typeid(txout_to_key) || tx.vout[n].target.type() == typeid(txout_htlc)) { uint64_t amount = tx.vout[n].amount; gindices[amount] += 1; @@ -2367,11 +2801,6 @@ namespace currency return CURRENCY_MAX_BLOCK_SIZE; } //----------------------------------------------------------------------------------------------- - size_t get_max_tx_size() - { - return CURRENCY_MAX_TRANSACTION_BLOB_SIZE; - } - //----------------------------------------------------------------------------------------------- uint64_t get_base_block_reward(bool is_pos, const boost::multiprecision::uint128_t& already_generated_coins, uint64_t height) { if (!height) @@ -2416,7 +2845,7 @@ namespace currency div128_32(product_hi, product_lo, static_cast(median_size), &reward_hi, &reward_lo); div128_32(reward_hi, reward_lo, static_cast(median_size), &reward_hi, &reward_lo); CHECK_AND_ASSERT_MES(0 == reward_hi, false, "0 == reward_hi"); - CHECK_AND_ASSERT_MES(reward_lo < base_reward, false, "reward_lo < base_reward, reward: " << reward << ", base_reward: " << base_reward << ", current_block_size: " << current_block_size << ", median_size: " << median_size); + CHECK_AND_ASSERT_MES(reward_lo < base_reward, false, "reward_lo < base_reward, reward_lo: " << reward_lo << ", base_reward: " << base_reward << ", current_block_size: " << current_block_size << ", median_size: " << median_size); reward = reward_lo; return true; @@ -2443,19 +2872,31 @@ namespace currency return true; } //----------------------------------------------------------------------- - bool is_payment_id_size_ok(const std::string& payment_id) + bool is_payment_id_size_ok(const payment_id_t& payment_id) { return payment_id.size() <= BC_PAYMENT_ID_SERVICE_SIZE_MAX; } //----------------------------------------------------------------------- std::string get_account_address_as_str(const account_public_address& addr) { - return tools::base58::encode_addr(CURRENCY_PUBLIC_ADDRESS_BASE58_PREFIX, t_serializable_object_to_blob(addr)); + if (addr.flags == 0) + return tools::base58::encode_addr(CURRENCY_PUBLIC_ADDRESS_BASE58_PREFIX, t_serializable_object_to_blob(addr.to_old())); // classic Zano address + + if (addr.flags & ACCOUNT_PUBLIC_ADDRESS_FLAG_AUDITABLE) + return tools::base58::encode_addr(CURRENCY_PUBLIC_AUDITABLE_ADDRESS_BASE58_PREFIX, t_serializable_object_to_blob(addr)); // new format Zano address (auditable) + + return tools::base58::encode_addr(CURRENCY_PUBLIC_ADDRESS_BASE58_PREFIX, t_serializable_object_to_blob(addr)); // new format Zano address (normal) } //----------------------------------------------------------------------- - std::string get_account_address_and_payment_id_as_str(const account_public_address& addr, const std::string& payment_id) + std::string get_account_address_and_payment_id_as_str(const account_public_address& addr, const payment_id_t& payment_id) { - return tools::base58::encode_addr(CURRENCY_PUBLIC_INTEG_ADDRESS_BASE58_PREFIX, t_serializable_object_to_blob(addr) + payment_id); + if (addr.flags == 0) + return tools::base58::encode_addr(CURRENCY_PUBLIC_INTEG_ADDRESS_BASE58_PREFIX, t_serializable_object_to_blob(addr.to_old()) + payment_id); // classic integrated Zano address + + if (addr.flags & ACCOUNT_PUBLIC_ADDRESS_FLAG_AUDITABLE) + return tools::base58::encode_addr(CURRENCY_PUBLIC_AUDITABLE_INTEG_ADDRESS_BASE58_PREFIX, t_serializable_object_to_blob(addr) + payment_id); // new format integrated Zano address (auditable) + + return tools::base58::encode_addr(CURRENCY_PUBLIC_INTEG_ADDRESS_V2_BASE58_PREFIX, t_serializable_object_to_blob(addr) + payment_id); // new format integrated Zano address (normal) } //----------------------------------------------------------------------- bool get_account_address_from_str(account_public_address& addr, const std::string& str) @@ -2464,9 +2905,9 @@ namespace currency return get_account_address_and_payment_id_from_str(addr, integrated_payment_id, str); } //----------------------------------------------------------------------- - bool get_account_address_and_payment_id_from_str(account_public_address& addr, std::string& payment_id, const std::string& str) + bool get_account_address_and_payment_id_from_str(account_public_address& addr, payment_id_t& payment_id, const std::string& str) { - static const size_t addr_blob_size = sizeof(account_public_address); + payment_id.clear(); blobdata blob; uint64_t prefix; if (!tools::base58::decode_addr(str, prefix, blob)) @@ -2475,56 +2916,97 @@ namespace currency return false; } - if (blob.size() < addr_blob_size) + if (blob.size() < sizeof(account_public_address_old)) { - LOG_PRINT_L1("Address " << str << " has invalid format: blob size is " << blob.size() << " which is less, than expected " << addr_blob_size); + LOG_PRINT_L1("Address " << str << " has invalid format: blob size is " << blob.size() << " which is less, than expected " << sizeof(account_public_address_old)); return false; } - if (blob.size() > addr_blob_size + BC_PAYMENT_ID_SERVICE_SIZE_MAX) + if (blob.size() > sizeof(account_public_address) + BC_PAYMENT_ID_SERVICE_SIZE_MAX) { - LOG_PRINT_L1("Address " << str << " has invalid format: blob size is " << blob.size() << " which is more, than allowed " << addr_blob_size + BC_PAYMENT_ID_SERVICE_SIZE_MAX); + LOG_PRINT_L1("Address " << str << " has invalid format: blob size is " << blob.size() << " which is more, than allowed " << sizeof(account_public_address) + BC_PAYMENT_ID_SERVICE_SIZE_MAX); return false; } + bool parse_as_old_format = false; + if (prefix == CURRENCY_PUBLIC_ADDRESS_BASE58_PREFIX) { - // nothing + // normal address + if (blob.size() == sizeof(account_public_address_old)) + { + parse_as_old_format = true; + } + else if (blob.size() == sizeof(account_public_address)) + { + parse_as_old_format = false; + } + else + { + LOG_PRINT_L1("Account public address cannot be parsed from \"" << str << "\", incorrect size"); + return false; + } + } + else if (prefix == CURRENCY_PUBLIC_AUDITABLE_ADDRESS_BASE58_PREFIX) + { + // auditable, parse as new format + parse_as_old_format = false; } else if (prefix == CURRENCY_PUBLIC_INTEG_ADDRESS_BASE58_PREFIX) { - payment_id = blob.substr(addr_blob_size); - blob = blob.substr(0, addr_blob_size); + payment_id = blob.substr(sizeof(account_public_address_old)); + blob = blob.substr(0, sizeof(account_public_address_old)); + parse_as_old_format = true; + } + else if (prefix == CURRENCY_PUBLIC_AUDITABLE_INTEG_ADDRESS_BASE58_PREFIX || prefix == CURRENCY_PUBLIC_INTEG_ADDRESS_V2_BASE58_PREFIX) + { + payment_id = blob.substr(sizeof(account_public_address)); + blob = blob.substr(0, sizeof(account_public_address)); + parse_as_old_format = false; } else { - LOG_PRINT_L1("Address " << str << " has wrong prefix " << prefix << ", expected " << CURRENCY_PUBLIC_ADDRESS_BASE58_PREFIX << " or " << CURRENCY_PUBLIC_INTEG_ADDRESS_BASE58_PREFIX); + LOG_PRINT_L1("Address " << str << " has wrong prefix " << prefix); return false; } - if (!::serialization::parse_binary(blob, addr)) + if (parse_as_old_format) { - LOG_PRINT_L1("Account public address keys can't be parsed for address \"" << str << "\""); + account_public_address_old addr_old = AUTO_VAL_INIT(addr_old); + if (!::serialization::parse_binary(blob, addr_old)) + { + LOG_PRINT_L1("Account public address (old) cannot be parsed from \"" << str << "\""); + return false; + } + addr = account_public_address::from_old(addr_old); + } + else + { + if (!::serialization::parse_binary(blob, addr)) + { + LOG_PRINT_L1("Account public address cannot be parsed from \"" << str << "\""); + return false; + } + } + + if (payment_id.size() > BC_PAYMENT_ID_SERVICE_SIZE_MAX) + { + LOG_PRINT_L1("Failed to parse address from \"" << str << "\": payment id size exceeded: " << payment_id.size()); return false; } - if (!crypto::check_key(addr.m_spend_public_key) || !crypto::check_key(addr.m_view_public_key)) + if (!crypto::check_key(addr.spend_public_key) || !crypto::check_key(addr.view_public_key)) { - LOG_PRINT_L1("Failed to validate address keys for address \"" << str << "\""); + LOG_PRINT_L1("Failed to validate address keys for public address \"" << str << "\""); return false; } return true; } - //------------------------------------------------------------------ - bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median) + //--------------------------------------------------------------- + bool parse_payment_id_from_hex_str(const std::string& payment_id_str, payment_id_t& payment_id) { - /// tx expiration condition (tx is ok if the following is true) - /// tx_expiration_time - TX_EXPIRATION_MEDIAN_SHIFT > get_last_n_blocks_timestamps_median(TX_EXPIRATION_TIMESTAMP_CHECK_WINDOW) - uint64_t expiration_time = get_tx_expiration_time(tx); - if (expiration_time == 0) - return false; // 0 means it never expires - return expiration_time <= expiration_ts_median + TX_EXPIRATION_MEDIAN_SHIFT; + return epee::string_tools::parse_hexstr_to_binbuff(payment_id_str, payment_id); } //-------------------------------------------------------------------------------- crypto::hash prepare_prefix_hash_for_sign(const transaction& tx, uint64_t in_index, const crypto::hash& tx_id) @@ -2624,6 +3106,7 @@ namespace currency return o << "<" << r.n << ":" << r.tx_id << ">"; } //-------------------------------------------------------------------------------- +#ifndef ANDROID_BUILD const std::locale& utf8_get_conversion_locale() { static std::locale loc = boost::locale::generator().generate("en_US.UTF-8"); @@ -2643,6 +3126,7 @@ namespace currency return true; return utf8_to_lower(s).find(utf8_to_lower(match), 0) != std::string::npos; } +#endif //-------------------------------------------------------------------------------- bool operator ==(const currency::transaction& a, const currency::transaction& b) { return currency::get_transaction_hash(a) == currency::get_transaction_hash(b); @@ -2659,21 +3143,41 @@ namespace currency return false; } + + boost::multiprecision::uint1024_t get_a_to_b_relative_cumulative_difficulty(const wide_difficulty_type& difficulty_pos_at_split_point, + const wide_difficulty_type& difficulty_pow_at_split_point, + const difficulties& a_diff, + const difficulties& b_diff ) + { + static const wide_difficulty_type difficulty_starter = DIFFICULTY_STARTER; + const wide_difficulty_type& a_pos_cumulative_difficulty = a_diff.pos_diff > 0 ? a_diff.pos_diff : difficulty_starter; + const wide_difficulty_type& b_pos_cumulative_difficulty = b_diff.pos_diff > 0 ? b_diff.pos_diff : difficulty_starter; + const wide_difficulty_type& a_pow_cumulative_difficulty = a_diff.pow_diff > 0 ? a_diff.pow_diff : difficulty_starter; + const wide_difficulty_type& b_pow_cumulative_difficulty = b_diff.pow_diff > 0 ? b_diff.pow_diff : difficulty_starter; + + boost::multiprecision::uint1024_t basic_sum = boost::multiprecision::uint1024_t(a_pow_cumulative_difficulty) + (boost::multiprecision::uint1024_t(a_pos_cumulative_difficulty)*difficulty_pow_at_split_point) / difficulty_pos_at_split_point; + boost::multiprecision::uint1024_t res = + (basic_sum * a_pow_cumulative_difficulty * a_pos_cumulative_difficulty) / (boost::multiprecision::uint1024_t(b_pow_cumulative_difficulty)*b_pos_cumulative_difficulty); + +// if (res > boost::math::tools::max_value()) +// { +// ASSERT_MES_AND_THROW("[INTERNAL ERROR]: Failed to get_a_to_b_relative_cumulative_difficulty, res = " << res << ENDL +// << ", difficulty_pos_at_split_point: " << difficulty_pos_at_split_point << ENDL +// << ", difficulty_pow_at_split_point:" << difficulty_pow_at_split_point << ENDL +// << ", a_pos_cumulative_difficulty:" << a_pos_cumulative_difficulty << ENDL +// << ", b_pos_cumulative_difficulty:" << b_pos_cumulative_difficulty << ENDL +// << ", a_pow_cumulative_difficulty:" << a_pow_cumulative_difficulty << ENDL +// << ", b_pow_cumulative_difficulty:" << b_pow_cumulative_difficulty << ENDL +// ); +// } + TRY_ENTRY(); +// wide_difficulty_type short_res = res.convert_to(); + return res; + CATCH_ENTRY_WITH_FORWARDING_EXCEPTION(); + } + + + } // namespace currency -bool parse_hash256(const std::string str_hash, crypto::hash& hash) -{ - std::string buf; - bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf); - if (!res || buf.size() != sizeof(crypto::hash)) - { - std::cout << "invalid hash format: <" << str_hash << '>' << std::endl; - return false; - } - else - { - buf.copy(reinterpret_cast(&hash), sizeof(crypto::hash)); - return true; - } -} diff --git a/src/currency_core/currency_format_utils.h b/src/currency_core/currency_format_utils.h index f8b27bd..6474307 100644 --- a/src/currency_core/currency_format_utils.h +++ b/src/currency_core/currency_format_utils.h @@ -21,13 +21,15 @@ #include "crypto/crypto.h" #include "crypto/hash.h" #include "difficulty.h" -//#include "offers_services_helpers.h" #include "rpc/core_rpc_server_commands_defs.h" #include "bc_payments_id_service.h" #include "bc_attachments_helpers_basic.h" #include "blockchain_storage_basic.h" #include "currency_format_utils_blocks.h" #include "currency_format_utils_transactions.h" +#include "core_runtime_config.h" +#include "wallet/wallet_public_structs_defs.h" + // ------ get_tx_type_definition ------------- #define GUI_TX_TYPE_NORMAL 0 @@ -43,9 +45,8 @@ #define GUI_TX_TYPE_ESCROW_RELEASE_BURN 10 #define GUI_TX_TYPE_ESCROW_CANCEL_PROPOSAL 11 #define GUI_TX_TYPE_ESCROW_RELEASE_CANCEL 12 - - - +#define GUI_TX_TYPE_HTLC_DEPOSIT 13 +#define GUI_TX_TYPE_HTLC_REDEEM 14 @@ -57,36 +58,8 @@ namespace currency bool operator ==(const currency::extra_attachment_info& a, const currency::extra_attachment_info& b); - typedef boost::multiprecision::uint128_t uint128_tl; - struct tx_source_entry - { - typedef std::pair output_entry; // txout_v is either global output index or ref_by_id; public_key - is output ephemeral pub key - std::vector outputs; //index + key - uint64_t real_output; //index in outputs vector of real output_entry - crypto::public_key real_out_tx_key; //real output's transaction's public key - size_t real_output_in_tx_index; //index in transaction outputs vector - uint64_t amount; //money - crypto::hash multisig_id; //if txin_multisig: multisig output id - size_t ms_sigs_count; //if txin_multisig: must be equal to output's minimum_sigs - size_t ms_keys_count; //if txin_multisig: must be equal to size of output's keys container - bool separately_signed_tx_complete; //for separately signed tx only: denotes the last source entry in complete tx to explicitly mark the final step of tx creation - - bool is_multisig() const { return ms_sigs_count > 0; } - }; - - struct tx_destination_entry - { - uint64_t amount; //money - std::list addr; //destination address, in case of 1 address - txout_to_key, in case of more - txout_multisig - size_t minimum_sigs; // if txout_multisig: minimum signatures that are required to spend this output (minimum_sigs <= addr.size()) IF txout_to_key - not used - uint64_t amount_to_provide; //amount money that provided by initial creator of tx, used with partially created transactions - - tx_destination_entry() : amount(0), minimum_sigs(0), amount_to_provide(0){} - tx_destination_entry(uint64_t a, const account_public_address& ad) : amount(a), addr(1, ad), minimum_sigs(0), amount_to_provide(0){} - tx_destination_entry(uint64_t a, const std::list& addr) : amount(a), addr(addr), minimum_sigs(addr.size()), amount_to_provide(0){} - }; struct tx_extra_info { @@ -162,6 +135,62 @@ namespace currency END_KV_SERIALIZE_MAP() }; + struct htlc_info + { + bool hltc_our_out_is_before_expiration; + }; + + + struct finalize_tx_param + { + uint64_t unlock_time; + std::vector extra; + std::vector attachments; + currency::account_public_address crypt_address; + uint8_t tx_outs_attr; + bool shuffle; + uint8_t flags; + crypto::hash multisig_id; + std::vector sources; + std::vector selected_transfers; + std::vector prepared_destinations; + uint64_t expiration_time; + crypto::public_key spend_pub_key; // only for validations + + BEGIN_SERIALIZE_OBJECT() + FIELD(unlock_time) + FIELD(extra) + FIELD(attachments) + FIELD(crypt_address) + FIELD(tx_outs_attr) + FIELD(shuffle) + FIELD(flags) + FIELD(multisig_id) + FIELD(sources) + FIELD(selected_transfers) + FIELD(prepared_destinations) + FIELD(expiration_time) + FIELD(spend_pub_key) + END_SERIALIZE() + }; + + struct finalized_tx + { + currency::transaction tx; + crypto::secret_key one_time_key; + finalize_tx_param ftp; + std::string htlc_origin; + std::vector> outs_key_images; // pairs (out_index, key_image) for each change output + + BEGIN_SERIALIZE_OBJECT() + FIELD(tx) + FIELD(one_time_key) + FIELD(ftp) + FIELD(htlc_origin) + FIELD(outs_key_images) + END_SERIALIZE() + }; + //--------------------------------------------------------------- bool construct_miner_tx(size_t height, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, @@ -187,35 +216,11 @@ namespace currency //--------------------------------------------------------------- - template - uint64_t get_tx_x_detail(const transaction& tx) - { - extra_type_t e = AUTO_VAL_INIT(e); - get_type_in_variant_container(tx.extra, e); - return e.v; - } - template - void set_tx_x_detail(transaction& tx, uint64_t v) - { - extra_type_t e = AUTO_VAL_INIT(e); - e.v = v; - update_or_add_field_to_extra(tx.extra, e); - } - - inline uint64_t get_tx_unlock_time(const transaction& tx){ return get_tx_x_detail(tx);} - inline uint64_t get_tx_flags(const transaction& tx){ return get_tx_x_detail(tx); } - inline uint64_t get_tx_expiration_time(const transaction& tx){ return get_tx_x_detail(tx); } - inline void set_tx_unlock_time(transaction& tx, uint64_t v){ set_tx_x_detail(tx, v); } - inline void set_tx_flags(transaction& tx, uint64_t v){ set_tx_x_detail(tx, v); } - inline void set_tx_expiration_time(transaction& tx, uint64_t v){ set_tx_x_detail(tx, v); } - account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations); - - bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median); - - uint64_t get_string_uint64_hash(const std::string& str); - bool construct_tx_out(const tx_destination_entry& de, const crypto::secret_key& tx_sec_key, size_t output_index, transaction& tx, std::set& deriv_cache, uint8_t tx_outs_attr = CURRENCY_TO_KEY_OUT_RELAXED); + bool construct_tx_out(const tx_destination_entry& de, const crypto::secret_key& tx_sec_key, size_t output_index, transaction& tx, std::set& deriv_cache, const account_keys& self, finalized_tx& result, uint8_t tx_outs_attr = CURRENCY_TO_KEY_OUT_RELAXED); + bool construct_tx_out(const tx_destination_entry& de, const crypto::secret_key& tx_sec_key, size_t output_index, transaction& tx, std::set& deriv_cache, const account_keys& self, uint8_t tx_outs_attr = CURRENCY_TO_KEY_OUT_RELAXED); bool validate_alias_name(const std::string& al); + bool validate_password(const std::string& password); void get_attachment_extra_info_details(const std::vector& attachment, extra_attachment_info& eai); bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, @@ -236,6 +241,7 @@ namespace currency uint8_t tx_outs_attr = CURRENCY_TO_KEY_OUT_RELAXED, bool shuffle = true, uint64_t flags = 0); + bool construct_tx(const account_keys& sender_account_keys, const std::vector& sources, const std::vector& destinations, @@ -250,6 +256,9 @@ namespace currency bool shuffle = true, uint64_t flags = 0); + bool construct_tx(const account_keys& sender_account_keys, const finalize_tx_param& param, finalized_tx& result); + + bool sign_multisig_input_in_tx(currency::transaction& tx, size_t ms_input_index, const currency::account_keys& keys, const currency::transaction& source_tx, bool *p_is_input_fully_signed = nullptr); bool sign_extra_alias_entry(extra_alias_entry& ai, const crypto::public_key& pkey, const crypto::secret_key& skey); @@ -265,6 +274,7 @@ namespace currency bool is_out_to_acc(const account_keys& acc, const txout_to_key& out_key, const crypto::key_derivation& derivation, size_t output_index); bool is_out_to_acc(const account_keys& acc, const txout_multisig& out_multisig, const crypto::key_derivation& derivation, size_t output_index); bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered, crypto::key_derivation& derivation); + bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector& outs, uint64_t& money_transfered, crypto::key_derivation& derivation, std::list& htlc_info_list); bool lookup_acc_outs(const account_keys& acc, const transaction& tx, std::vector& outs, uint64_t& money_transfered, crypto::key_derivation& derivation); bool get_tx_fee(const transaction& tx, uint64_t & fee); uint64_t get_tx_fee(const transaction& tx); @@ -275,10 +285,13 @@ namespace currency std::string short_hash_str(const crypto::hash& h); bool is_mixattr_applicable_for_fake_outs_counter(uint8_t mix_attr, uint64_t fake_attr_count); bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t current_blockchain_size, uint64_t current_time); + crypto::key_derivation get_encryption_key_derivation(bool is_income, const transaction& tx, const account_keys& acc_keys); bool decrypt_payload_items(bool is_income, const transaction& tx, const account_keys& acc_keys, std::vector& decrypted_items); void encrypt_attachments(transaction& tx, const account_keys& sender_keys, const account_public_address& destination_addr, const keypair& tx_random_key); bool is_derivation_used_to_encrypt(const transaction& tx, const crypto::key_derivation& derivation); + void load_wallet_transfer_info_flags(tools::wallet_public::wallet_transfer_info& x); uint64_t get_tx_type(const transaction& tx); + uint64_t get_tx_type_ex(const transaction& tx, tx_out& htlc_out, txin_htlc& htlc_in); size_t get_multisig_out_index(const std::vector& outs); size_t get_multisig_in_index(const std::vector& inputs); @@ -294,6 +307,8 @@ namespace currency bool check_inputs_types_supported(const transaction& tx); bool check_outs_valid(const transaction& tx); bool parse_amount(uint64_t& amount, const std::string& str_amount); + bool parse_tracking_seed(const std::string& tracking_seed, account_public_address& address, crypto::secret_key& view_sec_key, uint64_t& creation_timestamp); + bool unserialize_block_complete_entry(const COMMAND_RPC_GET_BLOCKS_FAST::response& serialized, @@ -308,8 +323,8 @@ namespace currency bool check_inputs_overflow(const transaction& tx); uint64_t get_block_height(const transaction& coinbase); uint64_t get_block_height(const block& b); - std::vector relative_output_offsets_to_absolute(const std::vector& off); - std::vector absolute_output_offsets_to_relative(const std::vector& off); + std::vector relative_output_offsets_to_absolute(const std::vector& off); + std::vector absolute_output_offsets_to_relative(const std::vector& off); // prints amount in format "3.14", "0.0" std::string print_money_brief(uint64_t amount); @@ -320,7 +335,7 @@ namespace currency bool set_payment_id_to_tx(std::vector& att, const std::string& payment_id); bool add_padding_to_tx(transaction& tx, size_t count); bool is_service_tx(const transaction& tx); - bool is_mixin_tx(const transaction& tx); + bool does_tx_have_only_mixin_inputs(const transaction& tx); bool is_showing_sender_addres(const transaction& tx); uint64_t get_amount_for_zero_pubkeys(const transaction& tx); //std::string get_comment_from_tx(const transaction& tx); @@ -344,9 +359,9 @@ namespace currency bool fill_tx_rpc_details(tx_rpc_extended_info& tei, const transaction& tx, const transaction_chain_entry* ptce, const crypto::hash& h, uint64_t timestamp, bool is_short = false); bool fill_block_rpc_details(block_rpc_extended_info& pei_rpc, const block_extended_info& bei_chain, const crypto::hash& h); void append_per_block_increments_for_tx(const transaction& tx, std::unordered_map& gindices); - std::string get_word_from_timstamp(uint64_t timestamp); - uint64_t get_timstamp_from_word(std::string word); - + std::string get_word_from_timstamp(uint64_t timestamp, bool use_password); + uint64_t get_timstamp_from_word(std::string word, bool& password_used); + std::string generate_origin_for_htlc(const txout_htlc& htlc, const account_keys& acc_keys); template typename std::conditional::value, const std::vector, std::vector >::type& get_txin_etc_options(t_txin_v& in) { @@ -357,6 +372,8 @@ namespace currency if (in.type() == typeid(txin_to_key)) return boost::get(in).etc_details; + else if (in.type() == typeid(txin_htlc)) + return boost::get(in).etc_details; else if (in.type() == typeid(txin_multisig)) return boost::get(in).etc_details; else @@ -383,11 +400,12 @@ namespace currency size_t get_max_tx_size(); bool get_block_reward(bool is_pos, size_t median_size, size_t current_block_size, const boost::multiprecision::uint128_t& already_generated_coins, uint64_t &reward, uint64_t height); uint64_t get_base_block_reward(bool is_pos, const boost::multiprecision::uint128_t& already_generated_coins, uint64_t height); - bool is_payment_id_size_ok(const std::string& payment_id); + bool is_payment_id_size_ok(const payment_id_t& payment_id); std::string get_account_address_as_str(const account_public_address& addr); - std::string get_account_address_and_payment_id_as_str(const account_public_address& addr, const std::string& payment_id); + std::string get_account_address_and_payment_id_as_str(const account_public_address& addr, const payment_id_t& payment_id); bool get_account_address_from_str(account_public_address& addr, const std::string& str); - bool get_account_address_and_payment_id_from_str(account_public_address& addr, std::string& payment_id, const std::string& str); + bool get_account_address_and_payment_id_from_str(account_public_address& addr, payment_id_t& payment_id, const std::string& str); + bool parse_payment_id_from_hex_str(const std::string& payment_id_str, payment_id_t& payment_id); bool is_coinbase(const transaction& tx); bool is_coinbase(const transaction& tx, bool& pos_coinbase); bool have_attachment_service_in_container(const std::vector& av, const std::string& service_id, const std::string& instruction); @@ -398,7 +416,7 @@ namespace currency bool is_out_to_acc(const account_keys& acc, const tx_out_t& out_key, const crypto::public_key& tx_pub_key, size_t output_index) { crypto::key_derivation derivation; - generate_key_derivation(tx_pub_key, acc.m_view_secret_key, derivation); + generate_key_derivation(tx_pub_key, acc.view_secret_key, derivation); return is_out_to_acc(acc, out_key, derivation, output_index); } //---------------------------------------------------------------------------------------------------- @@ -485,19 +503,32 @@ namespace currency ai.m_alias = ard.alias; return true; } - - + //--------------------------------------------------------------- + template + add_type_t& get_or_add_field_to_variant_vector(container_type& container) + { + for (auto& ev : container) + { + if (ev.type() == typeid(add_type_t)) + return boost::get(ev); + } + container.push_back(add_type_t()); + return boost::get(container.back()); + } + //--------------------------------------------------------------- template extra_t& get_or_add_field_to_extra(std::vector& extra) { - for (auto& ev : extra) - { - if (ev.type() == typeid(extra_t)) - return boost::get(ev); - } - extra.push_back(extra_t()); - return boost::get(extra.back()); +// for (auto& ev : extra) +// { +// if (ev.type() == typeid(extra_t)) +// return boost::get(ev); +// } +// extra.push_back(extra_t()); +// return boost::get(extra.back()); + return get_or_add_field_to_variant_vector(extra); } + //--------------------------------------------------------------- template void update_or_add_field_to_extra(std::vector& variant_container, const variant_type_t& v) { @@ -511,7 +542,22 @@ namespace currency } variant_container.push_back(v); } - + //--------------------------------------------------------------- + template + void remove_field_of_type_from_extra(std::vector& variant_container) + { + for (size_t i = 0; i != variant_container.size();) + { + if (variant_container[i].type() == typeid(variant_type_t)) + { + variant_container.erase(variant_container.begin()+i); + } + else + { + i++; + } + } + } //--------------------------------------------------------------- template bool get_payment_id_from_tx(const t_container& att, std::string& payment_id) @@ -607,6 +653,7 @@ namespace currency size_t operator()(const txin_gen& /*txin*/) const { return 0; } size_t operator()(const txin_to_key& txin) const { return txin.key_offsets.size(); } size_t operator()(const txin_multisig& txin) const { return txin.sigs_count; } + size_t operator()(const txin_htlc& txin) const { return 1; } }; return boost::apply_visitor(txin_signature_size_visitor(), tx_in); @@ -616,6 +663,8 @@ namespace currency { if (in.type().hash_code() == typeid(txin_to_key).hash_code()) return &boost::get(in).etc_details; + if (in.type().hash_code() == typeid(txin_htlc).hash_code()) + return &boost::get(in).etc_details; if (in.type().hash_code() == typeid(txin_multisig).hash_code()) return &boost::get(in).etc_details; return nullptr; @@ -625,6 +674,8 @@ namespace currency { if (in.type().hash_code() == typeid(txin_to_key).hash_code()) return &boost::get(in).etc_details; + if (in.type().hash_code() == typeid(txin_htlc).hash_code()) + return &boost::get(in).etc_details; if (in.type().hash_code() == typeid(txin_multisig).hash_code()) return &boost::get(in).etc_details; return nullptr; @@ -643,13 +694,70 @@ namespace currency return boost::apply_visitor(input_amount_getter(), v); } //--------------------------------------------------------------- + template + void create_and_add_tx_payer_to_container_from_address(container_t& container, const account_public_address& addr, uint64_t top_block_height, const core_runtime_config& crc) + { + if (top_block_height > crc.hard_fork_02_starts_after_height) + { + // after hardfork 2 + tx_payer result = AUTO_VAL_INIT(result); + result.acc_addr = addr; + container.push_back(result); + } + else + { + // before hardfork 2 -- add only if addr is not auditable + if (!addr.is_auditable()) + { + tx_payer_old result = AUTO_VAL_INIT(result); + result.acc_addr = addr.to_old(); + container.push_back(result); + } + } + } + //--------------------------------------------------------------- + template + void create_and_add_tx_receiver_to_container_from_address(container_t& container, const account_public_address& addr, uint64_t top_block_height, const core_runtime_config& crc) + { + if (top_block_height > crc.hard_fork_02_starts_after_height) + { + // after hardfork 2 + tx_receiver result = AUTO_VAL_INIT(result); + result.acc_addr = addr; + container.push_back(result); + } + else + { + // before hardfork 2 -- add only if addr is not auditable + if (!addr.is_auditable()) + { + tx_receiver_old result = AUTO_VAL_INIT(result); + result.acc_addr = addr.to_old(); + container.push_back(result); + } + } + } + //--------------------------------------------------------------- + //--------------------------------------------------------------- std::ostream& operator <<(std::ostream& o, const ref_by_id& r); //--------------------------------------------------------------- +#ifndef ANDROID_BUILD std::string utf8_to_upper(const std::string& s); std::string utf8_to_lower(const std::string& s); bool utf8_substring_test_case_insensitive(const std::string& match, const std::string& s); // Returns true is 's' contains 'match' (case-insensitive) +#endif + + struct difficulties + { + wide_difficulty_type pos_diff; + wide_difficulty_type pow_diff; + }; + + boost::multiprecision::uint1024_t get_a_to_b_relative_cumulative_difficulty(const wide_difficulty_type& difficulty_pos_at_split_point, + const wide_difficulty_type& difficulty_pow_at_split_point, + const difficulties& a_diff, + const difficulties& b_diff + ); + } // namespace currency - - - diff --git a/src/currency_core/currency_format_utils_abstract.h b/src/currency_core/currency_format_utils_abstract.h index 142740a..451b46e 100644 --- a/src/currency_core/currency_format_utils_abstract.h +++ b/src/currency_core/currency_format_utils_abstract.h @@ -68,12 +68,12 @@ namespace currency return t_unserializable_object_from_blob(b, b_blob); } //--------------------------------------------------------------- - template + template bool have_type_in_variant_container(const variant_t_container& av) { for (auto& ai : av) { - if (ai.type() == typeid(specic_type_t)) + if (ai.type() == typeid(specific_type_t)) { return true; } @@ -81,32 +81,71 @@ namespace currency return false; } //--------------------------------------------------------------- - template + template size_t count_type_in_variant_container(const variant_t_container& av) { size_t result = 0; for (auto& ai : av) { - if (ai.type() == typeid(specic_type_t)) + if (ai.type() == typeid(specific_type_t)) ++result; } return result; } //--------------------------------------------------------------- - template - bool get_type_in_variant_container(const variant_t_container& av, specic_type_t& a) + template + bool get_type_in_variant_container(const variant_t_container& av, specific_type_t& a) { for (auto& ai : av) { - if (ai.type() == typeid(specic_type_t)) + if (ai.type() == typeid(specific_type_t)) { - a = boost::get(ai); + a = boost::get(ai); return true; } } return false; } //--------------------------------------------------------------- + // callback should return true to continue iterating through the container + template + bool handle_2_alternative_types_in_variant_container(const container_t& container, callback_t cb) + { + bool found = false; + for (auto& item : container) + { + if (item.type() == typeid(A)) + { + found = true; + if (!cb(boost::get(item))) + break; + } + else if (item.type() == typeid(B)) + { + found = true; + if (!cb(boost::get(item))) + break; + } + } + return found; + } + inline + const txin_to_key& get_to_key_input_from_txin_v(const txin_v& in_v) + { + if (in_v.type() == typeid(txin_to_key)) + { + return boost::get(in_v); + } + else if (in_v.type() == typeid(txin_htlc)) + { + const txin_htlc& in = boost::get(in_v); + return static_cast(in); + } + else { + ASSERT_MES_AND_THROW("[get_to_key_input_from_txin_v] Wrong type " << in_v.type().name()); + } + } + //--------------------------------------------------------------- template bool check_allowed_types_in_variant_container(const variant_container_t& container, const std::unordered_set& allowed_types, bool elements_must_be_unique = true) { diff --git a/src/currency_core/currency_format_utils_blocks.cpp b/src/currency_core/currency_format_utils_blocks.cpp index b1ec69b..30d7f26 100644 --- a/src/currency_core/currency_format_utils_blocks.cpp +++ b/src/currency_core/currency_format_utils_blocks.cpp @@ -6,6 +6,7 @@ #include "currency_format_utils_blocks.h" #include "serialization/serialization.h" +#include "currency_format_utils.h" #include "currency_format_utils_abstract.h" #include "currency_format_utils_transactions.h" namespace currency diff --git a/src/currency_core/currency_format_utils_transactions.cpp b/src/currency_core/currency_format_utils_transactions.cpp index 533313a..264f604 100644 --- a/src/currency_core/currency_format_utils_transactions.cpp +++ b/src/currency_core/currency_format_utils_transactions.cpp @@ -6,11 +6,127 @@ #include "currency_format_utils_transactions.h" #include "serialization/serialization.h" -#include "currency_format_utils_abstract.h" #include "currency_format_utils.h" +#include "currency_format_utils_abstract.h" namespace currency { + //--------------------------------------------------------------- + account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations) + { + for (const auto& de : destinations) + { + if (de.addr.size() == 1 && sender_account_keys.account_address != de.addr.back()) + return de.addr.back(); // return the first destination address that is non-multisig and not equal to the sender's address + } + return sender_account_keys.account_address; // otherwise, fallback to sender's address + } + //------------------------------------------------------------------ + bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median) + { + /// tx expiration condition (tx is ok if the following is true) + /// tx_expiration_time - TX_EXPIRATION_MEDIAN_SHIFT > get_last_n_blocks_timestamps_median(TX_EXPIRATION_TIMESTAMP_CHECK_WINDOW) + uint64_t expiration_time = get_tx_expiration_time(tx); + if (expiration_time == 0) + return false; // 0 means it never expires + return expiration_time <= expiration_ts_median + TX_EXPIRATION_MEDIAN_SHIFT; + } + //--------------------------------------------------------------- + uint64_t get_burned_amount(const transaction& tx) + { + uint64_t res = 0; + for (auto& o : tx.vout) + { + if (o.target.type() == typeid(txout_to_key)) + { + if (boost::get(o.target).key == null_pkey) + res += o.amount; + } + } + return res; + } + //--------------------------------------------------------------- + uint64_t get_tx_max_unlock_time(const transaction& tx) + { + // etc_tx_details_unlock_time have priority over etc_tx_details_unlock_time2 + uint64_t v = get_tx_x_detail(tx); + if (v) + return v; + + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(tx.extra, ut2); + if (!ut2.unlock_time_array.size()) + return 0; + + uint64_t max_unlock_time = 0; + CHECK_AND_ASSERT_THROW_MES(ut2.unlock_time_array.size() == tx.vout.size(), "unlock_time_array.size=" << ut2.unlock_time_array.size() + << " is not the same as tx.vout.size =" << tx.vout.size() << " in tx: " << get_transaction_hash(tx)); + for (size_t i = 0; i != tx.vout.size(); i++) + { + if (ut2.unlock_time_array[i] > max_unlock_time) + max_unlock_time = ut2.unlock_time_array[i]; + } + + return max_unlock_time; + } + + //--------------------------------------------------------------- + uint64_t get_tx_unlock_time(const transaction& tx, uint64_t o_i) + { + // etc_tx_details_expiration_time have priority over etc_tx_details_expiration_time2 + uint64_t v = get_tx_x_detail(tx); + if (v) + return v; + + CHECK_AND_ASSERT_THROW_MES(tx.vout.size() > o_i, "tx.vout.size=" << tx.vout.size() + << " is not bigger then o_i=" << o_i << " in tx: " << get_transaction_hash(tx)); + + + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(tx.extra, ut2); + if (!ut2.unlock_time_array.size()) + return 0; + + CHECK_AND_ASSERT_THROW_MES(ut2.unlock_time_array.size() > o_i, "unlock_time_array.size=" << ut2.unlock_time_array.size() + << " is less or equal to o_i=" << o_i << " in tx: " << get_transaction_hash(tx)); + + return ut2.unlock_time_array[o_i]; + } + //--------------------------------------------------------------- + bool get_tx_max_min_unlock_time(const transaction& tx, uint64_t& max_unlock_time, uint64_t& min_unlock_time) + { + max_unlock_time = min_unlock_time = 0; + // etc_tx_details_expiration_time have priority over etc_tx_details_expiration_time2 + uint64_t v = get_tx_x_detail(tx); + if (v) + { + max_unlock_time = min_unlock_time = v; + return true; + } + + etc_tx_details_unlock_time2 ut2 = AUTO_VAL_INIT(ut2); + get_type_in_variant_container(tx.extra, ut2); + if (!ut2.unlock_time_array.size()) + { + return true; + } + CHECK_AND_ASSERT_THROW_MES(ut2.unlock_time_array.size() == tx.vout.size(), "unlock_time_array.size=" << ut2.unlock_time_array.size() + << " is not equal tx.vout.size()=" << tx.vout.size() << " in tx: " << get_transaction_hash(tx)); + if (ut2.unlock_time_array.size()) + { + max_unlock_time = min_unlock_time = ut2.unlock_time_array[0]; + for (size_t i = 1; i != ut2.unlock_time_array.size(); i++) + { + if (ut2.unlock_time_array[i] > max_unlock_time) + max_unlock_time = ut2.unlock_time_array[i]; + if (ut2.unlock_time_array[i] < min_unlock_time) + min_unlock_time = ut2.unlock_time_array[i]; + } + } + + + return true; + } //--------------------------------------------------------------- void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h) { @@ -77,6 +193,16 @@ namespace currency return get_object_blobsize(t, tx_blob_size); } //--------------------------------------------------------------- + size_t get_objects_blobsize(const std::list& ls) + { + size_t total = 0; + for (const auto& tx : ls) + { + total += get_object_blobsize(tx); + } + return total; + } + //--------------------------------------------------------------- size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size) { size_t tx_blob_size = prefix_blob_size; @@ -130,5 +256,20 @@ namespace currency { return t_serializable_object_to_blob(tx, b_blob); } + //--------------------------------------------------------------- + bool read_keyimages_from_tx(const transaction& tx, std::list& kil) + { + std::unordered_set ki; + BOOST_FOREACH(const auto& in, tx.vin) + { + if (in.type() == typeid(txin_to_key) || in.type() == typeid(txin_htlc)) + { + + if (!ki.insert(get_to_key_input_from_txin_v(in).k_image).second) + return false; + } + } + return true; + } } \ No newline at end of file diff --git a/src/currency_core/currency_format_utils_transactions.h b/src/currency_core/currency_format_utils_transactions.h index a4937fa..337d40d 100644 --- a/src/currency_core/currency_format_utils_transactions.h +++ b/src/currency_core/currency_format_utils_transactions.h @@ -9,10 +9,112 @@ #include "crypto/crypto.h" #include "currency_core/currency_basic.h" #include "currency_protocol/blobdatatype.h" +#include "currency_core/account.h" namespace currency { + struct tx_source_entry + { + typedef serializable_pair output_entry; // txout_v is either global output index or ref_by_id; public_key - is output ephemeral pub key + + std::vector outputs; //index + key + uint64_t real_output; //index in outputs vector of real output_entry + crypto::public_key real_out_tx_key; //real output's transaction's public key + size_t real_output_in_tx_index; //index in transaction outputs vector + uint64_t amount; //money + uint64_t transfer_index; //money + crypto::hash multisig_id; //if txin_multisig: multisig output id + size_t ms_sigs_count; //if txin_multisig: must be equal to output's minimum_sigs + size_t ms_keys_count; //if txin_multisig: must be equal to size of output's keys container + bool separately_signed_tx_complete; //for separately signed tx only: denotes the last source entry in complete tx to explicitly mark the final step of tx creation + std::string htlc_origin; //for htlc, specify origin + + bool is_multisig() const { return ms_sigs_count > 0; } + + BEGIN_SERIALIZE_OBJECT() + FIELD(outputs) + FIELD(real_output) + FIELD(real_out_tx_key) + FIELD(real_output_in_tx_index) + FIELD(amount) + FIELD(transfer_index) + FIELD(multisig_id) + FIELD(ms_sigs_count) + FIELD(ms_keys_count) + FIELD(separately_signed_tx_complete) + FIELD(htlc_origin) + END_SERIALIZE() + }; + + + //if this struct is present, then creating htlc out, expiration -> number of blocks that htlc proposal is active + struct destination_option_htlc_out + { + uint64_t expiration; + crypto::hash htlc_hash; + + BEGIN_SERIALIZE_OBJECT() + FIELD(expiration) + FIELD(htlc_hash) + END_SERIALIZE() + }; + + + struct tx_destination_entry + { + uint64_t amount; //money + std::list addr; //destination address, in case of 1 address - txout_to_key, in case of more - txout_multisig + size_t minimum_sigs; //if txout_multisig: minimum signatures that are required to spend this output (minimum_sigs <= addr.size()) IF txout_to_key - not used + uint64_t amount_to_provide; //amount money that provided by initial creator of tx, used with partially created transactions + uint64_t unlock_time; + destination_option_htlc_out htlc_options; //htlc options + + + tx_destination_entry() : amount(0), minimum_sigs(0), amount_to_provide(0), unlock_time(0), htlc_options(destination_option_htlc_out()){} + tx_destination_entry(uint64_t a, const account_public_address& ad) : amount(a), addr(1, ad), minimum_sigs(0), amount_to_provide(0), unlock_time(0), htlc_options(destination_option_htlc_out()) {} + tx_destination_entry(uint64_t a, const account_public_address& ad, uint64_t ut) : amount(a), addr(1, ad), minimum_sigs(0), amount_to_provide(0), unlock_time(ut), htlc_options(destination_option_htlc_out()) {} + tx_destination_entry(uint64_t a, const std::list& addr) : amount(a), addr(addr), minimum_sigs(addr.size()), amount_to_provide(0), unlock_time(0), htlc_options(destination_option_htlc_out()) {} + + BEGIN_SERIALIZE_OBJECT() + FIELD(amount) + FIELD(addr) + FIELD(minimum_sigs) + FIELD(amount_to_provide) + FIELD(unlock_time) + FIELD(htlc_options) + END_SERIALIZE() + }; + + template + uint64_t get_tx_x_detail(const transaction& tx) + { + extra_type_t e = AUTO_VAL_INIT(e); + get_type_in_variant_container(tx.extra, e); + return e.v; + } + template + void set_tx_x_detail(transaction& tx, uint64_t v) + { + extra_type_t e = AUTO_VAL_INIT(e); + e.v = v; + update_or_add_field_to_extra(tx.extra, e); + } + + uint64_t get_tx_unlock_time(const transaction& tx, uint64_t o_i); + uint64_t get_tx_max_unlock_time(const transaction& tx); + bool get_tx_max_min_unlock_time(const transaction& tx, uint64_t& max_unlock_time, uint64_t& min_unlock_time); + inline bool should_unlock_value_be_treated_as_block_height(uint64_t v) { return v < CURRENCY_MAX_BLOCK_NUMBER; } + inline uint64_t get_tx_flags(const transaction& tx) { return get_tx_x_detail(tx); } + inline uint64_t get_tx_expiration_time(const transaction& tx) {return get_tx_x_detail(tx); } + inline void set_tx_unlock_time(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } + inline void set_tx_flags(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } + inline void set_tx_expiration_time(transaction& tx, uint64_t v) { set_tx_x_detail(tx, v); } + account_public_address get_crypt_address_from_destinations(const account_keys& sender_account_keys, const std::vector& destinations); + //----------------------------------------------------------------------------------------------- + + bool is_tx_expired(const transaction& tx, uint64_t expiration_ts_median); + uint64_t get_burned_amount(const transaction& tx); void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h); crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash); @@ -21,7 +123,11 @@ namespace currency bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res, uint64_t& blob_size); size_t get_object_blobsize(const transaction& t); + size_t get_objects_blobsize(const std::list& ls); size_t get_object_blobsize(const transaction& t, uint64_t prefix_blob_size); blobdata tx_to_blob(const transaction& b); bool tx_to_blob(const transaction& b, blobdata& b_blob); -} \ No newline at end of file + bool read_keyimages_from_tx(const transaction& tx, std::list& kil); + + +} diff --git a/src/currency_core/difficulty.cpp b/src/currency_core/difficulty.cpp index f2a1254..68d82c1 100644 --- a/src/currency_core/difficulty.cpp +++ b/src/currency_core/difficulty.cpp @@ -24,9 +24,9 @@ namespace currency { using std::uint64_t; using std::vector; -#if defined(_MSC_VER) -#include -#include +//#if defined(_MSC_VER) +//#include +//#include static inline void mul(uint64_t a, uint64_t b, uint64_t &low, uint64_t &high) { boost::multiprecision::uint128_t res = boost::multiprecision::uint128_t(a) * b; @@ -36,7 +36,7 @@ namespace currency { //low = UnsignedMultiply128(a, b, &high); } -#else +/* #else static inline void mul(uint64_t a, uint64_t b, uint64_t &low, uint64_t &high) { typedef unsigned __int128 uint128_t; @@ -45,7 +45,7 @@ namespace currency { high = (uint64_t)(res >> 64); } -#endif +#endif */ static inline bool cadd(uint64_t a, uint64_t b) { return a + b < a; @@ -179,9 +179,9 @@ namespace currency { return res.convert_to(); } - wide_difficulty_type next_difficulty(vector& timestamps, vector& cumulative_difficulties, size_t target_seconds) + wide_difficulty_type next_difficulty_1(vector& timestamps, vector& cumulative_difficulties, size_t target_seconds) { - TIME_MEASURE_START_PD(target_calculating_enum_blocks); + // timestamps - first is latest, back - is oldest timestamps if (timestamps.size() > DIFFICULTY_WINDOW) { @@ -220,4 +220,40 @@ namespace currency { } return summ / devider; } + + wide_difficulty_type next_difficulty_2(vector& timestamps, vector& cumulative_difficulties, size_t target_seconds) + { + + // timestamps - first is latest, back - is oldest timestamps + if (timestamps.size() > DIFFICULTY_WINDOW) + { + timestamps.resize(DIFFICULTY_WINDOW); + cumulative_difficulties.resize(DIFFICULTY_WINDOW); + } + + + size_t length = timestamps.size(); + CHECK_AND_ASSERT_MES(length == cumulative_difficulties.size(), 0, "Check \"length == cumulative_difficulties.size()\" failed"); + if (length <= 1) + { + return DIFFICULTY_STARTER; + } + static_assert(DIFFICULTY_WINDOW >= 2, "Window is too small"); + + CHECK_AND_ASSERT_MES(length <= DIFFICULTY_WINDOW, 0, "length <= DIFFICULTY_WINDOW check failed, length=" << length); + + sort(timestamps.begin(), timestamps.end(), std::greater()); + + static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Cut length is too large"); + wide_difficulty_type dif_slow = get_adjustment_for_zone(timestamps, cumulative_difficulties, target_seconds, DIFFICULTY_WINDOW, DIFFICULTY_CUT / 2, DIFFICULTY_CUT / 2); + wide_difficulty_type dif_medium = get_adjustment_for_zone(timestamps, cumulative_difficulties, target_seconds, DIFFICULTY_WINDOW / 3, DIFFICULTY_CUT / 8, DIFFICULTY_CUT / 12); + uint64_t devider = 1; + wide_difficulty_type summ = dif_slow; + if (dif_medium != 0) + { + summ += dif_medium; + ++devider; + } + return summ / devider; + } } diff --git a/src/currency_core/difficulty.h b/src/currency_core/difficulty.h index 529389e..ab97f14 100644 --- a/src/currency_core/difficulty.h +++ b/src/currency_core/difficulty.h @@ -19,7 +19,8 @@ namespace currency typedef boost::multiprecision::uint128_t wide_difficulty_type; bool check_hash(const crypto::hash &hash, wide_difficulty_type difficulty); - wide_difficulty_type next_difficulty(std::vector& timestamps, std::vector& cumulative_difficulties, size_t target_seconds); + wide_difficulty_type next_difficulty_1(std::vector& timestamps, std::vector& cumulative_difficulties, size_t target_seconds); + wide_difficulty_type next_difficulty_2(std::vector& timestamps, std::vector& cumulative_difficulties, size_t target_seconds); uint64_t difficulty_to_boundary(wide_difficulty_type difficulty); void difficulty_to_boundary_long(wide_difficulty_type difficulty, crypto::hash& result); } diff --git a/src/currency_core/genesis.cpp b/src/currency_core/genesis.cpp index fba792a..a58dcb4 100644 --- a/src/currency_core/genesis.cpp +++ b/src/currency_core/genesis.cpp @@ -7,8 +7,14 @@ namespace currency { +#ifndef TESTNET const genesis_tx_raw_data ggenesis_tx_raw = { { - 0xa080800e00000101,0x800326b0b4a0f2fd,0x46f236cb0efcd16c,0x547fbb0ff9468d85,0x1da1229cbceeeddf,0x00509f1127de5f6d,0xb0b4a0f2fda08080,0x19fbc70d41860326,0xb0afa73cfe1209ca,0x0d299d4e62aa2f1f,0x7071d8e322c2ebfc,0xf2fda0808000ed15,0x9e5a660326b0b4a0,0x2928590bf14c0f33,0x3dfd3f48e046a8c2,0xfa969b2fa09ad0ed,0x808000acfcbb24be,0x0326b0b4a0f2fda0,0x682ecad7a79177d8,0xc9e501939827d6ec,0x7f55e8f25beacf76,0xc3a22fd82ddcb367,0xb4f89aecdce08000,0x6aeaf9b0de0326b0,0x631393eedf0fc04d,0x8ed9b961192f541a,0x5088f34df1470474,0x93dc80800011241e,0x30b80307d0ffc2e0,0xdcf3a14a0ac108e5,0xd508a0ec648f342d,0x0c1ac8310dcce994,0x8000346b43733822,0x05c6c5bc97b1a080,0x44663811c2802f03,0x5456a1cc803e5b0c,0x9a7c995f82c5bb18,0xe95939d19b769901,0xbc97b1a0808000e9,0xb0624da20305c6c5,0xe7921e8df615d26f,0x27abce5d4d975bc6,0xecb92e6224ce0952,0xa080800085eb1099,0xb10305c6c5bc97b1,0xea6de475ef2cdaaf,0xa47b3fe828343c89,0x8ec5057c0bf7dd44,0x000f2403abe0ade8,0xc6c5bc97b1a08080,0x60e1a72d445b0305,0x8bb2fc2bfd63e41b,0x7772ae843713073b,0xe1c2db8d00c414ff,0x97b1a0808000b5c2,0xd69ad60305c6c5bc,0x8b0b54a7a541d22c,0x312a3c33f20800c1,0xe401b39ce755f512,0x808000d176bfb84a,0x0305c6c5bc97b1a0,0x7711834fbded84b6,0x1d4dca20946a1b23,0x80bfed89b730469e,0x641a20bd14e5d7cf,0xc5bc97b1a0808000,0xca32ddfd2c0305c6,0xf662200d93c916ca,0x0ca700521b6ece14,0x1a14d2365bb10a8e,0xbd88808000fd386e,0xc6e40304dbfb86ad,0xaba7f00aede6da7a,0xc5a36eac7d327196,0x5237aa32dafc085e,0x11003545013f1ed3,0xbd65d3fbab8aa616,0xb8c69175919e298e,0xe4fe3762fd534801,0xba7de375298d061d,0xf917cbb4170015e4,0x1747ce171e061770,0x05d7177e6b170d69,0x69170e891774ff17,0x17882e17c3621765,0x000a0ec1de17d7e4 }, - { 0x00 } }; + 0xa080801a00000101,0x800326b0b4a0f2fd,0xeebe5a6d44a03ed5,0x0e146a5322076dcf,0x992269ec1e34796e,0x003b14d1fe6c757e,0xb0b4a0f2fda08080,0xfd92adf982e70326,0xd8d4b6458b60e1a4,0xa69adb9475e808ed,0x4c383fcedfb6e20a,0xf2fda08080003458,0x8b177f0326b0b4a0,0xef9769ed70d152cd,0x04097d0daa65d123,0x9cd9f8e708f25bbc,0x8080005dfb23beac,0x0326b0b4a0f2fda0,0x6752077f8e75fc8a,0x437f68e0bf774836,0x5a38b52ff21c01c0,0x2d3727ec82ce1425,0xb4f89aecdce08000,0xa07d9fe35f0326b0,0x6c742533eb3b4261,0xfc2ed631332e5e16,0x3d025449393e538b,0x93dc80800015e433,0x70a20307d0ffc2e0,0xb81808dc5029bd46,0x04129413283e31f1,0x143e631cc81020b0,0x80008519d1377ae3,0x05c6c5bc97b1a080,0xf71887a841a72a03,0x681b659b8d2832d4,0x5677f9b15d11d1e6,0xffb2ad80c02a341c,0xbc97b1a08080003f,0xf9cebe7c0305c6c5,0xee223954dc682820,0x8194d2bac0dff6d6,0x86d8a55a30e30183,0xa08080006775f5f0,0x220305c6c5bc97b1,0x3c36e1ebdcee584a,0x4e9ed1a89532ef46,0xf0cb8b411bf6d579,0x00d0e6392ada64d4,0xc6c5bc97b1a08080,0x2fc2b05779450305,0xc9cf47618cc5283e,0xa9e088224807a77e,0xda854e29d2f49646,0x97b1a08080002e74,0xed35180305c6c5bc,0xa78d5545117b8293,0x5c3f8babc16e7062,0xef9324ecd7f86e39,0x808000231900ee9c,0x0305c6c5bc97b1a0,0x0c5bfd9450e89e30,0x194b86e8316970bc,0x5dd8c2e3c2af6ff1,0x4d2ba46f683df89c,0xc5bc97b1a0808000,0xc7cc22ad390305c6,0x891b500cb0799642,0xf5884473a7c01f07,0xeb88d74972d8e36f,0x91ed808000b5d239,0x535a0302fddecd95,0x791c7275cd15d685,0xc2536511d4132e01,0x0c9ad1ee9196aa77,0x80002d55a4efc7d3,0x018e8df2b7f0a080,0x0848b53f974a6a03,0x96b2572cb6015b7d,0xa71b18d2755de52c,0x075e4ca4bd0e4487,0xef93bf82808000ca,0xe23077bc730308f0,0x266a622b7bd9de26,0x4b80410b36c32203,0xb3026d0a2610916c,0xfe8480800084746d,0x6f34b90311e1dea6,0xeb38aee70a8febba,0x8b45df519f0df12e,0x258f0a71e83385da,0x8080000b85701a76,0xdf0308f0ef93bf82,0xb3170ab580f881a3,0x07f0a33f0756a3f4,0xf0721645b2b2bd7b,0x00b0077e03f43a85,0x08f0ef93bf828080,0xf91c1f6308c00f03,0x901a68f4adcc918b,0xad0346f5b7869662,0xd3ed49961fccd915,0xafeaa69a808000f0,0x2c99a01fe90301e3,0xd28642bcae6728d9,0xa6f38c4c630c2b6c,0x2c8a361de6b9294f,0xa69a8080002191e8,0xe134670301e3afea,0x6cf0798aeae985c8,0x4c9b90e1ff211b81,0xc32a954ce05a738b,0x8080009fd2412c6f,0x790301e3afeaa69a,0xb3b0062d6c27a6bb,0x12e133832172b705,0xf3f7d1dfdf336fb8,0x00922d0a879c6027,0x03c6dfd4ccb48080,0x7aa13f278feecf03,0x464f78f86a3e4553,0xa5a464e65c4cf651,0x18f07e7ed8bdd351,0xef93bf82808000dd,0x70b4e3ebae0308f0,0x74c452ecdce312d9,0xca3fb591982461fc,0x3e01aaf9b53ede69,0xfe84808000a4fa65,0x20e5ba0311e1dea6,0xb3e07ec0aabd06a5,0x7bf14a03bf83ccfd,0x6024154f95fd3220,0x808000c13077fa8b,0x5c580316deb183e9,0xc2c948248ab422c3,0xebd3db36bad27d52,0x5fe30392c1525a4e,0x1e00952287d66a6d,0x163df474d5ba8816,0x86f8892015449a71,0x22c93333d9ecb472,0x64fa5516bddfebb3,0x3234373061401303,0x3934353331623633,0x3063626435643130,0x6334636130633932,0x3831383439633530,0x3330336634336332,0x3963306232336630,0x3261623765656133,0x80c9170015633537,0x4c17fba117829117,0x17624a17d2f21711,0xfec81731f9178ced,0x9c17624117a36017,0x1786df17edda1708,0x9ffa17d6e1171b42,0x8b17aa69177ff417,0x179815170a83170c,0xc7e8171ce317da27 }, + { 0x17,0x86,0xd6,0x0e,0x0a,0x00,0x00 } }; +#else + const genesis_tx_raw_data ggenesis_tx_raw = {{ + 0xd880800500000101,0x0301f2ee9fa8ff8a,0xac88983c159856b0,0x6334c7d1b567f262,0x284f7f961a7b1266,0x8c0c68c45bab62fc,0xe1dea6fe84808000,0x337be98b45240311,0xab6cd1e4c66e5a33,0x70e889d98e70fd57,0xb97de43fb09861d4,0xf9f0cde08000d574,0x0270187703048dba,0xcac58027c0851473,0xaa10d864c4c87b46,0x820d371e2ba469e8,0xfea08000fce35acc,0x357903049598bddf,0x15df9e58bd0002aa,0xc940a97b60484e8d,0xf94f171e77d0b2d9,0x80003602c681487a,0x0304c38fbab1f8e0,0xc2529eba91cf7476,0x0bbee139aab295f9,0xf1cb8c58a828a2ca,0xcac8f5469af83932,0x5c8a1027cc160900,0x50bdcc9348baf32a,0xa7bd03751819d9fd,0xd6acc8dbbb0d9b29,0x3730614013368b02,0x3533316236333234,0x6264356431303934,0x6361306339323063,0x3834396335306334,0x3366343363323831,0x3062323366303330,0x6237656561333963,0x1700156335373261,0xce5017baa917a8f0,0x0a0eefcc17975617}, + {0x00,0x00}}; +#endif } diff --git a/src/currency_core/genesis.h b/src/currency_core/genesis.h index 73f8f94..8ea77d8 100644 --- a/src/currency_core/genesis.h +++ b/src/currency_core/genesis.h @@ -9,11 +9,20 @@ namespace currency { #pragma pack(push, 1) +#ifndef TESTNET struct genesis_tx_raw_data { - uint64_t const v[86]; - uint8_t const r[1]; + uint64_t const v[161]; + uint8_t const r[7]; }; + +#else + struct genesis_tx_raw_data + { + uint64_t const v[42]; + uint8_t const r[2]; + }; +#endif #pragma pack(pop) extern const genesis_tx_raw_data ggenesis_tx_raw; } diff --git a/src/currency_core/genesis_acc.cpp b/src/currency_core/genesis_acc.cpp index aac46ed..f49a9c0 100644 --- a/src/currency_core/genesis_acc.cpp +++ b/src/currency_core/genesis_acc.cpp @@ -9,24 +9,50 @@ namespace currency { - const std::string ggenesis_tx_pub_key_str = "a68aabfbd365bd8e299e917591c6b8014853fd6237fee41d068d2975e37dbae4"; +#ifndef TESTNET + const std::string ggenesis_tx_pub_key_str = "88bad574f43d16719a44152089f88672b4ecd93333c922b3ebdfbd1655fa6403"; const crypto::public_key ggenesis_tx_pub_key = epee::string_tools::parse_tpod_from_hex_string(ggenesis_tx_pub_key_str); - const genesis_tx_dictionary_entry ggenesis_dict[14] = { + const genesis_tx_dictionary_entry ggenesis_dict[26] = { { 898363347618325980ULL,7 }, { 1234271292339965434ULL,1 }, { 2785329203593578547ULL,12 }, + { 2912579291078040461ULL,18 }, + { 3515932779881697835ULL,17 }, { 4955366495399988463ULL,11 }, { 5233257582118330150ULL,5 }, + { 5931539148443336682ULL,24 }, + { 6436517662239927298ULL,19 }, { 6604452700210763953ULL,13 }, + { 7200550178847042641ULL,15 }, { 8712326356392296687ULL,9 }, { 8863158309745010598ULL,4 }, + { 9048445805125559105ULL,16 }, { 9527474759752332295ULL,2 }, + { 9647541513390373765ULL,20 }, { 9921730437908704447ULL,8 }, + { 10751885755236960099ULL,25 }, + { 11032572278436047420ULL,22 }, { 11109691972771859220ULL,0 }, + { 13554174209305230569ULL,23 }, { 14297297752337562678ULL,3 }, + { 15636081871140663679ULL,21 }, { 15951161519112687845ULL,6 }, + { 17146058209502212345ULL,14 }, { 17472133472787764818ULL,10 } }; +#else + const std::string ggenesis_tx_pub_key_str = "cc27108a5c2af3ba4893ccbd50fdd919187503bda7299b0dbbdbc8acd6028b36"; + const crypto::public_key ggenesis_tx_pub_key = epee::string_tools::parse_tpod_from_hex_string(ggenesis_tx_pub_key_str); + const genesis_tx_dictionary_entry ggenesis_dict[5] = { + { 4413532107669521528ULL, 2 }, + { 4848259848862559835ULL, 4 }, + { 4891306118630423916ULL, 1 }, + { 6536034028979999929ULL, 0 }, + { 15528122346224653564ULL, 3 } +}; +#endif + + } diff --git a/src/currency_core/genesis_acc.h b/src/currency_core/genesis_acc.h index 628c725..f26ee95 100644 --- a/src/currency_core/genesis_acc.h +++ b/src/currency_core/genesis_acc.h @@ -3,48 +3,46 @@ // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#pragma once +#pragma once #include #include #include "currency_format_utils.h" -namespace currency -{ - +namespace currency { #pragma pack(push, 1) - struct genesis_tx_dictionary_entry +struct genesis_tx_dictionary_entry { + uint64_t addr_hash; + uint64_t offset; + + bool operator<(const genesis_tx_dictionary_entry& s) const { - uint64_t addr_hash; - uint64_t offset; - - bool operator< (const genesis_tx_dictionary_entry& s) const - { - return addr_hash < s.addr_hash; - } - }; -#pragma pack(pop) - extern const genesis_tx_dictionary_entry ggenesis_dict[14]; - - extern const crypto::public_key ggenesis_tx_pub_key; - - inline bool get_account_genesis_offset_by_address(const std::string& addr, uint64_t& offset) - { - genesis_tx_dictionary_entry key_entry = AUTO_VAL_INIT(key_entry); - key_entry.addr_hash = get_string_uint64_hash(addr); - - const genesis_tx_dictionary_entry* pfirst = &ggenesis_dict[0]; - const genesis_tx_dictionary_entry* plast = &ggenesis_dict[sizeof(ggenesis_dict) / sizeof(ggenesis_dict[0])]; - const genesis_tx_dictionary_entry* plower = std::lower_bound(pfirst, plast, key_entry); - if (plower == plast) - return false; - if (plower->addr_hash != key_entry.addr_hash) - return false; - offset = plower->offset; - return true; + return addr_hash < s.addr_hash; } +}; +#pragma pack(pop) + +#ifndef TESTNET +extern const genesis_tx_dictionary_entry ggenesis_dict[26]; +#else +extern const genesis_tx_dictionary_entry ggenesis_dict[5]; +#endif + +extern const crypto::public_key ggenesis_tx_pub_key; + +inline bool get_account_genesis_offset_by_address(const std::string& addr, uint64_t& offset) +{ + genesis_tx_dictionary_entry key_entry = AUTO_VAL_INIT(key_entry); + key_entry.addr_hash = get_string_uint64_hash(addr); + + const genesis_tx_dictionary_entry* pfirst = &ggenesis_dict[0]; + const genesis_tx_dictionary_entry* plast = &ggenesis_dict[sizeof(ggenesis_dict) / sizeof(ggenesis_dict[0])]; + const genesis_tx_dictionary_entry* plower = std::lower_bound(pfirst, plast, key_entry); + if(plower == plast) + return false; + if(plower->addr_hash != key_entry.addr_hash) + return false; + offset = plower->offset; + return true; +} } - - - - diff --git a/src/currency_core/miner.cpp b/src/currency_core/miner.cpp index afb7705..005c274 100644 --- a/src/currency_core/miner.cpp +++ b/src/currency_core/miner.cpp @@ -41,7 +41,7 @@ namespace currency miner::miner(i_miner_handler* phandler, blockchain_storage& bc):m_stop(1), - m_bc(bc), + //m_bc(bc), m_template(boost::value_initialized()), m_template_no(0), m_diffic(0), @@ -56,13 +56,16 @@ namespace currency m_current_hash_rate(0), m_last_hr_merge_time(0), m_hashes(0), - m_config(AUTO_VAL_INIT(m_config)) + m_config(AUTO_VAL_INIT(m_config)), + m_mine_address{} { } //----------------------------------------------------------------------------------------------------- miner::~miner() { + TRY_ENTRY(); stop(); + CATCH_ENTRY_NO_RETURN(); } //----------------------------------------------------------------------------------------------------- bool miner::set_block_template(const block& bl, const wide_difficulty_type& di, uint64_t height) diff --git a/src/currency_core/miner.h b/src/currency_core/miner.h index 1e9d030..69a4bdc 100644 --- a/src/currency_core/miner.h +++ b/src/currency_core/miner.h @@ -24,6 +24,7 @@ namespace currency struct i_miner_handler { virtual bool handle_block_found(const block& b, block_verification_context* p_verification_result = nullptr) = 0; + virtual bool get_block_template(const create_block_template_params& params, create_block_template_response& resp) = 0; virtual bool get_block_template(block& b, const account_public_address& adr, const account_public_address& stakeholder_address, wide_difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce, bool pos = false, const pos_entry& pe = pos_entry()) = 0; protected: ~i_miner_handler(){}; @@ -105,7 +106,7 @@ namespace currency std::list m_threads; ::critical_section m_threads_lock; i_miner_handler* m_phandler; - blockchain_storage& m_bc; + //blockchain_storage& m_bc; account_public_address m_mine_address; math_helper::once_a_time_seconds<5> m_update_block_template_interval; math_helper::once_a_time_seconds<2> m_update_merge_hr_interval; diff --git a/src/currency_core/offers_service_basics.h b/src/currency_core/offers_service_basics.h index 0c73d4f..16d2990 100644 --- a/src/currency_core/offers_service_basics.h +++ b/src/currency_core/offers_service_basics.h @@ -19,7 +19,7 @@ namespace bc_services { //fields filled in UI - uint8_t offer_type; // OFFER_TYPE_PRIMARY_TO_TARGET - 0, OFFER_TYPE_TARGET_TO_PRIMARY - 1 etc. + uint8_t offer_type; // OFFER_TYPE_PRIMARY_TO_TARGET(SELL ORDER) - 0, OFFER_TYPE_TARGET_TO_PRIMARY(BUY ORDER) - 1 etc. uint64_t amount_primary; // amount of the currency uint64_t amount_target; // amount of other currency or goods std::string bonus; // @@ -125,4 +125,69 @@ namespace bc_services typedef boost::variant offers_attachment_t; + + inline std::string transform_double_to_string(const double& a) + { + return std::to_string(a); + } + + inline double transform_string_to_double(const std::string& d) + { + double n = 0; + epee::string_tools::get_xtype_from_string(n, d); + return n; + } + + + struct core_offers_filter + { + uint64_t order_by; + bool reverse; + uint64_t offset; + uint64_t limit; + //filter entry + uint64_t timestamp_start; + uint64_t timestamp_stop; + uint64_t offer_type_mask; + uint64_t amount_low_limit; + uint64_t amount_up_limit; + double rate_low_limit; + double rate_up_limit; + std::list payment_types; + std::string location_country; + std::string location_city; + std::string target; + std::string primary; + bool bonus; + std::string category; + std::string keyword; + bool fake; + uint64_t current_time; + + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(order_by) + KV_SERIALIZE(reverse) + KV_SERIALIZE(offset) + KV_SERIALIZE(limit) + KV_SERIALIZE(timestamp_start) + KV_SERIALIZE(timestamp_stop) + KV_SERIALIZE(offer_type_mask) + KV_SERIALIZE(amount_low_limit) + KV_SERIALIZE(amount_up_limit) + KV_SERIALIZE_CUSTOM(rate_low_limit, std::string, bc_services::transform_double_to_string, bc_services::transform_string_to_double) + KV_SERIALIZE_CUSTOM(rate_up_limit, std::string, bc_services::transform_double_to_string, bc_services::transform_string_to_double) + KV_SERIALIZE(payment_types) + KV_SERIALIZE(location_country) + KV_SERIALIZE(location_city) + KV_SERIALIZE(target) + KV_SERIALIZE(primary) + KV_SERIALIZE(bonus) + KV_SERIALIZE(category) + KV_SERIALIZE(keyword) + KV_SERIALIZE(fake) + END_KV_SERIALIZE_MAP() + }; + + } \ No newline at end of file diff --git a/src/currency_core/offers_services_helpers.cpp b/src/currency_core/offers_services_helpers.cpp index f73e339..d99bf4d 100644 --- a/src/currency_core/offers_services_helpers.cpp +++ b/src/currency_core/offers_services_helpers.cpp @@ -17,17 +17,7 @@ namespace bc_services { - std::string transform_double_to_string(const double& a) - { - return std::to_string(a); - } - double transform_string_to_double(const std::string& d) - { - double n = 0; - epee::string_tools::get_xtype_from_string(n, d); - return n; - } bool order_offers_by_timestamp(const offer_details_ex* a, const offer_details_ex* b) { @@ -66,7 +56,11 @@ namespace bc_services //------------------------------------------------------------------ bool order_offers_by_name(const offer_details_ex* a, const offer_details_ex* b) { +#ifndef ANDROID_BUILD return currency::utf8_to_lower(a->target) < currency::utf8_to_lower(b->target); +#else + return false; +#endif } //------------------------------------------------------------------ std::vector gsort_offers_predicates; @@ -156,14 +150,14 @@ namespace bc_services //check category if (!of.category.empty() && o.category.find(of.category) == std::string::npos) return false; - +#ifndef ANDROID_BUILD //check target condition if (of.target.size() && !currency::utf8_substring_test_case_insensitive(of.target, o.target)) return false; if (of.primary.size() && !currency::utf8_substring_test_case_insensitive(of.primary, o.primary)) return false; - +#endif //check payment_types condition (TODO: add more complicated algo here) if (of.payment_types.size()) @@ -179,11 +173,11 @@ namespace bc_services //check target condition if (of.location_country.size() && (of.location_country != o.location_country)) return false; - +#ifndef ANDROID_BUILD //check target condition if (of.location_city.size() && !currency::utf8_substring_test_case_insensitive(of.location_city, o.location_city)) return false; - +#endif //check amount if (of.amount_low_limit && (of.amount_low_limit > o.amount_primary)) return false; @@ -227,12 +221,12 @@ namespace bc_services //-------------------------------------------------------------------------------- bool filter_offers_list(std::list& offers, const bc_services::core_offers_filter& filter, uint64_t current_core_time) { - +#ifndef ANDROID_BUILD //filter offers.remove_if([&](bc_services::offer_details_ex& o) -> bool { return !is_offer_matched_by_filter(o, filter, current_core_time); }); - +#endif //sort CHECK_AND_ASSERT_MES(filter.order_by < gsort_offers_predicates.size(), false, "Wrong cof.order_by value"); auto cb = *gsort_offers_predicates[static_cast(filter.order_by)]; diff --git a/src/currency_core/offers_services_helpers.h b/src/currency_core/offers_services_helpers.h index 14d6bb9..fc2203f 100644 --- a/src/currency_core/offers_services_helpers.h +++ b/src/currency_core/offers_services_helpers.h @@ -18,60 +18,6 @@ namespace bc_services { - std::string transform_double_to_string(const double& a); - double transform_string_to_double(const std::string& d); - - - struct core_offers_filter - { - uint64_t order_by; - bool reverse; - uint64_t offset; - uint64_t limit; - //filter entry - uint64_t timestamp_start; - uint64_t timestamp_stop; - uint64_t offer_type_mask; - uint64_t amount_low_limit; - uint64_t amount_up_limit; - double rate_low_limit; - double rate_up_limit; - std::list payment_types; - std::string location_country; - std::string location_city; - std::string target; - std::string primary; - bool bonus; - std::string category; - std::string keyword; - bool fake; - uint64_t current_time; - - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(order_by) - KV_SERIALIZE(reverse) - KV_SERIALIZE(offset) - KV_SERIALIZE(limit) - KV_SERIALIZE(timestamp_start) - KV_SERIALIZE(timestamp_stop) - KV_SERIALIZE(offer_type_mask) - KV_SERIALIZE(amount_low_limit) - KV_SERIALIZE(amount_up_limit) - KV_SERIALIZE_CUSTOM(rate_low_limit, std::string, bc_services::transform_double_to_string, bc_services::transform_string_to_double) - KV_SERIALIZE_CUSTOM(rate_up_limit, std::string, bc_services::transform_double_to_string, bc_services::transform_string_to_double) - KV_SERIALIZE(payment_types) - KV_SERIALIZE(location_country) - KV_SERIALIZE(location_city) - KV_SERIALIZE(target) - KV_SERIALIZE(primary) - KV_SERIALIZE(bonus) - KV_SERIALIZE(category) - KV_SERIALIZE(keyword) - KV_SERIALIZE(fake) - END_KV_SERIALIZE_MAP() - }; - struct offer_id { @@ -245,8 +191,21 @@ namespace bc_services inline double extract_rate(const odeh& v) { return calculate_offer_rate(v); } inline size_t extract_payment_types(const odeh& v) { return v.payment_types.size(); } inline std::string extract_contacts(const odeh& v) { return v.contacts; } - inline std::string extract_location(const odeh& v) { return currency::utf8_to_lower(v.location_country + v.location_city); } - inline std::string extract_name(const odeh& v) { return currency::utf8_to_lower(v.target); } + inline std::string extract_location(const odeh& v) { + +#ifndef ANDROID_BUILD + return currency::utf8_to_lower(v.location_country + v.location_city); +#else + return "UNSUPORTED"; +#endif + } + inline std::string extract_name(const odeh& v) { +#ifndef ANDROID_BUILD + return currency::utf8_to_lower(v.target); +#else + return "UNSUPORTED"; +#endif + } template struct sort_id_to_type diff --git a/src/currency_core/tx_pool.cpp b/src/currency_core/tx_pool.cpp index 817f2a5..7a65990 100644 --- a/src/currency_core/tx_pool.cpp +++ b/src/currency_core/tx_pool.cpp @@ -9,7 +9,7 @@ #include #include -#include "common/db_backend_lmdb.h" +#include "common/db_backend_selector.h" #include "tx_pool.h" #include "currency_boost_serialization.h" #include "currency_core/currency_config.h" @@ -19,20 +19,23 @@ #include "misc_language.h" #include "warnings.h" #include "crypto/hash.h" -#include "offers_service_basics.h" #include "profile_tools.h" +#include "common/db_backend_selector.h" DISABLE_VS_WARNINGS(4244 4345 4503) //'boost::foreach_detail_::or_' : decorated name length exceeded, name was truncated #define TRANSACTION_POOL_CONTAINER_TRANSACTIONS "transactions" -#define TRANSACTION_POOL_CONTAINER_CANCEL_OFFER_HASH "cancel_offer_hash" #define TRANSACTION_POOL_CONTAINER_BLACK_TX_LIST "black_tx_list" #define TRANSACTION_POOL_CONTAINER_ALIAS_NAMES "alias_names" #define TRANSACTION_POOL_CONTAINER_ALIAS_ADDRESSES "alias_addresses" #define TRANSACTION_POOL_CONTAINER_KEY_IMAGES "key_images" #define TRANSACTION_POOL_CONTAINER_SOLO_OPTIONS "solo" +#define TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION 92 // DON'T CHANGE THIS, if you need to resync db! Change TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION instead! #define TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION BLOCKCHAIN_STORAGE_MAJOR_COMPATIBILITY_VERSION + 1 + +#define CONFLICT_KEY_IMAGE_SPENT_DEPTH_TO_REMOVE_TX_FROM_POOL 50 // if there's a conflict in key images between tx in the pool and in the blockchain this much depth in required to remove correspongin tx from pool + #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "tx_pool" ENABLE_CHANNEL_BY_DEFAULT("tx_pool"); @@ -43,15 +46,14 @@ namespace currency tx_memory_pool::tx_memory_pool(blockchain_storage& bchs, i_currency_protocol* pprotocol) : m_blockchain(bchs), m_pprotocol(pprotocol), - m_db(std::shared_ptr(new tools::db::lmdb_db_backend), m_dummy_rw_lock), + m_db(nullptr, m_dummy_rw_lock), m_db_transactions(m_db), - m_db_cancel_offer_hash(m_db), m_db_black_tx_list(m_db), m_db_solo_options(m_db), - m_db_key_images_set(m_db), +// m_db_key_images_set(m_db), m_db_alias_names(m_db), m_db_alias_addresses(m_db), - m_db_storage_major_compatibility_version(TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION, m_db_solo_options) + m_db_storage_major_compatibility_version(TRANSACTION_POOL_OPTIONS_ID_STORAGE_MAJOR_COMPATIBILITY_VERSION, m_db_solo_options) { } @@ -90,6 +92,24 @@ namespace currency //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(const transaction &tx, const crypto::hash &id, uint64_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool from_core) { + if (!kept_by_block && !from_core && m_blockchain.is_in_checkpoint_zone()) + { + // BCS is in CP zone, tx verification is impossible until it gets synchronized + tvc.m_added_to_pool = false; + tvc.m_should_be_relayed = false; + tvc.m_verification_failed = false; + tvc.m_verification_impossible = true; + return false; + } + + if (!m_blockchain.validate_tx_for_hardfork_specific_terms(tx, id)) + { + // + LOG_ERROR("Transaction " << id <<" doesn't fit current hardfork"); + tvc.m_verification_failed = true; + return false; + } + TIME_MEASURE_START_PD(tx_processing_time); TIME_MEASURE_START_PD(check_inputs_types_supported_time); if(!check_inputs_types_supported(tx)) @@ -157,19 +177,7 @@ namespace currency uint64_t tx_fee = inputs_amount - outputs_amount; if (tx_fee < m_blockchain.get_core_runtime_config().tx_pool_min_fee) { - //exception for cancel offer transactions - if (process_cancel_offer_rules(tx)) - { - // this tx has valid offer cansellation instructions and thus can go for free - // check soft size constrain - if (blob_size > CURRENCY_FREE_TX_MAX_BLOB_SIZE) - { - LOG_ERROR("Blob size (" << blob_size << ") << exceeds limit for transaction " << id << " that contains offer cancellation and has smaller fee (" << tx_fee << ") than expected"); - tvc.m_verification_failed = true; - return false; - } - } - else if (is_valid_contract_finalization_tx(tx)) + if (is_valid_contract_finalization_tx(tx)) { // that means tx has less fee then allowed by current tx pull rules, but this transaction is actually // a finalization of contract, and template of this contract finalization tx was prepared actually before @@ -177,8 +185,8 @@ namespace currency } else { - // this tx has no fee OR invalid offer cancellations instructions -- so the exceptions of zero fee is not applicable - LOG_ERROR("Transaction with id= " << id << " has too small fee: " << tx_fee << ", expected fee: " << m_blockchain.get_core_runtime_config().tx_pool_min_fee); + // this tx has no fee + LOG_PRINT_RED_L0("Transaction with id= " << id << " has too small fee: " << tx_fee << ", expected fee: " << m_blockchain.get_core_runtime_config().tx_pool_min_fee); tvc.m_verification_failed = false; tvc.m_should_be_relayed = false; tvc.m_added_to_pool = false; @@ -265,7 +273,7 @@ namespace currency td.receive_time = get_core_time(); m_db_transactions.set(id, td); - on_tx_add(tx, kept_by_block); + on_tx_add(id, tx, kept_by_block); TIME_MEASURE_FINISH_PD(update_db_time); return true; @@ -288,80 +296,7 @@ namespace currency CRITICAL_REGION_LOCAL(m_taken_txs_lock); m_taken_txs.clear(); } - //--------------------------------------------------------------------------------- - bool tx_memory_pool::process_cancel_offer_rules(const transaction& tx) - { - //TODO: this code doesn't take into account offer id in source tx - //TODO: add scan on tx size for free transaction here - m_db_transactions.begin_transaction(); - misc_utils::auto_scope_leave_caller seh = misc_utils::create_scope_leave_handler([&](){m_db_transactions.commit_transaction(); }); - - - size_t serv_att_count = 0; - std::list co_list; - for (const auto& a: tx.attachment) - { - if (a.type() == typeid(tx_service_attachment)) - { - const tx_service_attachment& srv_at = boost::get(a); - if (srv_at.service_id == BC_OFFERS_SERVICE_ID && srv_at.instruction == BC_OFFERS_SERVICE_INSTRUCTION_DEL) - { - if (!m_blockchain.validate_tx_service_attachmens_in_services(srv_at, serv_att_count, tx)) - { - LOG_ERROR("validate_tx_service_attachmens_in_services failed for an offer cancellation transaction"); - return false; - } - bc_services::extract_type_and_add(srv_at.body, co_list); - if (m_db_cancel_offer_hash.get(co_list.back().tx_id)) - { - LOG_ERROR("cancellation of offer " << co_list.back().tx_id << " has already been processed earlier; zero fee is disallowed"); - return false; - } - } - serv_att_count++; - } - } - if (!co_list.size()) - { - LOG_PRINT_L1("No cancel offers found"); - return false; - } - - for (auto co : co_list) - { - m_db_cancel_offer_hash.set(co.tx_id, true); - } - - return true; - } - //--------------------------------------------------------------------------------- - bool tx_memory_pool::unprocess_cancel_offer_rules(const transaction& tx) - { - m_db_transactions.begin_transaction(); - misc_utils::auto_scope_leave_caller seh = misc_utils::create_scope_leave_handler([&](){m_db_transactions.commit_transaction(); }); - - std::list co_list; - for (const auto& a : tx.attachment) - { - if (a.type() == typeid(tx_service_attachment)) - { - const tx_service_attachment& srv_at = boost::get(a); - if (srv_at.service_id == BC_OFFERS_SERVICE_ID && srv_at.instruction == BC_OFFERS_SERVICE_INSTRUCTION_DEL) - { - co_list.clear(); - bc_services::extract_type_and_add(srv_at.body, co_list); - if (!co_list.size()) - return false; - auto vptr = m_db_cancel_offer_hash.find(co_list.back().tx_id); - if (vptr == m_db_cancel_offer_hash.end()) - return false; - m_db_cancel_offer_hash.erase(co_list.back().tx_id); - } - } - } - return true; - } -// //--------------------------------------------------------------------------------- +//--------------------------------------------------------------------------------- bool tx_memory_pool::get_aliases_from_tx_pool(std::list& aliases)const { @@ -397,8 +332,9 @@ namespace currency { LOCAL_READONLY_TRANSACTION(); extra_alias_entry eai = AUTO_VAL_INIT(eai); - if (get_type_in_variant_container(tx.extra, eai)) - { + + bool r = false; + bool found = handle_2_alternative_types_in_variant_container(tx.extra, [this, &r, &tx, is_in_block](const extra_alias_entry& eai) { //check in blockchain extra_alias_entry eai2 = AUTO_VAL_INIT(eai2); bool already_have_alias_registered = m_blockchain.get_alias_info(eai.m_alias, eai2); @@ -407,7 +343,8 @@ namespace currency if (!is_in_block && !eai.m_sign.size() && already_have_alias_registered) { LOG_PRINT_L0("Alias \"" << eai.m_alias << "\" already registered in blockchain, transaction rejected"); - return false; + r = false; + return false; // stop handling } std::string prev_alias = m_blockchain.get_alias_by_address(eai.m_address); @@ -416,8 +353,9 @@ namespace currency { LOG_PRINT_L0("Address \"" << get_account_address_as_str(eai.m_address) << "\" already registered with \""<< prev_alias - << "\" aliass in blockchain (new alias: \"" << eai.m_alias << "\"), transaction rejected"); - return false; + << "\" alias in blockchain (new alias: \"" << eai.m_alias << "\"), transaction rejected"); + r = false; + return false; // stop handling } if (!is_in_block) @@ -425,24 +363,29 @@ namespace currency if (m_db_alias_names.get(eai.m_alias)) { LOG_PRINT_L0("Alias \"" << eai.m_alias << "\" already in transaction pool, transaction rejected"); - return false; + r = false; + return false; // stop handling } if (m_db_alias_addresses.get(eai.m_address)) { LOG_PRINT_L0("Alias \"" << eai.m_alias << "\" already in transaction pool by it's address(" << get_account_address_as_str(eai.m_address) << ") , transaction rejected"); - return false; + r = false; + return false; // stop handling } //validate alias reward if (!m_blockchain.prevalidate_alias_info(tx, eai)) { LOG_PRINT_L0("Alias \"" << eai.m_alias << "\" reward validation failed, transaction rejected"); - return false; + r = false; + return false; // stop handling } } + r = true; + return true; // continue handling + }); - } - return true; + return !found || r; // if found, r must be true for success } //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(const transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool from_core) @@ -467,9 +410,8 @@ namespace currency tx = txe_tr->tx; blob_size = txe_tr->blob_size; fee = txe_tr->fee; - unprocess_cancel_offer_rules(txe_tr->tx); m_db_transactions.erase(id); - on_tx_remove(tx, txe_tr->kept_by_block); + on_tx_remove(id, tx, txe_tr->kept_by_block); set_taken(id); return true; } @@ -512,28 +454,51 @@ namespace currency return true; // maximum age check - remove too old - uint64_t tx_age = get_core_time() - tx_entry.receive_time; + int64_t tx_age = get_core_time() - tx_entry.receive_time; if ((tx_age > CURRENCY_MEMPOOL_TX_LIVETIME )) { - - LOG_PRINT_L0("Tx " << h << " removed from tx pool, reason: outdated, age: " << tx_age); + LOG_PRINT_L0("tx " << h << " is about to be removed from tx pool, reason: outdated, age: " << tx_age << " = " << misc_utils::get_time_interval_string(tx_age)); to_delete.push_back(tx_to_delete_entry(h, tx_entry.tx, tx_entry.kept_by_block)); } // expiration time check - remove expired if (is_tx_expired(tx_entry.tx, tx_expiration_ts_median) ) { - LOG_PRINT_L0("Tx " << h << " removed from tx pool, reason: expired, expiration time: " << get_tx_expiration_time(tx_entry.tx) << ", blockchain median: " << tx_expiration_ts_median); + LOG_PRINT_L0("tx " << h << " is about to be removed from tx pool, reason: expired, expiration time: " << get_tx_expiration_time(tx_entry.tx) << ", blockchain median: " << tx_expiration_ts_median); to_delete.push_back(tx_to_delete_entry(h, tx_entry.tx, tx_entry.kept_by_block)); } + // if a tx has at least one key image already used in blockchain (deep enough) -- remove such tx, as it cannot be added to any block + // although it will be removed by the age check above, we consider desireable + // to remove it from the pool faster in order to unblock related key images used in the same tx + uint64_t should_be_spent_before_height = m_blockchain.get_current_blockchain_size() - 1; + if (should_be_spent_before_height > CONFLICT_KEY_IMAGE_SPENT_DEPTH_TO_REMOVE_TX_FROM_POOL) + { + should_be_spent_before_height -= CONFLICT_KEY_IMAGE_SPENT_DEPTH_TO_REMOVE_TX_FROM_POOL; + for (auto& in : tx_entry.tx.vin) + { + if (in.type() == typeid(txin_to_key)) + { + // if at least one key image is spent deep enought -- remove such tx + const crypto::key_image& ki = boost::get(in).k_image; + if (m_blockchain.have_tx_keyimg_as_spent(ki, should_be_spent_before_height)) + { + LOG_PRINT_L0("tx " << h << " is about to be removed from tx pool, reason: ki was spent in the blockchain before height " << should_be_spent_before_height << ", tx age: " << misc_utils::get_time_interval_string(tx_age)); + to_delete.push_back(tx_to_delete_entry(h, tx_entry.tx, tx_entry.kept_by_block)); + return true; + } + } + } + } + + return true; }); for (auto& e : to_delete) { m_db_transactions.erase(e.hash); - on_tx_remove(e.tx, e.kept_by_block); + on_tx_remove(e.hash, e.tx, e.kept_by_block); } @@ -674,8 +639,10 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id) + bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id, const std::list& bsk) { + + return true; } //--------------------------------------------------------------------------------- @@ -755,38 +722,33 @@ namespace currency return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::insert_key_images(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::insert_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block) { + CRITICAL_REGION_LOCAL(m_key_images_lock); for(const auto& in : tx.vin) { if (in.type() == typeid(txin_to_key)) { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - uint64_t count = 0; - auto ki_entry_ptr = m_db_key_images_set.get(tokey_in.k_image); - if (ki_entry_ptr.get()) - count = *ki_entry_ptr; - uint64_t count_before = count; - ++count; - m_db_key_images_set.set(tokey_in.k_image, count); - LOG_PRINT_L2("tx pool: key image added: " << tokey_in.k_image << ", from tx " << get_transaction_hash(tx) << ", counter: " << count_before << " -> " << count); + const txin_to_key& tokey_in = boost::get(in); + auto& id_set = m_key_images[tokey_in.k_image]; + size_t sz_before = id_set.size(); + id_set.insert(tx_id); + LOG_PRINT_L2("tx pool: key image added: " << tokey_in.k_image << ", from tx " << tx_id << ", counter: " << sz_before << " -> " << id_set.size()); } } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_tx_add(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::on_tx_add(crypto::hash tx_id, const transaction& tx, bool kept_by_block) { - if (!kept_by_block) - insert_key_images(tx, kept_by_block); // take into account only key images from txs that are not 'kept_by_block' + insert_key_images(tx_id, tx, kept_by_block); insert_alias_info(tx); return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::on_tx_remove(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::on_tx_remove(const crypto::hash &id, const transaction& tx, bool kept_by_block) { - if (!kept_by_block) - remove_key_images(tx, kept_by_block); // take into account only key images from txs that are not 'kept_by_block' + remove_key_images(id, tx, kept_by_block); remove_alias_info(tx); return true; } @@ -820,34 +782,33 @@ namespace currency return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::remove_key_images(const transaction& tx, bool kept_by_block) + bool tx_memory_pool::remove_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block) { + CRITICAL_REGION_LOCAL(m_key_images_lock); for(const auto& in : tx.vin) { if (in.type() == typeid(txin_to_key)) - { - CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail - uint64_t count = 0; - auto ki_entry_ptr = m_db_key_images_set.get(tokey_in.k_image); - if (!ki_entry_ptr.get() || *ki_entry_ptr == 0) - { - LOG_ERROR("INTERNAL_ERROR: for tx " << get_transaction_hash(tx) << " key image " << tokey_in.k_image << " not found"); - continue; - } - count = *ki_entry_ptr; - uint64_t count_before = count; - --count; - if (count) - m_db_key_images_set.set(tokey_in.k_image, count); - else - m_db_key_images_set.erase(tokey_in.k_image); - LOG_PRINT_L2("tx pool: key image removed: " << tokey_in.k_image << ", from tx " << get_transaction_hash(tx) << ", counter: " << count_before << " -> " << count); + { + const txin_to_key& tokey_in = boost::get(in); + + auto it_map = epee::misc_utils::it_get_or_insert_value_initialized(m_key_images, tokey_in.k_image); + auto& id_set = it_map->second; + size_t count_before = id_set.size(); + auto it_set = id_set.find(tx_id); + if(it_set != id_set.end()) + id_set.erase(it_set); + + size_t count_after = id_set.size(); + if (id_set.size() == 0) + m_key_images.erase(it_map); + + LOG_PRINT_L2("tx pool: key image removed: " << tokey_in.k_image << ", from tx " << tx_id << ", counter: " << count_before << " -> " << count_after); } } return false; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_key_images_from_tx_pool(std::unordered_set& key_images) const + bool tx_memory_pool::get_key_images_from_tx_pool(key_image_cache& key_images) const { m_db_transactions.enumerate_items([&](uint64_t i, const crypto::hash& h, const tx_details &tx_entry) @@ -856,7 +817,7 @@ namespace currency { if (in.type() == typeid(txin_to_key)) { - key_images.insert(boost::get(in).k_image); + key_images[boost::get(in).k_image].insert(h); } } return true; @@ -867,9 +828,9 @@ namespace currency //--------------------------------------------------------------------------------- bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im)const { - - auto ptr = m_db_key_images_set.find(key_im); - if (ptr) + CRITICAL_REGION_LOCAL(m_key_images_lock); + auto it = m_key_images.find(key_im); + if (it != m_key_images.end()) return true; return false; } @@ -889,20 +850,18 @@ namespace currency m_db.begin_transaction(); m_db_transactions.clear(); - m_db_key_images_set.clear(); - m_db_cancel_offer_hash.clear(); m_db.commit_transaction(); // should m_db_black_tx_list be cleared here? + CIRITCAL_OPERATION(m_key_images,clear()); } //--------------------------------------------------------------------------------- void tx_memory_pool::clear() { m_db.begin_transaction(); m_db_transactions.clear(); - m_db_cancel_offer_hash.clear(); m_db_black_tx_list.clear(); - m_db_key_images_set.clear(); m_db.commit_transaction(); + CIRITCAL_OPERATION(m_key_images,clear()); } //--------------------------------------------------------------------------------- bool tx_memory_pool::is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const @@ -944,8 +903,11 @@ namespace currency } } //if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure - if(m_blockchain.have_tx_keyimges_as_spent(txd.tx)) + if (m_blockchain.have_tx_keyimges_as_spent(txd.tx)) + { return false; + } + if (!check_tx_multisig_ins_and_outs(txd.tx, false)) return false; @@ -1049,7 +1011,9 @@ namespace currency const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, - uint64_t height) + uint64_t height, + const std::list& explicit_txs + ) { LOCAL_READONLY_TRANSACTION(); //typedef transactions_container::value_type txv; @@ -1058,12 +1022,12 @@ namespace currency std::vector txs_v; txs_v.reserve(m_db_transactions.size()); - std::vector txs; + std::vector txs; // selected transactions, vector of indices of txs_v - //std::transform(m_transactions.begin(), m_transactions.end(), txs.begin(), [](txv &a) -> txv * { return &a; }); - //keep getting it as a values cz db items cache will keep it as unserialied object stored by shared ptrs + + //keep getting it as a values cz db items cache will keep it as unserialised object stored by shared ptrs m_db_transactions.enumerate_keys([&](uint64_t i, crypto::hash& k){txs_v.resize(i + 1); txs_v[i].first = k; return true;}); - txs.resize(txs_v.size(), nullptr); + txs.resize(txs_v.size(), SIZE_MAX); for (uint64_t i = 0; i != txs_v.size(); i++) { @@ -1072,21 +1036,23 @@ namespace currency if (!txs_v[i].second) { LOG_ERROR("Internal tx pool db error: key " << k << " was enumerated as key but couldn't get value"); - continue; + return false; } - txs[i] = &txs_v[i]; + txs[i] = i; } - std::sort(txs.begin(), txs.end(), [](txv *a, txv *b) -> bool { + std::sort(txs.begin(), txs.end(), [&txs_v](size_t a, size_t b) -> bool { boost::multiprecision::uint128_t a_, b_; - a_ = boost::multiprecision::uint128_t(a->second->fee) * b->second->blob_size; - b_ = boost::multiprecision::uint128_t(b->second->fee) * a->second->blob_size; + a_ = boost::multiprecision::uint128_t(txs_v[a].second->fee) * txs_v[b].second->blob_size; + b_ = boost::multiprecision::uint128_t(txs_v[b].second->fee) * txs_v[a].second->blob_size; return a_ > b_; }); - size_t current_size = 0; + + size_t explicit_total_size = get_objects_blobsize(explicit_txs); + size_t current_size = explicit_total_size; uint64_t current_fee = 0; uint64_t best_money; if (!get_block_reward(pos, median_size, CURRENCY_COINBASE_BLOB_RESERVED_SIZE, already_generated_coins, best_money, height)) { @@ -1100,10 +1066,10 @@ namespace currency // scan txs for alias reg requests - if there are such requests, don't process alias updates bool alias_regs_exist = false; - for (auto txp : txs) + for (auto txi : txs) { tx_extra_info ei = AUTO_VAL_INIT(ei); - bool r = parse_and_validate_tx_extra(txp->second->tx, ei); + bool r = parse_and_validate_tx_extra(txs_v[txi].second->tx, ei); CHECK_AND_ASSERT_MES(r, false, "parse_and_validate_tx_extra failed while looking up the tx pool"); if (!ei.m_alias.m_alias.empty() && !ei.m_alias.m_sign.size()) { alias_regs_exist = true; @@ -1117,12 +1083,12 @@ namespace currency for (size_t i = 0; i < txs.size(); i++) { - txv &tx(*txs[i]); + txv &tx(txs_v[txs[i]]); // expiration time check -- skip expired transactions if (is_tx_expired(tx.second->tx, tx_expiration_ts_median)) { - txs[i] = nullptr; + txs[i] = SIZE_MAX; continue; } @@ -1136,7 +1102,7 @@ namespace currency if ((alias_count >= MAX_ALIAS_PER_BLOCK) || // IF this tx registers/updates an alias AND alias per block threshold exceeded (update_an_alias && alias_regs_exist)) // OR this tx updates an alias AND there are alias reg requests... { - txs[i] = NULL; // ...skip this tx + txs[i] = SIZE_MAX; // ...skip this tx continue; } } @@ -1154,7 +1120,7 @@ namespace currency } if (!is_tx_ready_to_go_result || have_key_images(k_images, tx.second->tx)) { - txs[i] = NULL; + txs[i] = SIZE_MAX; continue; } append_key_images(k_images, tx.second->tx); @@ -1181,80 +1147,150 @@ namespace currency for (size_t i = 0; i != txs.size(); i++) { - if (txs[i]) + if (txs[i] != SIZE_MAX) { + txv &tx(txs_v[txs[i]]); if (i < best_position) { - bl.tx_hashes.push_back(txs[i]->first); + bl.tx_hashes.push_back(tx.first); } - else if (have_attachment_service_in_container(txs[i]->second->tx.attachment, BC_OFFERS_SERVICE_ID, BC_OFFERS_SERVICE_INSTRUCTION_DEL)) + else if (have_attachment_service_in_container(tx.second->tx.attachment, BC_OFFERS_SERVICE_ID, BC_OFFERS_SERVICE_INSTRUCTION_DEL)) { // BC_OFFERS_SERVICE_INSTRUCTION_DEL transactions has zero fee, so include them here regardless of reward effectiveness - bl.tx_hashes.push_back(txs[i]->first); - total_size += txs[i]->second->blob_size; + bl.tx_hashes.push_back(tx.first); + total_size += tx.second->blob_size; } } } + // add explicit transactions + for (const auto& tx : explicit_txs) + { + fee += get_tx_fee(tx); + bl.tx_hashes.push_back(get_transaction_hash(tx)); + } return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::initialize_db_solo_options_values() + void tx_memory_pool::store_db_solo_options_values() { m_db.begin_transaction(); m_db_storage_major_compatibility_version = TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION; m_db.commit_transaction(); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::init(const std::string& config_folder) + bool tx_memory_pool::init(const std::string& config_folder, const boost::program_options::variables_map& vm) { + tools::db::db_backend_selector dbbs; + dbbs.init(vm); + auto p_backend = dbbs.create_backend(); + if (!p_backend) + { + LOG_PRINT_RED_L0("Failed to create db engine"); + return false; + } + m_db.reset_backend(p_backend); + LOG_PRINT_L0("DB ENGINE USED BY POOL: " << m_db.get_backend()->name()); + m_config_folder = config_folder; - uint64_t cache_size = CACHE_SIZE; - LOG_PRINT_GREEN("Using pool db file cache size(L1): " << cache_size, LOG_LEVEL_0); + uint64_t cache_size_l1 = CACHE_SIZE; + LOG_PRINT_GREEN("Using pool db file cache size(L1): " << cache_size_l1, LOG_LEVEL_0); - LOG_PRINT_L0("Loading blockchain..."); - const std::string folder_name = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME; - tools::create_directories_if_necessary(folder_name); - bool res = m_db.open(folder_name, cache_size); - CHECK_AND_ASSERT_MES(res, false, "Failed to initialize pool database in folder: " << folder_name); - - res = m_db_transactions.init(TRANSACTION_POOL_CONTAINER_TRANSACTIONS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_key_images_set.init(TRANSACTION_POOL_CONTAINER_KEY_IMAGES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_cancel_offer_hash.init(TRANSACTION_POOL_CONTAINER_CANCEL_OFFER_HASH); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_black_tx_list.init(TRANSACTION_POOL_CONTAINER_BLACK_TX_LIST); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_alias_names.init(TRANSACTION_POOL_CONTAINER_ALIAS_NAMES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_alias_addresses.init(TRANSACTION_POOL_CONTAINER_ALIAS_ADDRESSES); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - res = m_db_solo_options.init(TRANSACTION_POOL_CONTAINER_SOLO_OPTIONS); - CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); - - - m_db_transactions.set_cache_size(1000); - m_db_alias_names.set_cache_size(10000); - m_db_alias_addresses.set_cache_size(10000); - m_db_cancel_offer_hash.set_cache_size(1000); - m_db_black_tx_list.set_cache_size(1000); - - bool need_reinit = false; - if (m_db_storage_major_compatibility_version != TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION) - need_reinit = true; - - if (need_reinit) + // remove old incompatible DB + const std::string old_db_folder_path = m_config_folder + "/" CURRENCY_POOLDATA_FOLDERNAME_OLD; + if (boost::filesystem::exists(epee::string_encoding::utf8_to_wstring(old_db_folder_path))) { - clear(); - LOG_PRINT_MAGENTA("Tx Pool reinitialized.", LOG_LEVEL_0); + LOG_PRINT_YELLOW("Removing old DB in " << old_db_folder_path << "...", LOG_LEVEL_0); + boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(old_db_folder_path)); } - initialize_db_solo_options_values(); - LOG_PRINT_GREEN("TX_POOL Initialized ok. (" << m_db_transactions.size() << " transactions)", LOG_LEVEL_0); + const std::string db_folder_path = dbbs.get_pool_db_folder_path(); + + LOG_PRINT_L0("Loading blockchain from " << db_folder_path << "..."); + + bool db_opened_okay = false; + for(size_t loading_attempt_no = 0; loading_attempt_no < 2; ++loading_attempt_no) + { + bool res = m_db.open(db_folder_path, cache_size_l1); + if (!res) + { + // if DB could not be opened -- try to remove the whole folder and re-open DB + LOG_PRINT_YELLOW("Failed to initialize database in folder: " << db_folder_path << ", first attempt", LOG_LEVEL_0); + boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(db_folder_path)); + res = m_db.open(db_folder_path, cache_size_l1); + CHECK_AND_ASSERT_MES(res, false, "Failed to initialize database in folder: " << db_folder_path << ", second attempt"); + } + + res = m_db_transactions.init(TRANSACTION_POOL_CONTAINER_TRANSACTIONS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_black_tx_list.init(TRANSACTION_POOL_CONTAINER_BLACK_TX_LIST); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_alias_names.init(TRANSACTION_POOL_CONTAINER_ALIAS_NAMES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_alias_addresses.init(TRANSACTION_POOL_CONTAINER_ALIAS_ADDRESSES); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + res = m_db_solo_options.init(TRANSACTION_POOL_CONTAINER_SOLO_OPTIONS); + CHECK_AND_ASSERT_MES(res, false, "Unable to init db container"); + + m_db_transactions.set_cache_size(1000); + m_db_alias_names.set_cache_size(10000); + m_db_alias_addresses.set_cache_size(10000); + m_db_black_tx_list.set_cache_size(1000); + + bool need_reinit = false; + if (m_db_storage_major_compatibility_version > 0 && m_db_storage_major_compatibility_version != TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION) + { + need_reinit = true; + LOG_PRINT_MAGENTA("Tx pool DB needs reinit because it has major compatibility ver is " << m_db_storage_major_compatibility_version << ", expected: " << TRANSACTION_POOL_MAJOR_COMPATIBILITY_VERSION, LOG_LEVEL_0); + } + + if (need_reinit) + { + LOG_PRINT_L1("DB at " << db_folder_path << " is about to be deleted and re-created..."); + m_db_transactions.deinit(); +// m_db_key_images_set.deinit(); + m_db_black_tx_list.deinit(); + m_db_alias_names.deinit(); + m_db_alias_addresses.deinit(); + m_db_solo_options.deinit(); + m_db.close(); + size_t files_removed = boost::filesystem::remove_all(epee::string_encoding::utf8_to_wstring(db_folder_path)); + LOG_PRINT_L1(files_removed << " files at " << db_folder_path << " removed"); + + // try to re-create DB and re-init containers + continue; + } + + db_opened_okay = true; + break; + } + + CHECK_AND_ASSERT_MES(db_opened_okay, false, "All attempts to open DB at " << db_folder_path << " failed"); + + store_db_solo_options_values(); + + LOG_PRINT_GREEN("tx pool loaded ok from " << db_folder_path << ", loaded " << m_db_transactions.size() << " transactions", LOG_LEVEL_0); + if (epee::log_space::log_singletone::get_log_detalisation_level() >= LOG_LEVEL_2 && m_db_transactions.size() != 0) + { + std::stringstream ss; + m_db_transactions.enumerate_items([&](uint64_t i, const crypto::hash& h, const tx_details &tx_entry) + { + ss << h << " sz: " << std::setw(5) << tx_entry.blob_size << " rcv: " << misc_utils::get_time_interval_string(time(nullptr) - tx_entry.receive_time) << " ago" << ENDL; + return true; + }); + LOG_PRINT_L2(ss.str()); + } + + load_keyimages_cache(); + return true; } - + //--------------------------------------------------------------------------------- + bool tx_memory_pool::load_keyimages_cache() + { + CRITICAL_REGION_LOCAL(m_key_images_lock); + return get_key_images_from_tx_pool(m_key_images); + } //--------------------------------------------------------------------------------- bool tx_memory_pool::deinit() { diff --git a/src/currency_core/tx_pool.h b/src/currency_core/tx_pool.h index 9e524b4..4e660a6 100644 --- a/src/currency_core/tx_pool.h +++ b/src/currency_core/tx_pool.h @@ -20,6 +20,7 @@ using namespace epee; #include "math_helper.h" #include "common/db_abstract_accessor.h" +#include "common/command_line.h" #include "currency_format_utils.h" #include "verification_context.h" @@ -80,6 +81,7 @@ namespace currency epee::math_helper::average db_commit_time; }; + typedef std::unordered_map> key_image_cache; tx_memory_pool(blockchain_storage& bchs, i_currency_protocol* pprotocol); bool add_tx(const transaction &tx, const crypto::hash &id, uint64_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool from_core = false); @@ -98,7 +100,7 @@ namespace currency bool check_tx_multisig_ins_and_outs(const transaction& tx, bool check_against_pool_txs)const; - bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id); + bool on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id, const std::list& bsk); bool on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id); bool on_finalize_db_transaction(); bool add_transaction_to_black_list(const transaction& tx); @@ -114,9 +116,9 @@ namespace currency void clear(); // load/store operations - bool init(const std::string& config_folder); + bool init(const std::string& config_folder, const boost::program_options::variables_map& vm); bool deinit(); - bool fill_block_template(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height); + bool fill_block_template(block &bl, bool pos, size_t median_size, const boost::multiprecision::uint128_t& already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t height, const std::list& explicit_txs); bool get_transactions(std::list& txs) const; bool get_all_transactions_details(std::list& txs)const; bool get_all_transactions_brief_details(std::list& txs)const; @@ -134,35 +136,29 @@ namespace currency uint64_t get_core_time() const; bool get_aliases_from_tx_pool(std::list& aliases)const; bool get_aliases_from_tx_pool(std::map& aliases)const; - //crypto::hash get_last_core_hash() {return m_last_core_top_hash;} - //void set_last_core_hash(const crypto::hash& h) { m_last_core_top_hash = h; } - + + bool remove_stuck_transactions(); // made public to be called from coretests private: - bool on_tx_add(const transaction& tx, bool kept_by_block); - bool on_tx_remove(const transaction& tx, bool kept_by_block); - bool insert_key_images(const transaction& tx, bool kept_by_block); - bool remove_key_images(const transaction& tx, bool kept_by_block); + bool on_tx_add(crypto::hash tx_id, const transaction& tx, bool kept_by_block); + bool on_tx_remove(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block); + bool insert_key_images(const crypto::hash& tx_id, const transaction& tx, bool kept_by_block); + bool remove_key_images(const crypto::hash &tx_id, const transaction& tx, bool kept_by_block); bool insert_alias_info(const transaction& tx); bool remove_alias_info(const transaction& tx); bool is_valid_contract_finalization_tx(const transaction &tx)const; - void initialize_db_solo_options_values(); - bool remove_stuck_transactions(); + void store_db_solo_options_values(); bool is_transaction_ready_to_go(tx_details& txd, const crypto::hash& id)const; bool validate_alias_info(const transaction& tx, bool is_in_block)const; - bool get_key_images_from_tx_pool(std::unordered_set& key_images)const; - //bool push_alias_info(const transaction& tx); - //bool pop_alias_info(const transaction& tx); - bool process_cancel_offer_rules(const transaction& tx); - bool unprocess_cancel_offer_rules(const transaction& tx); + bool get_key_images_from_tx_pool(key_image_cache& key_images) const; bool check_is_taken(const crypto::hash& id) const; void set_taken(const crypto::hash& id); void reset_all_taken(); + bool load_keyimages_cache(); typedef tools::db::cached_key_value_accessor transactions_container; typedef tools::db::cached_key_value_accessor hash_container; - typedef tools::db::cached_key_value_accessor key_images_container; typedef tools::db::cached_key_value_accessor solo_options_container; typedef tools::db::cached_key_value_accessor aliases_container; typedef tools::db::cached_key_value_accessor address_to_aliases_container; @@ -174,14 +170,11 @@ namespace currency //containers transactions_container m_db_transactions; - hash_container m_db_cancel_offer_hash; hash_container m_db_black_tx_list; - key_images_container m_db_key_images_set; aliases_container m_db_alias_names; address_to_aliases_container m_db_alias_addresses; solo_options_container m_db_solo_options; tools::db::solo_db_value m_db_storage_major_compatibility_version; - //crypto::hash m_last_core_top_hash; epee::math_helper::once_a_time_seconds<30> m_remove_stuck_tx_interval; @@ -194,26 +187,10 @@ namespace currency //in memory containers mutable epee::critical_section m_taken_txs_lock; std::unordered_set m_taken_txs; - mutable epee::critical_section m_remove_stuck_txs_lock; - - /************************************************************************/ - /* */ - /************************************************************************/ - class amount_visitor: public boost::static_visitor - { - public: - uint64_t operator()(const txin_to_key& tx) const - { - return tx.amount; - } - uint64_t operator()(const txin_gen& /*tx*/) const - { - CHECK_AND_ASSERT_MES(false, 0, "coinbase transaction in memory pool"); - return 0; - } - uint64_t operator()(const txin_multisig& in) const { return in.amount; } - }; + mutable epee::critical_section m_key_images_lock; + key_image_cache m_key_images; + mutable epee::critical_section m_remove_stuck_txs_lock; }; } diff --git a/src/currency_core/tx_semantic_validation.cpp b/src/currency_core/tx_semantic_validation.cpp new file mode 100644 index 0000000..252c60b --- /dev/null +++ b/src/currency_core/tx_semantic_validation.cpp @@ -0,0 +1,100 @@ +// Copyright (c) 2018-2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + + + +#include "tx_semantic_validation.h" +#include "currency_format_utils.h" + +namespace currency +{ + //----------------------------------------------------------------------------------------------- + bool check_tx_extra(const transaction& tx) + { + tx_extra_info ei = AUTO_VAL_INIT(ei); + bool r = parse_and_validate_tx_extra(tx, ei); + if (!r) + return false; + return true; + } + //----------------------------------------------------------------------------------------------- + bool check_tx_inputs_keyimages_diff(const transaction& tx) + { + std::unordered_set ki; + BOOST_FOREACH(const auto& in, tx.vin) + { + if (in.type() == typeid(txin_to_key)) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false); + if (!ki.insert(tokey_in.k_image).second) + return false; + } + else if (in.type() == typeid(txin_htlc)) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_htlc, htlc_in, false); + if (!ki.insert(htlc_in.k_image).second) + return false; + } + } + return true; + } + //----------------------------------------------------------------------------------------------- + bool validate_tx_semantic(const transaction& tx, size_t tx_block_size) + { + if (!tx.vin.size()) + { + LOG_PRINT_RED_L0("tx with empty inputs, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_inputs_types_supported(tx)) + { + LOG_PRINT_RED_L0("unsupported input types for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_outs_valid(tx)) + { + LOG_PRINT_RED_L0("tx with invalid outputs, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (!check_money_overflow(tx)) + { + LOG_PRINT_RED_L0("tx has money overflow, rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + uint64_t amount_in = 0; + get_inputs_money_amount(tx, amount_in); + uint64_t amount_out = get_outs_money_amount(tx); + + if (amount_in < amount_out) + { + LOG_PRINT_RED_L0("tx with wrong amounts: ins " << amount_in << ", outs " << amount_out << ", rejected for tx id= " << get_transaction_hash(tx)); + return false; + } + + if (tx_block_size >= CURRENCY_MAX_TRANSACTION_BLOB_SIZE) + { + LOG_PRINT_RED_L0("tx has too big size " << tx_block_size << ", expected no bigger than " << CURRENCY_BLOCK_GRANTED_FULL_REWARD_ZONE); + return false; + } + + //check if tx use different key images + if (!check_tx_inputs_keyimages_diff(tx)) + { + LOG_PRINT_RED_L0("tx inputs have the same key images"); + return false; + } + + if (!check_tx_extra(tx)) + { + LOG_PRINT_RED_L0("tx has wrong extra, rejected"); + return false; + } + + return true; + } +} \ No newline at end of file diff --git a/src/currency_core/tx_semantic_validation.h b/src/currency_core/tx_semantic_validation.h new file mode 100644 index 0000000..2e6f479 --- /dev/null +++ b/src/currency_core/tx_semantic_validation.h @@ -0,0 +1,16 @@ +// Copyright (c) 2018-2019 Zano Project +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + + +#include "include_base_utils.h" +#include "currency_format_utils_transactions.h" + + +namespace currency +{ + //check correct values, amounts and all lightweight checks not related with database + bool validate_tx_semantic(const transaction& tx, size_t tx_block_size); +} diff --git a/src/currency_core/verification_context.h b/src/currency_core/verification_context.h index a9f1f64..85533cc 100644 --- a/src/currency_core/verification_context.h +++ b/src/currency_core/verification_context.h @@ -25,7 +25,11 @@ namespace currency bool m_verification_failed; //bad block, should drop connection bool m_marked_as_orphaned; bool m_already_exists; - bool added_to_altchain; - uint64_t height_difference; + bool m_added_to_altchain; + uint64_t m_height_difference; + //this is work like a first-level cache for transactions while block is getting handled. It lets transactions + //associated with the block to get handled directly to core without being handled by tx_pool(which makes full + //inputs validation, including signatures check) + transactions_map m_onboard_transactions; }; } diff --git a/src/currency_protocol/currency_protocol_defs.h b/src/currency_protocol/currency_protocol_defs.h index 73b85e1..48f09e2 100644 --- a/src/currency_protocol/currency_protocol_defs.h +++ b/src/currency_protocol/currency_protocol_defs.h @@ -13,6 +13,7 @@ #include "currency_core/connection_context.h" #include "currency_core/blockchain_storage_basic.h" #include "currency_protocol/blobdatatype.h" +#include "currency_core/basic_kv_structs.h" namespace currency { @@ -20,7 +21,7 @@ namespace currency #define BC_COMMANDS_POOL_BASE 2000 - + /************************************************************************/ /* */ /************************************************************************/ @@ -28,15 +29,21 @@ namespace currency { blobdata block; std::list txs; + std::vector coinbase_global_outs; + std::vector > > tx_global_outs; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block) KV_SERIALIZE(txs) + KV_SERIALIZE(coinbase_global_outs) + KV_SERIALIZE(tx_global_outs) END_KV_SERIALIZE_MAP() }; struct block_direct_data_entry { std::shared_ptr block_ptr; + std::shared_ptr coinbase_ptr; std::list > txs_ptr; }; diff --git a/src/currency_protocol/currency_protocol_handler.h b/src/currency_protocol/currency_protocol_handler.h index 795354f..eaf06bd 100644 --- a/src/currency_protocol/currency_protocol_handler.h +++ b/src/currency_protocol/currency_protocol_handler.h @@ -19,10 +19,10 @@ #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "currency_protocol" -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4355) -#define ASYNC_RELAY_MODE +#define ASYNC_RELAY_MODE // relay transactions asyncronously via m_relay_que namespace currency { @@ -66,7 +66,12 @@ namespace currency uint64_t get_max_seen_height(); virtual size_t get_synchronized_connections_count(); virtual size_t get_synchronizing_connections_count(); + int64_t get_net_time_delta_median(); + bool add_time_delta_and_check_time_sync(int64_t delta); + bool get_last_time_sync_difference(int64_t& last_median2local_time_difference, int64_t& last_ntp2local_time_difference); // returns true if differences in allowed bounds + //----------------------------------------------------------------------------------- + void set_to_debug_mode(uint32_t ip); private: //----------------- commands handlers ---------------------------------------------- @@ -93,7 +98,6 @@ namespace currency nodetool::p2p_endpoint_stub m_p2p_stub; nodetool::i_p2p_endpoint* m_p2p; - std::atomic m_syncronized_connections_count; std::atomic m_synchronized; std::atomic m_have_been_synchronized; std::atomic m_max_height_seen; @@ -108,26 +112,32 @@ namespace currency std::thread m_relay_que_thread; std::atomic m_want_stop; + std::deque m_time_deltas; + std::mutex m_time_deltas_lock; + int64_t m_last_median2local_time_difference; + int64_t m_last_ntp2local_time_difference; + uint32_t m_debug_ip_address; + template - bool post_notify(typename t_parametr::request& arg, currency_connection_context& context) - { - LOG_PRINT_L2("[POST]" << typeid(t_parametr).name()); - std::string blob; - epee::serialization::store_t_to_binary(arg, blob); - return m_p2p->invoke_notify_to_peer(t_parametr::ID, blob, context); - } + bool post_notify(typename t_parametr::request& arg, currency_connection_context& context) + { + LOG_PRINT_L3("[POST]" << typeid(t_parametr).name() << " to " << context); + std::string blob; + epee::serialization::store_t_to_binary(arg, blob); + return m_p2p->invoke_notify_to_peer(t_parametr::ID, blob, context); + } - template - bool relay_post_notify(typename t_parametr::request& arg, currency_connection_context& exlude_context) - { - std::string arg_buff; - epee::serialization::store_t_to_binary(arg, arg_buff); - std::list relayed_peers; - bool r = m_p2p->relay_notify_to_all(t_parametr::ID, arg_buff, exlude_context, relayed_peers); + template + bool relay_post_notify(typename t_parametr::request& arg, currency_connection_context& exlude_context) + { + std::string arg_buff; + epee::serialization::store_t_to_binary(arg, arg_buff); + std::list relayed_peers; + bool r = m_p2p->relay_notify_to_all(t_parametr::ID, arg_buff, exlude_context, relayed_peers); - LOG_PRINT_GREEN("[POST RELAY] " << typeid(t_parametr).name() << " relayed contexts list: " << ENDL << print_connection_context_list(relayed_peers), LOG_LEVEL_2); - return r; - } + LOG_PRINT_GREEN("[POST RELAY] " << typeid(t_parametr).name() << " to (" << relayed_peers.size() << "): " << print_connection_context_list(relayed_peers, ", "), LOG_LEVEL_2); + return r; + } }; } @@ -137,4 +147,4 @@ namespace currency #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL NULL -POP_WARNINGS +POP_VS_WARNINGS diff --git a/src/currency_protocol/currency_protocol_handler.inl b/src/currency_protocol/currency_protocol_handler.inl index 0485525..6599306 100644 --- a/src/currency_protocol/currency_protocol_handler.inl +++ b/src/currency_protocol/currency_protocol_handler.inl @@ -12,26 +12,27 @@ namespace currency //----------------------------------------------------------------------------------------------------------------------- template - t_currency_protocol_handler::t_currency_protocol_handler(t_core& rcore, nodetool::i_p2p_endpoint* p_net_layout):m_core(rcore), - m_p2p(p_net_layout), - m_syncronized_connections_count(0), - m_synchronized(false), - m_have_been_synchronized(false), - m_max_height_seen(0), - m_core_inital_height(0), - m_want_stop(false) - - + t_currency_protocol_handler::t_currency_protocol_handler(t_core& rcore, nodetool::i_p2p_endpoint* p_net_layout) + : m_core(rcore) + , m_p2p(p_net_layout) + , m_synchronized(false) + , m_have_been_synchronized(false) + , m_max_height_seen(0) + , m_core_inital_height(0) + , m_want_stop(false) + , m_last_median2local_time_difference(0) + , m_last_ntp2local_time_difference(0) + , m_debug_ip_address(0) { if(!m_p2p) m_p2p = &m_p2p_stub; } //----------------------------------------------------------------------------------------------------------------------- - template - t_currency_protocol_handler::~t_currency_protocol_handler() - { - deinit(); - } + template + t_currency_protocol_handler::~t_currency_protocol_handler() + { + deinit(); + } //----------------------------------------------------------------------------------------------------------------------- template bool t_currency_protocol_handler::init(const boost::program_options::variables_map& vm) @@ -93,22 +94,32 @@ namespace currency << std::setw(20) << "Peer id" << std::setw(25) << "Recv/Sent (idle,sec)" << std::setw(25) << "State" - << std::setw(20) << "Livetime(seconds)" + << std::setw(20) << "Livetime" << std::setw(20) << "Client version" << ENDL; + size_t incoming_count = 0, outgoing_count = 0; + std::multimap conn_map; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id) { - ss << std::setw(29) << std::left << std::string(cntxt.m_is_income ? "[INC]":"[OUT]") + + std::stringstream conn_ss; + time_t livetime = time(NULL) - cntxt.m_started; + conn_ss << std::setw(29) << std::left << std::string(cntxt.m_is_income ? "[INC]":"[OUT]") + string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) << std::setw(20) << std::hex << peer_id << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" << std::setw(25) << get_protocol_state_string(cntxt.m_state) - << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) + << std::setw(20) << epee::misc_utils::get_time_interval_string(livetime) << std::setw(20) << cntxt.m_remote_version << ENDL; + conn_map.insert(std::make_pair(livetime, conn_ss.str())); + (cntxt.m_is_income ? incoming_count : outgoing_count) += 1; return true; }); - LOG_PRINT_L0("Connections: " << ENDL << ss.str()); + + for(auto it = conn_map.rbegin(); it != conn_map.rend(); ++it) + ss << it->second; + + LOG_PRINT_L0("Connections (" << incoming_count << " in, " << outgoing_count << " out, " << incoming_count + outgoing_count << " total):" << ENDL << ss.str()); } //------------------------------------------------------------------------------------------------------------------------ template @@ -121,7 +132,23 @@ namespace currency if(context.m_state == currency_connection_context::state_befor_handshake && !is_inital) return true; - context.m_time_delta = m_core.get_blockchain_storage().get_core_runtime_config().get_core_time() - hshd.core_time; + uint64_t local_time = m_core.get_blockchain_storage().get_core_runtime_config().get_core_time(); + context.m_time_delta = local_time - hshd.core_time; + + // for outgoing connections -- check time difference + if (!context.m_is_income) + { + if (!add_time_delta_and_check_time_sync(context.m_time_delta)) + { + // serious time sync problem detected + i_critical_error_handler* ceh(m_core.get_critical_error_handler()); + if (ceh != nullptr && ceh->on_critical_time_sync_error()) + { + // error is handled by a callee, should not be ignored here, stop processing immideately + return true; + } + } + } if(context.m_state == currency_connection_context::state_synchronizing) return true; @@ -152,16 +179,17 @@ namespace currency && m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() < hshd.last_checkpoint_height && m_core.get_current_blockchain_size() < hshd.last_checkpoint_height ) { - LOG_PRINT_RED("Remote node have longer checkpoints zone( " << hshd.last_checkpoint_height << ") " << - "that local (" << m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() << ")" << - "That means that current software is outdated, please updated it." << - "Current heigh lay under checkpoints on remote host, so it is not possible validate this transactions on local host, disconnecting.", LOG_LEVEL_0); + LOG_PRINT_RED("Remote node has longer checkpoints zone (" << hshd.last_checkpoint_height << ") " << + "than local (" << m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() << "). " << + "It means that current software is outdated, please updated it! " << + "Current height lays under checkpoints zone on remote host, so it's impossible to validate remote transactions locally, disconnecting.", LOG_LEVEL_0); return false; - }else if (m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() < hshd.last_checkpoint_height) + } + else if (m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() < hshd.last_checkpoint_height) { - LOG_PRINT_MAGENTA("Remote node have longer checkpoints zone( " << hshd.last_checkpoint_height << ") " << - "that local (" << m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() << ")" << - "That means that current software is outdated, please updated it.", LOG_LEVEL_0); + LOG_PRINT_MAGENTA("Remote node has longer checkpoints zone (" << hshd.last_checkpoint_height << ") " << + "than local (" << m_core.get_blockchain_storage().get_checkpoints().get_top_checkpoint_height() << "). " << + "It means that current software is outdated, please updated it!", LOG_LEVEL_0); } context.m_state = currency_connection_context::state_synchronizing; @@ -214,10 +242,13 @@ namespace currency template int t_currency_protocol_handler::handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, currency_connection_context& context) { + //do not process requests if it comes from node wich is debugged + if (m_debug_ip_address != 0 && context.m_remote_ip == m_debug_ip_address) + return 1; + if(context.m_state != currency_connection_context::state_normal) return 1; - //check if block already exists block b = AUTO_VAL_INIT(b); block_verification_context bvc = AUTO_VAL_INIT(bvc); @@ -227,6 +258,7 @@ namespace currency m_p2p->drop_connection(context); return 1; } + crypto::hash block_id = get_block_hash(b); LOG_PRINT_GREEN("[HANDLE]NOTIFY_NEW_BLOCK " << block_id << " HEIGHT " << get_block_height(b) << " (hop " << arg.hop << ")", LOG_LEVEL_2); @@ -235,7 +267,7 @@ namespace currency if (it != m_blocks_id_que.end()) { //already have this block handler in que - LOG_PRINT("Block " << block_id << " already in processing que", LOG_LEVEL_2); + LOG_PRINT("Block " << block_id << " already in processing que", LOG_LEVEL_3); return 1; } else @@ -258,6 +290,11 @@ namespace currency } //pre-validate block here, and propagate it to network asap to avoid latency of handling big block (tx flood) + //######################################################## + /* + problem with prevalidation: in case of pre_validate_block() is passed but handle_incoming_tx() is failed + network got spammed with notifications about this broken block and then connections got closed. + temporary disabled to more investigation bool prevalidate_relayed = false; if (m_core.pre_validate_block(b, bvc, block_id) && bvc.m_added_to_main_chain) { @@ -266,18 +303,28 @@ namespace currency relay_block(arg, context); prevalidate_relayed = true; } + */ + //######################################################## //now actually process block for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) { - currency::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, true); - if(tvc.m_verification_failed) + if (tx_blob_it->size() > CURRENCY_MAX_TRANSACTION_BLOB_SIZE) { - LOG_PRINT_L0("Block verification failed: transaction verification failed, dropping connection"); + LOG_ERROR("WRONG TRANSACTION BLOB, too big size " << tx_blob_it->size() << ", rejected"); m_p2p->drop_connection(context); return 1; } + + crypto::hash tx_hash = null_hash; + transaction tx; + if (!parse_and_validate_tx_from_blob(*tx_blob_it, tx, tx_hash)) + { + LOG_ERROR("WRONG TRANSACTION BLOB, Failed to parse, rejected"); + m_p2p->drop_connection(context); + return 1; + } + bvc.m_onboard_transactions[tx_hash] = tx; } m_core.pause_mine(); @@ -289,15 +336,15 @@ namespace currency m_p2p->drop_connection(context); return 1; } - LOG_PRINT_GREEN("[HANDLE]NOTIFY_NEW_BLOCK EXTRA: id: " << block_id - << ",bvc.m_added_to_main_chain " << bvc.m_added_to_main_chain - << ",prevalidate_result " << prevalidate_relayed - << ",bvc.added_to_altchain " << bvc.added_to_altchain - << ",bvc.m_marked_as_orphaned " << bvc.m_marked_as_orphaned, LOG_LEVEL_2); + LOG_PRINT_GREEN("[HANDLE]NOTIFY_NEW_BLOCK EXTRA " << block_id + << " bvc.m_added_to_main_chain=" << bvc.m_added_to_main_chain + //<< ", prevalidate_result=" << prevalidate_relayed + << ", bvc.added_to_altchain=" << bvc.m_added_to_altchain + << ", bvc.m_marked_as_orphaned=" << bvc.m_marked_as_orphaned, LOG_LEVEL_2); - if (bvc.m_added_to_main_chain || (bvc.added_to_altchain && bvc.height_difference < 2)) + if (bvc.m_added_to_main_chain || (bvc.m_added_to_altchain && bvc.m_height_difference < 2)) { - if (!prevalidate_relayed) + if (true/*!prevalidate_relayed*/) { // pre-validation failed prevoiusly, but complete check was success, not an alternative block ++arg.hop; @@ -321,6 +368,10 @@ namespace currency template int t_currency_protocol_handler::handle_notify_new_transactions(int command, NOTIFY_NEW_TRANSACTIONS::request& arg, currency_connection_context& context) { + //do not process requests if it comes from node wich is debugged + if (m_debug_ip_address != 0 && context.m_remote_ip == m_debug_ip_address) + return 1; + if(context.m_state != currency_connection_context::state_normal) return 1; uint64_t inital_tx_count = arg.txs.size(); @@ -358,6 +409,10 @@ namespace currency template int t_currency_protocol_handler::handle_request_get_objects(int command, NOTIFY_REQUEST_GET_OBJECTS::request& arg, currency_connection_context& context) { + //do not process requests if it comes from node wich is debugged + if (m_debug_ip_address != 0 && context.m_remote_ip == m_debug_ip_address) + return 1; + LOG_PRINT_L2("[HANDLE]NOTIFY_REQUEST_GET_OBJECTS: arg.blocks.size() = " << arg.blocks.size() << ", arg.txs.size()="<< arg.txs.size()); LOG_PRINT_L3("[HANDLE]NOTIFY_REQUEST_GET_OBJECTS: " << ENDL << currency::print_kv_structure(arg)); @@ -403,6 +458,10 @@ namespace currency template int t_currency_protocol_handler::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, currency_connection_context& context) { + //do not process requests if it comes from node wich is debugged + if (m_debug_ip_address != 0 && context.m_remote_ip == m_debug_ip_address) + return 1; + LOG_PRINT_L2("[HANDLE]NOTIFY_RESPONSE_GET_OBJECTS: arg.blocks.size()=" << arg.blocks.size() << ", arg.missed_ids.size()=" << arg.missed_ids.size() << ", arg.txs.size()=" << arg.txs.size()); LOG_PRINT_L3("[HANDLE]NOTIFY_RESPONSE_GET_OBJECTS: " << ENDL << currency::print_kv_structure(arg)); if(context.m_last_response_height > arg.current_blockchain_height) @@ -486,27 +545,36 @@ namespace currency { CHECK_STOP_FLAG__DROP_AND_RETURN_IF_SET(1, "Blocks processing interrupted, connection dropped"); + block_verification_context bvc = boost::value_initialized(); //process transactions TIME_MEASURE_START(transactions_process_time); for (const auto& tx_blob : block_entry.txs) { CHECK_STOP_FLAG__DROP_AND_RETURN_IF_SET(1, "Block txs processing interrupted, connection dropped"); - - tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(tx_blob, tvc, true); - if(tvc.m_verification_failed) + crypto::hash tx_id = null_hash; + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_id)) { - LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " + LOG_ERROR_CCONTEXT("failed to parse tx: " << string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); m_p2p->drop_connection(context); return 1; } + bvc.m_onboard_transactions[tx_id] = tx; +// tx_verification_context tvc = AUTO_VAL_INIT(tvc); +// m_core.handle_incoming_tx(tx_blob, tvc, true); +// if(tvc.m_verification_failed) +// { +// LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " +// << string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); +// m_p2p->drop_connection(context); +// return 1; +// } } TIME_MEASURE_FINISH(transactions_process_time); //process block TIME_MEASURE_START(block_process_time); - block_verification_context bvc = boost::value_initialized(); m_core.handle_incoming_block(block_entry.block, bvc, false); if (count > 2 && bvc.m_already_exists) @@ -554,25 +622,19 @@ namespace currency template bool t_currency_protocol_handler::on_idle() { - bool have_synced_conn = false; - m_p2p->for_each_connection([&](currency_connection_context& context, nodetool::peerid_type peer_id)->bool{ - if (context.m_state == currency_connection_context::state_normal) - { - have_synced_conn = true; - return false; - } - return true; - }); + size_t synchronized_connections_count = get_synchronized_connections_count(); + size_t total_connections_count = m_p2p->get_connections_count(); + bool have_enough_synchronized_connections = synchronized_connections_count > total_connections_count / 2; - if (have_synced_conn && !m_synchronized) + if (have_enough_synchronized_connections && !m_synchronized) { on_connection_synchronized(); m_synchronized = true; - LOG_PRINT_MAGENTA("Synchronized set to TRUE (idle)", LOG_LEVEL_0); + LOG_PRINT_MAGENTA("Synchronized set to TRUE (" << synchronized_connections_count << " of " << total_connections_count << " conn. synced)", LOG_LEVEL_0); } - else if (!have_synced_conn && m_synchronized) + else if (!have_enough_synchronized_connections && m_synchronized) { - LOG_PRINT_MAGENTA("Synchronized set to FALSE (idle)", LOG_LEVEL_0); + LOG_PRINT_MAGENTA("Synchronized set to FALSE (" << synchronized_connections_count << " of " << total_connections_count << " conn. synced)", LOG_LEVEL_0); m_synchronized = false; } @@ -582,6 +644,10 @@ namespace currency template int t_currency_protocol_handler::handle_request_chain(int command, NOTIFY_REQUEST_CHAIN::request& arg, currency_connection_context& context) { + //do not process requests if it comes from node wich is debugged + if (m_debug_ip_address != 0 && context.m_remote_ip == m_debug_ip_address) + return 1; + LOG_PRINT_L2("[HANDLE]NOTIFY_REQUEST_CHAIN: block_ids.size()=" << arg.block_ids.size()); LOG_PRINT_L3("[HANDLE]NOTIFY_REQUEST_CHAIN: " << print_kv_structure(arg)); NOTIFY_RESPONSE_CHAIN_ENTRY::request r; @@ -666,8 +732,7 @@ namespace currency { std::list local_que; { - std::unique_lock lk(m_relay_que_lock); - //m_relay_que_cv.wait(lk); + CRITICAL_REGION_LOCAL(m_relay_que_lock); local_que.swap(m_relay_que); } if (local_que.size()) @@ -699,13 +764,14 @@ namespace currency if (req.txs.size()) { post_notify(req, cc); - print_connection_context_short(cc, debug_ss); - debug_ss << ": " << req.txs.size() << ENDL; + + if (debug_ss.tellp()) + debug_ss << ", "; + debug_ss << cc << ": " << req.txs.size(); } } TIME_MEASURE_FINISH_MS(ms); - LOG_PRINT_GREEN("[POST RELAY] NOTIFY_NEW_TRANSACTIONS relayed (" << ms << "ms)contexts list: " << debug_ss.str(), LOG_LEVEL_2); - + LOG_PRINT_GREEN("[POST RELAY] NOTIFY_NEW_TRANSACTIONS relayed (" << ms << "ms) to: " << debug_ss.str(), LOG_LEVEL_2); } //------------------------------------------------------------------------------------------------------------------------ template @@ -744,9 +810,82 @@ namespace currency return epee::misc_utils::median(deltas); } //------------------------------------------------------------------------------------------------------------------------ + #define TIME_SYNC_DELTA_RING_BUFFER_SIZE 8 + #define TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE (60 * 5) // max acceptable difference between time delta median among peers and local time (seconds) + #define TIME_SYNC_NTP_TO_LOCAL_MAX_DIFFERENCE (60 * 5) // max acceptable difference between NTP time and local time (seconds) + + template + bool t_currency_protocol_handler::add_time_delta_and_check_time_sync(int64_t time_delta) + { + CRITICAL_REGION_LOCAL(m_time_deltas_lock); + + auto get_core_time = [this] { return m_core.get_blockchain_storage().get_core_runtime_config().get_core_time(); }; + + m_time_deltas.push_back(time_delta); + while (m_time_deltas.size() > TIME_SYNC_DELTA_RING_BUFFER_SIZE) + m_time_deltas.pop_front(); + + if (m_time_deltas.size() < TIME_SYNC_DELTA_RING_BUFFER_SIZE) + return true; // not enough data + + std::vector time_deltas_copy(m_time_deltas.begin(), m_time_deltas.end()); + + m_last_median2local_time_difference = epee::misc_utils::median(time_deltas_copy); + LOG_PRINT_MAGENTA("TIME: network time difference is " << m_last_median2local_time_difference << " (max is " << TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE << ")", ((m_last_median2local_time_difference >= 3) ? LOG_LEVEL_2 : LOG_LEVEL_3)); + if (std::abs(m_last_median2local_time_difference) > TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE) + { + int64_t ntp_time = tools::get_ntp_time(); + LOG_PRINT_L2("NTP: received time " << ntp_time << " (" << epee::misc_utils::get_time_str_v2(ntp_time) << "), diff: " << std::showpos << get_core_time() - ntp_time); + if (ntp_time == 0) + { + // error geting ntp time + LOG_PRINT_RED("TIME: network time difference is " << m_last_median2local_time_difference << " (max is " << TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE << ") but NTP servers did not respond", LOG_LEVEL_0); + return false; + } + + // got ntp time correctly + // update local time, because getting ntp time could be time consuming + uint64_t local_time_2 = get_core_time(); + m_last_ntp2local_time_difference = local_time_2 - ntp_time; + if (std::abs(m_last_ntp2local_time_difference) > TIME_SYNC_NTP_TO_LOCAL_MAX_DIFFERENCE) + { + // local time is out of sync + LOG_PRINT_RED("TIME: network time difference is " << m_last_median2local_time_difference << " (max is " << TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE << "), NTP time difference is " << + m_last_ntp2local_time_difference << " (max is " << TIME_SYNC_NTP_TO_LOCAL_MAX_DIFFERENCE << ")", LOG_LEVEL_0); + return false; + } + + // NTP time is OK + LOG_PRINT_YELLOW("TIME: network time difference is " << m_last_median2local_time_difference << " (max is " << TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE << "), NTP time difference is " << + m_last_ntp2local_time_difference << " (max is " << TIME_SYNC_NTP_TO_LOCAL_MAX_DIFFERENCE << ")", LOG_LEVEL_1); + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template + bool t_currency_protocol_handler::get_last_time_sync_difference(int64_t& last_median2local_time_difference, int64_t& last_ntp2local_time_difference) + { + CRITICAL_REGION_LOCAL(m_time_deltas_lock); + last_median2local_time_difference = m_last_median2local_time_difference; + last_ntp2local_time_difference = m_last_ntp2local_time_difference; + + return !(std::abs(m_last_median2local_time_difference) > TIME_SYNC_DELTA_TO_LOCAL_MAX_DIFFERENCE && std::abs(m_last_ntp2local_time_difference) > TIME_SYNC_NTP_TO_LOCAL_MAX_DIFFERENCE); + } + template + void t_currency_protocol_handler::set_to_debug_mode(uint32_t ip) + { + m_debug_ip_address = ip; + LOG_PRINT_L0("debug mode is set for IP " << epee::string_tools::get_ip_string_from_int32(m_debug_ip_address)); + } + //------------------------------------------------------------------------------------------------------------------------ template int t_currency_protocol_handler::handle_response_chain_entry(int command, NOTIFY_RESPONSE_CHAIN_ENTRY::request& arg, currency_connection_context& context) { + //do not process requests if it comes from node wich is debugged + if (m_debug_ip_address != 0 && context.m_remote_ip == m_debug_ip_address) + return 1; + LOG_PRINT_L2("[HANDLE]NOTIFY_RESPONSE_CHAIN_ENTRY: m_block_ids.size()=" << arg.m_block_ids.size() << ", m_start_height=" << arg.start_height << ", m_total_height=" << arg.total_height); LOG_PRINT_L3("[HANDLE]NOTIFY_RESPONSE_CHAIN_ENTRY: " << ENDL << currency::print_kv_structure(arg)); @@ -799,7 +938,7 @@ namespace currency { #ifdef ASYNC_RELAY_MODE { - std::unique_lock lk(m_relay_que_lock); + CRITICAL_REGION_LOCAL(m_relay_que_lock); m_relay_que.push_back(AUTO_VAL_INIT(relay_que_entry())); m_relay_que.back().first = arg; m_relay_que.back().second = exclude_context; diff --git a/src/currency_protocol/currency_protocol_handler_common.h b/src/currency_protocol/currency_protocol_handler_common.h index ccbd88b..2bff506 100644 --- a/src/currency_protocol/currency_protocol_handler_common.h +++ b/src/currency_protocol/currency_protocol_handler_common.h @@ -36,4 +36,20 @@ namespace currency } }; + + /************************************************************************/ + /* */ + /************************************************************************/ + struct i_critical_error_handler + { + // called by currency protocol when the time is critically out of sync + // return true if the error is not ignored and the called should not proceed + virtual bool on_critical_time_sync_error() = 0; + + virtual bool on_critical_low_free_space(uint64_t available, uint64_t required) = 0; + + virtual bool on_immediate_stop_requested() = 0; + }; + + } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index b80d163..301b4cd 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying @@ -32,10 +32,11 @@ using namespace epee; #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL "p2p" +ENABLE_CHANNEL_BY_DEFAULT(LOG_DEFAULT_CHANNEL); #define CURRENT_P2P_STORAGE_ARCHIVE_VER (CURRENCY_FORMATION_VERSION+13) -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4355) namespace nodetool @@ -70,7 +71,17 @@ namespace nodetool m_alert_mode(0), m_maintainers_entry_local(AUTO_VAL_INIT(m_maintainers_entry_local)), m_maintainers_info_local(AUTO_VAL_INIT(m_maintainers_info_local)), - m_startup_time(time(nullptr)) + m_startup_time(time(nullptr)), + m_config{}, + m_have_address(false), + m_first_connection_maker_call(false), + m_listenning_port{}, + m_external_port{}, + m_ip_address{}, + m_last_stat_request_time{}, + m_use_only_priority_peers(false), + m_peer_livetime{}, + m_debug_requests_enabled(false) {} static void init_options(boost::program_options::options_description& desc); @@ -121,11 +132,14 @@ namespace nodetool HANDLE_INVOKE_T2(COMMAND_TIMED_SYNC, &node_server::handle_timed_sync) HANDLE_INVOKE_T2(COMMAND_PING, &node_server::handle_ping) #ifdef ALLOW_DEBUG_COMMANDS - HANDLE_INVOKE_T2(COMMAND_REQUEST_STAT_INFO, &node_server::handle_get_stat_info) - HANDLE_INVOKE_T2(COMMAND_REQUEST_NETWORK_STATE, &node_server::handle_get_network_state) - HANDLE_INVOKE_T2(COMMAND_REQUEST_PEER_ID, &node_server::handle_get_peer_id) - HANDLE_INVOKE_T2(COMMAND_REQUEST_LOG, &node_server::handle_request_log) - HANDLE_INVOKE_T2(COMMAND_SET_LOG_LEVEL, &node_server::handle_set_log_level) + if (m_debug_requests_enabled) + { + HANDLE_INVOKE_T2(COMMAND_REQUEST_STAT_INFO, &node_server::handle_get_stat_info) + HANDLE_INVOKE_T2(COMMAND_REQUEST_NETWORK_STATE, &node_server::handle_get_network_state) + HANDLE_INVOKE_T2(COMMAND_REQUEST_PEER_ID, &node_server::handle_get_peer_id) + HANDLE_INVOKE_T2(COMMAND_REQUEST_LOG, &node_server::handle_request_log) + HANDLE_INVOKE_T2(COMMAND_SET_LOG_LEVEL, &node_server::handle_set_log_level) + } #endif CHAIN_INVOKE_MAP_TO_OBJ_FORCE_CONTEXT(m_payload_handler, typename t_payload_net_handler::connection_context&) END_INVOKE_MAP2() @@ -185,10 +199,11 @@ namespace nodetool bool make_new_connection_from_peerlist(bool use_white_list); bool try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist = false, uint64_t last_seen_stamp = 0, bool white = true); size_t get_random_index_with_fixed_probability(size_t max_index); + bool is_peer_id_used(const peerid_type id); bool is_peer_used(const peerlist_entry& peer); bool is_addr_connected(const net_address& peer); template - bool try_ping(basic_node_data& node_data, p2p_connection_context& context, t_callback cb); + bool try_ping(basic_node_data& node_data, p2p_connection_context& context, const t_callback& cb); bool make_expected_connections_count(bool white_list, size_t expected_connections); void cache_connect_fail_info(const net_address& addr); bool is_addr_recently_failed(const net_address& addr); @@ -229,8 +244,10 @@ namespace nodetool bool m_allow_local_ip; bool m_hide_my_port; bool m_offline_mode; + bool m_debug_requests_enabled; uint64_t m_startup_time; + //critical_section m_connections_lock; //connections_indexed_container m_connections; @@ -286,4 +303,4 @@ namespace nodetool #undef LOG_DEFAULT_CHANNEL #define LOG_DEFAULT_CHANNEL NULL -POP_WARNINGS +POP_VS_WARNINGS diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index c7e18da..5042548 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying @@ -36,6 +36,7 @@ namespace nodetool const command_line::arg_descriptor > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"}; const command_line::arg_descriptor arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; const command_line::arg_descriptor arg_p2p_offline_mode = { "offline-mode", "Don't connect to any node and reject any connections", false, true }; + const command_line::arg_descriptor arg_p2p_disable_debug_reqs = { "disable-debug-p2p-requests", "Disable p2p debug requests", false, true }; } //----------------------------------------------------------------------------------- @@ -51,6 +52,7 @@ namespace nodetool command_line::add_arg(desc, arg_p2p_seed_node); command_line::add_arg(desc, arg_p2p_hide_my_port); command_line::add_arg(desc, arg_p2p_offline_mode); + command_line::add_arg(desc, arg_p2p_disable_debug_reqs); command_line::add_arg(desc, arg_p2p_use_only_priority_nodes); } //----------------------------------------------------------------------------------- @@ -98,9 +100,9 @@ namespace nodetool if (m_offline_mode) return false; - //@#@ workaround + //@#@ temporary workaround return true; - +#if 0 CRITICAL_REGION_LOCAL(m_blocked_ips_lock); auto it = m_blocked_ips.find(addr); if(it == m_blocked_ips.end()) @@ -112,6 +114,7 @@ namespace nodetool return true; } return false; +#endif } //----------------------------------------------------------------------------------- template @@ -158,6 +161,7 @@ namespace nodetool m_external_port = command_line::get_arg(vm, arg_p2p_external_port); m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip); m_offline_mode = command_line::get_arg(vm, arg_p2p_offline_mode); + m_debug_requests_enabled = !command_line::get_arg(vm, arg_p2p_disable_debug_reqs); if (m_offline_mode) { @@ -190,8 +194,6 @@ namespace nodetool } if(command_line::has_arg(vm, arg_p2p_use_only_priority_nodes)) m_use_only_priority_peers = true; - else - m_use_only_priority_peers = false; if (command_line::has_arg(vm, arg_p2p_seed_node)) @@ -275,13 +277,20 @@ namespace nodetool #ifndef TESTNET //TODO: //ADD_HARDCODED_SEED_NODE(std::string("0.0.0.0:") + std::to_string(P2P_DEFAULT_PORT)); - ADD_HARDCODED_SEED_NODE("207.154.237.82", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("207.154.240.198", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("207.154.255.10", P2P_DEFAULT_PORT); - ADD_HARDCODED_SEED_NODE("207.154.228.141", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.43.225", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("94.130.137.230", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.42.247", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("94.130.160.115", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("195.201.107.230", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.46.49", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("159.69.76.144", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("144.76.183.143", P2P_DEFAULT_PORT); #else //TODO: - //ADD_HARDCODED_SEED_NODE(std::string("0.0.0.0:") + std::to_string(P2P_DEFAULT_PORT)); + ADD_HARDCODED_SEED_NODE("95.217.43.225", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("94.130.137.230", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("95.217.42.247", P2P_DEFAULT_PORT); + ADD_HARDCODED_SEED_NODE("94.130.160.115", P2P_DEFAULT_PORT); #endif bool res = handle_command_line(vm); @@ -402,14 +411,14 @@ namespace nodetool template bool node_server::on_maintainers_entry_update() { - LOG_PRINT_MAGENTA("Fresh maintainers info recieved(timestamp: " << m_maintainers_info_local.timestamp << ")", LOG_LEVEL_0); + LOG_PRINT_CHANNEL_COLOR2(NULL, NULL, "Fresh maintainers info recieved(timestamp: " << m_maintainers_info_local.timestamp << ")", LOG_LEVEL_0, epee::log_space::console_color_magenta); if(PROJECT_VERSION_BUILD_NO < m_maintainers_info_local.build_no) { - LOG_PRINT_MAGENTA("Newer version avaliable: " << static_cast(m_maintainers_info_local.ver_major) << + LOG_PRINT_CHANNEL_COLOR2(NULL, NULL, "Newer version avaliable: " << static_cast(m_maintainers_info_local.ver_major) << "." << static_cast(m_maintainers_info_local.ver_minor) << "." << static_cast(m_maintainers_info_local.ver_revision) << "." << static_cast(m_maintainers_info_local.build_no) << - ", current version: " << PROJECT_VERSION_LONG, LOG_LEVEL_0); + ", current version: " << PROJECT_VERSION_LONG, LOG_LEVEL_0, epee::log_space::console_color_magenta); } handle_alert_conditions(); @@ -475,6 +484,12 @@ namespace nodetool return; } + if (!tools::check_remote_client_version(rsp.payload_data.client_version)) + { + LOG_ERROR_CCONTEXT("COMMAND_HANDSHAKE Failed, wrong client version: " << rsp.payload_data.client_version << ", closing connection."); + return; + } + if(!handle_maintainers_entry(rsp.maintrs_entry)) { LOG_ERROR_CCONTEXT("COMMAND_HANDSHAKE Failed, wrong maintainers entry!, closing connection."); @@ -497,6 +512,13 @@ namespace nodetool return; } + if (is_peer_id_used(rsp.node_data.peer_id)) + { + LOG_PRINT_L0("It seems that peer " << std::hex << rsp.node_data.peer_id << " has already been connected, dropping connection"); + hsh_result = false; + return; + } + pi = context.peer_id = rsp.node_data.peer_id; m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_ip, context.m_remote_port); @@ -521,7 +543,7 @@ namespace nodetool if(!hsh_result) { - LOG_PRINT_CC_L0(context_, "COMMAND_HANDSHAKE Failed"); + LOG_PRINT_CC_L0(context_, "COMMAND_HANDSHAKE Failed, closing connection"); m_net_server.get_config_object().close(context_.m_connection_id); } @@ -583,6 +605,26 @@ namespace nodetool } //----------------------------------------------------------------------------------- template + bool node_server::is_peer_id_used(const peerid_type id) + { + if (id == m_config.m_peer_id) + return true; // ourself + + bool used = false; + m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (id == cntxt.peer_id) + { + used = true; + return false; // stop enumerating + } + return true; + }); + + return used; + } + //----------------------------------------------------------------------------------- + template bool node_server::is_peer_used(const peerlist_entry& peer) { @@ -624,7 +666,7 @@ namespace nodetool template bool node_server::try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, bool white) { - LOG_PRINT_L0("Connecting to " << string_tools::get_ip_string_from_int32(na.ip) << ":" << string_tools::num_to_string_fast(na.port) << "(white=" << white << ", last_seen: " << (last_seen_stamp?misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never" ) << ")..."); + LOG_PRINT_L1("Connecting to " << string_tools::get_ip_string_from_int32(na.ip) << ":" << string_tools::num_to_string_fast(na.port) << "(white=" << white << ", last_seen: " << (last_seen_stamp?misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never" ) << ")..."); typename net_server::t_connection_context con = AUTO_VAL_INIT(con); bool res = m_net_server.connect(string_tools::get_ip_string_from_int32(na.ip), @@ -633,7 +675,7 @@ namespace nodetool con); if(!res) { - LOG_PRINT_L0("Connect failed to " + LOG_PRINT_L1("Connect failed to " << string_tools::get_ip_string_from_int32(na.ip) << ":" << string_tools::num_to_string_fast(na.port) /*<< ", try " << try_count*/); @@ -647,13 +689,15 @@ namespace nodetool LOG_PRINT_CC_L0(con, "Failed to HANDSHAKE with peer " << string_tools::get_ip_string_from_int32(na.ip) << ":" << string_tools::num_to_string_fast(na.port) - /*<< ", try " << try_count*/); + << ", closing connection"); + m_net_server.get_config_object().close(con.m_connection_id); return false; } + if(just_take_peerlist) { m_net_server.get_config_object().close(con.m_connection_id); - LOG_PRINT_CC_GREEN(con, "CONNECTION HANDSHAKED OK AND CLOSED.", LOG_LEVEL_2); + LOG_PRINT_CC_GREEN(con, "CONNECTION HANDSHAKED OK AND CLOSED with peer " << string_tools::get_ip_string_from_int32(na.ip) << ":" << string_tools::num_to_string_fast(na.port), LOG_LEVEL_2); return true; } @@ -664,7 +708,7 @@ namespace nodetool m_peerlist.append_with_peer_white(pe_local); //update last seen and push it to peerlist manager - LOG_PRINT_CC_GREEN(con, "CONNECTION HANDSHAKED OK.", LOG_LEVEL_2); + LOG_PRINT_CC_GREEN(con, "CONNECTION HANDSHAKED OK with peer " << string_tools::get_ip_string_from_int32(na.ip) << ":" << string_tools::num_to_string_fast(na.port), LOG_LEVEL_2); return true; } //----------------------------------------------------------------------------------- @@ -793,7 +837,10 @@ namespace nodetool if(is_addr_connected(na)) continue; - try_to_connect_and_handshake_with_new_peer(na); + if (!try_to_connect_and_handshake_with_new_peer(na)) + { + LOG_PRINT_L0("connection to priority node " << string_tools::get_ip_string_from_int32(na.ip) << ":" << string_tools::num_to_string_fast(na.port) << " failed"); + } } if(m_use_only_priority_peers) return true; @@ -893,7 +940,7 @@ namespace nodetool if(m_alert_mode != ALERT_TYPE_CALM) return true; - LOG_PRINT_L0("This software is old, please update."); + LOG_PRINT_CHANNEL2(NULL, NULL, "This software is outdated, please update.", LOG_LEVEL_0); return true; } //----------------------------------------------------------------------------------- @@ -903,7 +950,7 @@ namespace nodetool if(m_alert_mode != ALERT_TYPE_URGENT) return true; - LOG_PRINT_CYAN("[URGENT]:This software is old, please update.", LOG_LEVEL_0); + LOG_PRINT_CHANNEL_COLOR2(NULL, NULL, "[URGENT]:This software is dramatically outdated, please update to latest version.", LOG_LEVEL_0, epee::log_space::console_color_cyan); return true; } //----------------------------------------------------------------------------------- @@ -913,7 +960,7 @@ namespace nodetool if(m_alert_mode != ALERT_TYPE_CRITICAL) return true; - LOG_PRINT_RED("[CRITICAL]:This software is old, please update.", LOG_LEVEL_0); + LOG_PRINT_CHANNEL_COLOR2(NULL, NULL, "[CRITICAL]:This software is critically outdated, please update to latest version.", LOG_LEVEL_0, epee::log_space::console_color_red); return true; } //----------------------------------------------------------------------------------- @@ -1182,7 +1229,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template template - bool node_server::try_ping(basic_node_data& node_data, p2p_connection_context& context, t_callback cb) + bool node_server::try_ping(basic_node_data& node_data, p2p_connection_context& context, const t_callback& cb) { if(!node_data.my_port) return false; @@ -1296,6 +1343,21 @@ namespace nodetool return 1; } + if (is_peer_id_used(arg.node_data.peer_id)) + { + LOG_PRINT_CCONTEXT_L1("COMMAND_HANDSHAKE came, but seems that peer " << std::hex << arg.node_data.peer_id << " has already been connected to this node, dropping connection"); + drop_connection(context); + return 1; + } + + if (!tools::check_remote_client_version(arg.payload_data.client_version)) + { + LOG_PRINT_CCONTEXT_L2("COMMAND_HANDSHAKE: wrong client version: " << arg.payload_data.client_version << ", closing connection."); + drop_connection(context); + add_ip_fail(context.m_remote_ip); + return 1; + } + if(!handle_maintainers_entry(arg.maintrs_entry)) { LOG_ERROR_CCONTEXT("COMMAND_HANDSHAKE Failed, wrong maintainers entry!, closing connection."); @@ -1334,7 +1396,7 @@ namespace nodetool get_local_node_data(rsp.node_data); m_payload_handler.get_payload_sync_data(rsp.payload_data); fill_maintainers_entry(rsp.maintrs_entry); - LOG_PRINT_GREEN("COMMAND_HANDSHAKE", LOG_LEVEL_1); + LOG_PRINT_GREEN("COMMAND_HANDSHAKE: v" << arg.payload_data.client_version << " top: " << epee::string_tools::pod_to_hex(arg.payload_data.top_id).substr(0, 6) << " @ " << arg.payload_data.current_height - 1, LOG_LEVEL_1); return 1; } //----------------------------------------------------------------------------------- @@ -1380,6 +1442,7 @@ namespace nodetool std::string s = ss.str(); return s; } + //----------------------------------------------------------------------------------- template void node_server::on_connection_new(p2p_connection_context& context) diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index 6fdf437..5cdfa0e 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2018 Zano Project +// Copyright (c) 2014-2019 Zano Project // Copyright (c) 2014-2018 The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying @@ -140,7 +140,7 @@ namespace nodetool friend class boost::serialization::access; epee::critical_section m_peerlist_lock; std::string m_config_folder; - bool m_allow_local_ip; + bool m_allow_local_ip = false; peers_indexed m_peers_gray; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index d28de56..6238bed 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -72,13 +72,13 @@ namespace currency return true; } #define check_core_ready() check_core_ready_(LOCAL_FUNCTION_DEF__) -#define CHECK_CORE_READY() if(!check_core_ready()){res.status = CORE_RPC_STATUS_BUSY;return true;} +#define CHECK_CORE_READY() if(!check_core_ready()){res.status = API_RETURN_CODE_BUSY;return true;} //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, connection_context& cntx) { CHECK_CORE_READY(); res.height = m_core.get_current_blockchain_size(); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -100,7 +100,7 @@ namespace currency res.current_max_allowed_block_size = m_core.get_blockchain_storage().get_current_comulative_blocksize_limit(); if (!res.outgoing_connections_count) res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_connecting; - else if (res.synchronized_connections_count > total_conn/2 ) /* m_p2p.get_payload_object().is_synchronized()*/ + else if (m_p2p.get_payload_object().is_synchronized()) res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_online; else res.daemon_network_state = COMMAND_RPC_GET_INFO::daemon_network_state_synchronizing; @@ -116,7 +116,11 @@ namespace currency //conditional values if (req.flags&COMMAND_RPC_GET_INFO_FLAG_NET_TIME_DELTA_MEDIAN) - res.net_time_delta_median = m_p2p.get_payload_object().get_net_time_delta_median(); + { + int64_t last_median2local_time_diff, last_ntp2local_time_diff; + if (!m_p2p.get_payload_object().get_last_time_sync_difference(last_median2local_time_diff, last_ntp2local_time_diff)) + res.net_time_delta_median = 1; + } if (req.flags&COMMAND_RPC_GET_INFO_FLAG_CURRENT_NETWORK_HASHRATE_50) res.current_network_hashrate_50 = m_core.get_blockchain_storage().get_current_hashrate(50); if (req.flags&COMMAND_RPC_GET_INFO_FLAG_CURRENT_NETWORK_HASHRATE_350) @@ -252,7 +256,7 @@ namespace currency - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -263,14 +267,21 @@ namespace currency if (req.block_ids.back() != m_core.get_blockchain_storage().get_block_id_by_height(0)) { //genesis mismatch, return specific - res.status = CORE_RPC_STATUS_GENESIS_MISMATCH; + res.status = API_RETURN_CODE_GENESIS_MISMATCH; + return true; + } + + if (req.minimum_height >= m_core.get_blockchain_storage().get_current_blockchain_size()) + { + //wrong minimum_height + res.status = API_RETURN_CODE_BAD_ARG; return true; } blockchain_storage::blocks_direct_container bs; - if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height, req.need_global_indexes)) { - res.status = CORE_RPC_STATUS_FAILED; + res.status = API_RETURN_CODE_FAIL; return false; } @@ -279,12 +290,13 @@ namespace currency res.blocks.resize(res.blocks.size()+1); res.blocks.back().block_ptr = b.first; res.blocks.back().txs_ptr = std::move(b.second); + res.blocks.back().coinbase_ptr = b.third; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } - //------------------------------------------------------------------------------------------------------------------------------ + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx) { CHECK_CORE_READY(); @@ -292,28 +304,48 @@ namespace currency if (req.block_ids.back() != m_core.get_blockchain_storage().get_block_id_by_height(0)) { //genesis mismatch, return specific - res.status = CORE_RPC_STATUS_GENESIS_MISMATCH; + res.status = API_RETURN_CODE_GENESIS_MISMATCH; return true; } - std::list > > bs; - if(!m_core.find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if (req.minimum_height >= m_core.get_blockchain_storage().get_current_blockchain_size()) { - res.status = CORE_RPC_STATUS_FAILED; + //wrong minimum_height + res.status = API_RETURN_CODE_BAD_ARG; + return true; + } + + blockchain_storage::blocks_direct_container bs; + if (!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT, req.minimum_height, req.need_global_indexes)) + { + res.status = API_RETURN_CODE_FAIL; return false; } - BOOST_FOREACH(auto& b, bs) + for (auto& b : bs) { res.blocks.resize(res.blocks.size()+1); - res.blocks.back().block = block_to_blob(b.first); + res.blocks.back().block = block_to_blob(b.first->bl); + if (req.need_global_indexes) + { + CHECK_AND_ASSERT_MES(b.third.get(), false, "Internal error on handling COMMAND_RPC_GET_BLOCKS_FAST: b.third is empty, ie coinbase info is not prepared"); + res.blocks.back().coinbase_global_outs = b.third->m_global_output_indexes; + res.blocks.back().tx_global_outs.resize(b.second.size()); + } + size_t i = 0; + BOOST_FOREACH(auto& t, b.second) { - res.blocks.back().txs.push_back(tx_to_blob(t)); + res.blocks.back().txs.push_back(tx_to_blob(t->tx)); + if (req.need_global_indexes) + { + res.blocks.back().tx_global_outs[i].v = t->m_global_output_indexes; + } + i++; } } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -326,7 +358,7 @@ namespace currency return true; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; std::stringstream ss; typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount outs_for_amount; typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; @@ -342,21 +374,26 @@ namespace currency }); std::string s = ss.str(); LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, connection_context& cntx) { CHECK_CORE_READY(); - bool r = m_core.get_tx_outputs_gindexs(req.txid, res.o_indexes); - if(!r) + res.tx_global_outs.resize(req.txids.size()); + size_t i = 0; + for (auto& txid : req.txids) { - res.status = "Failed"; - return true; + bool r = m_core.get_tx_outputs_gindexs(txid, res.tx_global_outs[i].v); + if (!r) + { + res.status = API_RETURN_CODE_FAIL; + return true; + } + i++; } - res.status = CORE_RPC_STATUS_OK; - LOG_PRINT_L2("COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES: [" << res.o_indexes.size() << "]"); + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -368,7 +405,7 @@ namespace currency return true; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -382,13 +419,17 @@ namespace currency return true; } + res.tx_expiration_ts_median = m_core.get_blockchain_storage().get_tx_expiration_median(); + + for(auto& tx: txs) { res.txs.push_back(t_serializable_object_to_blob(tx)); } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_scan_pos(const COMMAND_RPC_SCAN_POS::request& req, COMMAND_RPC_SCAN_POS::response& res, connection_context& cntx) { CHECK_CORE_READY(); @@ -399,7 +440,7 @@ namespace currency bool core_rpc_server::on_check_keyimages(const COMMAND_RPC_CHECK_KEYIMAGES::request& req, COMMAND_RPC_CHECK_KEYIMAGES::response& res, connection_context& cntx) { m_core.get_blockchain_storage().check_keyimages(req.images, res.images_stat); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -442,28 +483,28 @@ namespace currency res.missed_tx.push_back(string_tools::pod_to_hex(miss_tx)); } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ -// bool core_rpc_server::on_rpc_get_all_offers(const COMMAND_RPC_GET_ALL_OFFERS::request& req, COMMAND_RPC_GET_ALL_OFFERS::response& res, connection_context& cntx) -// { -// m_core.get_blockchain_storage().get_all_offers(res.offers); -// res.status = CORE_RPC_STATUS_OK; -// return true; -// } + bool core_rpc_server::on_get_offers_ex(const COMMAND_RPC_GET_OFFERS_EX::request& req, COMMAND_RPC_GET_OFFERS_EX::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + { + m_of.get_offers_ex(req.filter, res.offers, res.total_offers, m_core.get_blockchain_storage().get_core_runtime_config().get_core_time()); + res.status = API_RETURN_CODE_OK; + return true; + } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_pos_mining_details(const COMMAND_RPC_GET_POS_MINING_DETAILS::request& req, COMMAND_RPC_GET_POS_MINING_DETAILS::response& res, connection_context& cntx) { if (!m_p2p.get_connections_count()) { - res.status = CORE_RPC_STATUS_DISCONNECTED; + res.status = API_RETURN_CODE_DISCONNECTED; return true; } res.pos_mining_allowed = m_core.get_blockchain_storage().is_pos_allowed(); if (!res.pos_mining_allowed) { - res.status = CORE_RPC_STATUS_NOT_FOUND; + res.status = API_RETURN_CODE_NOT_FOUND; return true; } @@ -472,21 +513,21 @@ namespace currency //TODO: need atomic operation with build_stake_modifier() res.starter_timestamp = m_core.get_blockchain_storage().get_last_timestamps_check_window_median(); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_current_core_tx_expiration_median(const COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN::request& req, COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN::response& res, connection_context& cntx) { res.expiration_median = m_core.get_blockchain_storage().get_tx_expiration_median(); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_rpc_get_blocks_details(const COMMAND_RPC_GET_BLOCKS_DETAILS::request& req, COMMAND_RPC_GET_BLOCKS_DETAILS::response& res, connection_context& cntx) { m_core.get_blockchain_storage().get_main_blocks_rpc_details(req.height_start, req.count, req.ignore_transactions, res.blocks); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -509,7 +550,7 @@ namespace currency } } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -520,25 +561,25 @@ namespace currency return false; m_core.get_blockchain_storage().search_by_id(id, res.types_found); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_out_info(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES_BY_AMOUNT::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES_BY_AMOUNT::response& res, connection_context& cntx) { if (!m_core.get_blockchain_storage().get_global_index_details(req, res)) - res.status = CORE_RPC_STATUS_NOT_FOUND; + res.status = API_RETURN_CODE_NOT_FOUND; else - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_multisig_info(const COMMAND_RPC_GET_MULTISIG_INFO::request& req, COMMAND_RPC_GET_MULTISIG_INFO::response& res, connection_context& cntx) { if (!m_core.get_blockchain_storage().get_multisig_id_details(req, res)) - res.status = CORE_RPC_STATUS_NOT_FOUND; + res.status = API_RETURN_CODE_NOT_FOUND; else - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -552,7 +593,7 @@ namespace currency { m_core.get_tx_pool().get_transactions_details(req.ids, res.txs); } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -566,14 +607,14 @@ namespace currency { m_core.get_tx_pool().get_transactions_brief_details(req.ids, res.txs); } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_all_pool_tx_list(const COMMAND_RPC_GET_ALL_POOL_TX_LIST::request& req, COMMAND_RPC_GET_ALL_POOL_TX_LIST::response& res, connection_context& cntx) { m_core.get_tx_pool().get_all_transactions_list(res.ids); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -585,7 +626,7 @@ namespace currency { res.aliases_que.push_back(alias_info_to_rpc_alias_info(a)); } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -597,7 +638,7 @@ namespace currency error_resp.message = "the requested block has not been found"; return false; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -609,14 +650,14 @@ namespace currency error_resp.message = "the requested block has not been found"; return false; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_alt_blocks_details(const COMMAND_RPC_GET_ALT_BLOCKS_DETAILS::request& req, COMMAND_RPC_GET_ALT_BLOCKS_DETAILS::response& res, connection_context& cntx) { m_core.get_blockchain_storage().get_alt_blocks_rpc_details(req.offset, req.count, res.blocks); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -635,7 +676,7 @@ namespace currency if (!m_p2p.get_payload_object().get_synchronized_connections_count()) { LOG_PRINT_L0("[on_send_raw_tx]: Failed to send, daemon not connected to net"); - res.status = CORE_RPC_STATUS_DISCONNECTED; + res.status = API_RETURN_CODE_DISCONNECTED; return true; } @@ -667,7 +708,7 @@ namespace currency r.txs.push_back(tx_blob); m_core.get_protocol()->relay_transactions(r, fake_context); //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -691,7 +732,7 @@ namespace currency currency_connection_context fake_context = AUTO_VAL_INIT(fake_context); bool call_res = m_core.get_protocol()->relay_transactions(r, fake_context); if (call_res) - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return call_res; } //------------------------------------------------------------------------------------------------------------------------------ @@ -710,7 +751,7 @@ namespace currency res.status = "Failed, mining not started"; return true; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -722,7 +763,7 @@ namespace currency res.status = "Failed, mining not stopped"; return true; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -730,7 +771,7 @@ namespace currency { CHECK_CORE_READY(); res.count = m_core.get_current_blockchain_size(); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -789,31 +830,49 @@ namespace currency return false; } - block b = AUTO_VAL_INIT(b); - wide_difficulty_type dt = 0; - currency::pos_entry pe = AUTO_VAL_INIT(pe); - pe.amount = req.pos_amount; - pe.index = req.pos_index; - //pe.keyimage key image will be set in the wallet - //pe.wallet_index is not included in serialization map, TODO: refactoring here - if (!m_core.get_block_template(b, miner_address, stakeholder_address, dt, res.height, req.extra_text, req.pos_block, pe)) + create_block_template_params params = AUTO_VAL_INIT(params); + params.miner_address = miner_address; + params.stakeholder_address = stakeholder_address; + params.ex_nonce = req.extra_text; + params.pos = req.pos_block; + params.pe.amount = req.pos_amount; + params.pe.index = req.pos_index; + params.pe.stake_unlock_time = req.stake_unlock_time; + //params.pe.keyimage key image will be set in the wallet + //params.pe.wallet_index is not included in serialization map, TODO: refactoring here + params.pcustom_fill_block_template_func = nullptr; + if (req.explicit_transaction.size()) + { + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(req.explicit_transaction, tx)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Wrong parameters: explicit_transaction is invalid"; + LOG_ERROR("Failed to parse explicit_transaction blob"); + return false; + } + params.explicit_txs.push_back(tx); + } + + create_block_template_response resp = AUTO_VAL_INIT(resp); + if (!m_core.get_block_template(params, resp)) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Internal error: failed to create block template"; LOG_ERROR("Failed to create block template"); return false; } - res.difficulty = dt.convert_to(); - blobdata block_blob = t_serializable_object_to_blob(b); + res.difficulty = resp.diffic.convert_to(); + blobdata block_blob = t_serializable_object_to_blob(resp.b); res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); - res.prev_hash = string_tools::pod_to_hex(b.prev_id); - + res.prev_hash = string_tools::pod_to_hex(resp.b.prev_id); + res.height = resp.height; //calculate epoch seed res.seed = currency::ethash_epoch_to_seed(currency::ethash_height_to_epoch(res.height)); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } @@ -846,7 +905,7 @@ namespace currency block_verification_context bvc = AUTO_VAL_INIT(bvc); if(!m_core.handle_block_found(b, &bvc)) { - if (bvc.added_to_altchain) + if (bvc.m_added_to_altchain) { error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_ADDED_AS_ALTERNATIVE; error_resp.message = "Block added as alternative"; @@ -858,10 +917,67 @@ namespace currency } //@#@ //temporary double check timestamp - if (time(NULL) - get_actual_timestamp(b) > 5) + if (time(NULL) - static_cast(get_actual_timestamp(b)) > 5) { LOG_PRINT_RED_L0("Found block (" << get_block_hash(b) << ") timestamp (" << get_actual_timestamp(b) - << ") is suspiciously less (" << time(NULL) - get_actual_timestamp(b) << ") then curren time( " << time(NULL) << ")"); + << ") is suspiciously less (" << time(NULL) - static_cast(get_actual_timestamp(b)) << ") than current time ( " << time(NULL) << ")"); + //mark node to make it easier to find it via scanner + m_core.get_blockchain_storage().get_performnce_data().epic_failure_happend = true; + } + // + + + res.status = "OK"; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_submitblock2(const COMMAND_RPC_SUBMITBLOCK2::request& req, COMMAND_RPC_SUBMITBLOCK2::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + { + CHECK_CORE_READY(); + + + block b = AUTO_VAL_INIT(b); + if (!parse_and_validate_block_from_blob(req.b, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + + block_verification_context bvc = AUTO_VAL_INIT(bvc); + for (const auto& txblob : req.explicit_txs) + { + + crypto::hash tx_hash = AUTO_VAL_INIT(tx_hash); + transaction tx = AUTO_VAL_INIT(tx); + if (!parse_and_validate_tx_from_blob(txblob.blob, tx, tx_hash)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong explicit tx blob"; + return false; + } + bvc.m_onboard_transactions[tx_hash] = tx; + } + + + if (!m_core.handle_block_found(b, &bvc)) + { + if (bvc.m_added_to_altchain) + { + error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_ADDED_AS_ALTERNATIVE; + error_resp.message = "Block added as alternative"; + return false; + } + error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_NOT_ACCEPTED; + error_resp.message = "Block not accepted"; + return false; + } + //@#@ + //temporary double check timestamp + if (time(NULL) - static_cast(get_actual_timestamp(b)) > 5) + { + LOG_PRINT_RED_L0("Found block (" << get_block_hash(b) << ") timestamp (" << get_actual_timestamp(b) + << ") is suspiciously less (" << time(NULL) - static_cast(get_actual_timestamp(b)) << ") than current time ( " << time(NULL) << ")"); //mark node to make it easier to find it via scanner m_core.get_blockchain_storage().get_performnce_data().epic_failure_happend = true; } @@ -921,7 +1037,7 @@ namespace currency error_resp.message = "Internal error: can't produce valid response."; return false; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -962,7 +1078,7 @@ namespace currency error_resp.message = "Internal error: can't produce valid response."; return false; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -989,7 +1105,7 @@ namespace currency error_resp.message = "Internal error: can't produce valid response."; return false; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1009,7 +1125,7 @@ namespace currency } if(!m_core.get_blockchain_storage().get_alias_info(req.alias, aib)) { - res.status = CORE_RPC_STATUS_NOT_FOUND; + res.status = API_RETURN_CODE_NOT_FOUND; return true; } res.alias_details.address = currency::get_account_address_as_str(aib.m_address); @@ -1017,7 +1133,7 @@ namespace currency if (aib.m_view_key.size()) res.alias_details.tracking_key = string_tools::pod_to_hex(aib.m_view_key.back()); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1038,7 +1154,7 @@ namespace currency return true; }); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } @@ -1057,7 +1173,7 @@ namespace currency alias_info_to_rpc_alias_info(alias, ai, res.aliases.back()); }, req.offset, req.count); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } @@ -1122,7 +1238,7 @@ namespace currency { if(!check_core_ready()) { - res.status = CORE_RPC_STATUS_BUSY; + res.status = API_RETURN_CODE_BUSY; return true; } @@ -1141,7 +1257,7 @@ namespace currency } } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1149,7 +1265,7 @@ namespace currency { if(!check_core_ready()) { - res.status = CORE_RPC_STATUS_BUSY; + res.status = API_RETURN_CODE_BUSY; return true; } @@ -1174,9 +1290,20 @@ namespace currency res.reward = get_alias_coast_from_fee(req.alias, std::max(default_tx_fee, current_median_fee)); if (res.reward) - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; else - res.status = CORE_RPC_STATUS_NOT_FOUND; + res.status = API_RETURN_CODE_NOT_FOUND; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_est_height_from_date(const COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& req, COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& res, connection_context& cntx) + { + bool r = m_core.get_blockchain_storage().get_est_height_from_date(req.timestamp, res.h); + + if (r) + res.status = API_RETURN_CODE_OK; + else + res.status = API_RETURN_CODE_NOT_FOUND; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1185,7 +1312,7 @@ namespace currency account_public_address addr = AUTO_VAL_INIT(addr); if (!get_account_address_from_str(addr, req)) { - res.status = CORE_RPC_STATUS_FAILED; + res.status = API_RETURN_CODE_FAIL; return true; } //res.alias = m_core.get_blockchain_storage().get_alias_by_address(addr); @@ -1195,20 +1322,20 @@ namespace currency req2.alias = m_core.get_blockchain_storage().get_alias_by_address(addr); if (!req2.alias.size()) { - res.status = CORE_RPC_STATUS_NOT_FOUND; + res.status = API_RETURN_CODE_NOT_FOUND; return true; } bool r = this->on_get_alias_details(req2, res2, error_resp, cntx); - if (!r || res2.status != CORE_RPC_STATUS_OK) + if (!r || res2.status != API_RETURN_CODE_OK) { - res.status = CORE_RPC_STATUS_FAILED; + res.status = API_RETURN_CODE_FAIL; return true; } res.alias_info.details = res2.alias_details; res.alias_info.alias = req2.alias; - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1216,7 +1343,7 @@ namespace currency { if(!check_core_ready()) { - res.status = CORE_RPC_STATUS_BUSY; + res.status = API_RETURN_CODE_BUSY; return true; } block b = AUTO_VAL_INIT(b); @@ -1234,7 +1361,7 @@ namespace currency LOG_ERROR("Submited block not accepted"); return true; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1242,19 +1369,19 @@ namespace currency { if (!check_core_ready()) { - res.status = CORE_RPC_STATUS_BUSY; + res.status = API_RETURN_CODE_BUSY; return true; } - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_reset_transaction_pool(const COMMAND_RPC_RESET_TX_POOL::request& req, COMMAND_RPC_RESET_TX_POOL::response& res, connection_context& cntx) { m_core.get_tx_pool().purge_transactions(); - res.status = CORE_RPC_STATUS_OK; + res.status = API_RETURN_CODE_OK; return true; } } diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index c74e31b..d95dd52 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -39,6 +39,7 @@ namespace currency bool init(const boost::program_options::variables_map& vm); bool on_get_blocks_direct(const COMMAND_RPC_GET_BLOCKS_DIRECT::request& req, COMMAND_RPC_GET_BLOCKS_DIRECT::response& res, connection_context& cntx); + bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, connection_context& cntx); bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx); @@ -55,7 +56,7 @@ namespace currency bool on_scan_pos(const COMMAND_RPC_SCAN_POS::request& req, COMMAND_RPC_SCAN_POS::response& res, connection_context& cntx); bool on_rpc_get_blocks_details(const COMMAND_RPC_GET_BLOCKS_DETAILS::request& req, COMMAND_RPC_GET_BLOCKS_DETAILS::response& res, connection_context& cntx); bool on_force_relaey_raw_txs(const COMMAND_RPC_FORCE_RELAY_RAW_TXS::request& req, COMMAND_RPC_FORCE_RELAY_RAW_TXS::response& res, connection_context& cntx); - //bool on_rpc_get_all_offers(const COMMAND_RPC_GET_ALL_OFFERS::request& req, COMMAND_RPC_GET_ALL_OFFERS::response& res, connection_context& cntx); + bool on_get_offers_ex(const COMMAND_RPC_GET_OFFERS_EX::request& req, COMMAND_RPC_GET_OFFERS_EX::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); //json_rpc @@ -63,6 +64,7 @@ namespace currency bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); + bool on_submitblock2(const COMMAND_RPC_SUBMITBLOCK2::request& req, COMMAND_RPC_SUBMITBLOCK2::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); @@ -87,6 +89,8 @@ namespace currency bool on_get_main_block_details(const COMMAND_RPC_GET_BLOCK_DETAILS::request& req, COMMAND_RPC_GET_BLOCK_DETAILS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_alt_block_details(const COMMAND_RPC_GET_BLOCK_DETAILS::request& req, COMMAND_RPC_GET_BLOCK_DETAILS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); bool on_get_alt_blocks_details(const COMMAND_RPC_GET_ALT_BLOCKS_DETAILS::request& req, COMMAND_RPC_GET_ALT_BLOCKS_DETAILS::response& res, connection_context& cntx); + bool on_get_est_height_from_date(const COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::request& req, COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE::response& res, connection_context& cntx); + @@ -125,12 +129,14 @@ namespace currency MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) + MAP_JON_RPC_WE("submitblock2", on_submitblock2, COMMAND_RPC_SUBMITBLOCK2) MAP_JON_RPC_WE("getlastblockheader", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("getblockheaderbyhash", on_get_block_header_by_hash, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH) MAP_JON_RPC_WE("getblockheaderbyheight", on_get_block_header_by_height, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT) MAP_JON_RPC_WE("get_alias_details", on_get_alias_details, COMMAND_RPC_GET_ALIAS_DETAILS) MAP_JON_RPC_WE("get_alias_by_address", on_alias_by_address, COMMAND_RPC_GET_ALIASES_BY_ADDRESS) MAP_JON_RPC_WE("get_alias_reward", on_get_alias_reward, COMMAND_RPC_GET_ALIAS_REWARD) + MAP_JON_RPC ("get_est_height_from_date", on_get_est_height_from_date, COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE) //block explorer api MAP_JON_RPC ("get_blocks_details", on_rpc_get_blocks_details, COMMAND_RPC_GET_BLOCKS_DETAILS) MAP_JON_RPC_WE("get_tx_details", on_get_tx_details, COMMAND_RPC_GET_TX_DETAILS) @@ -151,6 +157,8 @@ namespace currency // MAP_JON_RPC ("reset_transaction_pool", on_reset_transaction_pool, COMMAND_RPC_RESET_TX_POOL) MAP_JON_RPC ("get_current_core_tx_expiration_median", on_get_current_core_tx_expiration_median, COMMAND_RPC_GET_CURRENT_CORE_TX_EXPIRATION_MEDIAN) + // + MAP_JON_RPC_WE("marketplace_global_get_offers_ex", on_get_offers_ex, COMMAND_RPC_GET_OFFERS_EX) //remote miner rpc MAP_JON_RPC_N(on_login, mining::COMMAND_RPC_LOGIN) MAP_JON_RPC_N(on_getjob, mining::COMMAND_RPC_GETJOB) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index fe14304..2ef811a 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -5,6 +5,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #pragma once +#include "serialization/keyvalue_hexemizer.h" #include "currency_protocol/currency_protocol_defs.h" #include "currency_core/currency_basic.h" #include "currency_core/difficulty.h" @@ -14,17 +15,10 @@ #include "storages/portable_storage_base.h" #include "currency_core/offers_service_basics.h" #include "currency_core/basic_api_response_codes.h" - +#include "common/error_codes.h" namespace currency { //----------------------------------------------- -#define CORE_RPC_STATUS_OK BASIC_RESPONSE_STATUS_OK -#define CORE_RPC_STATUS_BUSY BASIC_RESPONSE_STATUS_BUSY -#define CORE_RPC_STATUS_NOT_FOUND BASIC_RESPONSE_STATUS_NOT_FOUND -#define CORE_RPC_STATUS_FAILED BASIC_RESPONSE_STATUS_FAILED -#define CORE_RPC_STATUS_GENESIS_MISMATCH "GENESIS_MISMATCH" -#define CORE_RPC_STATUS_DISCONNECTED "DISCONNECTED" - struct alias_rpc_details_base { @@ -115,9 +109,13 @@ namespace currency struct request { + bool need_global_indexes; + uint64_t minimum_height; std::list block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(need_global_indexes) + KV_SERIALIZE(minimum_height) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) END_KV_SERIALIZE_MAP() }; @@ -141,8 +139,6 @@ namespace currency typedef COMMAND_RPC_GET_BLOCKS_FAST_T COMMAND_RPC_GET_BLOCKS_FAST; typedef COMMAND_RPC_GET_BLOCKS_FAST_T COMMAND_RPC_GET_BLOCKS_DIRECT; - - //----------------------------------------------- struct COMMAND_RPC_GET_TRANSACTIONS { @@ -169,6 +165,30 @@ namespace currency END_KV_SERIALIZE_MAP() }; }; + + //----------------------------------------------- + struct COMMAND_RPC_GET_EST_HEIGHT_FROM_DATE + { + struct request + { + uint64_t timestamp; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(timestamp) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t h; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(h) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; //----------------------------------------------- struct COMMAND_RPC_GET_TX_POOL { @@ -181,10 +201,12 @@ namespace currency struct response { std::list txs; //transactions blobs + uint64_t tx_expiration_ts_median; std::string status; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) + KV_SERIALIZE(tx_expiration_ts_median) KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; @@ -216,19 +238,21 @@ namespace currency { struct request { - crypto::hash txid; + std::list txids; + BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_VAL_POD_AS_BLOB(txid) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(txids) END_KV_SERIALIZE_MAP() }; struct response { - std::vector o_indexes; + std::vector > > tx_global_outs; + //std::vector o_indexes; std::string status; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(o_indexes) + KV_SERIALIZE(tx_global_outs) KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() }; @@ -614,7 +638,8 @@ namespace currency daemon_network_state_online = 2, daemon_network_state_loading_core = 3, daemon_network_state_internal_error = 4, - daemon_network_state_unloading_core = 5 + daemon_network_state_unloading_core = 5, + daemon_network_state_downloading_database = 6 }; struct response @@ -771,21 +796,24 @@ namespace currency { struct request { - //uint64_t reserve_size; //max 255 bytes + blobdata explicit_transaction; std::string extra_text; std::string wallet_address; std::string stakeholder_address; bool pos_block; //is pos block uint64_t pos_amount; // uint64_t pos_index; // + uint64_t stake_unlock_time; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(explicit_transaction) KV_SERIALIZE(extra_text) KV_SERIALIZE(wallet_address) KV_SERIALIZE(stakeholder_address); KV_SERIALIZE(pos_block) KV_SERIALIZE(pos_amount) KV_SERIALIZE(pos_index) + KV_SERIALIZE(stake_unlock_time) END_KV_SERIALIZE_MAP() }; @@ -823,6 +851,29 @@ namespace currency }; }; + struct COMMAND_RPC_SUBMITBLOCK2 + { + struct request + { + std::string b; //hex encoded block blob + std::list explicit_txs; //hex encoded tx blobs + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_BLOB_AS_HEX_STRING(b) + KV_SERIALIZE(explicit_txs) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + struct block_header_response { uint8_t major_version; @@ -1084,6 +1135,7 @@ namespace currency uint64_t starter_timestamp; crypto::hash last_block_hash; bool is_pos_allowed; + uint64_t iterations_processed; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) @@ -1091,6 +1143,7 @@ namespace currency KV_SERIALIZE(block_timestamp) KV_SERIALIZE(height) KV_SERIALIZE(is_pos_allowed) + KV_SERIALIZE(iterations_processed) KV_SERIALIZE(starter_timestamp) KV_SERIALIZE_VAL_POD_AS_BLOB(last_block_hash); END_KV_SERIALIZE_MAP() @@ -1145,10 +1198,12 @@ namespace currency { uint64_t amount; uint64_t multisig_count; + std::string htlc_origin; std::string kimage_or_ms_id; std::vector global_indexes; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) + KV_SERIALIZE(htlc_origin) KV_SERIALIZE(kimage_or_ms_id) KV_SERIALIZE(global_indexes) KV_SERIALIZE(multisig_count) @@ -1501,12 +1556,13 @@ namespace currency }; - struct COMMAND_RPC_GET_ALL_OFFERS + struct COMMAND_RPC_GET_OFFERS_EX { struct request { - + bc_services::core_offers_filter filter; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(filter) END_KV_SERIALIZE_MAP() }; diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 16ac207..b951a10 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -16,7 +16,7 @@ #include "common/varint.h" #include "warnings.h" -PUSH_WARNINGS +PUSH_VS_WARNINGS DISABLE_VS_WARNINGS(4244) DISABLE_VS_WARNINGS(4100) @@ -172,4 +172,4 @@ struct binary_archive : public binary_archive_base } }; -POP_WARNINGS +POP_VS_WARNINGS diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index 425ca51..a7f6625 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -1,3 +1,4 @@ +// Copyright (c) 2014-2019 The Zano Project // Copyright (c) 2014-2017 The The Louisdor Project // Copyright (c) 2012-2013 The Cryptonote developers // Distributed under the MIT/X11 software license, see the accompanying @@ -73,7 +74,7 @@ struct json_archive; template <> struct json_archive : public json_archive_base { - json_archive(stream_type &s, bool indent = false) : base_type(s, indent) { } + json_archive(stream_type &s, bool indent = false) : base_type(s, indent), inner_array_size_{} { } template static auto promote_to_printable_integer_type(T v) -> decltype(+v) diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 138a037..ff29a0e 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -98,18 +98,43 @@ do { \ if (!ar.stream().good()) return false; \ } while (0); -#define DEFINE_SERIALIZATION_VERSION(v) inline static uint32_t get_serialization_veraion(){ return v; } +#define DEFINE_SERIALIZATION_VERSION(v) inline static uint32_t get_serialization_version() { return v; } #define VERSION_ENTRY(f) \ do { \ ar.tag(#f); \ if (ar.is_saving_arch()) \ - f = this->get_serialization_veraion(); \ + f = this->get_serialization_version(); \ bool r = ::do_serialize(ar, f); \ if (!r || !ar.stream().good()) return false; \ } while (0); +template +class serializable_pair : public std::pair +{ + typedef std::pair base; +public: + serializable_pair() + {} + serializable_pair(const first_type& a, const second_type& b) :std::pair(a, b) + {} + serializable_pair(const serializable_pair& sp) :std::pair(sp.first, sp.second) + {} + + BEGIN_SERIALIZE_OBJECT() + FIELD(base::first) + FIELD(base::second) + END_SERIALIZE() +}; + +template +serializable_pair make_serializable_pair(const first_type& first_value, const second_type& second_value) +{ + return serializable_pair(first_value, second_value); +} + + namespace serialization { namespace detail { diff --git a/src/serialization/stl_containers.h b/src/serialization/stl_containers.h index ae4d22a..34981ed 100644 --- a/src/serialization/stl_containers.h +++ b/src/serialization/stl_containers.h @@ -265,4 +265,4 @@ bool do_serialize(Archive &ar, std::vector &v) } ar.end_array(); return true; -} +} \ No newline at end of file