leiless
commented
11 months ago FYI, I happened to bump the LMDB lib version to 0.9.70 (based on the HEAD commit 30288c72573ceac719627183f1058cad1dd08b74) and rebuild the lib/py-lmdb/env-copy-txn.patch.
The py-lmdb package was successfully build on Windows 11 and Ubuntu 22.04 LTS.
$ pip3.exe install -U patch-ng
$ python.exe ./setup.py build
...
cl : Command line warning D9025 : overriding '/DNDEBUG' with '/UNDEBUG'
cl : Command line warning D9025 : overriding '/W3' with '/w'
cpython.c
"C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.38.33130\bin\HostX86\x64\link.exe" /nologo /INCREMENTAL:NO /LTCG /DLL /MANIFEST:EMBED,ID=2 /MANIFESTUAC:NO /LIBPATH:C:\Python311\libs /LIBPATH:C:\Python311 /LIBPATH:C:\Python311\PCbuild\amd64 "/LIBPATH:C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.38.33130\lib\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.22000.0\ucrt\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\\lib\10.0.22000.0\\um\x64" Advapi32.lib /EXPORT:PyInit_cpython build\temp.win-amd64-cpython-311\Release\build/lib/mdb.obj build\temp.win-amd64-cpython-311\Release\build/lib/midl.obj build\temp.win-amd64-cpython-311\Release\lmdb/cpython.obj /OUT:build\lib.win-amd64-cpython-311\lmdb\cpython.cp311-win_amd64.pyd /IMPLIB:build\temp.win-amd64-cpython-311\Release\build/lib\cpython.cp311-win_amd64.lib
Creating library build\temp.win-amd64-cpython-311\Release\build/lib\cpython.cp311-win_amd64.lib and object build\temp.win-amd64-cpython-311\Release\build/lib\cpython.cp311-win_amd64.exp
Generating code
Finished generating code>>> import lmdb
>>> lmdb.version(True)
(0, 9, 70, 1)Don't know if any tests can be carried on?
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 28cfc36f53..5b32750031 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -722,9 +722,13 @@ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
* consumes more CPU and runs more slowly than the default.
* Currently it fails if the environment has suffered a page leak.
* </ul>
+ * @param[in] txn Transaction used for the copy. If NULL, a temporary
+ * transaction will be used. This is only valid if the #MDB_CP_COMPACT
+ * flag is set.
+ *
* @return A non-zero error value on failure and 0 on success.
*/
-int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
+int mdb_env_copy3(MDB_env *env, const char *path, unsigned int flags, MDB_txn *txn);
/** @brief Copy an LMDB environment to the specified file descriptor,
* with options.
@@ -741,9 +745,12 @@ int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
* have already been opened for Write access.
* @param[in] flags Special options for this operation.
* See #mdb_env_copy2() for options.
+ * @param[in] txn Transaction used for the copy. If NULL, a temporary
+ * transaction will be used. This is only valid if the #MDB_CP_COMPACT
+ * flag is set.
* @return A non-zero error value on failure and 0 on success.
*/
-int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags);
+int mdb_env_copyfd3(MDB_env *env, mdb_filehandle_t fd, unsigned int flags, MDB_txn *txn);
/** @brief Return statistics about the LMDB environment.
*
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 965ba2a2b8..f32a681baa 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10375,12 +10375,12 @@ done:
/** Copy environment with compaction. */
static int ESECT
-mdb_env_copyfd1(MDB_env *env, HANDLE fd)
+mdb_env_copyfd1(MDB_env *env, HANDLE fd, MDB_txn *txn)
{
MDB_meta *mm;
MDB_page *mp;
mdb_copy my = {0};
- MDB_txn *txn = NULL;
+ MDB_txn *orig_txn = txn;
pthread_t thr;
pgno_t root, new_root;
int rc = MDB_SUCCESS;
@@ -10426,9 +10426,11 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
if (rc)
goto done;
- rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
- if (rc)
- goto finish;
+ if (!txn) {
+ rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+ if (rc)
+ goto finish;
+ }
mp = (MDB_page *)my.mc_wbuf[0];
memset(mp, 0, NUM_METAS * env->me_psize);
@@ -10488,7 +10490,8 @@ finish:
my.mc_error = rc;
mdb_env_cthr_toggle(&my, 1 | MDB_EOF);
rc = THREAD_FINISH(thr);
- mdb_txn_abort(txn);
+ if (!orig_txn)
+ mdb_txn_abort(txn);
done:
#ifdef _WIN32
@@ -10605,12 +10608,22 @@ leave:
}
int ESECT
-mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags)
+mdb_env_copyfd3(MDB_env *env, HANDLE fd, unsigned int flags, MDB_txn *txn)
{
if (flags & MDB_CP_COMPACT)
- return mdb_env_copyfd1(env, fd);
+ return mdb_env_copyfd1(env, fd, txn);
else
+ {
+ if (txn) /* may only use txn with compact */
+ return EINVAL;
return mdb_env_copyfd0(env, fd);
+ }
+}
+
+int ESECT
+mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags)
+{
+ return mdb_env_copyfd3(env, fd, flags, NULL);
}
int ESECT
@@ -10621,6 +10634,12 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd)
int ESECT
mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
+{
+ return mdb_env_copy3(env, path, flags, NULL);
+}
+
+int ESECT
+mdb_env_copy3(MDB_env *env, const char *path, unsigned int flags, MDB_txn *txn)
{
int rc;
MDB_name fname;
@@ -10632,7 +10651,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
mdb_fname_destroy(fname);
}
if (rc == MDB_SUCCESS) {
- rc = mdb_env_copyfd2(env, newfd, flags);
+ rc = mdb_env_copyfd3(env, newfd, flags, txn);
if (close(newfd) < 0 && rc == MDB_SUCCESS)
rc = ErrCode();
}
ghost
jnwatson
https://github.com/LMDB/lmdb/blob/mdb.master/libraries/liblmdb/lmdb.h#L227-L232
The upstream LMDB is already 0.9.70 and has many bug fixes. py-lmdb binding is currently 0.9.29 (Mar 17, 2021) (This version does not support sparse DB file on Windows).
Please consider upgrading the LMDB lib source code.