From 20b2d23218648e12fa10d8bd4ab9250c49944945 Mon Sep 17 00:00:00 2001 From: Corey Kaylor Date: Sun, 5 Jul 2026 18:07:33 -0500 Subject: [PATCH] Add wasm support --- .github/workflows/wasm.yml | 66 + README.md | 84 +- lmdb/compile-lmdb-macos.sh | 9 +- src/LightningDB.Tests/BrowserStorageTests.cs | 24 + src/LightningDB.Tests/EncryptionTests.cs | 20 + src/LightningDB.WasmTest/App.razor | 6 + .../Layout/MainLayout.razor | 3 + .../LightningDB.WasmTest.csproj | 37 + src/LightningDB.WasmTest/Pages/Home.razor | 545 + src/LightningDB.WasmTest/Pages/NotFound.razor | 5 + src/LightningDB.WasmTest/Program.cs | 11 + .../Properties/launchSettings.json | 25 + src/LightningDB.WasmTest/README.md | 92 + src/LightningDB.WasmTest/_Imports.razor | 10 + src/LightningDB.WasmTest/wwwroot/css/app.css | 89 + src/LightningDB.WasmTest/wwwroot/icon-192.png | Bin 0 -> 2626 bytes src/LightningDB.WasmTest/wwwroot/index.html | 33 + src/LightningDB/AesGcmCipher.cs | 6 +- src/LightningDB/LightningBrowserStorage.cs | 154 + src/LightningDB/LightningChecksum.cs | 20 + src/LightningDB/LightningDB.Wasm.targets | 32 + src/LightningDB/LightningDB.csproj | 5 + src/LightningDB/LightningDB.targets | 5 +- src/LightningDB/LightningEnvironment.cs | 69 +- src/LightningDB/Native/Lmdb.cs | 46 + .../NativeChaCha20Poly1305Cipher.cs | 64 + src/LightningDB/wasm/MONOCYPHER-LICENCE.md | 167 + src/LightningDB/wasm/README.md | 30 + src/LightningDB/wasm/lmdb.c | 29 + src/LightningDB/wasm/lmdb.h | 1934 +++ src/LightningDB/wasm/lmdb_wasm_crypto.c | 107 + src/LightningDB/wasm/mdb.c | 12856 ++++++++++++++++ src/LightningDB/wasm/midl.c | 452 + src/LightningDB/wasm/midl.h | 210 + src/LightningDB/wasm/monocypher.c | 2956 ++++ src/LightningDB/wasm/monocypher.h | 321 + 36 files changed, 20514 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/wasm.yml create mode 100644 src/LightningDB.Tests/BrowserStorageTests.cs create mode 100644 src/LightningDB.WasmTest/App.razor create mode 100644 src/LightningDB.WasmTest/Layout/MainLayout.razor create mode 100644 src/LightningDB.WasmTest/LightningDB.WasmTest.csproj create mode 100644 src/LightningDB.WasmTest/Pages/Home.razor create mode 100644 src/LightningDB.WasmTest/Pages/NotFound.razor create mode 100644 src/LightningDB.WasmTest/Program.cs create mode 100644 src/LightningDB.WasmTest/Properties/launchSettings.json create mode 100644 src/LightningDB.WasmTest/README.md create mode 100644 src/LightningDB.WasmTest/_Imports.razor create mode 100644 src/LightningDB.WasmTest/wwwroot/css/app.css create mode 100644 src/LightningDB.WasmTest/wwwroot/icon-192.png create mode 100644 src/LightningDB.WasmTest/wwwroot/index.html create mode 100644 src/LightningDB/LightningBrowserStorage.cs create mode 100644 src/LightningDB/LightningDB.Wasm.targets create mode 100644 src/LightningDB/NativeChaCha20Poly1305Cipher.cs create mode 100644 src/LightningDB/wasm/MONOCYPHER-LICENCE.md create mode 100644 src/LightningDB/wasm/README.md create mode 100644 src/LightningDB/wasm/lmdb.c create mode 100644 src/LightningDB/wasm/lmdb.h create mode 100644 src/LightningDB/wasm/lmdb_wasm_crypto.c create mode 100644 src/LightningDB/wasm/mdb.c create mode 100644 src/LightningDB/wasm/midl.c create mode 100644 src/LightningDB/wasm/midl.h create mode 100644 src/LightningDB/wasm/monocypher.c create mode 100644 src/LightningDB/wasm/monocypher.h diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml new file mode 100644 index 0000000..3e8a476 --- /dev/null +++ b/.github/workflows/wasm.yml @@ -0,0 +1,66 @@ +name: WASM + +on: + push: + branches: [ main ] + pull_request: + paths: + - 'src/LightningDB/**' + - 'src/LightningDB.WasmTest/**' + - '.github/workflows/wasm.yml' + +jobs: + wasm: + runs-on: ubuntu-latest # Chrome preinstalled + steps: + - name: Checkout Lightning.NET + uses: actions/checkout@v5 + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + - name: Install wasm-tools workload + run: dotnet workload install wasm-tools + # The publish alone catches native-link and pinvoke-table regressions. + - name: Publish wasm test app + run: dotnet publish src/LightningDB.WasmTest -c Release -o ./wasm-publish + - name: Serve app + run: | + (cd ./wasm-publish/wwwroot && python3 -m http.server 8080 &) + sleep 2 + # The test page publishes its final result in document.title, polled via + # the DevTools /json/list HTTP endpoint. (--virtual-time-budget with + # --dump-dom cannot be used: virtual time stalls on IndexedDB.) + # run_page URL EXPECTED_TITLE_PREFIX PROFILE_DIR -> prints "result: " + - name: Define harness + run: | + cat > /tmp/run_page.sh <<'EOF' + run_page() { + local title="" pid + google-chrome --headless=new --disable-gpu --no-first-run \ + --user-data-dir="$3" --remote-debugging-port=9222 "$1" >/dev/null 2>&1 & + pid=$! + for i in $(seq 1 60); do + sleep 3 + title=$(curl -s http://localhost:9222/json/list | python3 -c \ + "import sys,json; ts=json.load(sys.stdin); print(next((t['title'] for t in ts if t['type']=='page'), ''))" \ + 2>/dev/null || true) + case "$title" in DONE:*|PHASE-*) break;; esac + done + kill $pid 2>/dev/null; wait $pid 2>/dev/null || true + echo "result: $title" + case "$title" in "$2"*) return 0;; *) return 1;; esac + } + EOF + - name: Run test suite in headless Chrome + run: | + source /tmp/run_page.sh + run_page "http://localhost:8080/" "DONE:" "$(mktemp -d)" | tee /tmp/suite.txt + grep -qE 'DONE: [0-9]+ passed, 0 failed' /tmp/suite.txt + - name: Cross-reload IndexedDB persistence (two browser processes, shared profile) + run: | + source /tmp/run_page.sh + PROFILE=$(mktemp -d) + run_page "http://localhost:8080/?phase=write" "PHASE-WRITE-OK" "$PROFILE" + sleep 2 + run_page "http://localhost:8080/?phase=verify" "PHASE-VERIFY-OK" "$PROFILE" diff --git a/README.md b/README.md index edec132..d641f9b 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,86 @@ simulator slice requires iOS 14.0): release/App Store builds. On older Unity versions, use the device slice (`ios-arm64/lmdb.framework`) on its own instead. +### Browser (WebAssembly) + +LightningDB has experimental support for browser-wasm (Blazor WebAssembly, +Uno Platform). There is no prebuilt binary for the browser — the .NET +WebAssembly runtime statically links native code at app build time — so the +package ships LMDB as compilable C source and wires it up automatically when +your app's `RuntimeIdentifier` is `browser-wasm`. The only prerequisite: + +```bash +dotnet workload install wasm-tools +``` + +The first build performs a one-time native relink (~15 s, cached afterwards). + +**Required open flags.** Each is load-bearing on the browser's in-memory file +system (MEMFS); any other configuration is broken in ways verified by the +test suite in `src/LightningDB.WasmTest`: + +```csharp +using var env = new LightningEnvironment("/db", new EnvironmentConfiguration { MapSize = 16 * 1024 * 1024 }); +env.Open(EnvironmentOpenFlags.WriteMap | // MEMFS mmaps are copies; without WriteMap, + EnvironmentOpenFlags.NoLock | // committed data is silently invisible to readers + EnvironmentOpenFlags.NoThreadLocalStorage | // no file locking / robust mutexes in the sandbox + EnvironmentOpenFlags.NoSync); // emscripten msync only accepts mapping base + // addresses; sync-at-commit would poison the env +// ... transactions as usual ... +env.Flush(force: true); // the durability point (replaces sync-at-commit) +``` + +**Persistence with IndexedDB.** The browser file system (MEMFS) is in-memory — +by itself it does not survive a page reload. `LightningBrowserStorage` mounts a +directory backed by IndexedDB (emscripten IDBFS; the package links it +automatically, opt out with the `WasmEnableIDBFS=false` MSBuild property): + +```csharp +await LightningBrowserStorage.MountAsync("/persist"); // restore from IndexedDB — BEFORE opening +using var env = new LightningEnvironment("/persist/db", new EnvironmentConfiguration { MapSize = 16 * 1024 * 1024 }); +env.Open(/* the required flags above */); +// ... transactions ... +env.Flush(force: true); // LMDB -> browser file system +await LightningBrowserStorage.PersistAsync(); // browser file system -> IndexedDB +``` + +- This is **checkpoint durability**: data survives a reload after each + successful `PersistAsync()` (always preceded by `Flush(true)`), not after + every commit. IndexedDB writes are asynchronous and explicit by nature. +- The environment must be opened only after `MountAsync` completes — the + restore pass makes the mounted directory mirror IndexedDB, deleting files + created under the mount point beforehand. +- One tab at a time: concurrent tabs share the IndexedDB store but not the + in-memory copy, and the last persist wins wholesale. +- IndexedDB is best-effort storage; browsers may evict it under pressure. + Consider requesting `navigator.storage.persist()`. `PersistAsync` faults if + the browser rejects the write (e.g. quota exceeded). +- Strict `script-src` content security policies block the embedded `data:` + module import; host `LightningBrowserStorage.JavaScriptModuleSource` + yourself and set `LightningBrowserStorage.JavaScriptModuleUrl`. + +**What to expect:** + +- Single process, single writer; the page size is 64 KB (emscripten), and the + address space is 32-bit — keep map sizes modest. +- `AesGcmCipher` and `Sha256Checksum` are unavailable in the browser (no + platform AEAD, and managed page callbacks can't cross the wasm boundary). + Use the wasm-only `NativeChaCha20Poly1305Cipher` (RFC 8439, 32-byte key, + 16-byte tag) and `NativeBlake2bChecksum` (keyed BLAKE2b-256 when combined + with encryption); the crypto is compiled into the native library and runs + at native speed. +- **Encryption is experimental** with a known limitation: data committed in + one transaction is not visible to later read transactions in the same + environment session (encrypted environments bypass WriteMap). The supported + encrypted pattern is write → `Flush(true)` → dispose → reopen → read; reads + inside the writing transaction work normally. Unencrypted environments have + no such limitation. +- Encrypted database files are not portable to 64-bit desktop builds (raw + LMDB files never are across 32/64-bit). + +`src/LightningDB.WasmTest` is a runnable Blazor sample and the verification +suite for all of the above. + ## Basic Usage Here's a simple example demonstrating how to create an environment, open a database, and perform basic put and get operations: @@ -224,7 +304,9 @@ New capabilities exposed by LightningDB: Every page can be encrypted and/or checksummed. LMDB itself ships no cipher — the application supplies one; LightningDB includes hardware-accelerated AES-256-GCM and SHA-256 implementations out of the box (custom `LightningCipher`/`LightningChecksum` -implementations are also supported, and required on netstandard2.0 or browser-wasm): +implementations are also supported, and required on netstandard2.0; on browser-wasm +use `NativeChaCha20Poly1305Cipher`/`NativeBlake2bChecksum` — see +[Browser (WebAssembly)](#browser-webassembly)): ```csharp var config = new EnvironmentConfiguration diff --git a/lmdb/compile-lmdb-macos.sh b/lmdb/compile-lmdb-macos.sh index 201d310..b4854b4 100755 --- a/lmdb/compile-lmdb-macos.sh +++ b/lmdb/compile-lmdb-macos.sh @@ -14,6 +14,14 @@ git checkout -f 7ed58053d01cd21bd5ba07ed888d3583d4561ab4 # drop the patch once fixed upstream. git apply ../../../mingw-win32-fixes.patch +# browser-wasm ships as compilable source in the nupkg (consumers' wasm-tools +# workload compiles it; no prebuilt binary works there — mono-wasm cannot load +# side modules). Refresh the committed copies from the pinned+patched tree. +# See src/LightningDB/wasm/README.md. +WASM_SRC=../../../../src/LightningDB/wasm +cp mdb.c midl.c lmdb.h midl.h "$WASM_SRC/" +echo "wasm sources refreshed from $(git rev-parse --short HEAD) + mingw patch" + declare -A build_outputs declare -A supported_targets=( [ios-arm64/native/lmdb.dylib]="make CC='xcrun --sdk iphoneos clang -arch arm64 -miphoneos-version-min=12.0' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'" @@ -31,7 +39,6 @@ declare -A supported_targets=( [android-arm/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/armv7a-linux-androideabi21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'" [android-x86/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/i686-linux-android21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'" [android-x64/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/x86_64-linux-android21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'" - [browser-wasm/native/liblmdb.wasm]="emcc -O2 -pthread -fPIC -DNDEBUG -sSIDE_MODULE=1 -o liblmdb.so mdb.c midl.c module.c" ) function compile_lib() { diff --git a/src/LightningDB.Tests/BrowserStorageTests.cs b/src/LightningDB.Tests/BrowserStorageTests.cs new file mode 100644 index 0000000..b480b3a --- /dev/null +++ b/src/LightningDB.Tests/BrowserStorageTests.cs @@ -0,0 +1,24 @@ +using System; +using System.Threading.Tasks; +using Shouldly; + +namespace LightningDB.Tests; + +public class BrowserStorageTests +{ + public async Task mount_throws_off_browser() + { +#pragma warning disable CA1416 // intentionally calling browser-only API off-browser + await Should.ThrowAsync<PlatformNotSupportedException>( + () => LightningBrowserStorage.MountAsync("/persist")); +#pragma warning restore CA1416 + } + + public async Task persist_throws_off_browser() + { +#pragma warning disable CA1416 + await Should.ThrowAsync<PlatformNotSupportedException>( + () => LightningBrowserStorage.PersistAsync()); +#pragma warning restore CA1416 + } +} diff --git a/src/LightningDB.Tests/EncryptionTests.cs b/src/LightningDB.Tests/EncryptionTests.cs index c5e2cee..238bbff 100644 --- a/src/LightningDB.Tests/EncryptionTests.cs +++ b/src/LightningDB.Tests/EncryptionTests.cs @@ -143,4 +143,24 @@ public void checksum_environment_round_trip() Encoding.UTF8.GetString(value.CopyToNewArray()).ShouldBe("verified"); } } + + public void native_cipher_throws_off_browser() + { + // NativeChaCha20Poly1305Cipher is implemented inside the wasm build of the + // native library; configuring it anywhere else must fail clearly. + Should.Throw<PlatformNotSupportedException>(() => + CreateEnvironment(TempPath(), new EnvironmentConfiguration + { + Encryption = new EncryptionConfiguration(new NativeChaCha20Poly1305Cipher(), Key()) + })); + } + + public void native_checksum_throws_off_browser() + { + Should.Throw<PlatformNotSupportedException>(() => + CreateEnvironment(TempPath(), new EnvironmentConfiguration + { + Checksum = new NativeBlake2bChecksum() + })); + } } diff --git a/src/LightningDB.WasmTest/App.razor b/src/LightningDB.WasmTest/App.razor new file mode 100644 index 0000000..a8a79e5 --- /dev/null +++ b/src/LightningDB.WasmTest/App.razor @@ -0,0 +1,6 @@ +<Router AppAssembly="@typeof(App).Assembly" NotFoundPage="typeof(Pages.NotFound)"> + <Found Context="routeData"> + <RouteView RouteData="@routeData" DefaultLayout="@typeof(MainLayout)"/> + <FocusOnNavigate RouteData="@routeData" Selector="h1" /> + </Found> +</Router> diff --git a/src/LightningDB.WasmTest/Layout/MainLayout.razor b/src/LightningDB.WasmTest/Layout/MainLayout.razor new file mode 100644 index 0000000..e1a9a75 --- /dev/null +++ b/src/LightningDB.WasmTest/Layout/MainLayout.razor @@ -0,0 +1,3 @@ +@inherits LayoutComponentBase + +@Body diff --git a/src/LightningDB.WasmTest/LightningDB.WasmTest.csproj b/src/LightningDB.WasmTest/LightningDB.WasmTest.csproj new file mode 100644 index 0000000..7f7add2 --- /dev/null +++ b/src/LightningDB.WasmTest/LightningDB.WasmTest.csproj @@ -0,0 +1,37 @@ +<Project Sdk="Microsoft.NET.Sdk.BlazorWebAssembly"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <Nullable>enable</Nullable> + <ImplicitUsings>enable</ImplicitUsings> + <OverrideHtmlAssetPlaceholders>true</OverrideHtmlAssetPlaceholders> + <!-- WasmBuildNative is intentionally NOT set: the SDK enables it + automatically when a NativeFileReference is present, which is the + behavior nupkg consumers rely on. --> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly" Version="10.0.9" /> + <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="10.0.9" PrivateAssets="all" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="../LightningDB/LightningDB.csproj" /> + <!-- Direct wire-up of the packaged wasm source: buildTransitive targets in + the nupkg do this for package consumers, but do not apply to + ProjectReferences. --> + <NativeFileReference Include="../LightningDB/wasm/lmdb.c" /> + </ItemGroup> + + <!-- Mirrors _LightningDBEnableIdbfs from LightningDB.Wasm.targets: nupkg + consumers get it from buildTransitive, but that does not apply to + ProjectReferences, so the test app wires the IDBFS link flag itself. --> + <Target Name="_LightningDBEnableIdbfs" + Condition="'$(WasmEnableIDBFS)' != 'false'" + BeforeTargets="_WasmWriteRspFilesForLinking;_BrowserWasmWriteRspForLinking"> + <PropertyGroup Condition="!$(EmccExtraLDFlags.Contains('-lidbfs.js'))"> + <EmccExtraLDFlags>$(EmccExtraLDFlags) -lidbfs.js</EmccExtraLDFlags> + </PropertyGroup> + </Target> + +</Project> diff --git a/src/LightningDB.WasmTest/Pages/Home.razor b/src/LightningDB.WasmTest/Pages/Home.razor new file mode 100644 index 0000000..ae13839 --- /dev/null +++ b/src/LightningDB.WasmTest/Pages/Home.razor @@ -0,0 +1,545 @@ +@page "/" +@using System.Text +@using LightningDB +@inject NavigationManager Nav + +@* The title carries the final result so headless harnesses can poll it via the + DevTools /json/list endpoint (virtual-time budgets stall on IndexedDB). *@ +<PageTitle>@(_done ? _doneText : "LightningDB WASM Tests")</PageTitle> + +<h1>LightningDB WASM Tests</h1> + +<ul id="results"> + @foreach (var r in _results) + { + <li>@r</li> + } +</ul> + +@if (_done) +{ + <div id="done">@_doneText</div> +} + +@code { + private readonly List<string> _results = new(); + private bool _done; + private string _doneText = ""; + private int _passCount; + private int _failCount; + + private const EnvironmentOpenFlags WasmFlags = + EnvironmentOpenFlags.WriteMap | + EnvironmentOpenFlags.NoLock | + EnvironmentOpenFlags.NoThreadLocalStorage | + EnvironmentOpenFlags.NoSync; + + private const EnvironmentOpenFlags WasmFlagsSync = + EnvironmentOpenFlags.WriteMap | + EnvironmentOpenFlags.NoLock | + EnvironmentOpenFlags.NoThreadLocalStorage; + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender) return; + + // ?phase=write|verify runs the cross-reload IndexedDB persistence halves + // (see README); no query string runs the full suite. + var query = new Uri(Nav.Uri).Query.TrimStart('?'); + var phase = query.Split('&', StringSplitOptions.RemoveEmptyEntries) + .Select(p => p.Split('=', 2)) + .Where(kv => kv[0] == "phase" && kv.Length == 2) + .Select(kv => kv[1]) + .FirstOrDefault(); + if (phase is "write" or "verify") + { + await RunPhase(phase); + return; + } + + await RunTest("0-version", TestVersion); + await RunTest("a-open-put-get-same-txn", TestOpenPutGet); + await RunTest("b-read-after-commit-new-txn", TestReadAfterCommit); + await RunTest("c-cursor-1000", TestCursor1000); + await RunTest("d-map-growth", TestMapGrowth); + await RunTest("e-flush-reopen", TestFlushReopen); + await RunTest("f-default-mode-breaks-as-expected", TestDefaultMode); + await RunTest("g-del-stats", TestDelStats); + await RunTest("h-sync-commit-breaks-as-expected", TestSyncCommit); + await RunTest("i-encrypt-roundtrip", TestEncryptRoundtrip); + await RunTest("i2-encrypt-same-env-read-breaks-as-expected", TestEncryptReadSameEnv); + await RunTest("j-encrypt-wrong-key", TestEncryptWrongKey); + await RunTest("k-encrypt-with-checksum", TestEncryptWithChecksum); + await RunTest("l-checksum-only", TestChecksumOnly); + await RunTestAsync("m-idbfs-roundtrip", TestIdbfsRoundtrip); + _done = true; + _doneText = $"DONE: {_passCount} passed, {_failCount} failed"; + Console.WriteLine("SPIKE-" + _doneText); + StateHasChanged(); + } + + private async Task RunPhase(string phase) + { + try + { + _doneText = phase == "write" ? await PhaseWrite() : await PhaseVerify(); + } + catch (Exception ex) + { + _doneText = $"PHASE-{phase.ToUpperInvariant()}-FAIL: {ex.GetType().Name}: {ex.Message}"; + } + _done = true; + Console.WriteLine("SPIKE " + _doneText); + StateHasChanged(); + } + + private Task RunTest(string name, Func<string> test) + => RunTestAsync(name, () => Task.FromResult(test())); + + private async Task RunTestAsync(string name, Func<Task<string>> test) + { + string line; + try + { + var detail = await test(); + _passCount++; + line = $"PASS: {name} — {detail}"; + } + catch (Exception ex) + { + _failCount++; + line = $"FAIL: {name} — {ex.GetType().Name}: {ex.Message} | {ex.StackTrace?.Replace('\n', ' ')}"; + } + _results.Add(line); + Console.WriteLine("SPIKE " + line); + StateHasChanged(); + await Task.Delay(1); // let the UI paint before a potential native abort in the next test + } + + private static LightningEnvironment OpenEnv(string path, long mapSize = 16 * 1024 * 1024, + EnvironmentOpenFlags flags = WasmFlags) + { + var env = new LightningEnvironment(path, new EnvironmentConfiguration + { + MapSize = mapSize, + MaxDatabases = 2 + }); + env.Open(flags); + return env; + } + + private static string TestVersion() + { + using var env = new LightningEnvironment("/db-version"); + var v = env.Version; + return $"lmdb {v.Major}.{v.Minor}.{v.Patch}"; + } + + private static string TestOpenPutGet() + { + using var env = OpenEnv("/db"); + using var tx = env.BeginTransaction(); + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + tx.Put(db, "k1"u8.ToArray(), "v1"u8.ToArray()).ThrowOnError(); + var (rc, _, value) = tx.Get(db, "k1"u8.ToArray()); + rc.ThrowOnError(); + var got = Encoding.UTF8.GetString(value.CopyToNewArray()); + if (got != "v1") throw new Exception($"expected v1, got {got}"); + tx.Commit().ThrowOnError(); + return "put/get within txn ok, commit ok"; + } + + private static string TestReadAfterCommit() + { + // MAP_SHARED coherence make-or-break: commit in one txn, read in a brand-new txn + using var env = OpenEnv("/db"); + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + tx.Put(db, "k2"u8.ToArray(), "v2"u8.ToArray()).ThrowOnError(); + tx.Commit().ThrowOnError(); + } + using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + { + using var db = tx.OpenDatabase(); + var (rc, _, value) = tx.Get(db, "k2"u8.ToArray()); + rc.ThrowOnError(); + var got = Encoding.UTF8.GetString(value.CopyToNewArray()); + if (got != "v2") throw new Exception($"expected v2, got {got}"); + } + return "read-after-commit in new read txn ok"; + } + + private static string TestCursor1000() + { + using var env = OpenEnv("/db-cursor"); + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + for (var i = 0; i < 1000; i++) + { + tx.Put(db, Encoding.UTF8.GetBytes($"key{i:D5}"), Encoding.UTF8.GetBytes($"val{i}")).ThrowOnError(); + } + tx.Commit().ThrowOnError(); + } + using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + { + using var db = tx.OpenDatabase(); + using var cursor = tx.CreateCursor(db); + var count = 0; + string? prev = null; + var (rc, key, _) = cursor.First(); + while (rc == MDBResultCode.Success) + { + var k = Encoding.UTF8.GetString(key.CopyToNewArray()); + if (prev != null && string.CompareOrdinal(prev, k) >= 0) + throw new Exception($"order violation: {prev} >= {k}"); + prev = k; + count++; + (rc, key, _) = cursor.Next(); + } + if (count != 1000) throw new Exception($"expected 1000 entries, got {count}"); + } + return "1000 entries, ordered iteration ok"; + } + + private static string TestMapGrowth() + { + var detail = new StringBuilder(); + using var env = OpenEnv("/db-grow", mapSize: 1024 * 1024); + var value = new byte[4096]; + var i = 0; + var full = false; + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + for (; i < 1000; i++) + { + var rc = tx.Put(db, Encoding.UTF8.GetBytes($"grow{i:D5}"), value); + if (rc == MDBResultCode.MapFull) { full = true; break; } + rc.ThrowOnError(); + } + if (full) tx.Abort(); else tx.Commit().ThrowOnError(); + } + if (!full) throw new Exception($"never hit MapFull after {i} x 4KB puts in 1MB map"); + detail.Append($"MapFull at put #{i}; "); + + // grow via mdb_env_set_mapsize post-open + env.MapSize = 8 * 1024 * 1024; + detail.Append("set_mapsize(8MB) ok; "); + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + for (var j = 0; j < 400; j++) + { + tx.Put(db, Encoding.UTF8.GetBytes($"grow2-{j:D5}"), value).ThrowOnError(); + } + tx.Commit().ThrowOnError(); + } + detail.Append("400 x 4KB puts after growth ok"); + return detail.ToString(); + } + + private static string TestFlushReopen() + { + using (var env = OpenEnv("/db-reopen")) + { + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + tx.Put(db, "persist"u8.ToArray(), "yes"u8.ToArray()).ThrowOnError(); + tx.Commit().ThrowOnError(); + } + env.Flush(force: true).ThrowOnError(); // msync — the only MEMFS write-back + } + using (var env = OpenEnv("/db-reopen")) + { + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + var (rc, _, value) = tx.Get(db, "persist"u8.ToArray()); + rc.ThrowOnError(); + var got = Encoding.UTF8.GetString(value.CopyToNewArray()); + if (got != "yes") throw new Exception($"expected yes, got {got}"); + } + return "flush + dispose + reopen + read ok"; + } + + private static string TestDefaultMode() + { + // Control test: default (non-WriteMap) mode is expected broken on MEMFS — + // reads go via mmap but commits via pwrite, which MEMFS does not reflect + // into existing mappings. PASS means it broke in the documented way. + using var env = OpenEnv("/db-default", flags: EnvironmentOpenFlags.NoLock | EnvironmentOpenFlags.NoThreadLocalStorage); + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + tx.Put(db, "dk"u8.ToArray(), "dv"u8.ToArray()).ThrowOnError(); + tx.Commit().ThrowOnError(); + } + using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + { + using var db = tx.OpenDatabase(); + var (rc, _, value) = tx.Get(db, "dk"u8.ToArray()); + if (rc == MDBResultCode.NotFound) + return "confirmed: committed key invisible to new txn without WriteMap (stale mmap)"; + var got = rc == MDBResultCode.Success ? Encoding.UTF8.GetString(value.CopyToNewArray()) : rc.ToString(); + if (got != "dv") return $"confirmed broken (stale read: '{got}')"; + throw new Exception("default (non-WriteMap) mode unexpectedly works — docs may need updating"); + } + } + + private static string TestSyncCommit() + { + // Control test: without NoSync, commit-time msync hits emscripten's + // base-address-only msync when LMDB syncs the alternating meta page, + // failing with EINVAL and poisoning the env. PASS means it broke in + // the documented way. + var detail = new StringBuilder(); + using var env = OpenEnv("/db-sync", flags: WasmFlagsSync); + var sawFailure = false; + for (var attempt = 1; attempt <= 2 && !sawFailure; attempt++) + { + try + { + using var tx = env.BeginTransaction(); + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + tx.Put(db, Encoding.UTF8.GetBytes($"s{attempt}"), "v"u8.ToArray()).ThrowOnError(); + var rc = tx.Commit(); + detail.Append($"commit#{attempt}={rc}; "); + sawFailure |= rc != MDBResultCode.Success; + } + catch (LightningException ex) + { + detail.Append($"commit#{attempt} threw {ex.Message}; "); + sawFailure = true; + } + } + if (!sawFailure) + throw new Exception($"sync-at-commit unexpectedly works ({detail}) — docs may need updating"); + return $"confirmed: sync-at-commit fails without NoSync ({detail.ToString().TrimEnd(' ', ';')})"; + } + + private static readonly byte[] EncKey = Enumerable.Range(0, 32).Select(i => (byte)(i * 7 + 1)).ToArray(); + + private static LightningEnvironment OpenEncryptedEnv(string path, byte[] key, LightningChecksum? checksum = null) + { + var env = new LightningEnvironment(path, new EnvironmentConfiguration + { + MapSize = 16 * 1024 * 1024, + MaxDatabases = 2, + Encryption = new EncryptionConfiguration(new NativeChaCha20Poly1305Cipher(), key), + Checksum = checksum + }); + env.Open(WasmFlags); + return env; + } + + private static string EncryptRoundtrip(string path, LightningChecksum? checksumA = null, LightningChecksum? checksumB = null) + { + var big = new byte[100_000]; // > one 64 KB page: overflow pages exercise mdb_rpage_decrypt + for (var i = 0; i < big.Length; i++) big[i] = (byte)(i % 251); + + using (var env = OpenEncryptedEnv(path, EncKey, checksumA)) + { + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + for (var i = 0; i < 50; i++) + tx.Put(db, Encoding.UTF8.GetBytes($"enc{i:D3}"), Encoding.UTF8.GetBytes($"value-{i}")).ThrowOnError(); + tx.Put(db, "big"u8.ToArray(), big).ThrowOnError(); + tx.Commit().ThrowOnError(); + } + env.Flush(force: true).ThrowOnError(); + } + + using (var env = OpenEncryptedEnv(path, EncKey, checksumB)) + { + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + for (var i = 0; i < 50; i++) + { + var (rc, _, value) = tx.Get(db, Encoding.UTF8.GetBytes($"enc{i:D3}")); + rc.ThrowOnError(); + var got = Encoding.UTF8.GetString(value.CopyToNewArray()); + if (got != $"value-{i}") throw new Exception($"enc{i:D3}: expected value-{i}, got {got}"); + } + var (rcBig, _, bigVal) = tx.Get(db, "big"u8.ToArray()); + rcBig.ThrowOnError(); + var gotBig = bigVal.CopyToNewArray(); + if (!gotBig.AsSpan().SequenceEqual(big)) + throw new Exception($"big value mismatch (len {gotBig.Length} vs {big.Length})"); + } + return "50 entries + 100 KB overflow value roundtripped through reopen"; + } + + private static string TestEncryptRoundtrip() => EncryptRoundtrip("/db-enc"); + + private static string TestEncryptReadSameEnv() + { + // Control test: read-after-commit within the SAME env session under + // encryption is expected broken. Encrypted envs run LMDB's chunked + // page-cache I/O model (WriteMap is ignored); meta pages are read + // through a mapping created at open, and MEMFS mappings never reflect + // later pwrites — so a post-commit read txn sees the pre-commit meta. + // The supported encrypted pattern is write → Flush(true) → dispose → + // reopen → read (tests i/k). PASS means it broke in the documented way. + using var env = OpenEncryptedEnv("/db-enc2", EncKey); + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + tx.Put(db, "same"u8.ToArray(), "env"u8.ToArray()).ThrowOnError(); + tx.Commit().ThrowOnError(); + } + using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + { + using var db = tx.OpenDatabase(); + var (rc, _, value) = tx.Get(db, "same"u8.ToArray()); + if (rc == MDBResultCode.NotFound) + return "confirmed: committed key invisible to same-session read txn under encryption (stale meta mapping)"; + if (rc != MDBResultCode.Success) + return $"confirmed broken (rc={rc})"; + var got = Encoding.UTF8.GetString(value.CopyToNewArray()); + throw new Exception($"same-session encrypted read unexpectedly works (got '{got}') — docs may need updating"); + } + } + + private static string TestEncryptWrongKey() + { + // /db-enc was created by test i with EncKey; a different key must not read it. + var wrongKey = new byte[32]; + for (var i = 0; i < 32; i++) wrongKey[i] = (byte)(EncKey[i] ^ 0xFF); + try + { + using var env = OpenEncryptedEnv("/db-enc", wrongKey); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + var (rc, _, value) = tx.Get(db, "enc000"u8.ToArray()); + if (rc == MDBResultCode.Success) + throw new Exception($"wrong key unexpectedly read plaintext '{Encoding.UTF8.GetString(value.CopyToNewArray())}'"); + return $"confirmed: wrong key rejected (rc={rc})"; + } + catch (LightningException ex) + { + return $"confirmed: wrong key rejected ({ex.Message})"; + } + } + + private static string TestEncryptWithChecksum() + => EncryptRoundtrip("/db-enc-sum", new NativeBlake2bChecksum(), new NativeBlake2bChecksum()); + + private static string TestChecksumOnly() + { + using (var env = new LightningEnvironment("/db-sum", new EnvironmentConfiguration + { + MapSize = 16 * 1024 * 1024, + MaxDatabases = 2, + Checksum = new NativeBlake2bChecksum() + })) + { + env.Open(WasmFlags); + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + for (var i = 0; i < 20; i++) + tx.Put(db, Encoding.UTF8.GetBytes($"sum{i:D3}"), Encoding.UTF8.GetBytes($"value-{i}")).ThrowOnError(); + tx.Commit().ThrowOnError(); + } + env.Flush(force: true).ThrowOnError(); + } + + using (var env = new LightningEnvironment("/db-sum", new EnvironmentConfiguration + { + MapSize = 16 * 1024 * 1024, + MaxDatabases = 2, + Checksum = new NativeBlake2bChecksum() + })) + { + env.Open(WasmFlags); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + var (rc, _, value) = tx.Get(db, "sum007"u8.ToArray()); + rc.ThrowOnError(); + var got = Encoding.UTF8.GetString(value.CopyToNewArray()); + if (got != "value-7") throw new Exception($"expected value-7, got {got}"); + } + return "20 checksummed entries roundtripped through reopen"; + } + + private static string TestDelStats() + { + using var env = OpenEnv("/db"); + using var tx = env.BeginTransaction(); + using var db = tx.OpenDatabase(); + tx.Delete(db, "k1"u8.ToArray()).ThrowOnError(); + var (rc, _, _) = tx.Get(db, "k1"u8.ToArray()); + if (rc != MDBResultCode.NotFound) throw new Exception($"expected NotFound after delete, got {rc}"); + var stats = tx.GetStats(db); + tx.Commit().ThrowOnError(); + return $"delete ok; entries={stats.Entries}, depth={stats.BTreeDepth}, pageSize={stats.PageSize}"; + } + + private static string PersistedDbPath(string mount) => mount + "/db"; + + private static void WriteMarkerData(string dbPath, int count) + { + using var env = OpenEnv(dbPath); + using (var tx = env.BeginTransaction()) + { + using var db = tx.OpenDatabase(configuration: new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create }); + for (var i = 0; i < count; i++) + tx.Put(db, Encoding.UTF8.GetBytes($"p{i:D3}"), Encoding.UTF8.GetBytes($"persisted-{i}")).ThrowOnError(); + tx.Commit().ThrowOnError(); + } + env.Flush(force: true).ThrowOnError(); // LMDB -> MEMFS; PersistAsync then moves MEMFS -> IndexedDB + } + + private static int ReadMarkerData(string dbPath, int count) + { + using var env = OpenEnv(dbPath); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + for (var i = 0; i < count; i++) + { + var (rc, _, value) = tx.Get(db, Encoding.UTF8.GetBytes($"p{i:D3}")); + rc.ThrowOnError(); + var got = Encoding.UTF8.GetString(value.CopyToNewArray()); + if (got != $"persisted-{i}") throw new Exception($"p{i:D3}: expected persisted-{i}, got {got}"); + } + return count; + } + + private static async Task<string> TestIdbfsRoundtrip() + { + // Full IndexedDB cycle within one page load: write + persist, wipe the + // in-memory copy, restore from IndexedDB, read back. + const string mount = "/persist-rt"; + await LightningBrowserStorage.MountAsync(mount); + WriteMarkerData(PersistedDbPath(mount), 25); + await LightningBrowserStorage.PersistAsync(); + + Directory.Delete(PersistedDbPath(mount), recursive: true); + if (Directory.Exists(PersistedDbPath(mount))) + throw new Exception("MEMFS copy still present after delete"); + + await LightningBrowserStorage.MountAsync(mount); // idempotent: restore only + var n = ReadMarkerData(PersistedDbPath(mount), 25); + return $"{n} entries persisted to IndexedDB, MEMFS wiped, restored and read back"; + } + + private async Task<string> PhaseWrite() + { + const string mount = "/persist"; + await LightningBrowserStorage.MountAsync(mount); + WriteMarkerData(PersistedDbPath(mount), 25); + await LightningBrowserStorage.PersistAsync(); + return "PHASE-WRITE-OK: 25 entries persisted"; + } + + private async Task<string> PhaseVerify() + { + const string mount = "/persist"; + await LightningBrowserStorage.MountAsync(mount); + var n = ReadMarkerData(PersistedDbPath(mount), 25); + return $"PHASE-VERIFY-OK: {n} entries restored from IndexedDB"; + } +} diff --git a/src/LightningDB.WasmTest/Pages/NotFound.razor b/src/LightningDB.WasmTest/Pages/NotFound.razor new file mode 100644 index 0000000..917ada1 --- /dev/null +++ b/src/LightningDB.WasmTest/Pages/NotFound.razor @@ -0,0 +1,5 @@ +@page "/not-found" +@layout MainLayout + +<h3>Not Found</h3> +<p>Sorry, the content you are looking for does not exist.</p> \ No newline at end of file diff --git a/src/LightningDB.WasmTest/Program.cs b/src/LightningDB.WasmTest/Program.cs new file mode 100644 index 0000000..df5c5ae --- /dev/null +++ b/src/LightningDB.WasmTest/Program.cs @@ -0,0 +1,11 @@ +using Microsoft.AspNetCore.Components.Web; +using Microsoft.AspNetCore.Components.WebAssembly.Hosting; +using LightningDB.WasmTest; + +var builder = WebAssemblyHostBuilder.CreateDefault(args); +builder.RootComponents.Add<App>("#app"); +builder.RootComponents.Add<HeadOutlet>("head::after"); + +builder.Services.AddScoped(sp => new HttpClient { BaseAddress = new Uri(builder.HostEnvironment.BaseAddress) }); + +await builder.Build().RunAsync(); diff --git a/src/LightningDB.WasmTest/Properties/launchSettings.json b/src/LightningDB.WasmTest/Properties/launchSettings.json new file mode 100644 index 0000000..1eff386 --- /dev/null +++ b/src/LightningDB.WasmTest/Properties/launchSettings.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "inspectUri": "{wsProtocol}://{url.hostname}:{url.port}/_framework/debug/ws-proxy?browser={browserInspectUri}", + "applicationUrl": "http://localhost:5203", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "inspectUri": "{wsProtocol}://{url.hostname}:{url.port}/_framework/debug/ws-proxy?browser={browserInspectUri}", + "applicationUrl": "https://localhost:7025;http://localhost:5203", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/LightningDB.WasmTest/README.md b/src/LightningDB.WasmTest/README.md new file mode 100644 index 0000000..e06e0f5 --- /dev/null +++ b/src/LightningDB.WasmTest/README.md @@ -0,0 +1,92 @@ +# LightningDB.WasmTest + +A Blazor WebAssembly app that verifies LightningDB's browser-wasm support +end-to-end. It doubles as the sample for using LMDB in the browser. + +This project is intentionally **not** in the solution: building it requires +the `wasm-tools` workload, which contributors working on the core library +shouldn't need. + +## Running + +```sh +dotnet workload install wasm-tools # one-time +dotnet run --project src/LightningDB.WasmTest +``` + +Open the printed localhost URL. The page runs the test suite automatically and +renders one `PASS`/`FAIL` line per test plus a final marker: + +``` +DONE: 13 passed, 0 failed +``` + +Headless (what CI does): + +```sh +google-chrome --headless=new --disable-gpu --dump-dom \ + --virtual-time-budget=90000 http://localhost:5203/ | grep 'DONE:' +``` + +## What the tests prove + +| test | proves | +|---|---| +| 0 | pinvoke table generated; native LMDB linked (version reported) | +| a, b | put/get and read-after-commit coherence under the required flag set | +| c | cursor iteration (1000 ordered entries) | +| d | `MDB_MAP_FULL` surfaces and `MapSize` growth works post-open | +| e | `Flush(true)` + dispose + reopen preserves data within the page session | +| f (control) | default mode (no `WriteMap`) silently loses commits on MEMFS — this test PASSES when the failure occurs as documented | +| g | delete + stats (also shows the 64 KB emscripten page size) | +| h (control) | sync-at-commit (no `NoSync`) fails — emscripten's `msync` only accepts mapping base addresses; LMDB's alternating meta page hits this | +| i | encryption (`NativeChaCha20Poly1305Cipher`): 50 entries + a 100 KB overflow value roundtrip through reopen-with-key | +| i2 (control) | encrypted envs: committed data invisible to later read txns in the same env session (REMAP_CHUNKS + stale MEMFS meta mapping) — PASSES when the documented limitation reproduces | +| j | wrong key is rejected (`MDB_CRYPTO_FAIL`) | +| k | encryption + keyed `NativeBlake2bChecksum` roundtrip | +| l | checksum-only roundtrip | +| m | IndexedDB persistence: write + `Flush(true)` + `PersistAsync`, wipe the MEMFS copy, restore via `MountAsync`, read back | + +## Cross-reload persistence test + +`?phase=write` mounts `/persist`, writes 25 entries, flushes and persists to +IndexedDB, then renders/titles `PHASE-WRITE-OK`. `?phase=verify` mounts +(restoring from IndexedDB) and reads them back (`PHASE-VERIFY-OK`). Run the two +phases in separate browser processes sharing a profile to prove data survives a +full cold start: + +```sh +PROFILE=$(mktemp -d) +google-chrome --headless=new --user-data-dir=$PROFILE --remote-debugging-port=9222 'http://localhost:5203/?phase=write' & +# poll curl http://localhost:9222/json/list until the page title is PHASE-WRITE-OK, kill chrome, then repeat with ?phase=verify +``` + +Note for harnesses: the page publishes its final result in `document.title` +because Chrome's `--virtual-time-budget`/`--dump-dom` stalls on IndexedDB — +poll the DevTools `/json/list` endpoint instead (see `.github/workflows/wasm.yml`). +Persistence is **checkpoint durability**: it survives reload after each +successful `Flush(true)` + `PersistAsync()` pair, not after every commit. + +## Required environment flags on browser-wasm + +```csharp +env.Open(EnvironmentOpenFlags.WriteMap | + EnvironmentOpenFlags.NoLock | + EnvironmentOpenFlags.NoThreadLocalStorage | + EnvironmentOpenFlags.NoSync); +// durability point: +env.Flush(force: true); +``` + +Every flag is load-bearing; see the "Browser (WebAssembly)" section of the +main README for the reasons and the storage/persistence caveats. + +## How the native library gets here + +`LightningDB.WasmTest.csproj` adds `../LightningDB/wasm/lmdb.c` as a +`NativeFileReference`; the wasm-tools workload compiles it with its bundled +emscripten and statically links it into `dotnet.native.wasm`. NuGet consumers +get the same wiring automatically from the package's +`buildTransitive/LightningDB.targets` when `RuntimeIdentifier` is +`browser-wasm` — the direct reference here exists only because buildTransitive +targets don't apply to ProjectReferences. diff --git a/src/LightningDB.WasmTest/_Imports.razor b/src/LightningDB.WasmTest/_Imports.razor new file mode 100644 index 0000000..22ef3be --- /dev/null +++ b/src/LightningDB.WasmTest/_Imports.razor @@ -0,0 +1,10 @@ +@using System.Net.Http +@using System.Net.Http.Json +@using Microsoft.AspNetCore.Components.Forms +@using Microsoft.AspNetCore.Components.Routing +@using Microsoft.AspNetCore.Components.Web +@using Microsoft.AspNetCore.Components.Web.Virtualization +@using Microsoft.AspNetCore.Components.WebAssembly.Http +@using Microsoft.JSInterop +@using LightningDB.WasmTest +@using LightningDB.WasmTest.Layout diff --git a/src/LightningDB.WasmTest/wwwroot/css/app.css b/src/LightningDB.WasmTest/wwwroot/css/app.css new file mode 100644 index 0000000..3b9a456 --- /dev/null +++ b/src/LightningDB.WasmTest/wwwroot/css/app.css @@ -0,0 +1,89 @@ +.valid.modified:not([type=checkbox]) { + outline: 1px solid #26b050; +} + +.invalid { + outline: 1px solid red; +} + +.validation-message { + color: red; +} + +#blazor-error-ui { + color-scheme: light only; + background: lightyellow; + bottom: 0; + box-shadow: 0 -1px 2px rgba(0, 0, 0, 0.2); + box-sizing: border-box; + display: none; + left: 0; + padding: 0.6rem 1.25rem 0.7rem 1.25rem; + position: fixed; + width: 100%; + z-index: 1000; +} + + #blazor-error-ui .dismiss { + cursor: pointer; + position: absolute; + right: 0.75rem; + top: 0.5rem; + } + +.blazor-error-boundary { + background: url(data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTYiIGhlaWdodD0iNDkiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIG92ZXJmbG93PSJoaWRkZW4iPjxkZWZzPjxjbGlwUGF0aCBpZD0iY2xpcDAiPjxyZWN0IHg9IjIzNSIgeT0iNTEiIHdpZHRoPSI1NiIgaGVpZ2h0PSI0OSIvPjwvY2xpcFBhdGg+PC9kZWZzPjxnIGNsaXAtcGF0aD0idXJsKCNjbGlwMCkiIHRyYW5zZm9ybT0idHJhbnNsYXRlKC0yMzUgLTUxKSI+PHBhdGggZD0iTTI2My41MDYgNTFDMjY0LjcxNyA1MSAyNjUuODEzIDUxLjQ4MzcgMjY2LjYwNiA1Mi4yNjU4TDI2Ny4wNTIgNTIuNzk4NyAyNjcuNTM5IDUzLjYyODMgMjkwLjE4NSA5Mi4xODMxIDI5MC41NDUgOTIuNzk1IDI5MC42NTYgOTIuOTk2QzI5MC44NzcgOTMuNTEzIDI5MSA5NC4wODE1IDI5MSA5NC42NzgyIDI5MSA5Ny4wNjUxIDI4OS4wMzggOTkgMjg2LjYxNyA5OUwyNDAuMzgzIDk5QzIzNy45NjMgOTkgMjM2IDk3LjA2NTEgMjM2IDk0LjY3ODIgMjM2IDk0LjM3OTkgMjM2LjAzMSA5NC4wODg2IDIzNi4wODkgOTMuODA3MkwyMzYuMzM4IDkzLjAxNjIgMjM2Ljg1OCA5Mi4xMzE0IDI1OS40NzMgNTMuNjI5NCAyNTkuOTYxIDUyLjc5ODUgMjYwLjQwNyA1Mi4yNjU4QzI2MS4yIDUxLjQ4MzcgMjYyLjI5NiA1MSAyNjMuNTA2IDUxWk0yNjMuNTg2IDY2LjAxODNDMjYwLjczNyA2Ni4wMTgzIDI1OS4zMTMgNjcuMTI0NSAyNTkuMzEzIDY5LjMzNyAyNTkuMzEzIDY5LjYxMDIgMjU5LjMzMiA2OS44NjA4IDI1OS4zNzEgNzAuMDg4N0wyNjEuNzk1IDg0LjAxNjEgMjY1LjM4IDg0LjAxNjEgMjY3LjgyMSA2OS43NDc1QzI2Ny44NiA2OS43MzA5IDI2Ny44NzkgNjkuNTg3NyAyNjcuODc5IDY5LjMxNzkgMjY3Ljg3OSA2Ny4xMTgyIDI2Ni40NDggNjYuMDE4MyAyNjMuNTg2IDY2LjAxODNaTTI2My41NzYgODYuMDU0N0MyNjEuMDQ5IDg2LjA1NDcgMjU5Ljc4NiA4Ny4zMDA1IDI1OS43ODYgODkuNzkyMSAyNTkuNzg2IDkyLjI4MzcgMjYxLjA0OSA5My41Mjk1IDI2My41NzYgOTMuNTI5NSAyNjYuMTE2IDkzLjUyOTUgMjY3LjM4NyA5Mi4yODM3IDI2Ny4zODcgODkuNzkyMSAyNjcuMzg3IDg3LjMwMDUgMjY2LjExNiA4Ni4wNTQ3IDI2My41NzYgODYuMDU0N1oiIGZpbGw9IiNGRkU1MDAiIGZpbGwtcnVsZT0iZXZlbm9kZCIvPjwvZz48L3N2Zz4=) no-repeat 1rem/1.8rem, #b32121; + padding: 1rem 1rem 1rem 3.7rem; + color: white; +} + + .blazor-error-boundary::after { + content: "An error has occurred." + } + +.loading-progress { + position: absolute; + display: block; + width: 8rem; + height: 8rem; + inset: 20vh 0 auto 0; + margin: 0 auto 0 auto; +} + + .loading-progress circle { + fill: none; + stroke: #e0e0e0; + stroke-width: 0.6rem; + transform-origin: 50% 50%; + transform: rotate(-90deg); + } + + .loading-progress circle:last-child { + stroke: #1b6ec2; + stroke-dasharray: calc(3.141 * var(--blazor-load-percentage, 0%) * 0.8), 500%; + transition: stroke-dasharray 0.05s ease-in-out; + } + +.loading-progress-text { + position: absolute; + text-align: center; + font-weight: bold; + inset: calc(20vh + 3.25rem) 0 auto 0.2rem; +} + + .loading-progress-text:after { + content: var(--blazor-load-percentage-text, "Loading"); + } + +code { + color: #c02d76; +} + +.form-floating > .form-control-plaintext::placeholder, .form-floating > .form-control::placeholder { + color: var(--bs-secondary-color); + text-align: end; +} + +.form-floating > .form-control-plaintext:focus::placeholder, .form-floating > .form-control:focus::placeholder { + text-align: start; +} \ No newline at end of file diff --git a/src/LightningDB.WasmTest/wwwroot/icon-192.png b/src/LightningDB.WasmTest/wwwroot/icon-192.png new file mode 100644 index 0000000000000000000000000000000000000000..166f56da7612ea74df6a297154c8d281a4f28a14 GIT binary patch literal 2626 zcmV-I3cdA-P)<h;3K|Lk000e1NJLTq006)M006)U0{{R3WdfWv00024P)t-s{{H@4 zEs<X`lU*;8Vm6g!IhXhK`f5L$`uO|v@b}Tg;BZHu@9OmY`~7D;nV5UJZ9|>v<nVP( zqH;>0<J;_yaJ0Xz)UB1r=iu+z&gIR%-GpJRfLy8d^7!A@>A9xRwxP|bki~~&uFk>U z#<tk9oy&Mpq}<Zy)yd=R=kmO%(xQjJo`Sx_v)87K!mpUgmUy_6bheCavca#_%DUTo zRi=JfsO8=5oPWHEX|Nq!r?mh82~SBxK~#9!?AqB{+CUfv;P0Es$_@oY1w@J>P+PQh zyZ;-jwXKqnKbb6)@RaxQz@vm={%t~VbaZrdbaZrdbaeEeXj>~BG?&`J0XrqR#sSlO zg~N5iUk*15JibvlR1f^^1czzNKWvoJtc!Sj*G37QXbZ8LeD{Fzxgdv#Q{x}ytfZ5q z+^k#NaEp>zX_8~aSaZ`O%B9C&YLHb(mNtgGD&Kezd5S@&C=n~Uy1NWHM`<h#YU7lD zm9CdL9sbpa<Qup7D216a$je(sctQ!(I{aWkFlUq|_RhKErk*&$MY6DOP0fG`B6W2+ z7x0x~h;&1VgZ)_yNwdO!Yf)0czI?SAVSottlwDyT23M|mNc-1}vBp2<Exun<bcro( z#_HhT8DG6rZOu5+0ZgV)h{5c#*RdE!QzSQmd4oxi*J621#bVSD;0&1zx%4}V#Tm*y zPL?AbCoYTS%d@8p+0>t07VQP^MopUXki{2^#ryd94>UJMYW|(#4qV`kb7eD)Q=~NN zaVIRi@|TJ!Rni8J=5DOutQ#bEyMVr8*;HU|)MEKmVC+IOiDi9y)vz=rdtAUHW$yjt zrj3B7v(>exU=IrzC<+?AE=2vI;%<!%!-vt-(no?tlL32KK$B7QY5^mskK7p;s8<=G zF+&P#&E0^34!2pSNXNV6P8i(T<rZs(44zp$jU*no=x1c(U9)gV;%bIeePmW)(bC8N z5_xz-7OIMPTeq{3SD!_iBG!r_!#YG4_n+K`Z7JJ!tJ7~|NEV94`GPn<7;=e7&TY@; z7B%eqQd}11J7O#tQb7b~W>fafM}#ShGDZx=0Nus5QHKdyb9pw&4>4XCpa<A;)}jHs zk1bzVj1(n%fUt3chSb@P_CP`8C<Z(s#OpAHYHcSg_-fq2D)I>-o?P(Gnco1CGX|U> z$f+_tA3+V~<{MU^A%eP!8R*-sD9y<>Jc7A(;aC5hVbs;kX9&Sa$JMG!W_BLFQa*hM zri__C@0i0U1X#?)Y=)>JpvTnY6^s;fu#I}K9u>Old<?zFox>V}m<l>!Ch`d1Vs@v9 zb}w(!TvOmSzmMBa9gYvD4xo<T|5u(ZhD-uiWOj&F*W7SsP`P#ICgFvwpl&f+@L+O` zv9QTyloKcA(+DNJOq^2JqzShZ6L<v!_c7nyBePM6HsFR<Ab_~7>cL2r0ds6%Hs>Z& z#7#o9PGHDmfG%JQq`<YL45%<n%`?AU_AI<mX3j=5NBn|?AT-4wFYiEcg=lN(E=z`u zkVyQL>O5~dt|MAQN@2wyJw_@``7Giyy(yyk(m8U*kk5$X1^;3$a3}N^Lp6hE5!#8l z#~NYHmKAs6IAe&A;bvM8OochRmXN>`D`{N$%#dZCRxp4-dJ?*3P}}T`tYa3?zz5BA zTu7uE#GsDpZ$~j9q=Zq!L<NGm&@IPtDRfSe3h#ih!#~?Qv#zwEAdb&uA%rXt!zxmi z0s>YjLbZPXFILZK4?S)C-zE1(dC2d<7nO4-nSCbV#9E|E1MM|V<9>i4h?WX*r*ul1 z5#k6;po8z=fdMiVVz*h+iaTlz#WOYmU^SX5#97H~B32s-#4wk<1NTN#g?LrYieCu> zF7pbOLR;q2D#Q`^t%QcY06*X-jM+ei7%ZuanUTH#9Y%FBi*Z#22({_}3^=BboIsbg zR0#jJ>9QR8SnmtSS6x($?$}6$x+q)697#m${Z@G6Ujf=6iO^S}7P`q8DkH!IHd4lB zDzwxt3BHsPAcXFFY^Fj}(073>NL_$A%v2sUW(CRutd%{G`5ow?L`XYSO*Qu?x+Gzv zBtR}Y6`XF4xX7)Z04D+fH;TMapdQFFameUuHL34NN)r@aF4RO%x&NApeWGtr#mG~M z6sEIZS;Uj1HB1*0hh=O@0q1=Ia@<dFq&^5=<DO0~lxa6Gan0?4ewBC#_gV<kS)r+q znGyyld#LtCjkR*v7xhNkEan@j5rM)Ndq*Niweouscy&Dogz14(I#`D&1`@QyucqO) z`k*XINRb2Ro%BR|C$XnE#GRxlt(mLFbDdD^5H<oYk?c_h4ap9`0C+zT08~j-n<sPd zgo-TaPx=9~<L@r=$)ib-kOpXovyi#v(Zapx-kCJ<8IzM1Dfri|%6<JZ-5*Az&s4h$ zhgpHvU#0K%aZUXXy3!lrQpSqZn8}nN*RQ}ZO$WIzzhrd3WgbXk52Q^1#aQe~JjH|R zt0^CaEB=r^nHtnUvgY}uI$WKVIsZMCLTfL^i%CRyG5zEF#+W=a!fy~j$D~FwWp&$x zIk<em?^2O_#~^(}%zzh0@y16{_w?VNuQ&f6|ArwuDi2;-?;>L>-tETu-3n(op+97E z#&~2xgg<Hq<K@3Wp`SCt2|D>rl(LA|giII;RwBlX2^Q`B{_t}gxNL;iB11gEPC>v` zb4SJ;;BFOB!{chn>?c<t+;+Clfvm6>CeGDKuqI0+!skyWTn*k!WiPNBf=8rn;@y%( znhq%8fj2eAe<Zt1C4G%P+r+s~$Tq2@vKXdpi+n@Yz!l^G6jG)*W2D7o$Wc1pi6e8% zJp<jD1Z7+jZe)V8@YXjv{lga5aU(LOTb#%hX_Ykv{|W2J$mNmkV1T<2S;X1|tC%nW zvn8{R)TRv0dCKbKbK&XYG1$u10YCDSCj?~ZxGcj>?`A5mP;<Lvi&&=%vArF8&|Z(H zVMtP*T(H$&nnx}miTPo7v+Vb0vsM3%<$2T1NRjxr<4zl7lK~fq8dL#UZZo46PY<0( zvdv=v+=LN0V~8@cNc^p^09CmJ{2bcybW6VAg=8B%gLW5Wfo0Br=_OPVD}I+}vv?ow zaxhl5&&UO#X<5=ECBrs&GPNRNhXK({ztrdzYw8VEOk>TE&i<a%NJguL$3@x+RnLG$ zc#Cs35y&6E?8ZL205N?CvBxrkXg&TpU5;Mpjc~X*Pxr^?Jn&h7Go%k=&(`rz^pS)& zY?bs(A5G?1Jbp%>LEmQ^xV<fX7Oj*`3HtP@WJb2&46fh{h?*;pOR=6^!Rdd~{O)V2 zefoM;-yilycwVz2(P4O{-)9@2z6&kF&5yg$)u;{T>%-kmC-8mWao&EUK_^=GW-Y3z ksi~={si~={skwfB0gq6itke#r1ONa407*qoM6N<$g11Kq@c;k- literal 0 HcmV?d00001 diff --git a/src/LightningDB.WasmTest/wwwroot/index.html b/src/LightningDB.WasmTest/wwwroot/index.html new file mode 100644 index 0000000..c7d2923 --- /dev/null +++ b/src/LightningDB.WasmTest/wwwroot/index.html @@ -0,0 +1,33 @@ +<!DOCTYPE html> +<html lang="en"> + +<head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>WasmSpike + + + + + + + + +
+ + + + +
+
+ +
+ An unhandled error has occurred. + Reload + 🗙 +
+ + + + diff --git a/src/LightningDB/AesGcmCipher.cs b/src/LightningDB/AesGcmCipher.cs index 2b2963d..5879543 100644 --- a/src/LightningDB/AesGcmCipher.cs +++ b/src/LightningDB/AesGcmCipher.cs @@ -13,8 +13,8 @@ namespace LightningDB; /// /// The 12-byte GCM nonce is derived from LMDB's per-page IV (page number + transaction id, /// unique per page write) and the 16-byte tag is stored as the page's authentication data. -/// Not available on platforms where is false (e.g. -/// browser-wasm); supply a custom there. +/// Not available on platforms where is false; on +/// browser-wasm use instead. /// public sealed class AesGcmCipher : LightningCipher, IDisposable { @@ -29,7 +29,7 @@ public AesGcmCipher() : base(TagSize) { if (!AesGcm.IsSupported) throw new PlatformNotSupportedException( - "AES-GCM is not supported on this platform; supply a custom LightningCipher instead."); + "AES-GCM is not supported on this platform; on browser-wasm use NativeChaCha20Poly1305Cipher, otherwise supply a custom LightningCipher."); _aes = new ThreadLocal(trackAllValues: true); } diff --git a/src/LightningDB/LightningBrowserStorage.cs b/src/LightningDB/LightningBrowserStorage.cs new file mode 100644 index 0000000..72e9d22 --- /dev/null +++ b/src/LightningDB/LightningBrowserStorage.cs @@ -0,0 +1,154 @@ +#if NET8_0_OR_GREATER +using System; +using System.Runtime.InteropServices.JavaScript; +using System.Runtime.Versioning; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace LightningDB; + +/// +/// IndexedDB-backed persistence for browser-wasm environments, built on +/// emscripten's IDBFS file system. Only supported on browser-wasm; the +/// LightningDB package links IDBFS automatically (opt out with the +/// WasmEnableIDBFS=false MSBuild property). +/// +/// +/// +/// The browser file system (MEMFS) is in-memory and wiped on page reload. +/// mounts a directory backed by IndexedDB and +/// restores its previous contents; checkpoints the +/// current contents back to IndexedDB. This gives checkpoint durability: +/// data survives a reload after each successful persist, not after every +/// commit. The intended pattern: +/// +/// +/// await LightningBrowserStorage.MountAsync("/persist"); // BEFORE opening the environment +/// using var env = new LightningEnvironment("/persist/db", ...); +/// env.Open(/* required browser flags */); +/// // ... transactions ... +/// env.Flush(force: true); // LMDB -> browser file system +/// await LightningBrowserStorage.PersistAsync(); // browser file system -> IndexedDB +/// +/// +/// The environment must be opened only after +/// completes: the restore pass makes the mounted directory mirror IndexedDB, +/// deleting any files created under the mount point beforehand. +/// +/// +/// Only one tab may use a mounted store at a time — each tab has a private +/// in-memory copy over the same IndexedDB database (named after the mount +/// point), and the last persist wins wholesale. IndexedDB is best-effort +/// storage; browsers may evict it under pressure unless the origin holds +/// persistent-storage permission (navigator.storage.persist()). +/// faults if the browser rejects the write +/// (e.g. quota exceeded). +/// +/// +/// The supporting JavaScript is imported from an embedded data: module +/// URL. Sites with a strict script-src content security policy block +/// data: imports — host the module yourself and assign its URL to +/// before the first call; the module source +/// is available via . +/// +/// +[SupportedOSPlatform("browser")] +public static partial class LightningBrowserStorage +{ + private const string ModuleName = "LightningDB.IDBFS"; + + private static Task? _import; + private static readonly SemaphoreSlim SyncGate = new(1, 1); + + /// + /// The JavaScript module backing this API. Exposed so strict-CSP + /// applications can host it themselves (see ). + /// + public const string JavaScriptModuleSource = + """ + const FS = globalThis.getDotnetRuntime(0).Module.FS; + const mounted = new Set(); + export function mount(path) { + if (mounted.has(path)) return; + try { FS.mkdir(path); } catch (e) { if (e.errno !== 20 /* EEXIST */) throw e; } + FS.mount(FS.filesystems.IDBFS, {}, path); + mounted.add(path); + } + export function syncfs(populate) { + return new Promise((resolve, reject) => + FS.syncfs(populate, err => err ? reject(new Error(String(err))) : resolve())); + } + """; + + /// + /// Optional override for where the supporting JavaScript module is loaded + /// from. Defaults to an embedded data: URL; set this (before the + /// first / call) + /// to a URL serving when a strict + /// content security policy blocks data: module imports. + /// + public static string? JavaScriptModuleUrl { get; set; } + + /// + /// Mounts as an IndexedDB-backed directory and + /// restores its previous contents. Must complete before any + /// under the mount point is opened. + /// Idempotent: calling again for an already-mounted path performs only the + /// restore (which overwrites unpersisted in-memory changes with the last + /// persisted state). + /// + /// Absolute directory path, e.g. "/persist". + /// Also names the origin-scoped IndexedDB database. + public static async Task MountAsync(string mountPoint) + { + if (string.IsNullOrWhiteSpace(mountPoint)) + throw new ArgumentException("A mount point is required", nameof(mountPoint)); + await EnsureModuleAsync().ConfigureAwait(false); + Mount(mountPoint); + await GatedSyncFs(populate: true).ConfigureAwait(false); + } + + /// + /// Checkpoints all mounted IndexedDB-backed directories to IndexedDB. + /// Call after with force: true, + /// which is what moves committed LMDB data into the browser file system. + /// + public static async Task PersistAsync() + { + await EnsureModuleAsync().ConfigureAwait(false); + await GatedSyncFs(populate: false).ConfigureAwait(false); + } + + private static Task EnsureModuleAsync() + { + if (!OperatingSystem.IsBrowser()) + throw new PlatformNotSupportedException( + "LightningBrowserStorage is only supported on browser-wasm."); + return _import ??= JSHost.ImportAsync(ModuleName, + JavaScriptModuleUrl ?? "data:text/javascript;base64," + + Convert.ToBase64String(Encoding.UTF8.GetBytes(JavaScriptModuleSource))); + } + + // emscripten warns on (and can misbehave with) concurrent FS.syncfs calls, + // so all syncs funnel through one gate. + private static async Task GatedSyncFs(bool populate) + { + await SyncGate.WaitAsync().ConfigureAwait(false); + try + { + await SyncFs(populate).ConfigureAwait(false); + } + finally + { + SyncGate.Release(); + } + } + + [JSImport("mount", ModuleName)] + private static partial void Mount(string path); + + [JSImport("syncfs", ModuleName)] + private static partial Task SyncFs(bool populate); +} +#endif diff --git a/src/LightningDB/LightningChecksum.cs b/src/LightningDB/LightningChecksum.cs index 3c7d80f..07c7c5a 100644 --- a/src/LightningDB/LightningChecksum.cs +++ b/src/LightningDB/LightningChecksum.cs @@ -40,3 +40,23 @@ public override void Compute(ReadOnlySpan source, Span destination, => System.Security.Cryptography.SHA256.HashData(source, destination); } #endif + +/// +/// BLAKE2b-256 page checksum implemented inside the native LMDB library. Only +/// functional on browser-wasm, where managed page callbacks are unavailable +/// (the .NET WebAssembly runtime cannot marshal delegates as native callbacks). +/// When combined with , the hash is +/// keyed with the encryption key. +/// +/// +/// is never invoked from managed code; the hashing runs +/// entirely inside the native library. +/// +public sealed class NativeBlake2bChecksum : LightningChecksum +{ + public override int Size => 32; + + public override void Compute(ReadOnlySpan source, Span destination, ReadOnlySpan key) + => throw new PlatformNotSupportedException( + $"{nameof(NativeBlake2bChecksum)} executes inside the native library and is only supported on browser-wasm."); +} diff --git a/src/LightningDB/LightningDB.Wasm.targets b/src/LightningDB/LightningDB.Wasm.targets new file mode 100644 index 0000000..42adcb0 --- /dev/null +++ b/src/LightningDB/LightningDB.Wasm.targets @@ -0,0 +1,32 @@ + + + + + + + + + + $(EmccExtraLDFlags) -lidbfs.js + + + diff --git a/src/LightningDB/LightningDB.csproj b/src/LightningDB/LightningDB.csproj index b3ef677..c978c28 100644 --- a/src/LightningDB/LightningDB.csproj +++ b/src/LightningDB/LightningDB.csproj @@ -32,6 +32,11 @@ runtimes + + + diff --git a/src/LightningDB/LightningDB.targets b/src/LightningDB/LightningDB.targets index 9042e2a..1c4f7ff 100644 --- a/src/LightningDB/LightningDB.targets +++ b/src/LightningDB/LightningDB.targets @@ -3,7 +3,10 @@ \..\ - + + + /// Installs the natively-implemented ChaCha20-Poly1305 page cipher (and optionally the + /// BLAKE2b-256 checksum) on the environment. Must be called before mdb_env_open. + /// Only available in browser-wasm builds, where the cipher is statically compiled into + /// the native library (see wasm/lmdb_wasm_crypto.c). + /// + /// The environment handle + /// The 32-byte encryption key (copied by the native library) + /// The key length; must be 32 + /// Non-zero to also install the keyed BLAKE2b-256 checksum + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode lmdb_setup_encryption(nint env, ref byte key, int keyLength, int withChecksum); + + /// + /// Installs the natively-implemented BLAKE2b-256 page checksum on the environment. + /// Must be called before mdb_env_open. Only available in browser-wasm builds. + /// + /// The environment handle + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode lmdb_setup_checksum(nint env); + /// /// Prepares (first phase of a two-phase commit) all the operations of a transaction. /// @@ -1111,6 +1136,27 @@ public static MDBResultCode mdb_env_copy2(nint env, string path, EnvironmentCopy [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] public static extern MDBResultCode mdb_env_set_checksum(nint env, ChecksumFunction func, uint size); + /// + /// Installs the natively-implemented ChaCha20-Poly1305 page cipher (and optionally the + /// BLAKE2b-256 checksum) on the environment. Only available in browser-wasm builds. + /// + /// The environment handle + /// The 32-byte encryption key (copied by the native library) + /// The key length; must be 32 + /// Non-zero to also install the keyed BLAKE2b-256 checksum + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode lmdb_setup_encryption(nint env, ref byte key, int keyLength, int withChecksum); + + /// + /// Installs the natively-implemented BLAKE2b-256 page checksum on the environment. + /// Only available in browser-wasm builds. + /// + /// The environment handle + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode lmdb_setup_checksum(nint env); + /// /// Prepares (first phase of a two-phase commit) all the operations of a transaction. /// diff --git a/src/LightningDB/NativeChaCha20Poly1305Cipher.cs b/src/LightningDB/NativeChaCha20Poly1305Cipher.cs new file mode 100644 index 0000000..396ec54 --- /dev/null +++ b/src/LightningDB/NativeChaCha20Poly1305Cipher.cs @@ -0,0 +1,64 @@ +using System; + +namespace LightningDB; + +/// +/// ChaCha20-Poly1305 (RFC 8439) page cipher implemented inside the native LMDB +/// library. Only functional on browser-wasm, where managed page callbacks are +/// unavailable (the .NET WebAssembly runtime cannot marshal delegates as native +/// callbacks) and the platform provides no AEAD cipher (AesGcm, +/// ChaCha20Poly1305 and friends all throw). Requires a 32-byte key and +/// reserves a 16-byte Poly1305 tag per page. +/// +/// +/// +/// Experimental: enabling encryption switches LMDB to its chunked page-cache +/// I/O model (MDB_REMAP_CHUNKS; WriteMap is ignored). Under +/// emscripten's in-memory file system this has a known limitation: data +/// committed in one transaction is not visible to later read transactions in +/// the same environment session (the meta-page mapping created at open never +/// reflects subsequent writes). The supported encrypted pattern is +/// write → Flush(force: true) → dispose → reopen → read; reads within +/// the writing transaction itself work normally. +/// +/// +/// On-disk format: pages are encrypted with the IETF ChaCha20 stream starting +/// at block counter 1, with the Poly1305 key derived from block 0 and the MAC +/// computed per RFC 8439 with empty additional data — the same detached-AEAD +/// construction as libsodium's crypto_aead_chacha20poly1305_ietf and +/// upstream LMDB's crypto.c example. The 12-byte nonce is +/// LE32(pgno) || LE32(txnid) || 0x00000000 (page number and transaction +/// id are 32-bit in wasm builds). +/// +/// +/// / are never invoked from managed +/// code; the crypto runs entirely inside the native library. +/// +/// +public sealed class NativeChaCha20Poly1305Cipher : LightningCipher +{ + /// + /// The required encryption key length in bytes. + /// + public const int KeySize = 32; + + /// + /// Creates the cipher marker. The actual cipher is installed natively when the + /// environment is created. + /// + public NativeChaCha20Poly1305Cipher() : base(16) + { + } + + /// + public override bool Encrypt(ReadOnlySpan source, Span destination, + ReadOnlySpan key, ReadOnlySpan iv, Span authData) + => throw new PlatformNotSupportedException( + $"{nameof(NativeChaCha20Poly1305Cipher)} executes inside the native library and is only supported on browser-wasm."); + + /// + public override bool Decrypt(ReadOnlySpan source, Span destination, + ReadOnlySpan key, ReadOnlySpan iv, ReadOnlySpan authData) + => throw new PlatformNotSupportedException( + $"{nameof(NativeChaCha20Poly1305Cipher)} executes inside the native library and is only supported on browser-wasm."); +} diff --git a/src/LightningDB/wasm/MONOCYPHER-LICENCE.md b/src/LightningDB/wasm/MONOCYPHER-LICENCE.md new file mode 100644 index 0000000..9c3fae9 --- /dev/null +++ b/src/LightningDB/wasm/MONOCYPHER-LICENCE.md @@ -0,0 +1,167 @@ +Monocypher as a whole is dual-licensed. Choose whichever licence you +want from the two licences listed below. + +The first licence is a regular 2-clause BSD licence. The second licence +is the CC-0 from Creative Commons. It is intended to release Monocypher +to the public domain. The BSD licence serves as a fallback option. + +See the individual files for specific information about who contributed +to what file during which years. See below for special notes. + +Licence 1 (2-clause BSD) +------------------------ + +Copyright (c) 2017-2023, Loup Vaillant +Copyright (c) 2017-2019, Michael Savage +Copyright (c) 2017-2023, Fabio Scotoni +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Licence 2 (CC-0) +---------------- + +> CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE +> LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN +> ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS +> INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES +> REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS +> PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM +> THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED +> HEREUNDER. + +### Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work +of authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without +fear of later claims of infringement build upon, modify, incorporate in +other works, reuse and redistribute as freely as possible in any form +whatsoever and for any purposes, including without limitation commercial +purposes. These owners may contribute to the Commons to promote the +ideal of a free culture and the further production of creative, cultural +and scientific works, or to gain reputation or greater distribution for +their Work in part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or +she is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under +its terms, with knowledge of his or her Copyright and Related Rights in +the Work and the meaning and intended legal effect of CC0 on those +rights. + +1. **Copyright and Related Rights.** A Work made available under CC0 may + be protected by copyright and related or neighboring rights + ("Copyright and Related Rights"). Copyright and Related Rights + include, but are not limited to, the following: + + - the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + - moral rights retained by the original author(s) and/or + performer(s); publicity and privacy rights pertaining to a person's + image or likeness depicted in a Work; + - rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + - rights protecting the extraction, dissemination, use and reuse of + data in a Work; + - database rights (such as those arising under Directive 96/9/EC of + the European Parliament and of the Council of 11 March 1996 on the + legal protection of databases, and under any national + implementation thereof, including any amended or successor version + of such directive); and + - other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. **Waiver.** To the greatest extent permitted by, but not in + contravention of, applicable law, Affirmer hereby overtly, fully, + permanently, irrevocably and unconditionally waives, abandons, and + surrenders all of Affirmer's Copyright and Related Rights and + associated claims and causes of action, whether now known or unknown + (including existing as well as future claims and causes of action), + in the Work (i) in all territories worldwide, (ii) for the maximum + duration provided by applicable law or treaty (including future time + extensions), (iii) in any current or future medium and for any number + of copies, and (iv) for any purpose whatsoever, including without + limitation commercial, advertising or promotional purposes (the + "Waiver"). Affirmer makes the Waiver for the benefit of each member + of the public at large and to the detriment of Affirmer's heirs and + successors, fully intending that such Waiver shall not be subject to + revocation, rescission, cancellation, termination, or any other legal + or equitable action to disrupt the quiet enjoyment of the Work by the + public as contemplated by Affirmer's express Statement of Purpose. + +3. **Public License Fallback.** Should any part of the Waiver for any + reason be judged legally invalid or ineffective under applicable law, + then the Waiver shall be preserved to the maximum extent permitted + taking into account Affirmer's express Statement of Purpose. In + addition, to the extent the Waiver is so judged Affirmer hereby + grants to each affected person a royalty-free, non transferable, non + sublicensable, non exclusive, irrevocable and unconditional license + to exercise Affirmer's Copyright and Related Rights in the Work (i) + in all territories worldwide, (ii) for the maximum duration provided + by applicable law or treaty (including future time extensions), (iii) + in any current or future medium and for any number of copies, and + (iv) for any purpose whatsoever, including without limitation + commercial, advertising or promotional purposes (the "License"). The + License shall be deemed effective as of the date CC0 was applied by + Affirmer to the Work. Should any part of the License for any reason + be judged legally invalid or ineffective under applicable law, such + partial invalidity or ineffectiveness shall not invalidate the + remainder of the License, and in such case Affirmer hereby affirms + that he or she will not (i) exercise any of his or her remaining + Copyright and Related Rights in the Work or (ii) assert any + associated claims and causes of action with respect to the Work, in + either case contrary to Affirmer's express Statement of Purpose. + +4. **Limitations and Disclaimers.** + + - No trademark or patent rights held by Affirmer are waived, + abandoned, surrendered, licensed or otherwise affected by this + document. + - Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, + or the present or absence of errors, whether or not discoverable, + all to the greatest extent permissible under applicable law. + - Affirmer disclaims responsibility for clearing rights of other + persons that may apply to the Work or any use thereof, including + without limitation any person's Copyright and Related Rights in the + Work. Further, Affirmer disclaims responsibility for obtaining any + necessary consents, permissions or other rights required for any + use of the Work. + - Affirmer understands and acknowledges that Creative Commons is not + a party to this document and has no duty or obligation with respect + to this CC0 or use of the Work. diff --git a/src/LightningDB/wasm/README.md b/src/LightningDB/wasm/README.md new file mode 100644 index 0000000..5d623ec --- /dev/null +++ b/src/LightningDB/wasm/README.md @@ -0,0 +1,30 @@ +# Browser-wasm native sources + +On browser-wasm, LMDB cannot ship as a prebuilt binary: the .NET WebAssembly +runtime statically links native dependencies into `dotnet.native.wasm` at app +build time (there is no dynamic loading of wasm side modules). This directory +therefore ships LMDB as **compilable C source**. The nupkg's +`buildTransitive/LightningDB.targets` adds `lmdb.c` as a `NativeFileReference` +when `RuntimeIdentifier == browser-wasm`, and the consumer's `wasm-tools` +workload compiles it with its own bundled emscripten — which also removes any +emscripten-version-matching concerns. + +## Files + +| file | provenance | +|---|---| +| `lmdb.c` | amalgamation entry point. The name is load-bearing: the wasm pinvoke module name is the file basename and must match `DllImport("lmdb")`. | +| `mdb.c`, `midl.c`, `lmdb.h`, `midl.h` | copied from the vendored OpenLDAP LMDB tree at `lmdb/lmdb/libraries/liblmdb/`, pinned commit `7ed58053` (mdb.master3 / LMDB 1.0 line) with `mingw-win32-fixes.patch` applied — the same tree all shipped native binaries are built from. | +| `monocypher.c`, `monocypher.h` | [Monocypher](https://monocypher.org) 4.0.2, verbatim. License: CC0 / BSD-2 (see `MONOCYPHER-LICENCE.md`). | +| `lmdb_wasm_crypto.c` | LightningDB shim: ChaCha20-Poly1305 (RFC 8439) page cipher + BLAKE2b-256 checksum, installed natively via `lmdb_setup_encryption` / `lmdb_setup_checksum` (no managed callbacks — mono-wasm cannot marshal delegates as native callbacks). | + +## Refreshing the LMDB copies + +`lmdb/compile-lmdb-macos.sh` refreshes `mdb.c`/`midl.c`/`lmdb.h`/`midl.h` here +from the pinned+patched tree as part of the native build run. Monocypher and +the shim/amalgamation are maintained manually. + +## Licenses + +- LMDB sources: OpenLDAP Public License (see `lmdb/lmdb/LICENSE` upstream). +- Monocypher: CC0-1.0 or BSD-2-Clause (`MONOCYPHER-LICENCE.md`). diff --git a/src/LightningDB/wasm/lmdb.c b/src/LightningDB/wasm/lmdb.c new file mode 100644 index 0000000..db3451a --- /dev/null +++ b/src/LightningDB/wasm/lmdb.c @@ -0,0 +1,29 @@ +/* Amalgamation for browser-wasm builds. + * + * The file name ("lmdb") is load-bearing: the .NET wasm build registers one + * pinvoke module per NativeFileReference file basename, and it must match + * the managed DllImport("lmdb") module name for the pinvoke table to be + * generated. + * + * The defines live here (not in EmccExtraCFlags) because NativeFileReference + * C sources are compiled during the emcc link step, which does not receive + * the extra compile flags. + * + * - MDB_USE_POSIX_MUTEX / MDB_USE_ROBUST=0: emscripten's libc has no robust + * mutex support; the lock table is bypassed at runtime anyway (browser + * environments must be opened with MDB_NOLOCK). + * - module.c is intentionally omitted: it only exports mdb_modload (never + * P/Invoked) and pulls in dlfcn, which wasm static linking cannot honor. + */ +#define MDB_USE_POSIX_MUTEX 1 +#define MDB_USE_ROBUST 0 + +#include "mdb.c" +#include "midl.c" + +/* mdb.c defines Z as a printf format modifier; it would clobber monocypher's + * ge struct fields. mdb.c's own uses are already expanded at this point. */ +#undef Z + +#include "monocypher.c" +#include "lmdb_wasm_crypto.c" diff --git a/src/LightningDB/wasm/lmdb.h b/src/LightningDB/wasm/lmdb.h new file mode 100644 index 0000000..5881527 --- /dev/null +++ b/src/LightningDB/wasm/lmdb.h @@ -0,0 +1,1934 @@ +/** @file lmdb.h + * @brief Lightning memory-mapped database library + * + * @mainpage Lightning Memory-Mapped Database Manager (LMDB) + * + * @section intro_sec Introduction + * LMDB is a Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. The entire database is exposed + * in a memory map, and all data fetches return data directly + * from the mapped memory, so no malloc's or memcpy's occur during + * data fetches. As such, the library is extremely simple because it + * requires no page caching layer of its own, and it is extremely high + * performance and memory-efficient. It is also fully transactional with + * full ACID semantics, and when the memory map is read-only, the + * database integrity cannot be corrupted by stray pointer writes from + * application code. + * + * See @ref upgrading if you used LMDB 0.9 previously. + * + * The library is fully thread-aware and supports concurrent read/write + * access from multiple processes and threads. Data pages use a copy-on- + * write strategy so no active data pages are ever overwritten, which + * also provides resistance to corruption and eliminates the need of any + * special recovery procedures after a system crash. Writes are fully + * serialized; only one write transaction may be active at a time, which + * guarantees that writers can never deadlock. The database structure is + * multi-versioned so readers run with no locks; writers cannot block + * readers, and readers don't block writers. + * + * Unlike other well-known database mechanisms which use either write-ahead + * transaction logs or append-only data writes, LMDB requires no maintenance + * during operation. Both write-ahead loggers and append-only databases + * require periodic checkpointing and/or compaction of their log or database + * files otherwise they grow without bound. LMDB tracks free pages within + * the database and re-uses them for new write operations, so the database + * size does not grow without bound in normal use. + * + * The memory map can be used as a read-only or read-write map. It is + * read-only by default as this provides total immunity to corruption. + * Using read-write mode offers much higher write performance, but adds + * the possibility for stray application writes thru pointers to silently + * corrupt the database. Of course if your application code is known to + * be bug-free (...) then this is not an issue. + * + * If this is your first time using a transactional embedded key/value + * store, you may find the \ref starting page to be helpful. + * + * @section caveats_sec Caveats + * Troubleshooting the lock file, plus semaphores on BSD systems: + * + * - A broken lockfile can cause sync issues. + * Stale reader transactions left behind by an aborted program + * cause further writes to grow the database quickly, and + * stale locks can block further operation. + * + * Fix: Check for stale readers periodically, using the + * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. + * Stale writers will be cleared automatically on most systems: + * - Windows - automatic + * - BSD, systems using SysV semaphores - automatic + * - Linux, systems using POSIX mutexes with Robust option - automatic + * Otherwise just make all programs using the database close it; + * the lockfile is always reset on first open of the environment. + * + * - On BSD systems or others configured with MDB_USE_SYSV_SEM or + * MDB_USE_POSIX_SEM, + * startup can fail due to semaphores owned by another userid. + * + * Fix: Open and close the database as the user which owns the + * semaphores (likely last user) or as root, while no other + * process is using the database. + * + * Restrictions/caveats (in addition to those listed for some functions): + * + * - Only the database owner should normally use the database on + * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. + * Multiple users can cause startup to fail later, as noted above. + * + * - There is normally no pure read-only mode, since readers need write + * access to locks and lock file. Exceptions: On read-only filesystems + * or with the #MDB_NOLOCK flag described under #mdb_env_open(). + * + * - An LMDB configuration will often reserve considerable \b unused + * memory address space and maybe file size for future growth. + * This does not use actual memory or disk space, but users may need + * to understand the difference so they won't be scared off. + * + * - By default, in versions before 0.9.10, unused portions of the data + * file might receive garbage data from memory freed by other code. + * (This does not happen when using the #MDB_WRITEMAP flag.) As of + * 0.9.10 the default behavior is to initialize such memory before + * writing to the data file. Since there may be a slight performance + * cost due to this initialization, applications may disable it using + * the #MDB_NOMEMINIT flag. Applications handling sensitive data + * which must not be written should not use this flag. This flag is + * irrelevant when using #MDB_WRITEMAP. + * + * - A thread can only use one transaction at a time, plus any child + * transactions. Each transaction belongs to one thread. See below. + * The #MDB_NOTLS flag changes this for read-only transactions. + * + * - Use an MDB_env* in the process which opened it, not after fork(). + * + * - Do not have open an LMDB database twice in the same process at + * the same time. Not even from a plain open() call - close()ing it + * breaks fcntl() advisory locking. (It is OK to reopen it after + * fork() - exec*(), since the lockfile has FD_CLOEXEC set.) + * + * - Avoid long-lived transactions. Read transactions prevent + * reuse of pages freed by newer write transactions, thus the + * database can grow quickly. Write transactions prevent + * other write transactions, since writes are serialized. + * + * - Avoid suspending a process with active transactions. These + * would then be "long-lived" as above. Also read transactions + * suspended when writers commit could sometimes see wrong data. + * + * ...when several processes can use a database concurrently: + * + * - Avoid aborting a process with an active transaction. + * The transaction becomes "long-lived" as above until a check + * for stale readers is performed or the lockfile is reset, + * since the process may not remove it from the lockfile. + * + * This does not apply to write transactions if the system clears + * stale writers, see above. + * + * - If you do that anyway, do a periodic check for stale readers. Or + * close the environment once in a while, so the lockfile can get reset. + * + * - Do not use LMDB databases on remote filesystems, even between + * processes on the same host. This breaks flock() on some OSes, + * possibly memory map sync, and certainly sync between programs + * on different hosts. + * + * - Opening a database can fail if another process is opening or + * closing it at exactly the same time. + * + * @author Howard Chu, Symas Corporation. + * + * @copyright Copyright 2011-2021 Howard Chu, Symas Corp. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * @par Derived From: + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _LMDB_H_ +#define _LMDB_H_ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Unix permissions for creating files, or dummy definition for Windows */ +#ifdef _MSC_VER +typedef int mdb_mode_t; +#else +typedef mode_t mdb_mode_t; +#endif + +#ifdef _WIN32 +# define MDB_FMT_Z "I" +#else +# define MDB_FMT_Z "z" /**< printf/scanf format modifier for size_t */ +#endif + +#if !defined(MDB_RPAGE_CACHE) || (defined(MDB_VL32) && !(MDB_RPAGE_CACHE)) +/** Support #MDB_REMAP_CHUNKS. Implied by MDB_VL32. Define as 0 to disable. */ +#define MDB_RPAGE_CACHE 1 +#endif + +#ifndef MDB_VL32 +/** Unsigned type used for mapsize, entry counts and page/transaction IDs. + * + * It is normally size_t, hence the name. Defining MDB_VL32 makes it + * uint64_t, but do not try this unless you know what you are doing. + */ +typedef size_t mdb_size_t; +# define MDB_SIZE_MAX SIZE_MAX /**< max #mdb_size_t */ +/** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */ +# define MDB_PRIy(t) MDB_FMT_Z #t +/** #mdb_size_t scanf formats, \b t = one of [dioux] without quotes */ +# define MDB_SCNy(t) MDB_FMT_Z #t +#else +typedef uint64_t mdb_size_t; +# define MDB_SIZE_MAX UINT64_MAX +# define MDB_PRIy(t) PRI##t##64 +# define MDB_SCNy(t) SCN##t##64 +# define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */ +#endif + +/** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#ifdef _WIN32 +typedef void *mdb_filehandle_t; +#else +typedef int mdb_filehandle_t; +#endif + +/** @defgroup mdb LMDB API + * @{ + * @brief Symas Lightning Memory-Mapped Database Manager + */ +/** @defgroup Version Version Macros + * @{ + */ +/** Library major version */ +#define MDB_VERSION_MAJOR 0 +/** Library minor version */ +#define MDB_VERSION_MINOR 9 +/** Library patch version */ +#define MDB_VERSION_PATCH 90 + +/** Combine args a,b,c into a single integer for easy version comparisons */ +#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) + +/** The full library version as a single integer */ +#define MDB_VERSION_FULL \ + MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) + +/** The release date of this library version */ +#define MDB_VERSION_DATE "May 1, 2017" + +/** A stringifier for the version info */ +#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" + +/** A helper for the stringifier macro */ +#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) + +/** The full library version as a C string */ +#define MDB_VERSION_STRING \ + MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) +/** @} */ + +/** @brief Opaque structure for a database environment. + * + * A DB environment supports multiple databases, all residing in the same + * shared-memory map. + */ +typedef struct MDB_env MDB_env; + +/** @brief Opaque structure for a transaction handle. + * + * All database operations require a transaction handle. Transactions may be + * read-only or read-write. + */ +typedef struct MDB_txn MDB_txn; + +/** @brief A handle for an individual database in the DB environment. */ +typedef unsigned int MDB_dbi; + +/** @brief Opaque structure for navigating through a database */ +typedef struct MDB_cursor MDB_cursor; + +/** @brief Generic structure used for passing keys and data in and out + * of the database. + * + * Values returned from the database are valid only until a subsequent + * update operation, or the end of the transaction. Do not modify or + * free them, they commonly point into the database itself. + * + * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. + * The same applies to data sizes in databases with the #MDB_DUPSORT flag. + * Other data items can in theory be from 0 to 0xffffffff bytes long. + */ +typedef struct MDB_val { + size_t mv_size; /**< size of the data item */ + void *mv_data; /**< address of the data item */ +} MDB_val; + +/** @brief A callback function used to compare two keys in a database */ +typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); + +/** @brief A callback function used to relocate a position-dependent data item + * in a fixed-address database. + * + * The \b newptr gives the item's desired address in + * the memory map, and \b oldptr gives its previous address. The item's actual + * data resides at the address in \b item. This callback is expected to walk + * through the fields of the record in \b item and modify any + * values based at the \b oldptr address to be relative to the \b newptr address. + * @param[in,out] item The item that is to be relocated. + * @param[in] oldptr The previous address. + * @param[in] newptr The new address to relocate to. + * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). + * @todo This feature is currently unimplemented. + */ +typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); + +#if MDB_RPAGE_CACHE +/** @brief A callback function used to encrypt/decrypt pages in the env. + * + * Encrypt or decrypt the data in src and store the result in dst using the + * provided key. The result must be the same number of bytes as the input. + * @param[in] src The input data to be transformed. + * @param[out] dst Storage for the result. + * @param[in] key An array of three values: key[0] is the encryption key, + * key[1] is the initialization vector, and key[2] is the authentication + * data, if any. + * @param[in] encdec 1 to encrypt, 0 to decrypt. + * @return A non-zero error value on failure and 0 on success. + */ +typedef int (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec); + +/** @brief A callback function used to checksum pages in the env. + * + * Compute the checksum of the data in src and store the result in dst, + * An optional key may be used with keyed hash algorithms. + * @param[in] src The input data to be transformed. + * @param[out] dst Storage for the result. + * @param[in] key An encryption key, if encryption was configured. This + * parameter will be NULL if there is no key. + */ +typedef void (MDB_sum_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key); +#endif + +/** @defgroup mdb_env Environment Flags + * @{ + */ + /** mmap at a fixed address (experimental) */ +#define MDB_FIXEDMAP 0x01 + /** encrypted DB - read-only flag, set by #mdb_env_set_encrypt() */ +#define MDB_ENCRYPT 0x2000U + /** no environment directory */ +#define MDB_NOSUBDIR 0x4000 + /** don't fsync after commit */ +#define MDB_NOSYNC 0x10000 + /** read only */ +#define MDB_RDONLY 0x20000 + /** don't fsync metapage after commit */ +#define MDB_NOMETASYNC 0x40000 + /** use writable mmap */ +#define MDB_WRITEMAP 0x80000 + /** use asynchronous msync when #MDB_WRITEMAP is used */ +#define MDB_MAPASYNC 0x100000 + /** tie reader locktable slots to #MDB_txn objects instead of to threads */ +#define MDB_NOTLS 0x200000 + /** don't do any locking, caller must manage their own locks */ +#define MDB_NOLOCK 0x400000 + /** don't do readahead (no effect on Windows) */ +#define MDB_NORDAHEAD 0x800000 + /** don't initialize malloc'd memory before writing to datafile */ +#define MDB_NOMEMINIT 0x1000000 + /** use the previous snapshot rather than the latest one */ +#define MDB_PREVSNAPSHOT 0x2000000 + /** don't use a single mmap, remap individual chunks (needs MDB_RPAGE_CACHE) */ +#define MDB_REMAP_CHUNKS 0x4000000 +/** @} */ + +/** @defgroup mdb_dbi_open Database Flags + * @{ + */ + /** use reverse string keys */ +#define MDB_REVERSEKEY 0x02 + /** use sorted duplicates */ +#define MDB_DUPSORT 0x04 + /** numeric keys in native byte order, either unsigned int or #mdb_size_t. + * (lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.) + * The keys must all be of the same size. */ +#define MDB_INTEGERKEY 0x08 + /** with #MDB_DUPSORT, sorted dup items have fixed size */ +#define MDB_DUPFIXED 0x10 + /** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */ +#define MDB_INTEGERDUP 0x20 + /** with #MDB_DUPSORT, use reverse string dups */ +#define MDB_REVERSEDUP 0x40 + /** create DB if not already existing */ +#define MDB_CREATE 0x40000 +/** @} */ + +/** @defgroup mdb_put Write Flags + * @{ + */ +/** For put: Don't write if the key already exists. */ +#define MDB_NOOVERWRITE 0x10 +/** Only for #MDB_DUPSORT
+ * For put: don't write if the key and data pair already exist.
+ * For mdb_cursor_del: remove all duplicate data items. + */ +#define MDB_NODUPDATA 0x20 +/** For mdb_cursor_put: overwrite the current key/data pair */ +#define MDB_CURRENT 0x40 +/** For put: Just reserve space for data, don't copy it. Return a + * pointer to the reserved space. + */ +#define MDB_RESERVE 0x10000 +/** Data is being appended, don't split full pages. */ +#define MDB_APPEND 0x20000 +/** Duplicate data is being appended, don't split full pages. */ +#define MDB_APPENDDUP 0x40000 +/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ +#define MDB_MULTIPLE 0x80000 +/* @} */ + +/** @defgroup mdb_copy Copy Flags + * @{ + */ +/** Compacting copy: Omit free space from copy, and renumber all + * pages sequentially. + */ +#define MDB_CP_COMPACT 0x01 +/* @} */ + +/** @brief Cursor Get operations. + * + * This is the set of all operations for retrieving data + * using a cursor. + */ +typedef enum MDB_cursor_op { + MDB_FIRST, /**< Position at first key/data item */ + MDB_FIRST_DUP, /**< Position at first data item of current key. + Only for #MDB_DUPSORT */ + MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ + MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ + MDB_GET_CURRENT, /**< Return key/data at current cursor position */ + MDB_GET_MULTIPLE, /**< Return up to a page of duplicate data items + from current cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_LAST, /**< Position at last key/data item */ + MDB_LAST_DUP, /**< Position at last data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT, /**< Position at next data item */ + MDB_NEXT_DUP, /**< Position at next data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT_MULTIPLE, /**< Return up to a page of duplicate data items + from next cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_NEXT_NODUP, /**< Position at first data item of next key */ + MDB_PREV, /**< Position at previous data item */ + MDB_PREV_DUP, /**< Position at previous data item of current key. + Only for #MDB_DUPSORT */ + MDB_PREV_NODUP, /**< Position at last data item of previous key */ + MDB_SET, /**< Position at specified key */ + MDB_SET_KEY, /**< Position at specified key, return key + data */ + MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */ + MDB_PREV_MULTIPLE /**< Position at previous page and return up to + a page of duplicate data items. Only for #MDB_DUPFIXED */ +} MDB_cursor_op; + +/** @defgroup errors Return Codes + * + * BerkeleyDB uses -30800 to -30999, we'll go under them + * @{ + */ + /** Successful result */ +#define MDB_SUCCESS 0 + /** key/data pair already exists */ +#define MDB_KEYEXIST (-30799) + /** key/data pair not found (EOF) */ +#define MDB_NOTFOUND (-30798) + /** Requested page not found - this usually indicates corruption */ +#define MDB_PAGE_NOTFOUND (-30797) + /** Located page was wrong type */ +#define MDB_CORRUPTED (-30796) + /** Update of meta page failed or environment had fatal error */ +#define MDB_PANIC (-30795) + /** Environment version mismatch */ +#define MDB_VERSION_MISMATCH (-30794) + /** File is not a valid LMDB file */ +#define MDB_INVALID (-30793) + /** Environment mapsize reached */ +#define MDB_MAP_FULL (-30792) + /** Environment maxdbs reached */ +#define MDB_DBS_FULL (-30791) + /** Environment maxreaders reached */ +#define MDB_READERS_FULL (-30790) + /** Too many TLS keys in use - Windows only */ +#define MDB_TLS_FULL (-30789) + /** Txn has too many dirty pages */ +#define MDB_TXN_FULL (-30788) + /** Cursor stack too deep - internal error */ +#define MDB_CURSOR_FULL (-30787) + /** Page has not enough space - internal error */ +#define MDB_PAGE_FULL (-30786) + /** Database contents grew beyond environment mapsize */ +#define MDB_MAP_RESIZED (-30785) + /** Operation and DB incompatible, or DB type changed. This can mean: + *
    + *
  • The operation expects an #MDB_DUPSORT / #MDB_DUPFIXED database. + *
  • Opening a named DB when the unnamed DB has #MDB_DUPSORT / #MDB_INTEGERKEY. + *
  • Accessing a data record as a database, or vice versa. + *
  • The database was dropped and recreated with different flags. + *
+ */ +#define MDB_INCOMPATIBLE (-30784) + /** Invalid reuse of reader locktable slot */ +#define MDB_BAD_RSLOT (-30783) + /** Transaction must abort, has a child, or is invalid */ +#define MDB_BAD_TXN (-30782) + /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ +#define MDB_BAD_VALSIZE (-30781) + /** The specified DBI was changed unexpectedly */ +#define MDB_BAD_DBI (-30780) + /** Unexpected problem - txn should abort */ +#define MDB_PROBLEM (-30779) + /** Page checksum incorrect */ +#define MDB_BAD_CHECKSUM (-30778) + /** Encryption/decryption failed */ +#define MDB_CRYPTO_FAIL (-30777) + /** Environment encryption mismatch */ +#define MDB_ENV_ENCRYPTION (-30776) + /** Transaction was already prepared */ +#define MDB_TXN_PENDING (-30775) + /** Environment can't rollback the last transaction */ +#define MDB_CANT_ROLLBACK (-30774) + /** Can't drop main DBI while other DBIs are open */ +#define MDB_DBIS_BUSY (-30773) + /** Write was incomplete */ +#define MDB_SHORT_WRITE (-30772) + /** Env is busy, can't use previous snapshot */ +#define MDB_ENV_BUSY (-30771) + /** Env or txn is read-only, can't write */ +#define MDB_IS_READONLY (-30770) + /** Requested map address is unavailable */ +#define MDB_ADDR_BUSY (-30769) + /** The last defined error code */ +#define MDB_LAST_ERRCODE MDB_ADDR_BUSY +/** @} */ + +/** @brief Statistics for a database in the environment */ +typedef struct MDB_stat { + unsigned int ms_psize; /**< Size of a database page. + This is currently the same for all databases. */ + unsigned int ms_depth; /**< Depth (height) of the B-tree */ + mdb_size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ + mdb_size_t ms_leaf_pages; /**< Number of leaf pages */ + mdb_size_t ms_overflow_pages; /**< Number of overflow pages */ + mdb_size_t ms_entries; /**< Number of data items */ +} MDB_stat; + +/** @brief Information about the environment */ +typedef struct MDB_envinfo { + void *me_mapaddr; /**< Address of map */ + mdb_size_t me_mapsize; /**< Size of the data memory map */ + mdb_size_t me_last_pgno; /**< ID of the last used page */ + mdb_size_t me_last_txnid; /**< ID of the last committed transaction */ + unsigned int me_maxreaders; /**< max reader slots in the environment */ + unsigned int me_numreaders; /**< max reader slots used in the environment */ +} MDB_envinfo; + + /** @brief Return the LMDB library version information. + * + * @param[out] major if non-NULL, the library major version number is copied here + * @param[out] minor if non-NULL, the library minor version number is copied here + * @param[out] patch if non-NULL, the library patch version number is copied here + * @retval "version string" The library version as a string + */ +char *mdb_version(int *major, int *minor, int *patch); + + /** @brief Return a string describing a given error code. + * + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) + * function. If the error code is greater than or equal to 0, then the string + * returned by the system function strerror(3) is returned. If the error code + * is less than 0, an error string corresponding to the LMDB library error is + * returned. See @ref errors for a list of LMDB-specific error codes. + * @param[in] err The error code + * @retval "error message" The description of the error + */ +char *mdb_strerror(int err); + + /** @brief Create an LMDB environment handle. + * + * This function allocates memory for a #MDB_env structure. To release + * the allocated memory and discard the handle, call #mdb_env_close(). + * Before the handle may be used, it must be opened using #mdb_env_open(). + * Various other options may also need to be set before opening the handle, + * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), + * depending on usage requirements. + * @param[out] env The address where the new handle will be stored + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_create(MDB_env **env); + + /** @brief Open an environment handle. + * + * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] path The directory in which the database files reside. This + * directory must already exist and be writable. + * @param[in] flags Special options for this environment. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * Flags set by mdb_env_set_flags() are also used. + *
    + *
  • #MDB_FIXEDMAP + * use a fixed address for the mmap region. This flag must be specified + * when creating the environment, and is stored persistently in the environment. + * If successful, the memory map will always reside at the same virtual address + * and pointers used to reference data items in the database will be constant + * across multiple invocations. This option may not always work, depending on + * how the operating system has allocated memory to shared libraries and other uses. + * The feature is highly experimental. + *
  • #MDB_NOSUBDIR + * By default, LMDB creates its environment in a directory whose + * pathname is given in \b path, and creates its data and lock files + * under that directory. With this option, \b path is used as-is for + * the database main data file. The database lock file is the \b path + * with "-lock" appended. + *
  • #MDB_RDONLY + * Open the environment in read-only mode. No write operations will be + * allowed. LMDB will still modify the lock file - except on read-only + * filesystems, where LMDB does not use locks. + *
  • #MDB_WRITEMAP + * Use a writeable memory map unless MDB_RDONLY is set. This uses + * fewer mallocs but loses protection from application bugs + * like wild pointer writes and other bad updates into the database. + * This may be slightly faster for DBs that fit entirely in RAM, but + * is slower for DBs larger than RAM. + * Incompatible with nested transactions. + * Do not mix processes with and without MDB_WRITEMAP on the same + * environment. This can defeat durability (#mdb_env_sync etc). + *
  • #MDB_NOMETASYNC + * Flush system buffers to disk only once per transaction, omit the + * metadata flush. Defer that until the system flushes files to disk, + * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization + * maintains database integrity, but a system crash may undo the last + * committed transaction. I.e. it preserves the ACI (atomicity, + * consistency, isolation) but not D (durability) database property. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_NOSYNC + * Don't flush system buffers to disk when committing a transaction. + * This optimization means a system crash can corrupt the database or + * lose the last transactions if buffers are not yet flushed to disk. + * The risk is governed by how often the system flushes dirty buffers + * to disk and how often #mdb_env_sync() is called. However, if the + * filesystem preserves write order and the #MDB_WRITEMAP flag is not + * used, transactions exhibit ACI (atomicity, consistency, isolation) + * properties and only lose D (durability). I.e. database integrity + * is maintained, but a system crash may undo the final transactions. + * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no + * hint for when to write transactions to disk, unless #mdb_env_sync() + * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_MAPASYNC + * When using #MDB_WRITEMAP, use asynchronous flushes to disk. + * As with #MDB_NOSYNC, a system crash can then corrupt the + * database or lose the last transactions. Calling #mdb_env_sync() + * ensures on-disk database integrity until next commit. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_NOTLS + * Don't use Thread-Local Storage. Tie reader locktable slots to + * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps + * the slot reserved for the #MDB_txn object. A thread may use parallel + * read-only transactions. A read-only transaction may span threads if + * the user synchronizes its use. Applications that multiplex many + * user threads over individual OS threads need this option. Such an + * application must also serialize the write transactions in an OS + * thread, since LMDB's write locking is unaware of the user threads. + *
  • #MDB_NOLOCK + * Don't do any locking. If concurrent access is anticipated, the + * caller must manage all concurrency itself. For proper operation + * the caller must enforce single-writer semantics, and must ensure + * that no readers are using old transactions while a writer is + * active. The simplest approach is to use an exclusive lock so that + * no readers may be active at all when a writer begins. + *
  • #MDB_NORDAHEAD + * Turn off readahead. Most operating systems perform readahead on + * read requests by default. This option turns it off if the OS + * supports it. Turning it off may help random read performance + * when the DB is larger than RAM and system RAM is full. + * The option is not implemented on Windows. + *
  • #MDB_NOMEMINIT + * Don't initialize malloc'd memory before writing to unused spaces + * in the data file. By default, memory for pages written to the data + * file is obtained using malloc. While these pages may be reused in + * subsequent transactions, freshly malloc'd pages will be initialized + * to zeroes before use. This avoids persisting leftover data from other + * code (that used the heap and subsequently freed the memory) into the + * data file. Note that many other system libraries may allocate + * and free memory from the heap for arbitrary uses. E.g., stdio may + * use the heap for file I/O buffers. This initialization step has a + * modest performance cost so some applications may want to disable + * it using this flag. This option can be a problem for applications + * which handle sensitive data like passwords, and it makes memory + * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, + * which writes directly to the mmap instead of using malloc for pages. The + * initialization is also skipped if #MDB_RESERVE is used; the + * caller is expected to overwrite all of the memory that was + * reserved in that case. + * This flag may be changed at any time using #mdb_env_set_flags(). + *
  • #MDB_PREVSNAPSHOT + * Open the environment with the previous snapshot rather than the latest + * one. This loses the latest transaction, but may help work around some + * types of corruption. If opened with write access, this must be the + * only process using the environment. This flag is automatically reset + * after a write transaction is successfully committed. + *
+ * @param[in] mode The UNIX permissions to set on created files and semaphores. + * This parameter is ignored on Windows. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the + * version that created the database environment. + *
  • #MDB_INVALID - the environment file headers are corrupted. + *
  • ENOENT - the directory specified by the path parameter doesn't exist. + *
  • EACCES - the user didn't have permission to access the environment files. + *
  • EAGAIN - the environment was locked by another process. + *
+ */ +int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); + + /** @brief Copy an LMDB environment to the specified path. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy(MDB_env *env, const char *path); + + /** @brief Copy an LMDB environment to the specified file descriptor. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); + + /** @brief Copy an LMDB environment to the specified path, with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_CP_COMPACT - Perform compaction while copying: omit free + * pages and sequentially renumber all pages in output. This option + * consumes more CPU and runs more slowly than the default. + * Currently it fails if the environment has suffered a page leak. + *
+ * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); + + /** @brief Copy an LMDB environment to the specified file descriptor, + * with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. See + * #mdb_env_copy2() for further details. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @param[in] flags Special options for this operation. + * See #mdb_env_copy2() for options. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); + + /** @brief Perform incremental dump of an LMDB environment to the + * specified file descriptor. + * + * This function may be used to make an incremental backup of an + * existing environment. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @param[in] txnid The transaction ID of a previous backup. It must + * be greater than zero. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_incr_dumpfd(MDB_env *env, mdb_filehandle_t fd, size_t txnid); + + /** @brief Perform incremental dump of an LMDB environment to the + * specified file. + * + * This function may be used to make an incremental backup of an + * existing environment. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The name of the file to write the copy to. It must + * not already exist. + * @param[in] txnid The transaction ID of a previous backup. It must + * be greater than zero. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_incr_dump(MDB_env *env, const char *path, size_t txnid); + + /** @brief Reload an incremental dump of an LMDB environment from the + * specified file descriptor. + * + * This function may be used to load an incremental backup of an + * existing environment. + * @note No other tasks may access the environment while this runs. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully, using #MDB_WRITEMAP. + * @param[in] fd The filedescriptor to read the backup from. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_incr_loadfd(MDB_env *env, mdb_filehandle_t fd); + + /** @brief Return statistics about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + */ +int mdb_env_stat(MDB_env *env, MDB_stat *stat); + + /** @brief Return information about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_envinfo structure + * where the information will be copied + */ +int mdb_env_info(MDB_env *env, MDB_envinfo *stat); + + /** @brief Flush the data buffers to disk. + * + * Data is always written to disk when #mdb_txn_commit() is called, + * but the operating system may keep it buffered. LMDB always flushes + * the OS buffers upon commit as well, unless the environment was + * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is + * not valid if the environment was opened with #MDB_RDONLY. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] force If non-zero, force a synchronous flush. Otherwise + * if the environment has the #MDB_NOSYNC flag set the flushes + * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EACCES - the environment is read-only. + *
  • EINVAL - an invalid parameter was specified. + *
  • EIO - an error occurred during synchronization. + *
+ */ +int mdb_env_sync(MDB_env *env, int force); + + /** @brief Close the environment and release the memory map. + * + * Only a single thread may call this function. All transactions, databases, + * and cursors must already be closed before calling this function. Attempts to + * use any such handles after calling this function will cause a SIGSEGV. + * The environment handle will be freed and must not be used again after this call. + * @param[in] env An environment handle returned by #mdb_env_create() + */ +void mdb_env_close(MDB_env *env); + + /** @brief Set environment flags. + * + * This may be used to set some flags in addition to those from + * #mdb_env_open(), or to unset these flags. If several threads + * change the flags at the same time, the result is undefined. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] flags The flags to change, bitwise OR'ed together + * @param[in] onoff A non-zero value sets the flags, zero clears them. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); + + /** @brief Get environment flags. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] flags The address of an integer to store the flags + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_flags(MDB_env *env, unsigned int *flags); + + /** @brief Return the path that was used in #mdb_env_open(). + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] path Address of a string pointer to contain the path. This + * is the actual string in the environment, not a copy. It should not be + * altered in any way. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_path(MDB_env *env, const char **path); + + /** @brief Return the filedescriptor for the given environment. + * + * This function may be called after fork(), so the descriptor can be + * closed before exec*(). Other LMDB file descriptors have FD_CLOEXEC. + * (Until LMDB 0.9.18, only the lockfile had that.) + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); + + /** @brief Set the size of the memory map to use for this environment. + * + * The size should be a multiple of the OS page size. The default is + * 10485760 bytes. The size of the memory map is also the maximum size + * of the database. The value should be chosen as large as possible, + * to accommodate future growth of the database. + * This function should be called after #mdb_env_create() and before #mdb_env_open(). + * It may be called at later times if no transactions are active in + * this process. Note that the library does not check for this condition, + * the caller must ensure it explicitly. + * + * The new size takes effect immediately for the current process but + * will not be persisted to any others until a write transaction has been + * committed by the current process. Also, only mapsize increases are + * persisted into the environment. + * + * If the mapsize is increased by another process, and data has grown + * beyond the range of the current mapsize, #mdb_txn_begin() will + * return #MDB_MAP_RESIZED. This function may be called with a size + * of zero to adopt the new size. + * + * Any attempt to set a size smaller than the space already consumed + * by the environment will be silently changed to the current size of the used space. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] size The size in bytes + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment has + * an active write transaction. + *
+ */ +int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size); + + + /** @brief Set the size of DB pages in bytes. + * + * The size defaults to the OS page size. Smaller or larger values may be + * desired depending on the size of keys and values being used. Also, an + * explicit size may need to be set when using filesystems like ZFS which + * don't use the OS page size. + */ +int mdb_env_set_pagesize(MDB_env *env, int size); + + /** @brief Set the maximum number of threads/reader slots for the environment. + * + * This defines the number of slots in the lock table that is used to track readers in the + * the environment. The default is 126. + * Starting a read-only transaction normally ties a lock table slot to the + * current thread until the environment closes or the thread exits. If + * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the + * MDB_txn object until it or the #MDB_env object is destroyed. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] readers The maximum number of reader lock table slots + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment is already open. + *
+ */ +int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); + + /** @brief Get the maximum number of threads/reader slots for the environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] readers Address of an integer to store the number of readers + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); + + /** @brief Set the maximum number of named databases for the environment. + * + * This function is only needed if multiple databases will be used in the + * environment. Simpler applications that use the environment as a single + * unnamed database can ignore this option. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * + * Currently a moderate number of slots are cheap but a huge number gets + * expensive: 7-120 words per transaction, and every #mdb_dbi_open() + * does a linear search of the opened slots. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbs The maximum number of databases + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified, or the environment is already open. + *
+ */ +int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); + + /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. + * + * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. + * See @ref MDB_val. + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The maximum size of a key we can write + */ +int mdb_env_get_maxkeysize(MDB_env *env); + + /** @brief Set application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_userctx(MDB_env *env, void *ctx); + + /** @brief Get the application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The pointer set by #mdb_env_set_userctx(). + */ +void *mdb_env_get_userctx(MDB_env *env); + + /** @brief A callback function for most LMDB assert() failures, + * called before printing the message and aborting. + * + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] msg The assertion message, not including newline. + */ +typedef void MDB_assert_func(MDB_env *env, const char *msg); + + /** Set or reset the assert() callback of the environment. + * Disabled if liblmdb is built with NDEBUG. + * @note This hack should become obsolete as lmdb's error handling matures. + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] func An #MDB_assert_func function, or 0. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); + +#if MDB_RPAGE_CACHE + /** @brief Set encryption on an environment. + * + * This must be called before #mdb_env_open(). + * It implicitly sets #MDB_REMAP_CHUNKS on the env. + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] func An #MDB_enc_func function. + * @param[in] key The encryption key. + * @param[in] size The size of authentication data in bytes, if any. + * Set this to zero for unauthenticated encryption mechanisms. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size); + + /** @brief Set checksums on an environment. + * + * This must be called before #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] func An #MDB_sum_func function. + * @param[in] size The size of computed checksum values, in bytes. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size); +#endif + + /** @brief Create a transaction for use with the environment. + * + * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). + * @note A transaction and its cursors must only be used by a single + * thread, and a thread may only have a single transaction at a time. + * If #MDB_NOTLS is in use, this does not apply to read-only transactions. + * @note Cursors may not span transactions. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] parent If this parameter is non-NULL, the new transaction + * will be a nested transaction, with the transaction indicated by \b parent + * as its parent. Transactions may be nested to any level. A parent + * transaction and its cursors may not issue any other operations than + * mdb_txn_commit and mdb_txn_abort while it has active child transactions. + * @note A parent transaction with read-only child transactions must not + * issue any operations while it has active child transactions, not even + * #mdb_txn_commit() nor #mdb_txn_abort(). Child transactions must be + * aborted prior to performing actions with the parent one. + * @param[in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_RDONLY + * This transaction will not perform any write operations. + *
  • #MDB_NOSYNC + * Don't flush system buffers to disk when committing this transaction. + *
  • #MDB_NOMETASYNC + * Flush system buffers but omit metadata flush when committing this transaction. + *
+ * @param[out] txn Address where the new #MDB_txn handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + *
  • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's + * mapsize and this environment's map must be resized as well. + * See #mdb_env_set_mapsize(). + *
  • #MDB_READERS_FULL - a read-only transaction was requested and + * the reader lock table is full. See #mdb_env_set_maxreaders(). + *
  • ENOMEM - out of memory. + *
+ */ +int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); + + /** @brief Returns the transaction's #MDB_env + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +MDB_env *mdb_txn_env(MDB_txn *txn); + + /** @brief Return the transaction's ID. + * + * This returns the identifier associated with this transaction. For a + * read-only transaction, this corresponds to the snapshot being read; + * concurrent readers will frequently have the same transaction ID. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A transaction ID, valid if input is an active transaction. + */ +mdb_size_t mdb_txn_id(MDB_txn *txn); + + /** @brief Retrieve the transaction's flags + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[out] flags Address where the flags will be returned. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_txn_flags(MDB_txn *txn, unsigned int *flags); + + /** @brief Commit all the operations of a transaction into the database. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
  • ENOSPC - no more disk space. + *
  • EIO - a low-level I/O error occurred while writing. + *
  • ENOMEM - out of memory. + *
+ */ +int mdb_txn_commit(MDB_txn *txn); + + /** @brief Prepare to commit all the operations of a transaction into the database. + * + * This function exists to support two-phase commit protocols. + * All writes in the transaction are persisted to storage, but the final + * metapage update is not performed. All cursors on the transaction will be + * closed. Only #mdb_txn_abort() or #mdb_txn_commit() are valid after this + * call. It is assumed that once the regular data pages are successfully written + * by this call, the metapage update from a subsequent commit cannot fail, but + * hardware-level media failures could still break this assumption. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
  • ENOSPC - no more disk space. + *
  • EIO - a low-level I/O error occurred while writing. + *
  • ENOMEM - out of memory. + *
  • #MDB_TXN_PENDING - the transaction has already been prepared. + * It can only be aborted or committed. + *
+ */ +int mdb_txn_prepare(MDB_txn *txn); + + /** @brief Rollback the last committed transaction in the environment. + * + * This function exists to support two-phase commit protocols. + * The metapage update for the last committed transaction will be zeroed, + * so its changes will be ignored. It should only be used when the local + * phase of a multi-phase transaction has fully committed, but some other + * remote phase which successfully prepared has failed to commit. + * This function may not be called twice in a row. No other operations + * may be performed on the environment, by any processes, between the + * preceding #mdb_txn_commit() and this call. + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] txnid The ID of the transaction to rollback, obtained from + * #mdb_txn_id() on the previous transaction. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
  • ENOSPC - no more disk space. + *
  • EIO - a low-level I/O error occurred while writing. + *
  • #MDB_CANT_ROLLBACK - a rollback has already been done, there is + * no other valid metapage to roll back to, or another transaction + * has already been committed over the specified txnid. + *
+ */ +int mdb_env_rollback(MDB_env *env, mdb_size_t txnid); + + /** @brief Abandon all the operations of the transaction instead of saving them. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_abort(MDB_txn *txn); + + /** @brief Reset a non-nested read-only transaction. + * + * Abort the transaction like #mdb_txn_abort(), but keep the transaction + * handle. #mdb_txn_renew() may reuse the handle. This saves allocation + * overhead if the process will start a new read-only transaction soon, + * and also locking overhead if #MDB_NOTLS is in use. The reader table + * lock is released, but the table slot stays tied to its thread or + * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free + * its lock table slot if MDB_NOTLS is in use. + * Cursors opened within the transaction must not be used + * again after this call, except with #mdb_cursor_renew(). + * Reader locks generally don't interfere with writers, but they keep old + * versions of database pages allocated. Thus they prevent the old pages + * from being reused when writers commit new data, and so under heavy load + * the database size may grow much more rapidly than otherwise. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_reset(MDB_txn *txn); + + /** @brief Renew a non-nested read-only transaction. + * + * This acquires a new reader lock for a transaction handle that had been + * released by #mdb_txn_reset(). It must be called before a reset transaction + * may be used again. Nested read-only transactions cannot be renewed. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_txn_renew(MDB_txn *txn); + +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) + + /** @brief Open a database in the environment. + * + * A database handle denotes the name and parameters of a database, + * independently of whether such a database exists. + * The database handle may be discarded by calling #mdb_dbi_close(). + * The old database handle is returned if the database was already open. + * The handle may only be closed once. + * + * The database handle will be private to the current transaction until + * the transaction is successfully committed. If the transaction is + * aborted the handle will be closed automatically. + * After a successful commit the handle will reside in the shared + * environment, and may be used by other transactions. + * + * This function must not be called from multiple concurrent + * transactions in the same process. A transaction that uses + * this function must finish (either commit or abort) before + * any other transaction in the process may use this function. + * + * To use named databases (with name != NULL), #mdb_env_set_maxdbs() + * must be called before opening the environment. Database names are + * keys in the unnamed database, and may be read but not written. + * @note Names are C strings and stored with their NUL terminator included. + * In LMDB 0.9 the NUL terminator was omitted. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] name The name of the database to open. If only a single + * database is needed in the environment, this value may be NULL. + * @param[in] flags Special options for this database. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_REVERSEKEY + * Keys are strings to be compared in reverse order, from the end + * of the strings to the beginning. By default, Keys are treated as strings and + * compared from beginning to end. + *
  • #MDB_DUPSORT + * Duplicate keys may be used in the database. (Or, from another perspective, + * keys may have multiple data items, stored in sorted order.) By default + * keys must be unique and may have only a single data item. + *
  • #MDB_INTEGERKEY + * Keys are binary integers in native byte order, either unsigned int + * or #mdb_size_t, and will be sorted as such. + * (lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.) + * The keys must all be of the same size. + *
  • #MDB_DUPFIXED + * This flag may only be used in combination with #MDB_DUPSORT. This option + * tells the library that the data items for this database are all the same + * size, which allows further optimizations in storage and retrieval. When + * all data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE + * and #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple + * items at once. + *
  • #MDB_INTEGERDUP + * This option specifies that duplicate data items are binary integers, + * similar to #MDB_INTEGERKEY keys. + *
  • #MDB_REVERSEDUP + * This option specifies that duplicate data items should be compared as + * strings in reverse order. + *
  • #MDB_CREATE + * Create the named database if it doesn't exist. This option is not + * allowed in a read-only transaction or a read-only environment. + *
+ * @param[out] dbi Address where the new #MDB_dbi handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - the specified database doesn't exist in the environment + * and #MDB_CREATE was not specified. + *
  • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). + *
+ */ +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); + + /** @brief Retrieve statistics for a database. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); + + /** @brief Retrieve the DB flags for a database handle. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] flags Address where the flags will be returned. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); + + /** @brief Close a database handle. Normally unnecessary. Use with care: + * + * This call is not mutex protected. Handles should only be closed by + * a single thread, and only if no other threads are going to reference + * the database handle or one of its cursors any further. Do not close + * a handle if an existing transaction has modified its database. + * Doing so can cause misbehavior from database corruption to errors + * like MDB_BAD_VALSIZE (since the DB name is gone). + * + * Closing a database handle is not necessary, but lets #mdb_dbi_open() + * reuse the handle value. Usually it's better to set a bigger + * #mdb_env_set_maxdbs(), unless that value would be large. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); + + /** @brief Empty or delete+close a database. + * + * See #mdb_dbi_close() for restrictions about closing the DB handle. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] del 0 to empty the DB, 1 to delete it from the + * environment and close the DB handle. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); + + /** @brief Set a custom key comparison function for a database. + * + * The comparison function is called whenever it is necessary to compare a + * key specified by the application with a key currently stored in the database. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating + * before longer keys. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. + * + * This comparison function is called whenever it is necessary to compare a data + * item specified by the application with a data item currently stored in the database. + * This function only takes effect if the database was opened with the #MDB_DUPSORT + * flag. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating + * before longer items. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a relocation function for a #MDB_FIXEDMAP database. + * + * @todo The relocation function is called whenever it is necessary to move the data + * of an item to a different position in the database (e.g. through tree + * balancing operations, shifts as a result of adds or deletes, etc.). It is + * intended to allow address/position-dependent data items to be stored in + * a database in an environment opened with the #MDB_FIXEDMAP option. + * Currently the relocation feature is unimplemented and setting + * this function has no effect. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] rel A #MDB_rel_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); + + /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. + * + * See #mdb_set_relfunc and #MDB_rel_func for more details. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * It will be passed to the callback function set by #mdb_set_relfunc + * as its \b relctx parameter whenever the callback is invoked. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); + + /** @brief Get items from a database. + * + * This function retrieves key/data pairs from the database. The address + * and length of the data associated with the specified \b key are returned + * in the structure to which \b data refers. + * If the database supports duplicate keys (#MDB_DUPSORT) then the + * first data item for the key will be returned. Retrieval of other + * items requires the use of #mdb_cursor_get(). + * + * @note The memory pointed to by the returned values is owned by the + * database. The caller need not dispose of the memory, and may not + * modify it in any way. For values returned in a read-only transaction + * any modification attempts will cause a SIGSEGV. + * @note Values returned from the database are valid only until a + * subsequent update operation, or the end of the transaction. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to search for in the database + * @param[out] data The data corresponding to the key + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - the key was not in the database. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Store items into a database. + * + * This function stores key/data pairs in the database. The default behavior + * is to enter the new key/data pair, replacing any previously existing key + * if duplicates are disallowed, or adding a duplicate data item if + * duplicates are allowed (#MDB_DUPSORT). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to store in the database + * @param[in,out] data The data to store + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + *
    + *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). The \b data + * parameter will be set to point to the existing item. + *
  • #MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later - before + * the next update operation or the transaction ends. This saves + * an extra memcpy if the data is being generated later. + * LMDB does nothing else with this memory, the caller is expected + * to modify all of the space requested. This flag must not be + * specified if the database was opened with #MDB_DUPSORT. + *
  • #MDB_APPEND - append the given key/data pair to the end of the + * database. This option allows fast bulk loading when keys are + * already known to be in the correct order. Loading unsorted keys + * with this flag will cause a #MDB_KEYEXIST error. + *
  • #MDB_APPENDDUP - as above, but for sorted dup data. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete items from a database. + * + * This function removes key/data pairs from the database. + * If the database does not support sorted duplicate data items + * (#MDB_DUPSORT) the data parameter is ignored. + * If the database supports sorted duplicates and the data parameter + * is NULL, all of the duplicate data items for the key will be + * deleted. Otherwise, if the data parameter is non-NULL + * only the matching data item will be deleted. + * This function will return #MDB_NOTFOUND if the specified key/data + * pair is not in the database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to delete from the database + * @param[in] data The data to delete + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Create a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * A cursor cannot be used when its database handle is closed. Nor + * when its transaction has ended, except with #mdb_cursor_renew(). + * It can be discarded with #mdb_cursor_close(). + * A cursor in a write-transaction can be closed before its transaction + * ends, and will otherwise be closed when its transaction ends. + * A cursor in a read-only transaction must be closed explicitly, before + * or after its transaction ends. It can be reused with + * #mdb_cursor_renew() before finally closing it. + * @note Earlier documentation said that cursors in every transaction + * were closed when the transaction committed or aborted. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] cursor Address where the new #MDB_cursor handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); + + /** @brief Close a cursor handle. + * + * The cursor handle will be freed and must not be used again after this call. + * Its transaction must still be live if it is a write-transaction. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +void mdb_cursor_close(MDB_cursor *cursor); + + /** @brief Renew a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * Cursors that are only used in read-only + * transactions may be re-used, to avoid unnecessary malloc/free overhead. + * The cursor may be associated with a new read-only transaction, and + * referencing the same database handle as it was created with. + * This may be done whether the previous transaction is live or dead. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); + + /** @brief Return the cursor's transaction handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); + + /** @brief Return the cursor's database handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); + + /** @brief Check if the cursor is pointing to a named database record. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @return 1 if current record is a named database, 0 otherwise. + */ +int mdb_cursor_is_db(MDB_cursor *cursor); + + /** @brief Retrieve by cursor. + * + * This function retrieves key/data pairs from the database. The address and length + * of the key are returned in the object to which \b key refers (except for the + * case of the #MDB_SET option, in which the \b key object is unchanged), and + * the address and length of the data are returned in the object to which \b data + * refers. + * See #mdb_get() for restrictions on using the output values. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in,out] key The key for a retrieved item + * @param[in,out] data The data of a retrieved item + * @param[in] op A cursor operation #MDB_cursor_op + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_NOTFOUND - no matching key found. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + MDB_cursor_op op); + + /** @brief Store by cursor. + * + * This function stores key/data pairs into the database. + * The cursor is positioned at the new item, or on failure usually near it. + * @note Earlier documentation incorrectly said errors would leave the + * state of the cursor unchanged. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] key The key operated on. + * @param[in] data The data operated on. + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + *
    + *
  • #MDB_CURRENT - replace the item at the current cursor position. + * The \b key parameter must still be provided, and must match it. + * If using sorted duplicates (#MDB_DUPSORT) the data item must still + * sort into the same place. This is intended to be used when the + * new data is the same size as the old. Otherwise it will simply + * perform a delete of the old record followed by an insert. + *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). + *
  • #MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later - before + * the next update operation or the transaction ends. This saves + * an extra memcpy if the data is being generated later. This flag + * must not be specified if the database was opened with #MDB_DUPSORT. + *
  • #MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * a #MDB_KEYEXIST error. + *
  • #MDB_APPENDDUP - as above, but for sorted dup data. + *
  • #MDB_MULTIPLE - store multiple contiguous data elements in a + * single request. This flag may only be specified if the database + * was opened with #MDB_DUPFIXED. The \b data argument must be an + * array of two MDB_vals. The mv_size of the first MDB_val must be + * the size of a single data element. The mv_data of the first MDB_val + * must point to the beginning of the array of contiguous data elements. + * The mv_size of the second MDB_val must be the count of the number + * of data elements to store. On return this field will be set to + * the count of the number of elements actually written. The mv_data + * of the second MDB_val is unused. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete current key/data pair + * + * This function deletes the key/data pair to which the cursor refers. + * This does not invalidate the cursor, so operations such as MDB_NEXT + * can still be used on it. + * Both MDB_NEXT and MDB_GET_CURRENT will return the same record after + * this operation. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + *
    + *
  • #MDB_NODUPDATA - delete all of the data items for the current key. + * This flag may only be specified if the database was opened with #MDB_DUPSORT. + *
+ * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EACCES - an attempt was made to write in a read-only transaction. + *
  • EINVAL - an invalid parameter was specified. + *
+ */ +int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); + + /** @brief Return count of duplicates for current key. + * + * This call is only valid on databases that support sorted duplicate + * data items #MDB_DUPSORT. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[out] countp Address where the count will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + *
    + *
  • EINVAL - cursor is not initialized, or an invalid parameter was specified. + *
+ */ +int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two data items were keys in the + * specified database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two items were data items of + * the specified database. The database must have the #MDB_DUPSORT flag. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief A callback function used to print a message from the library. + * + * @param[in] msg The string to be printed. + * @param[in] ctx An arbitrary context pointer for the callback. + * @return < 0 on failure, >= 0 on success. + */ +typedef int (MDB_msg_func)(const char *msg, void *ctx); + + /** @brief Dump the entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] func A #MDB_msg_func function + * @param[in] ctx Anything the message function needs + * @return < 0 on failure, >= 0 on success. + */ +int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); + + /** @brief Check for stale entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] dead Number of stale slots that were cleared + * @return 0 on success, non-zero on failure. + */ +int mdb_reader_check(MDB_env *env, int *dead); +/** @} */ + +/** @defgroup crypto LMDB Encryption Helper API + * @{ + * @brief Helpers for setting up encryption + */ + + /** @brief A function for converting a string into an encryption key. + * + * @param[in] passwd The string to be converted. + * @param[in,out] key The resulting key. The caller must + * provide the space for the key. + * @return 0 on success, non-zero on failure. + */ +typedef int (MDB_str2key_func)(const char *passwd, MDB_val *key); + + /** @brief A structure for dynamically loaded crypto modules. + * + * This is the information that the command line tools expect + * in order to operate on encrypted or checksummed environments. + */ +typedef struct MDB_crypto_funcs { + MDB_str2key_func *mcf_str2key; + MDB_enc_func *mcf_encfunc; + MDB_sum_func *mcf_sumfunc; + int mcf_keysize; /**< The size of an encryption key, in bytes */ + int mcf_esumsize; /**< The size of the MAC, for authenticated encryption */ + int mcf_sumsize; /**< The size of the checksum, for plain checksums */ +} MDB_crypto_funcs; + + /** @brief The function that returns the #MDB_crypto_funcs structure. + * + * The command line tools expect this function to be named "MDB_crypto". + * It must be exported by the dynamic module so that the tools can use it. + * @return A pointer to a #MDB_crypto_funcs structure. + */ +typedef MDB_crypto_funcs *(MDB_crypto_hooks)(void); + + /** @brief Load a dynamically loadable module. + * + * @param[in] file The pathname of the module to load. + * @param[in] symname The name of a symbol to resolve in the module. + * @param[out] mcf_ptr The crypto hooks returned from the module. + * @param[out] errmsg Messages for any errors from trying to load the module. + * @return The handle to the loadable module that can be unloaded by #mdb_modunload(), + * or NULL if loading failed. + */ +void *mdb_modload(const char *file, const char *symname, + MDB_crypto_funcs **mcf_ptr, char **errmsg); + + /** @brief Unload a dynamically loaded module. + * + * All environments that used the functions in the module must be closed + * before unloading the module. + * @param[in] handle The handle returned by #mdb_modload(). + */ +void mdb_modunload(void *handle); + + /** @brief Set an environment to use the given crypto functions. + * + * This is just a wrapper around #mdb_env_set_encrypt() to ease use of + * dynamically loaded crypto functions. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] mcf_ptr The crypto hooks retrieved by #mdb_modload(). + * @param[in] passphrase The secret used to generate the encryption key for the environment. + */ +void mdb_modsetup(MDB_env *env, MDB_crypto_funcs *mcf_ptr, const char *passphrase); + +/** @} */ + +#ifdef __cplusplus +} +#endif +/** @page tools LMDB Command Line Tools + The following describes the command line tools that are available for LMDB. + \li \ref mdb_copy_1 + \li \ref mdb_drop_1 + \li \ref mdb_dump_1 + \li \ref mdb_load_1 + \li \ref mdb_stat_1 +*/ + +#endif /* _LMDB_H_ */ diff --git a/src/LightningDB/wasm/lmdb_wasm_crypto.c b/src/LightningDB/wasm/lmdb_wasm_crypto.c new file mode 100644 index 0000000..330d717 --- /dev/null +++ b/src/LightningDB/wasm/lmdb_wasm_crypto.c @@ -0,0 +1,107 @@ +/* lmdb_wasm_crypto.c - native page cipher/checksum for browser-wasm builds. + * + * Cipher: ChaCha20-Poly1305 (RFC 8439, IETF variant, detached MAC, empty AAD) + * composed from Monocypher primitives. The on-disk format matches upstream + * LMDB's crypto.c (libsodium crypto_aead_chacha20poly1305_ietf_*_detached): + * data encrypted from block counter 1, Poly1305 key from block 0, 16-byte + * detached MAC stored in the page tail (key[2]). + * + * Nonce: upstream crypto.c builds LE32(pgno) || txnid from the 64-bit page + * header. wasm32 has 4-byte pgno_t/txnid_t (xsize == 8), so the 12-byte + * nonce here is LE32(pgno) || LE32(txnid) || 00 00 00 00. Encrypted files + * are therefore only readable by builds with the same layout (raw LMDB + * files are not portable across 32/64-bit builds anyway). + * + * Checksum: BLAKE2b-256 (keyed when combined with encryption, matching the + * upstream cryptoc.c pattern of a keyed checksum alongside the cipher). + * + * Installed statically via mdb_env_set_encrypt/mdb_env_set_checksum - the + * same pattern as upstream mtest_enc.c - because mono-wasm cannot marshal + * managed delegates as native callbacks. The entry points below are + * P/Invoked once from LightningDB before mdb_env_open. + */ +#include +#include + +#define LMDB_WASM_KEYBYTES 32 +#define LMDB_WASM_MACBYTES 16 +#define LMDB_WASM_SUMBYTES 32 + +static void lmdb_wasm_nonce(const MDB_val *iv, uint8_t nonce[12]) +{ + size_t tsz = iv->mv_size - sizeof(pgno_t); + memset(nonce, 0, 12); + memcpy(nonce, iv->mv_data, 4); /* low 32 bits of pgno */ + if (tsz > 8) tsz = 8; + memcpy(nonce + 4, (const char *)iv->mv_data + sizeof(pgno_t), tsz); +} + +/* RFC 8439 MAC over ciphertext with empty AAD: + * poly_key = ChaCha20 block 0; input = ct || pad16(ct) || LE64(0) || LE64(len) */ +static void lmdb_wasm_mac(uint8_t mac[16], const uint8_t *ct, size_t len, + const uint8_t *key, const uint8_t nonce[12]) +{ + uint8_t polykey[32], tail[16] = {0}; + crypto_poly1305_ctx ctx; + size_t i; + + crypto_chacha20_ietf(polykey, 0, 32, key, nonce, 0); + crypto_poly1305_init(&ctx, polykey); + crypto_poly1305_update(&ctx, ct, len); + crypto_poly1305_update(&ctx, tail, (16 - (len & 15)) & 15); + for (i = 0; i < 8; i++) + tail[8 + i] = (uint8_t)(len >> (8 * i)); + crypto_poly1305_update(&ctx, tail, 16); + crypto_poly1305_final(&ctx, mac); + crypto_wipe(polykey, sizeof(polykey)); +} + +static int lmdb_wasm_encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec) +{ + uint8_t nonce[12], mac[16]; + const uint8_t *k = key[0].mv_data; + + lmdb_wasm_nonce(&key[1], nonce); + if (encdec) { + crypto_chacha20_ietf(dst->mv_data, src->mv_data, src->mv_size, k, nonce, 1); + lmdb_wasm_mac(key[2].mv_data, dst->mv_data, src->mv_size, k, nonce); + } else if (key[2].mv_size) { + lmdb_wasm_mac(mac, src->mv_data, src->mv_size, k, nonce); + if (crypto_verify16(mac, key[2].mv_data)) + return -1; + crypto_chacha20_ietf(dst->mv_data, src->mv_data, src->mv_size, k, nonce, 1); + } else { + /* MAC-less header decrypt during incremental load (see crypto.c) */ + crypto_chacha20_ietf(dst->mv_data, src->mv_data, src->mv_size, k, nonce, 1); + } + return 0; +} + +static void lmdb_wasm_sumfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key) +{ + if (key && key->mv_size) + crypto_blake2b_keyed(dst->mv_data, LMDB_WASM_SUMBYTES, + key->mv_data, key->mv_size, src->mv_data, src->mv_size); + else + crypto_blake2b(dst->mv_data, LMDB_WASM_SUMBYTES, src->mv_data, src->mv_size); +} + +int lmdb_setup_encryption(MDB_env *env, const uint8_t *key, int key_len, int with_checksum) +{ + MDB_val enckey; + int rc; + + if (key_len != LMDB_WASM_KEYBYTES) + return EINVAL; + enckey.mv_data = (void *)key; + enckey.mv_size = (size_t)key_len; + rc = mdb_env_set_encrypt(env, lmdb_wasm_encfunc, &enckey, LMDB_WASM_MACBYTES); + if (rc) + return rc; + return with_checksum ? mdb_env_set_checksum(env, lmdb_wasm_sumfunc, LMDB_WASM_SUMBYTES) : 0; +} + +int lmdb_setup_checksum(MDB_env *env) +{ + return mdb_env_set_checksum(env, lmdb_wasm_sumfunc, LMDB_WASM_SUMBYTES); +} diff --git a/src/LightningDB/wasm/mdb.c b/src/LightningDB/wasm/mdb.c new file mode 100644 index 0000000..5c5ca12 --- /dev/null +++ b/src/LightningDB/wasm/mdb.c @@ -0,0 +1,12856 @@ +/** @file mdb.c + * @brief Lightning memory-mapped database library + * + * A Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. + */ +/* + * Copyright 2011-2021 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + * + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#if defined(MDB_VL32) || defined(__WIN64__) +#define _FILE_OFFSET_BITS 64 +#endif +#ifdef _WIN32 +#include +#include +#include /* get wcscpy() */ + +/* We use native NT APIs to setup the memory map, so that we can + * let the DB file grow incrementally instead of always preallocating + * the full size. These APIs are defined in and + * but those headers are meant for driver-level development and + * conflict with the regular user-level headers, so we explicitly + * declare them here. We get pointers to these functions from + * NTDLL.DLL at runtime, to avoid buildtime dependencies on any + * NTDLL import libraries. + */ +typedef NTSTATUS (WINAPI NtCreateSectionFunc) + (OUT PHANDLE sh, IN ACCESS_MASK acc, + IN void * oa OPTIONAL, + IN PLARGE_INTEGER ms OPTIONAL, + IN ULONG pp, IN ULONG aa, IN HANDLE fh OPTIONAL); + +static NtCreateSectionFunc *NtCreateSection; + +typedef enum _SECTION_INHERIT { + ViewShare = 1, + ViewUnmap = 2 +} SECTION_INHERIT; + +typedef NTSTATUS (WINAPI NtMapViewOfSectionFunc) + (IN PHANDLE sh, IN HANDLE ph, + IN OUT PVOID *addr, IN ULONG_PTR zbits, + IN SIZE_T cs, IN OUT PLARGE_INTEGER off OPTIONAL, + IN OUT PSIZE_T vs, IN SECTION_INHERIT ih, + IN ULONG at, IN ULONG pp); + +static NtMapViewOfSectionFunc *NtMapViewOfSection; + +typedef NTSTATUS (WINAPI NtCloseFunc)(HANDLE h); + +static NtCloseFunc *NtClose; + +/** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it + * as int64 which is wrong. MSVC doesn't define it at all, so just + * don't use it. + */ +#define MDB_PID_T int +#define MDB_THR_T DWORD +#include +#include +#ifdef __GNUC__ +# include +#else +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# define BYTE_ORDER LITTLE_ENDIAN +# ifndef SSIZE_MAX +# define SSIZE_MAX INT_MAX +# endif +#endif +#define MDB_OFF_T int64_t +#else +#include +#include +#define MDB_PID_T pid_t +#define MDB_THR_T pthread_t +#include +#include +#include +#ifdef HAVE_SYS_FILE_H +#include +#endif +#include +#define MDB_OFF_T off_t +#endif + +#if defined(__mips) && defined(__linux) +/* MIPS has cache coherency issues, requires explicit cache control */ +#include +#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) +#else +#define CACHEFLUSH(addr, bytes, cache) +#endif + +#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS) +/** fdatasync is broken on ext3/ext4fs on older kernels, see + * description in #mdb_env_open2 comments. You can safely + * define MDB_FDATASYNC_WORKS if this code will only be run + * on kernels 3.6 and newer. + */ +#define BROKEN_FDATASYNC +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#include +typedef SSIZE_T ssize_t; +#else +#include +#endif + +#if defined(__sun) || defined(__ANDROID__) +/* Most platforms have posix_memalign, older may only have memalign */ +#define HAVE_MEMALIGN 1 +#include +/* On Solaris, we need the POSIX sigwait function */ +#if defined (__sun) +# define _POSIX_PTHREAD_SEMANTICS 1 +#endif +#endif + +#ifdef _WIN32 +#define ALIGNED_FREE(x) _aligned_free(x) +#define MEMALIGN(rc, buf, align, size) do { buf=_aligned_malloc(size, align); rc = (buf == NULL) ? ERROR_NOT_ENOUGH_MEMORY : 0; } while(0) +#else +#define ALIGNED_FREE(x) free(x) +# ifdef HAVE_MEMALIGN +# define MEMALIGN(rc, buf, align, size) do { buf=memalign(align, size); rc = (buf == NULL) ? errno : 0; } while(0) +# else +# define MEMALIGN(rc, buf, align, size) rc = posix_memalign((void **)&buf, align, size) +# endif +#endif + +#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) +#include +#include /* defines BYTE_ORDER on HPUX and Solaris */ +#endif + +#if defined(__FreeBSD__) && defined(__FreeBSD_version) +# if __FreeBSD_version >= 1100110 +# define MDB_USE_POSIX_MUTEX 1 +# define MDB_USE_ROBUST 1 +# endif +# if __FreeBSD_version < 1101000 +# define MDB_FDATASYNC fsync +# endif +#elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__) +# if defined(__APPLE__) && !defined(MDB_USE_ROBUST) +# define MDB_USE_POSIX_SEM 1 +# endif +# if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM)) +# define MDB_USE_SYSV_SEM 1 +# endif +# if defined(__APPLE__) +# define MDB_FDATASYNC(fd) fcntl(fd, F_FULLFSYNC) +# else +# define MDB_FDATASYNC fsync +# endif +#elif defined(__ANDROID__) +# define MDB_FDATASYNC fsync +#elif defined(__HAIKU__) +# define MDB_USE_POSIX_SEM 1 +# define MDB_FDATASYNC fsync +#endif + +/* NetBSD does not define union semun in sys/sem.h */ +#if defined(__NetBSD__) && !defined(_SEM_SEMUN_UNDEFINED) +# define _SEM_SEMUN_UNDEFINED 1 +#endif + +#ifndef _WIN32 +#include +#include +#ifdef MDB_USE_POSIX_SEM +# define MDB_USE_HASH 1 +#include +#elif defined(MDB_USE_SYSV_SEM) +#include +#include +#ifdef _SEM_SEMUN_UNDEFINED +union semun { + int val; + struct semid_ds *buf; + unsigned short *array; +}; +#endif /* _SEM_SEMUN_UNDEFINED */ +#else +#define MDB_USE_POSIX_MUTEX 1 +#endif /* MDB_USE_POSIX_SEM */ +#endif /* !_WIN32 */ + +#if defined(_WIN32) + defined(MDB_USE_POSIX_SEM) + defined(MDB_USE_SYSV_SEM) \ + + defined(MDB_USE_POSIX_MUTEX) != 1 +# error "Ambiguous shared-lock implementation" +#endif + +#ifdef USE_VALGRIND +#include +#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) +#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) +#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) +#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) +#else +#define VGMEMP_CREATE(h,r,z) +#define VGMEMP_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) +#define VGMEMP_DESTROY(h) +#define VGMEMP_DEFINED(a,s) +#endif + +#ifndef BYTE_ORDER +# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) +/* Solaris just defines one or the other */ +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# ifdef _LITTLE_ENDIAN +# define BYTE_ORDER LITTLE_ENDIAN +# else +# define BYTE_ORDER BIG_ENDIAN +# endif +# else +# define BYTE_ORDER __BYTE_ORDER +# endif +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif + +#if (defined(__i386) || defined(__x86_64) || defined(_M_IX86)) && !defined(MISALIGNED_OK) +#define MISALIGNED_OK 1 +#endif + +#include "lmdb.h" +#include "midl.h" + +#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) +# error "Unknown or unsupported endianness (BYTE_ORDER)" +#elif (-6 & 5) || CHAR_BIT!=8 || UINT_MAX!=0xffffffff || MDB_SIZE_MAX%UINT_MAX +# error "Two's complement, reasonably sized integer types, please" +#endif + +#if (((__clang_major__ << 8) | __clang_minor__) >= 0x0302) || (((__GNUC__ << 8) | __GNUC_MINOR__) >= 0x0403) +/** Mark infrequently used env functions as cold. This puts them in a separate + * section, and optimizes them for size */ +#define ESECT __attribute__ ((cold)) +#else +/* On older compilers, use a separate section */ +# ifdef __GNUC__ +# ifdef __APPLE__ +# define ESECT __attribute__ ((section("__TEXT,text_env,regular,pure_instructions"))) +# else +# define ESECT __attribute__ ((section("text_env"))) +# endif +# else +# define ESECT +# endif +#endif + +#ifdef _WIN32 +#define CALL_CONV WINAPI +#else +#define CALL_CONV +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup compat Compatibility Macros + * A bunch of macros to minimize the amount of platform-specific ifdefs + * needed throughout the rest of the code. When the features this library + * needs are similar enough to POSIX to be hidden in a one-or-two line + * replacement, this macro approach is used. + * @{ + */ + + /** Features under development */ +#ifndef MDB_DEVEL +#define MDB_DEVEL 0 +#endif + + /** Wrapper around __func__, which is a C99 feature */ +#if __STDC_VERSION__ >= 199901L +# define mdb_func_ __func__ +#elif __GNUC__ >= 2 || _MSC_VER >= 1300 +# define mdb_func_ __FUNCTION__ +#else +/* If a debug message says (), update the #if statements above */ +# define mdb_func_ "" +#endif + +/* Internal error codes, not exposed outside liblmdb */ +#define MDB_NO_META (MDB_LAST_ERRCODE + 9) +#define MDB_NO_ROOT (MDB_LAST_ERRCODE + 10) +#ifdef _WIN32 +#define MDB_OWNERDEAD ((int) WAIT_ABANDONED) +#elif defined MDB_USE_SYSV_SEM +#define MDB_OWNERDEAD (MDB_LAST_ERRCODE + 11) +#elif defined(MDB_USE_POSIX_MUTEX) && defined(EOWNERDEAD) +#define MDB_OWNERDEAD EOWNERDEAD /**< #LOCK_MUTEX0() result if dead owner */ +#endif + +#ifdef __GLIBC__ +#define GLIBC_VER ((__GLIBC__ << 16 )| __GLIBC_MINOR__) +#endif +/** Some platforms define the EOWNERDEAD error code + * even though they don't support Robust Mutexes. + * Compile with -DMDB_USE_ROBUST=0, or use some other + * mechanism like -DMDB_USE_SYSV_SEM instead of + * -DMDB_USE_POSIX_MUTEX. (SysV semaphores are + * also Robust, but some systems don't support them + * either.) + */ +#ifndef MDB_USE_ROBUST +/* Android currently lacks Robust Mutex support. So does glibc < 2.4. */ +# if defined(MDB_USE_POSIX_MUTEX) && (defined(__ANDROID__) || \ + (defined(__GLIBC__) && GLIBC_VER < 0x020004)) +# define MDB_USE_ROBUST 0 +# else +# define MDB_USE_ROBUST 1 +# endif +#endif /* !MDB_USE_ROBUST */ + +#if defined(MDB_USE_POSIX_MUTEX) && (MDB_USE_ROBUST) +/* glibc < 2.12 only provided _np API */ +# if (defined(__GLIBC__) && GLIBC_VER < 0x02000c) || \ + (defined(PTHREAD_MUTEX_ROBUST_NP) && !defined(PTHREAD_MUTEX_ROBUST)) +# define PTHREAD_MUTEX_ROBUST PTHREAD_MUTEX_ROBUST_NP +# define pthread_mutexattr_setrobust(attr, flag) pthread_mutexattr_setrobust_np(attr, flag) +# define pthread_mutex_consistent(mutex) pthread_mutex_consistent_np(mutex) +# endif +#endif /* MDB_USE_POSIX_MUTEX && MDB_USE_ROBUST */ + +#if defined(MDB_OWNERDEAD) && (MDB_USE_ROBUST) +#define MDB_ROBUST_SUPPORTED 1 +#endif + +#ifdef _WIN32 +#define MDB_USE_HASH 1 +#define MDB_PIDLOCK 0 +#define THREAD_RET DWORD +#define pthread_t HANDLE +#define pthread_mutex_t HANDLE +#define pthread_cond_t HANDLE +typedef HANDLE mdb_mutex_t, mdb_mutexref_t; +#define pthread_key_t DWORD +#define pthread_self() GetCurrentThreadId() +#define pthread_key_create(x,y) \ + ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) +#define pthread_key_delete(x) TlsFree(x) +#define pthread_getspecific(x) TlsGetValue(x) +#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) +#define pthread_mutex_unlock(x) ReleaseMutex(*x) +#define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) +#define pthread_mutex_init(x,y) ((*x = CreateMutex(NULL, FALSE, NULL)) ? 0 : ErrCode()) +#define pthread_mutex_destroy(x) (CloseHandle(*x) ? 0 : ErrCode()) +#define pthread_cond_signal(x) SetEvent(*x) +#define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) +#define THREAD_CREATE(thr,start,arg) \ + (((thr) = CreateThread(NULL, 0, start, arg, 0, NULL)) ? 0 : ErrCode()) +#define THREAD_FINISH(thr) \ + (WaitForSingleObject(thr, INFINITE) ? ErrCode() : 0) +#define LOCK_MUTEX0(mutex) WaitForSingleObject(mutex, INFINITE) +#define UNLOCK_MUTEX(mutex) ReleaseMutex(mutex) +#define mdb_mutex_consistent(mutex) 0 +#define getpid() GetCurrentProcessId() +#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) +#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) +#define ErrCode() GetLastError() +#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} +#define close(fd) (CloseHandle(fd) ? 0 : -1) +#define munmap(ptr,len) UnmapViewOfFile(ptr) +#ifdef PROCESS_QUERY_LIMITED_INFORMATION +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION +#else +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 +#endif +#else +#define THREAD_RET void * +#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) +#define THREAD_FINISH(thr) pthread_join(thr,NULL) + + /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ +#define MDB_PIDLOCK 1 + +#ifdef MDB_USE_POSIX_SEM + +typedef sem_t *mdb_mutex_t, *mdb_mutexref_t; +#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) +#define UNLOCK_MUTEX(mutex) sem_post(mutex) + +static int +mdb_sem_wait(sem_t *sem) +{ + int rc; + while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; + return rc; +} + +#elif defined MDB_USE_SYSV_SEM + +typedef struct mdb_mutex { + int semid; + int semnum; + int *locked; +} mdb_mutex_t[1], *mdb_mutexref_t; + +#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) +#define UNLOCK_MUTEX(mutex) do { \ + struct sembuf sb = { 0, 1, SEM_UNDO }; \ + sb.sem_num = (mutex)->semnum; \ + *(mutex)->locked = 0; \ + semop((mutex)->semid, &sb, 1); \ +} while(0) + +static int +mdb_sem_wait(mdb_mutexref_t sem) +{ + int rc, *locked = sem->locked; + struct sembuf sb = { 0, -1, SEM_UNDO }; + sb.sem_num = sem->semnum; + do { + if (!semop(sem->semid, &sb, 1)) { + rc = *locked ? MDB_OWNERDEAD : MDB_SUCCESS; + *locked = 1; + break; + } + } while ((rc = errno) == EINTR); + return rc; +} + +#define mdb_mutex_consistent(mutex) 0 + +#else /* MDB_USE_POSIX_MUTEX: */ + /** Shared mutex/semaphore as the original is stored. + * + * Not for copies. Instead it can be assigned to an #mdb_mutexref_t. + * When mdb_mutexref_t is a pointer and mdb_mutex_t is not, then it + * is array[size 1] so it can be assigned to the pointer. + */ +typedef pthread_mutex_t mdb_mutex_t[1]; + /** Reference to an #mdb_mutex_t */ +typedef pthread_mutex_t *mdb_mutexref_t; + /** Lock the reader or writer mutex. + * Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX(). + */ +#define LOCK_MUTEX0(mutex) pthread_mutex_lock(mutex) + /** Unlock the reader or writer mutex. + */ +#define UNLOCK_MUTEX(mutex) pthread_mutex_unlock(mutex) + /** Mark mutex-protected data as repaired, after death of previous owner. + */ +#define mdb_mutex_consistent(mutex) pthread_mutex_consistent(mutex) +#endif /* MDB_USE_POSIX_SEM || MDB_USE_SYSV_SEM */ + + /** Get the error code for the last failed system function. + */ +#define ErrCode() errno + + /** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#define HANDLE int + + /** A value for an invalid file handle. + * Mainly used to initialize file variables and signify that they are + * unused. + */ +#define INVALID_HANDLE_VALUE (-1) + + /** Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. + */ +#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) +#endif + +#define Z MDB_FMT_Z /**< printf/scanf format modifier for size_t */ +#define Yu MDB_PRIy(u) /**< printf format for #mdb_size_t */ +#define Yd MDB_PRIy(d) /**< printf format for 'signed #mdb_size_t' */ + +#ifdef MDB_USE_SYSV_SEM +#define MNAME_LEN (sizeof(int)) +#else +#define MNAME_LEN (sizeof(pthread_mutex_t)) +#endif + +/** Initial part of #MDB_env.%me_mutexname[]. + * Changes to this code must be reflected in #MDB_LOCK_FORMAT. + */ +#ifdef _WIN32 +#define MUTEXNAME_PREFIX "Global\\MDB" +#elif defined MDB_USE_POSIX_SEM +#define MUTEXNAME_PREFIX "/MDB" +#endif + +/** @} */ + +#ifdef MDB_ROBUST_SUPPORTED + /** Lock mutex, handle any error, set rc = result. + * Return 0 on success, nonzero (not rc) on error. + */ +#define LOCK_MUTEX(rc, env, mutex) \ + (((rc) = LOCK_MUTEX0(mutex)) && \ + ((rc) = mdb_mutex_failed(env, mutex, rc))) +static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc); +#else +#define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex)) +#define mdb_mutex_failed(env, mutex, rc) (rc) +#endif + +#ifndef _WIN32 +/** A flag for opening a file and requesting synchronous data writes. + * This is only used when writing a meta page. It's not strictly needed; + * we could just do a normal write and then immediately perform a flush. + * But if this flag is available it saves us an extra system call. + * + * @note If O_DSYNC is undefined but exists in /usr/include, + * preferably set some compiler flag to get the definition. + */ +#ifndef MDB_DSYNC +# ifdef O_DSYNC +# define MDB_DSYNC O_DSYNC +# else +# define MDB_DSYNC O_SYNC +# endif +#endif +#endif + +/** Function for flushing the data of a file. Define this to fsync + * if fdatasync() is not supported. + */ +#ifndef MDB_FDATASYNC +# define MDB_FDATASYNC fdatasync +#endif + +#ifndef MDB_MSYNC +# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) +#endif + +#ifndef MS_SYNC +#define MS_SYNC 1 +#endif + +#ifndef MS_ASYNC +#define MS_ASYNC 0 +#endif + + /** A page number in the database. + * Note that 64 bit page numbers are overkill, since pages themselves + * already represent 12-13 bits of addressable memory, and the OS will + * always limit applications to a maximum of 63 bits of address space. + * + * @note In the #MDB_node structure, we only store 48 bits of this value, + * which thus limits us to only 60 bits of addressable data. + */ +typedef MDB_ID pgno_t; + + /** A transaction ID. + * See struct MDB_txn.mt_txnid for details. + */ +typedef MDB_ID txnid_t; + +/** @defgroup debug Debug Macros + * @{ + */ +#ifndef MDB_DEBUG + /** Enable debug output. Needs variable argument macros (a C99 feature). + * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs + * read from and written to the database (used for free space management). + */ +#define MDB_DEBUG 0 +#endif + +#define MDB_DBG_INFO 1 +#define MDB_DBG_TRACE 2 + +#if MDB_DEBUG +static int mdb_debug = MDB_DBG_TRACE; +static txnid_t mdb_debug_start; + + /** Print a debug message with printf formatting. + * Requires double parenthesis around 2 or more args. + */ +# define DPRINTF(args) ((void) ((mdb_debug & MDB_DBG_INFO) && DPRINTF0 args)) +# define DPRINTF0(fmt, ...) \ + fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__) + /** Trace info for replaying */ +# define MDB_TRACE(args) ((void) ((mdb_debug & MDB_DBG_TRACE) && DPRINTF1 args)) +# define DPRINTF1(fmt, ...) \ + fprintf(stderr, ">%d:%s: " fmt "\n", getpid(), mdb_func_, __VA_ARGS__) +#else +# define DPRINTF(args) ((void) 0) +# define MDB_TRACE(args) ((void) 0) +#endif + /** Print a debug string. + * The string is printed literally, with no format processing. + */ +#define DPUTS(arg) DPRINTF(("%s", arg)) + /** Debugging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +/** @} */ + + /** @brief The maximum size of a database page. + * + * It is 32k or 64k, since value-PAGEBASE must fit in + * #MDB_page.%mp_upper. + * + * LMDB will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. + */ +#define MAX_PAGESIZE (PAGEBASE ? 0x10000 : 0x8000) + + /** The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. + */ +#define MDB_MINKEYS 2 + + /** A stamp that identifies a file as an LMDB file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. + */ +#define MDB_MAGIC 0xBEEFC0DE + + /** The version number for a database's datafile format. */ +#define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 3) + /** The version number for a database's lockfile format. */ +#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 2) + /** Number of bits representing #MDB_LOCK_VERSION in #MDB_LOCK_FORMAT. + * The remaining bits must leave room for #MDB_lock_desc. + */ +#define MDB_LOCK_VERSION_BITS 12 + + /** @brief The max size of a key we can write, or 0 for computed max. + * + * This macro should normally be left alone or set to 0. + * Note that a database with big keys or dupsort data cannot be + * reliably modified by a liblmdb which uses a smaller max. + * The default is 511 for backwards compat, or 0 when #MDB_DEVEL. + * + * Other values are allowed, for backwards compat. However: + * A value bigger than the computed max can break if you do not + * know what you are doing, and liblmdb <= 0.9.10 can break when + * modifying a DB with keys/dupsort data bigger than its max. + * + * Data items in an #MDB_DUPSORT database are also limited to + * this size, since they're actually keys of a sub-DB. Keys and + * #MDB_DUPSORT data items must fit on a node in a regular page. + */ +#ifndef MDB_MAXKEYSIZE +#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) +#endif + + /** The maximum size of a key we can write to the environment. */ +#if MDB_MAXKEYSIZE +#define ENV_MAXKEY(env) (MDB_MAXKEYSIZE) +#else +#define ENV_MAXKEY(env) ((env)->me_maxkey) +#endif + + /** @brief The maximum size of a data item. + * + * We only store a 32 bit value for node sizes. + */ +#define MAXDATASIZE 0xffffffffUL + +#if MDB_DEBUG + /** Key size which fits in a #DKBUF. + * @ingroup debug + */ +#define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511) + /** A key buffer. + * @ingroup debug + * This is used for printing a hex dump of a key's contents. + */ +#define DKBUF char kbuf[DKBUF_MAXKEYSIZE*2+1] + /** A data value buffer. + * @ingroup debug + * This is used for printing a hex dump of a #MDB_DUPSORT value's contents. + */ +#define DDBUF char dbuf[DKBUF_MAXKEYSIZE*2+1+2] + /** Display a key in hex. + * @ingroup debug + * Invoke a function to display a key in hex. + */ +#define DKEY(x) mdb_dkey(x, kbuf) +#else +#define DKBUF +#define DDBUF +#define DKEY(x) 0 +#endif + + /** An invalid page number. + * Mainly used to denote an empty tree. + */ +#define P_INVALID (~(pgno_t)0) + + /** Test if the flags \b f are set in a flag word \b w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + + /** Round \b n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + + /** Least significant 1-bit of \b n. n must be of an unsigned type. */ +#define LOW_BIT(n) ((n) & (-(n))) + + /** (log2(\b p2) % \b n), for p2 = power of 2 and 0 < n < 8. */ +#define LOG2_MOD(p2, n) (7 - 86 / ((p2) % ((1U<<(n))-1) + 11)) + /* Explanation: Let p2 = 2**(n*y + x), x> (CACHELINE>64), 5)) + + 6 * (sizeof(MDB_PID_T)/4 % 3) /* legacy(2) to word(4/8)? */ + + 18 * (sizeof(pthread_t)/4 % 5) /* can be struct{id, active data} */ + + 90 * (sizeof(MDB_txbody) / CACHELINE % 3) + + 270 * (MDB_LOCK_TYPE % 120) + /* The above is < 270*120 < 2**15 */ + + ((sizeof(txnid_t) == 8) << 15) /* 32bit/64bit */ + + ((sizeof(MDB_reader) > CACHELINE) << 16) + /* Not really needed - implied by MDB_LOCK_TYPE != (_WIN32 locking) */ + + (((MDB_PIDLOCK) != 0) << 17) + /* 18 bits total: Must be <= (32 - MDB_LOCK_VERSION_BITS). */ +}; +/** @} */ + +/** Common header for all page types. The page type depends on #mp_flags. + * + * #P_BRANCH and #P_LEAF pages have unsorted '#MDB_node's at the end, with + * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages + * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header. + * + * #P_OVERFLOW records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of #F_BIGDATA nodes, + * and the node stores the pgno and number of pages used by the record. + * + * #P_SUBP sub-pages are small leaf "pages" with duplicate data. + * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page. + * (Duplicate data can also go in sub-databases, which use normal pages.) + * + * #P_META pages contain #MDB_meta, the start point of an LMDB snapshot. + * + * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once + * in the snapshot: Either used by a database or listed in a freeDB record. + */ +typedef struct MDB_page_header { +#define mp_pgno mp_p.p_pgno +#define mp_next mp_p.p_next + union { + pgno_t p_pgno; /**< page number */ + struct MDB_page *p_next; /**< for in-memory list of freed pages */ + } mh_p; + + /** If page is clean: snapshot txnid, dirty: txn workid, metapage: unused. + * + * The value indicates which snapshot/#MDB_txn the page belongs to. + * Tested with #IS_MUTABLE(), #IS_WRITABLE(), #IS_DIRTY_NW(). + * + * (clean page).mp_txnid == txnid of creator < txnid of later txns. + * (dirty page).mp_txnid >= mt_workid of creator txn. + * (dirty page).mt_txnid < mt_workid of children of creator txn. + * + * Thus an #MDB_txn can write to pages with mp_txnid >= txn.mt_workid. + * A page with smaller mp_txnid is dirty in an ancestor txn or clean. + * + * Non-#MDB_WRITEMAP sets txn.mt_workid > txn.mt_txnid, to tell apart + * spilled and dirty pages. WRITEMAP sets mt_workid = mt_txnid, since + * it does not copy/spill pages. Thus (page.mp_txnid == txn.mt_txnid) + * says "spilled page" without WRITEMAP, "dirty page" with WRITEMAP. + * + * Finally, ((dirty page).mp_txnid & #MDB_PGTXNID_FLAGMASK) can be used + * for flags with non-WRITEMAP; it keeps low bits in workid = 0. + */ + txnid_t mh_txnid; + + uint16_t mh_pad; /**< key size if this is a LEAF2 page */ +/** @defgroup mdb_page Page Flags + * @ingroup internal + * Flags for the page headers. + * @{ + */ +#define P_BRANCH 0x01 /**< branch page */ +#define P_LEAF 0x02 /**< leaf page */ +#define P_OVERFLOW 0x04 /**< overflow page */ +#define P_META 0x08 /**< meta page */ +#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ +#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ +#define P_DIRTY_OVF 0x2000 /**< page has dirty overflow nodes */ +#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /**< leave this page alone during spill */ +/** Persistent flags for page administration rather than page contents */ +#define P_ADM_FLAGS 0 /* later... */ +/** @} */ + uint16_t mh_flags; /**< @ref mdb_page */ +#define mp_lower mp_pb.pb.pb_lower +#define mp_upper mp_pb.pb.pb_upper +#define mp_pages mp_pb.pb_pages + union { + struct { + indx_t pb_lower; /**< lower bound of free space */ + indx_t pb_upper; /**< upper bound of free space */ + } pb; + uint32_t pb_pages; /**< number of overflow pages */ + } mh_pb; +} MDB_page_header; + +typedef struct MDB_page { + MDB_page_header mp_hdr; +#define mp_p mp_hdr.mh_p +#define mp_txnid mp_hdr.mh_txnid +#define mp_pad mp_hdr.mh_pad +#define mp_flags mp_hdr.mh_flags +#define mp_pb mp_hdr.mh_pb + indx_t mp_ptrs[0]; /**< dynamic size */ +} MDB_page; + +/** Alternate page header, for 2-byte aligned access */ +typedef struct MDB_page2 { + uint16_t mp2_p[(sizeof(pgno_t)+sizeof(txnid_t))/2]; + uint16_t mp2_pad; + uint16_t mp2_flags; + indx_t mp2_lower; + indx_t mp2_upper; + indx_t mp2_ptrs[0]; +} MDB_page2; + +#define MP_PGNO(p) (((MDB_page2 *)(void *)(p))->mp2_p) +#define MP_PAD(p) (((MDB_page2 *)(void *)(p))->mp2_pad) +#define MP_FLAGS(p) (((MDB_page2 *)(void *)(p))->mp2_flags) +#define MP_LOWER(p) (((MDB_page2 *)(void *)(p))->mp2_lower) +#define MP_UPPER(p) (((MDB_page2 *)(void *)(p))->mp2_upper) +#define MP_PTRS(p) (((MDB_page2 *)(void *)(p))->mp2_ptrs) + + /** Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned)sizeof(MDB_page_header)) + + /** Address of first usable data byte in a page, after the header */ +#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) + + /** ITS#7713, change PAGEBASE to handle 65536 byte pages */ +#define PAGEBASE PAGEHDRSZ + + /** Number of nodes on a page */ +#define NUMKEYS(p) ((MP_LOWER(p) - (PAGEHDRSZ-PAGEBASE)) >> 1) + + /** The amount of space remaining in the page */ +#define SIZELEFT(p) (indx_t)(MP_UPPER(p) - MP_LOWER(p)) + + /** The percentage of space used in the page, in tenths of a percent. */ +#define PAGEFILL(env, p) (1000L * ((env)->me_pagespace - SIZELEFT(p)) / \ + (env)->me_pagespace) + /** The minimum page fill factor, in tenths of a percent. + * Pages emptier than this are candidates for merging. + */ +#define FILL_THRESHOLD 250 + + /** Test if a page is a leaf page */ +#define IS_LEAF(p) F_ISSET(MP_FLAGS(p), P_LEAF) + /** Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) F_ISSET(MP_FLAGS(p), P_LEAF2) + /** Test if a page is a branch page */ +#define IS_BRANCH(p) F_ISSET(MP_FLAGS(p), P_BRANCH) + /** Test if a page is an overflow page */ +#define IS_OVERFLOW(p) F_ISSET(MP_FLAGS(p), P_OVERFLOW) + /** Test if a page is a sub page */ +#define IS_SUBP(p) F_ISSET(MP_FLAGS(p), P_SUBP) + + /** Test if (this non-sub page is dirty && env is non-#MDB_WRITEMAP) */ +#define IS_DIRTY_NW(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) + /** Test if this non-sub page belongs to the current snapshot */ +#define IS_MUTABLE(txn, p) ((p)->mp_txnid >= (txn)->mt_txnid) + /** Test if this non-sub page is writable in this txn (not an ancestor) */ +#define IS_WRITABLE(txn, p) ((p)->mp_txnid >= (txn)->mt_workid) + + /** Info about overflow page, stored in an F_BIGDATA node */ +typedef struct MDB_ovpage { + pgno_t op_pgno; + txnid_t op_txnid; + mdb_size_t op_pages; +} MDB_ovpage; + +#if OVERFLOW_NOTYET + /** Header for a dirty overflow page in memory */ +typedef struct MDB_dovpage { + MDB_page_header mp_hdr; + void *mp_ptr; +} MDB_dovpage; +#endif + + /** The number of overflow pages needed to store the given size. */ +#if MDB_RPAGE_CACHE +#define OVPAGES(env, size) ((PAGEHDRSZ-1 + (size) + (env)->me_sumsize + (env)->me_esumsize) / ((env)->me_psize) + 1) +#else +#define OVPAGES(env, size) ((PAGEHDRSZ-1 + (size)) / (env->me_psize) + 1) +#endif + + /** Link in #MDB_txn.%mt_loose_pgs list. + * Kept outside the page header, which is needed when reusing the page. + */ +#define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) + + /** Mark the page as writable by this txn */ +#ifndef MDB_TEST +#define SET_PGTXNID(txn, mp) ((mp)->mp_txnid = (txn)->mt_workid) +#else +#define SET_PGTXNID(txn, mp) \ + ((mp)->mp_txnid = (txn)->mt_workid \ + /* random unused "flags" added when not WRITEMAP for debugging */ \ + | (((txn)->mt_flags & MDB_TXN_WRITEMAP) ? 0 : \ + (MDB_RAND((size_t)(txn)) >> (32-MDB_PGTXNID_FLAGBITS)))) +#define MDB_RAND(x) (mdb_rnd = (mdb_rnd + (unsigned)(x)) * 987654321 + 54321) +static volatile unsigned mdb_rnd; +#endif + + /** mp_txnid bits reserved in dirty pages for flags. + * TODO: For future code with header-free ovpages, if we omit mp_flags + * from the "header" kept elsewhere. Otherwise, drop this code. + */ +#define MDB_PGTXNID_FLAGBITS 4 +#define MDB_PGTXNID_STEP ((txnid_t)1 << MDB_PGTXNID_FLAGBITS) +#define MDB_PGTXNID_FLAGMASK (MDB_PGTXNID_STEP-1) + + /** Header for a single key/data pair within a page. + * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. + * We guarantee 2-byte alignment for 'MDB_node's. + * + * #mn_lo and #mn_hi are used for data size on leaf nodes, and for child + * pgno on branch nodes. On 64 bit platforms, #mn_flags is also used + * for pgno. (Branch nodes have no flags). Lo and hi are in host byte + * order in case some accesses can be optimized to 32-bit word access. + * + * Leaf node flags describe node contents. #F_BIGDATA says the node's + * data part is an MDB_ovpage struct pointing to a page with actual data. + * #F_DUPDATA and #F_SUBDATA can be combined giving duplicate data in + * a sub-page/sub-database, and named databases (just #F_SUBDATA). + */ +typedef struct MDB_node { + /** part of data size or pgno + * @{ */ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short mn_lo, mn_hi; +#else + unsigned short mn_hi, mn_lo; +#endif + /** @} */ +/** @defgroup mdb_node Node Flags + * @ingroup internal + * Flags for node headers. + * @{ + */ +#define F_BIGDATA 0x01 /**< data put on overflow page */ +#define F_SUBDATA 0x02 /**< data is a sub-database */ +#define F_DUPDATA 0x04 /**< data has duplicates */ + +/** valid flags for #mdb_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) + +/** @} */ + unsigned short mn_flags; /**< @ref mdb_node */ + unsigned short mn_ksize; /**< key size */ + char mn_data[1]; /**< key and data are appended here */ +} MDB_node; + + /** Size of the node header, excluding dynamic data at the end */ +#define NODESIZE offsetof(MDB_node, mn_data) + + /** Bit position of top word in page number, for shifting mn_flags */ +#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) + + /** Size of a node in a branch page with a given key. + * This is just the node header plus the key, there is no data. + */ +#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) + + /** Size of a node in a leaf page with a given key and data. + * This is node header plus key plus data size. + */ +#define LEAFSIZE(k, d) (NODESIZE + EVEN((k)->mv_size) + EVEN((d)->mv_size)) + + /** Address of node \b i in page \b p */ +#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + MP_PTRS(p)[i] + PAGEBASE)) + + /** Address of the key for the node */ +#define NODEKEY(node) (void *)((node)->mn_data) + + /** Address of the data for a node */ +#define NODEDATA(node) (void *)((char *)(node)->mn_data + EVEN((node)->mn_ksize)) + + /** Get the page number pointed to by a branch node */ +#define NODEPGNO(node) \ + ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ + (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) + /** Set the page number in a branch node */ +#define SETPGNO(node,pgno) do { \ + (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ + if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) + + /** Get the size of the data in a leaf node */ +#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) + /** Set the size of the data for a leaf node */ +#define SETDSZ(node,size) do { \ + (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) + /** The size of a key in a node */ +#define NODEKSZ(node) ((node)->mn_ksize) + + /** Copy a page number from src to dst */ +#if MISALIGNED_OK +#define COPY_PGNO(dst,src) dst = src +#undef MP_PGNO +#define MP_PGNO(p) ((p)->mp_pgno) +#else +#if MDB_SIZE_MAX > 0xffffffffU +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d++ = *s++; \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#else +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#endif +#endif + /** The address of a key in a LEAF2 page. + * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. + * There are no node headers, keys are stored contiguously. + */ +#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) + + /** Set the \b node's key into \b keyptr, if requested. */ +#define MDB_GET_KEY(node, keyptr) { if ((keyptr) != NULL) { \ + (keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } } + + /** Set the \b node's key into \b key. */ +#define MDB_GET_KEY2(node, key) { key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); } + + /** Information about a single database in the environment. */ +typedef struct MDB_db { + uint32_t md_pad; /**< also ksize for LEAF2 pages */ + uint16_t md_flags; /**< @ref mdb_dbi_open */ + uint16_t md_depth; /**< depth of this tree */ + pgno_t md_branch_pages; /**< number of internal pages */ + pgno_t md_leaf_pages; /**< number of leaf pages */ + pgno_t md_overflow_pages; /**< number of overflow pages */ + mdb_size_t md_entries; /**< number of data items */ + pgno_t md_root; /**< the root page of this tree */ +} MDB_db; + +#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ +#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) + /** #mdb_dbi_open() flags */ +#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ + MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) + + /** Handle for the DB used to track free pages. */ +#define FREE_DBI 0 + /** Handle for the default DB. */ +#define MAIN_DBI 1 + /** Number of DBs in metapage (free and main) - also hardcoded elsewhere */ +#define CORE_DBS 2 + + /** Number of meta pages - also hardcoded elsewhere */ +#define NUM_METAS 2 + + /** Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). + */ +typedef struct MDB_meta { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mm_magic; + /** Version number of this file. Must be set to #MDB_DATA_VERSION. */ + uint32_t mm_version; +#ifdef MDB_VL32 + union { /* always zero since we don't support fixed mapping in MDB_VL32 */ + MDB_ID mmun_ull; + void *mmun_address; + } mm_un; +#define mm_address mm_un.mmun_address +#else + void *mm_address; /**< address for fixed mapping */ +#endif + mdb_size_t mm_mapsize; /**< size of mmap region */ + MDB_db mm_dbs[CORE_DBS]; /**< first is free space, 2nd is main db */ + /** The size of pages used in this DB */ +#define mm_psize mm_dbs[FREE_DBI].md_pad + /** Any persistent environment flags. @ref mdb_env */ +#define mm_flags mm_dbs[FREE_DBI].md_flags + /** Last used page in the datafile. + * Actually the file may be shorter if the freeDB lists the final pages. + */ + pgno_t mm_last_pg; + volatile txnid_t mm_txnid; /**< txnid that committed this page */ +} MDB_meta; + + /** Buffer for a stack-allocated meta page. + * The members define size and alignment, and silence type + * aliasing warnings. They are not used directly; that could + * mean incorrectly using several union members in parallel. + */ +typedef union MDB_metabuf { + MDB_page mb_page; + struct { + char mm_pad[PAGEHDRSZ]; + MDB_meta mm_meta; + } mb_metabuf; +} MDB_metabuf; + + /** Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. + */ +typedef struct MDB_dbx { + MDB_val md_name; /**< name of the database */ + MDB_cmp_func *md_cmp; /**< function for comparing keys */ + MDB_cmp_func *md_dcmp; /**< function for comparing data items */ + MDB_rel_func *md_rel; /**< user relocate function */ + void *md_relctx; /**< user-provided context for md_rel */ +} MDB_dbx; + + /** A database transaction. + * Every operation requires a transaction handle. + */ +struct MDB_txn { + MDB_txn *mt_parent; /**< parent of a nested txn */ + /** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */ + MDB_txn *mt_child; + /** The count of nested RDONLY txns under this txn */ + unsigned int mt_rdonly_child_count; + /** Mutex protecting mt_rdonly_child_count */ + pthread_mutex_t mt_child_mutex; + pgno_t mt_next_pgno; /**< next unallocated page */ +#if MDB_RPAGE_CACHE + pgno_t mt_last_pgno; /**< last written page */ +#endif + /** The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. + */ + txnid_t mt_txnid; + + /** Written to mp_txnid of dirty pages, to be fixed by #mdb_page_flush(). + * + * Value >= WRITEMAP ? txnid : 1 + (parent ? parent.last_workid : txnid). + * See #MDB_page.%mp_txnid. + * + * An MDB_txn can write to a page when page.mp_txnid >= txn.mt_workid. + * New children get bigger workid than pages dirty in their parent + * (i.e. bigger than parent.mt_last_workid). When children commit, + * they copy #mt_last_workid to the parent to match their pages. + */ + txnid_t mt_workid; + /** Current max mp_txnid of the MDB_txn's dirty pages: Starts as + * #mt_workid, then grows as it is copied from children who commit. + */ + txnid_t mt_last_workid; + + MDB_env *mt_env; /**< the DB environment */ + /** The list of pages that became unused during this transaction. + */ + MDB_IDL mt_free_pgs; + /** The list of loose pages that became unused and may be reused + * in this transaction, linked through #NEXT_LOOSE_PAGE(page). + */ + MDB_page *mt_loose_pgs; + /** Number of loose pages (#mt_loose_pgs) */ + int mt_loose_count; + /** The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. + * Unused with #MDB_WRITEMAP, which does not use a dirty list. + */ + MDB_IDL mt_spill_pgs; + union { + /** For write txns: Modified pages, sorted. Unused when MDB_WRITEMAP. */ + MDB_ID2L dirty_list; + /** For read txns: This thread/txn's reader table slot, or NULL. */ + MDB_reader *reader; + } mt_u; +#if OVERFLOW_NOTYET + /** The sorted list of dirty overflow pages. */ + MDB_ID2L mt_dirty_ovs; +#endif + /** Array of records for each DB known in the environment. */ + MDB_dbx *mt_dbxs; + /** Array of MDB_db records for each known DB */ + MDB_db *mt_dbs; + /** Array of sequence numbers for each DB handle */ + unsigned int *mt_dbiseqs; +/** @defgroup mt_dbflag Transaction DB Flags + * @ingroup internal + * @{ + */ +#define DB_DIRTY 0x01 /**< DB was written in this txn */ +#define DB_STALE 0x02 /**< Named-DB record is older than txnID */ +#define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ +#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ +#define DB_USRVALID 0x10 /**< As #DB_VALID, but not set for #FREE_DBI */ +#define DB_DUPDATA 0x20 /**< DB is #MDB_DUPSORT data */ +/** @} */ + /** In write txns, array of cursors for each DB */ + MDB_cursor **mt_cursors; + /** Array of flags for each DB */ + unsigned char *mt_dbflags; +#if MDB_RPAGE_CACHE + /** List of read-only pages (actually chunks) */ + MDB_ID3L mt_rpages; + /** We map chunks of 16 pages. Even though Windows uses 4KB pages, all + * mappings must begin on 64KB boundaries. So we round off all pgnos to + * a chunk boundary. We do the same on Linux for symmetry, and also to + * reduce the frequency of mmap/munmap calls. + */ +#define MDB_RPAGE_CHUNK 16 +#define MDB_TRPAGE_SIZE 4096 /**< size of #mt_rpages array of chunks */ +#define MDB_TRPAGE_MAX (MDB_TRPAGE_SIZE-1) /**< maximum chunk index */ + unsigned int mt_rpcheck; /**< threshold for reclaiming unref'd chunks */ +#endif + /** Number of DB records in use, or 0 when the txn is finished. + * This number only ever increments until the txn finishes; we + * don't decrement it when individual DB handles are closed. + */ + MDB_dbi mt_numdbs; + +/** @defgroup mdb_txn Transaction Flags + * @ingroup internal + * @{ + */ + /** #mdb_txn_begin() flags */ +#define MDB_TXN_BEGIN_FLAGS (MDB_NOMETASYNC|MDB_NOSYNC|MDB_RDONLY) +#define MDB_TXN_NOMETASYNC MDB_NOMETASYNC /**< don't sync meta for this txn on commit */ +#define MDB_TXN_NOSYNC MDB_NOSYNC /**< don't sync this txn on commit */ +#define MDB_TXN_RDONLY MDB_RDONLY /**< read-only transaction */ + /* internal txn flags */ +#define MDB_TXN_WRITEMAP MDB_WRITEMAP /**< copy of #MDB_env flag in writers */ +#define MDB_TXN_FINISHED 0x01 /**< txn is finished or never began */ +#define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ +#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ +#define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ +#define MDB_TXN_HAS_CHILD 0x10 /**< txn has an #MDB_txn.%mt_child */ +#define MDB_TXN_DIRTYNUM 0x20 /**< dirty list uses nump list */ +#define MDB_TXN_PREPARE 0x40 /**< prepare txn, don't fully commit */ +#define MDB_TXN_DROPPED 0x80 /**< main DBI has been dropped, env will reset */ + /** most operations on the txn are currently illegal */ +#define MDB_TXN_BLOCKED (MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD|MDB_TXN_PREPARE|MDB_TXN_DROPPED) +/** @} */ + unsigned int mt_flags; /**< @ref mdb_txn */ + /** #dirty_list room: Array size - \#dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirty_list into mt_parent after freeing hidden mt_parent pages. + * When #MDB_WRITEMAP, it is nonzero but otherwise irrelevant. + */ + unsigned int mt_dirty_room; +}; + +/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. + */ +#define CURSOR_STACK 32 + +struct MDB_xcursor; + + /** Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. #MDB_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a #P_SUBP page can be stale. + * (A node with #F_DUPDATA but no #F_SUBDATA contains a subpage). + */ +struct MDB_cursor { + /** Next cursor on this DB in this txn */ + MDB_cursor *mc_next; + /** Backup of the original cursor if this cursor is a shadow */ + MDB_cursor *mc_backup; + /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ + struct MDB_xcursor *mc_xcursor; + /** The transaction that owns this cursor */ + MDB_txn *mc_txn; + /** The database handle this cursor operates on */ + MDB_dbi mc_dbi; + /** The database record for this cursor */ + MDB_db *mc_db; + /** The database auxiliary record for this cursor */ + MDB_dbx *mc_dbx; + /** The @ref mt_dbflag for this database */ + unsigned char *mc_dbflag; + unsigned short mc_snum; /**< number of pushed pages */ + unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ +/** @defgroup mdb_cursor Cursor Flags + * @ingroup internal + * Cursor state flags. + * @{ + */ +#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ +#define C_EOF 0x02 /**< No more data */ +#define C_SUB 0x04 /**< Cursor is a sub-cursor */ +#define C_DEL 0x08 /**< last op was a cursor_del */ +#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ +#define C_WRITEMAP MDB_TXN_WRITEMAP /**< Copy of txn flag */ +/** Read-only cursor into the txn's original snapshot in the map. + * Set for read-only txns, and in #mdb_page_alloc() for #FREE_DBI when + * #MDB_DEVEL & 2. Only implements code which is necessary for this. + */ +#define C_ORIG_RDONLY MDB_TXN_RDONLY +/** @} */ + unsigned int mc_flags; /**< @ref mdb_cursor */ + MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ +#if MDB_RPAGE_CACHE + MDB_page *mc_ovpg; /**< a referenced overflow page */ +# define MC_OVPG(mc) ((mc)->mc_ovpg) +# define MC_SET_OVPG(mc, pg) \ + ((mc)->mc_ovpg = \ + (MDB_REMAPPING((mc)->mc_txn->mt_env->me_flags) ? (pg) : NULL)) +#else +# define MC_OVPG(mc) ((MDB_page *)0) +# define MC_SET_OVPG(mc, pg) ((void)0) +#endif +}; + + /** Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. + */ +typedef struct MDB_xcursor { + /** A sub-cursor for traversing the Dup DB */ + MDB_cursor mx_cursor; + /** The database record for this Dup DB */ + MDB_db mx_db; + /** The auxiliary DB record for this Dup DB */ + MDB_dbx mx_dbx; + /** The @ref mt_dbflag for this Dup DB */ + unsigned char mx_dbflag; +} MDB_xcursor; + + /** Check if there is an inited xcursor */ +#define XCURSOR_INITED(mc) \ + ((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + + /** Update the xcursor's sub-page pointer, if any, in \b mc. Needed + * when the node which contains the sub-page may have moved. Called + * with leaf page \b mp = mc->mc_pg[\b top]. + */ +#define XCURSOR_REFRESH(mc, top, mp) do { \ + MDB_page *xr_pg = (mp); \ + MDB_node *xr_node; \ + if (!XCURSOR_INITED(mc) || (mc)->mc_ki[top] >= NUMKEYS(xr_pg)) break; \ + xr_node = NODEPTR(xr_pg, (mc)->mc_ki[top]); \ + if ((xr_node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) \ + (mc)->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(xr_node); \ +} while (0) + + /** State of FreeDB old pages, stored in the MDB_env */ +typedef struct MDB_pgstate { + pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ + txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ +} MDB_pgstate; + + /** The database environment. */ +struct MDB_env { + HANDLE me_fd; /**< The main data file */ + HANDLE me_lfd; /**< The lock file */ + HANDLE me_mfd; /**< For writing and syncing the meta pages */ +#ifdef _WIN32 +#if MDB_RPAGE_CACHE + HANDLE me_fmh; /**< File Mapping handle */ +#endif + HANDLE me_ovfd; /**< Overlapped/async with write-through file handle */ +#endif /* _WIN32 */ + /** Failed to update the meta page. Probably an I/O error. */ +#define MDB_FATAL_ERROR 0x80000000U + /** using a raw block device */ +#define MDB_RAWPART 0x40000000U + /** Some fields are initialized. */ +#define MDB_ENV_ACTIVE 0x20000000U + /** me_txkey is set */ +#define MDB_ENV_TXKEY 0x10000000U + /** fdatasync is unreliable */ +#define MDB_FSYNCONLY 0x08000000U + uint32_t me_flags; /**< @ref mdb_env */ + unsigned int me_psize; /**< DB page size, inited from me_os_psize */ + unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ + unsigned int me_maxreaders; /**< size of the reader table */ + /** Max #MDB_txninfo.%mti_numreaders of interest to #mdb_env_close() */ + volatile int me_close_readers; + MDB_dbi me_numdbs; /**< number of DBs opened */ + MDB_dbi me_maxdbs; /**< size of the DB table */ + MDB_PID_T me_pid; /**< process ID of this env */ + char *me_path; /**< path to the DB files */ + char *me_map; /**< the memory map of the data file */ + MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ + MDB_meta *me_metas[NUM_METAS]; /**< pointers to the two meta pages */ + void *me_pbuf; /**< scratch area for DUPSORT put() */ + MDB_txn *me_txn; /**< current write transaction */ + MDB_txn *me_txn0; /**< prealloc'd write transaction */ + mdb_size_t me_mapsize; /**< size of the data memory map */ + MDB_OFF_T me_size; /**< current file size */ + pgno_t me_maxpg; /**< me_mapsize / me_psize */ + MDB_dbx *me_dbxs; /**< array of static DB info */ + uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ + unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ + pthread_key_t me_txkey; /**< thread-key for readers */ + txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ + MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ +# define me_pglast me_pgstate.mf_pglast +# define me_pghead me_pgstate.mf_pghead + MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ + /** IDL of pages that became unused in a write txn */ + MDB_IDL me_free_pgs; + /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. + * Unused except for a dummy element when #MDB_WRITEMAP. + */ + MDB_ID2L me_dirty_list; + int *me_dirty_nump; + /** Max number of freelist items that can fit in a single overflow page */ + int me_maxfree_1pg; + /** Max size of a node on a page */ + unsigned int me_nodemax; +#if !(MDB_MAXKEYSIZE) + unsigned int me_maxkey; /**< max size of a key */ +#endif + int me_live_reader; /**< have liveness lock in reader table */ +#ifdef _WIN32 + int me_pidquery; /**< Used in OpenProcess */ + OVERLAPPED *me_ov; /**< Used for overlapping I/O requests */ + int me_ovs; /**< Count of MDB_overlaps */ +#endif + int me_pagespace; /**< usable space in a page */ +#ifdef MDB_USE_POSIX_MUTEX /* Posix mutexes reside in shared mem */ +# define me_rmutex me_txns->mti_rmutex /**< Shared reader lock */ +# define me_wmutex me_txns->mti_wmutex /**< Shared writer lock */ +#else + mdb_mutex_t me_rmutex; + mdb_mutex_t me_wmutex; +# if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + /** Half-initialized name of mutexes, to be completed by #MUTEXNAME() */ + char me_mutexname[sizeof(MUTEXNAME_PREFIX) + 11]; +# endif +#endif +#if MDB_RPAGE_CACHE + MDB_ID3L me_rpages; /**< like #mt_rpages, but global to env */ + pthread_mutex_t me_rpmutex; /**< control access to #me_rpages */ + MDB_sum_func *me_sumfunc; /**< checksum env data */ + unsigned short me_sumsize; /**< size of per-page checksums */ +#define MDB_ERPAGE_SIZE 16384 +#define MDB_ERPAGE_MAX (MDB_ERPAGE_SIZE-1) + unsigned short me_esumsize; /**< size of per-page authentication data */ + unsigned int me_rpcheck; + + MDB_enc_func *me_encfunc; /**< encrypt env data */ + MDB_val me_enckey; /**< key for env encryption */ +#endif + void *me_userctx; /**< User-settable context */ + MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ +}; + + /** Nested transaction */ +typedef struct MDB_ntxn { + MDB_txn mnt_txn; /**< the transaction */ + MDB_pgstate mnt_pgstate; /**< parent transaction's saved freestate */ +} MDB_ntxn; + + /** max number of pages to commit in one writev() call */ +#define MDB_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES +#undef MDB_COMMIT_PAGES +#define MDB_COMMIT_PAGES IOV_MAX +#endif + + /** max bytes to write in one call */ +#define MAX_WRITE (0x40000000U >> (sizeof(ssize_t) == 4)) + + /** Check \b txn and \b dbi arguments to a function */ +#define TXN_DBI_EXIST(txn, dbi, validity) \ + ((txn) && (dbi)<(txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & (validity))) + + /** Check for misused \b dbi handles */ +#define TXN_DBI_CHANGED(txn, dbi) \ + ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) + +static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); +static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); +static int mdb_page_touch(MDB_cursor *mc); + +#define MDB_END_NAMES {"committed", "empty-commit", "abort", "reset", \ + "reset-tmp", "fail-begin", "fail-beginchild"} +enum { + /* mdb_txn_end operation number, for logging */ + MDB_END_COMMITTED, MDB_END_EMPTY_COMMIT, MDB_END_ABORT, MDB_END_RESET, + MDB_END_RESET_TMP, MDB_END_FAIL_BEGIN, MDB_END_FAIL_BEGINCHILD +}; +#define MDB_END_OPMASK 0x0F /**< mask for #mdb_txn_end() operation number */ +#define MDB_END_UPDATE 0x10 /**< update env state (DBIs) */ +#define MDB_END_FREE 0x20 /**< free txn unless it is #MDB_env.%me_txn0 */ +#define MDB_END_SLOT MDB_NOTLS /**< release any reader slot if #MDB_NOTLS */ +static void mdb_txn_end(MDB_txn *txn, unsigned mode); + +#if MDB_RPAGE_CACHE +#define MDB_PAGE_GET(mc, pg, numpgs, mp) mdb_page_get(mc, pg, numpgs, mp) +static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3); +#else +/* Drop unused numpgs argument when !MDB_RPAGE_CACHE */ +#define MDB_PAGE_GET(mc, pg, numpgs, mp) mdb_page_get(mc, pg, mp) +#endif +static int MDB_PAGE_GET(MDB_cursor *mc, pgno_t pgno, int numpgs, MDB_page **mp); + +static int mdb_page_search_root(MDB_cursor *mc, + MDB_val *key, int modify); +#define MDB_PS_MODIFY 1 +#define MDB_PS_ROOTONLY 2 +#define MDB_PS_FIRST 4 +#define MDB_PS_LAST 8 +static int mdb_page_search(MDB_cursor *mc, + MDB_val *key, int flags); +static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); + +#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ +static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, + pgno_t newpgno, unsigned int nflags); + +static int mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta); +static MDB_meta *mdb_env_pick_meta(const MDB_env *env); +static int mdb_env_write_meta(MDB_txn *txn); +#if defined(MDB_USE_POSIX_MUTEX) && !defined(MDB_ROBUST_SUPPORTED) /* Drop unused excl arg */ +# define mdb_env_close_active(env, excl) mdb_env_close1(env) +#endif +static void mdb_env_close_active(MDB_env *env, int excl); + +static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); +static int mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); +static void mdb_node_del(MDB_cursor *mc, int ksize); +static void mdb_node_shrink(MDB_page *mp, indx_t indx); +static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft); +static int mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data); +static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); +static size_t mdb_branch_size(MDB_env *env, MDB_val *key); + +static int mdb_rebalance(MDB_cursor *mc); +static int mdb_update_key(MDB_cursor *mc, MDB_val *key); + +static void mdb_cursor_pop(MDB_cursor *mc); +static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); + +static int _mdb_cursor_del(MDB_cursor *mc, unsigned int flags); +static int _mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, unsigned int flags); + +static int mdb_cursor_del0(MDB_cursor *mc); +static int mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags); +static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); +static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, + int *exactp); +static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); +static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); + +static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); +static void mdb_xcursor_init0(MDB_cursor *mc); +static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); +static void mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int force); + +static int mdb_drop0(MDB_cursor *mc, int subs); +static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); +static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); + +/** @cond */ +static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; +/** @endcond */ + +/** Compare two items pointing at '#mdb_size_t's of unknown alignment. */ +#ifdef MISALIGNED_OK +# define mdb_cmp_clong mdb_cmp_long +#else +# define mdb_cmp_clong mdb_cmp_cint +#endif + +/** True if we need #mdb_cmp_clong() instead of \b cmp for #MDB_INTEGERDUP */ +#define NEED_CMP_CLONG(cmp, ksize) \ + (UINT_MAX < MDB_SIZE_MAX && \ + (cmp) == mdb_cmp_int && (ksize) == sizeof(mdb_size_t)) + +#ifdef _WIN32 +static SECURITY_DESCRIPTOR mdb_null_sd; +static SECURITY_ATTRIBUTES mdb_all_sa; +static int mdb_sec_inited; + +struct MDB_name; +static int utf8_to_utf16(const char *src, struct MDB_name *dst, int xtra); +#endif + +/** Return the library version info. */ +char * ESECT +mdb_version(int *major, int *minor, int *patch) +{ + if (major) *major = MDB_VERSION_MAJOR; + if (minor) *minor = MDB_VERSION_MINOR; + if (patch) *patch = MDB_VERSION_PATCH; + return MDB_VERSION_STRING; +} + +/** Table of descriptions for LMDB @ref errors */ +static char *const mdb_errstr[] = { + "MDB_KEYEXIST: Key/data pair already exists", + "MDB_NOTFOUND: No matching key/data pair found", + "MDB_PAGE_NOTFOUND: Requested page not found", + "MDB_CORRUPTED: Located page was wrong type", + "MDB_PANIC: Update of meta page failed or environment had fatal error", + "MDB_VERSION_MISMATCH: Database environment version mismatch", + "MDB_INVALID: File is not an LMDB file", + "MDB_MAP_FULL: Environment mapsize limit reached", + "MDB_DBS_FULL: Environment maxdbs limit reached", + "MDB_READERS_FULL: Environment maxreaders limit reached", + "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", + "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", + "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", + "MDB_PAGE_FULL: Internal error - page has no more space", + "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", + "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", + "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", + "MDB_BAD_TXN: Transaction must abort, has a child, or is invalid", + "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", + "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", + "MDB_PROBLEM: Unexpected problem - txn should abort", + "MDB_BAD_CHECKSUM: Page checksum mismatch", + "MDB_CRYPTO_FAIL: Page encryption or decryption failed", + "MDB_ENV_ENCRYPTION: Environment encryption mismatch", + "MDB_TXN_PENDING: Transaction already prepared, must abort or commit", + "MDB_CANT_ROLLBACK: Environment can't rollback last transaction", + "MDB_DBIS_BUSY: Can't drop main DBI while other DBIs are open", + "MDB_SHORT_WRITE: Fewer bytes were written than requested", + "MDB_ENV_BUSY: Environment is busy, can't use previous snapshot", + "MDB_IS_READONLY: Can't write in readonly txn or environment", + "MDB_ADDR_BUSY: Requested map address is unavailable", +}; + +char * +mdb_strerror(int err) +{ +#ifdef _WIN32 + /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. + * This works as long as no function between the call to mdb_strerror + * and the actual use of the message uses more than 4K of stack. + */ +#define MSGSIZE 1024 +#define PADSIZE 4096 + char buf[MSGSIZE+PADSIZE], *ptr = buf; +#endif + int i; + if (!err) + return ("Successful return: 0"); + + if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { + i = err - MDB_KEYEXIST; + return mdb_errstr[i]; + } + +#ifdef _WIN32 + /* These are the C-runtime error codes we use. The comment indicates + * their numeric value, and the Win32 error they would correspond to + * if the error actually came from a Win32 API. A major mess, we should + * have used LMDB-specific error codes for everything. + */ + switch(err) { + case ENOMEM: /* 12, INVALID_ACCESS */ + case EINVAL: /* 22, BAD_COMMAND */ + return strerror(err); + default: + ; + } + buf[0] = 0; + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, 0, ptr, MSGSIZE, NULL); + return ptr; +#else + return strerror(err); +#endif +} + +/** assert(3) variant in cursor context */ +#define mdb_cassert(mc, expr) mdb_assert0((mc)->mc_txn->mt_env, expr, #expr) +/** assert(3) variant in transaction context */ +#define mdb_tassert(txn, expr) mdb_assert0((txn)->mt_env, expr, #expr) +/** assert(3) variant in environment context */ +#define mdb_eassert(env, expr) mdb_assert0(env, expr, #expr) + +#ifndef NDEBUG +# define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \ + mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__)) + +static void ESECT +mdb_assert_fail(MDB_env *env, const char *expr_txt, + const char *func, const char *file, int line) +{ + char buf[400]; + sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()", + file, line, expr_txt, func); + if (env->me_assert_func) + env->me_assert_func(env, buf); + fprintf(stderr, "%s\n", buf); + abort(); +} +#else +# define mdb_assert0(env, expr, expr_txt) ((void) 0) +#endif /* NDEBUG */ + +#if MDB_DEBUG +/** Return the page number of \b mp which may be sub-page, for debug output */ +static pgno_t +mdb_dbg_pgno(MDB_page *mp) +{ + pgno_t ret; + COPY_PGNO(ret, MP_PGNO(mp)); + return ret; +} + +/** Display a key in hexadecimal and return the address of the result. + * @param[in] key the key to display + * @param[in] buf the buffer to write into. Should always be #DKBUF. + * @return The key in hexadecimal form. + */ +char * +mdb_dkey(MDB_val *key, char *buf) +{ + char *ptr = buf; + unsigned char *c = key->mv_data; + unsigned int i; + + if (!key) + return ""; + + if (key->mv_size > DKBUF_MAXKEYSIZE) + return "MDB_MAXKEYSIZE"; + /* may want to make this a dynamic check: if the key is mostly + * printable characters, print it as-is instead of converting to hex. + */ +#if 1 + buf[0] = '\0'; + for (i=0; imv_size; i++) + ptr += sprintf(ptr, "%02x", *c++); +#else + sprintf(buf, "%.*s", key->mv_size, key->mv_data); +#endif + return buf; +} + +static char * +mdb_dval(MDB_txn *txn, MDB_dbi dbi, MDB_val *data, char *buf) +{ + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { + mdb_dkey(data, buf+1); + if (data->mv_size < DKBUF_MAXKEYSIZE - 2) { + *buf = '['; + strcpy(buf + data->mv_size * 2 + 1, "]"); + } + } else + *buf = '\0'; + return buf; +} + +static const char * +mdb_leafnode_type(MDB_node *n) +{ + static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; + return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" : + tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)]; +} + +/** Display all the keys in the page. */ +void +mdb_page_list(MDB_page *mp) +{ + pgno_t pgno = mdb_dbg_pgno(mp); + const char *type; + MDB_node *node; + unsigned int i, nkeys, nsize, total = 0; + MDB_val key; + DKBUF; + + switch (MP_FLAGS(mp) & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { + case P_BRANCH: type = "Branch page"; break; + case P_LEAF: type = "Leaf page"; break; + case P_LEAF|P_SUBP: type = "Sub-page"; break; + case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; + case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; + case P_OVERFLOW: + fprintf(stderr, "Overflow page %"Yu" pages %u\n", pgno, mp->mp_pages); + return; + case P_META: + fprintf(stderr, "Meta-page %"Yu" txnid %"Yu"\n", + pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); + return; + default: + fprintf(stderr, "Bad page %"Yu" flags 0x%X\n", pgno, MP_FLAGS(mp)); + return; + } + + nkeys = NUMKEYS(mp); + fprintf(stderr, "%s %"Yu" numkeys %d\n", type, pgno, nkeys); + + for (i=0; imp_pad; + key.mv_data = LEAF2KEY(mp, i, nsize); + total += nsize; + fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); + continue; + } + node = NODEPTR(mp, i); + key.mv_size = node->mn_ksize; + key.mv_data = node->mn_data; + nsize = NODESIZE + key.mv_size; + if (IS_BRANCH(mp)) { + fprintf(stderr, "key %d: page %"Yu", %s\n", i, NODEPGNO(node), + DKEY(&key)); + total += nsize; + } else { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + nsize += sizeof(MDB_ovpage); + else + nsize += NODEDSZ(node); + total += nsize; + nsize += sizeof(indx_t); + fprintf(stderr, "key %d: nsize %d, %s%s\n", + i, nsize, DKEY(&key), mdb_leafnode_type(node)); + } + total = EVEN(total); + } + fprintf(stderr, "Total: header %d + contents %d + unused %d\n", + IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + MP_LOWER(mp), total, SIZELEFT(mp)); +} + +void +mdb_cursor_chk(MDB_cursor *mc) +{ + unsigned int i; + MDB_node *node; + MDB_page *mp; + + if (!mc->mc_snum || !(mc->mc_flags & C_INITIALIZED)) return; + for (i=0; imc_top; i++) { + mp = mc->mc_pg[i]; + node = NODEPTR(mp, mc->mc_ki[i]); + if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) + printf("oops!\n"); + } + if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) + printf("ack!\n"); + if (XCURSOR_INITED(mc)) { + node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) && + mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) { + printf("blah!\n"); + } + } +} +#endif + +#if (MDB_DEBUG) > 2 +/** Count all the pages in each DB and in the freelist + * and make sure it matches the actual number of pages + * being used. + * All named DBs must be open for a correct count. + */ +static void mdb_audit(MDB_txn *txn) +{ + MDB_cursor mc; + MDB_val key, data; + MDB_ID freecount, count; + MDB_dbi i; + int rc; + + freecount = 0; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + mdb_tassert(txn, rc == MDB_NOTFOUND); + + count = 0; + for (i = 0; imt_numdbs; i++) { + MDB_xcursor mx; + if (!(txn->mt_dbflags[i] & DB_VALID)) + continue; + mdb_cursor_init(&mc, txn, i, &mx); + if (txn->mt_dbs[i].md_root == P_INVALID) + continue; + count += txn->mt_dbs[i].md_branch_pages + + txn->mt_dbs[i].md_leaf_pages + + txn->mt_dbs[i].md_overflow_pages; + if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST); + for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) { + unsigned j; + MDB_page *mp; + mp = mc.mc_pg[mc.mc_top]; + for (j=0; jmn_flags & F_SUBDATA) { + MDB_db db; + memcpy(&db, NODEDATA(leaf), sizeof(db)); + count += db.md_branch_pages + db.md_leaf_pages + + db.md_overflow_pages; + } + } + } + mdb_tassert(txn, rc == MDB_NOTFOUND); + } + } + if (freecount + count + NUM_METAS != txn->mt_next_pgno) { + fprintf(stderr, "audit: %"Yu" freecount: %"Yu" count: %"Yu" total: %"Yu" next_pgno: %"Yu"\n", + txn->mt_txnid, freecount, count+NUM_METAS, + freecount+count+NUM_METAS, txn->mt_next_pgno); + } +} +#endif + +int +mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_cmp(a, b); +} + +int +mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp; + if (NEED_CMP_CLONG(dcmp, a->mv_size)) + dcmp = mdb_cmp_clong; + return dcmp(a, b); +} + +/** Allocate memory for a page. + * Re-use old malloc'd pages first for singletons, otherwise just malloc. + * Set #MDB_TXN_ERROR on failure. + */ +static MDB_page * +mdb_page_malloc(MDB_txn *txn, unsigned num, int init) +{ + MDB_env *env = txn->mt_env; + MDB_page *ret = env->me_dpages; + size_t psize = env->me_psize, sz = psize, off; + /* For ! #MDB_NOMEMINIT, psize counts how much to init. + * For a single page alloc, we init everything after the page header. + * For multi-page, we init the final page; if the caller needed that + * many pages they will be filling in at least up to the last page. + */ + if (num == 1) { + psize -= off = PAGEHDRSZ; + if (ret) { + VGMEMP_ALLOC(env, ret, sz); + VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); + env->me_dpages = ret->mp_next; + goto init; + } + } else { + sz *= num; + off = sz - psize; + } + if ((ret = malloc(sz)) != NULL) { + VGMEMP_ALLOC(env, ret, sz); +init: + if (init && !(env->me_flags & MDB_NOMEMINIT)) { + memset((char *)ret + off, 0, psize); + ret->mp_pad = 0; + } + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + ret->mp_flags = 0; + return ret; +} +/** Free a single page. + * Saves single pages to a list, for future reuse. + * (This is not used for multi-page overflow pages.) + */ +static void +mdb_page_free(MDB_env *env, MDB_page *mp) +{ + mp->mp_next = env->me_dpages; + VGMEMP_FREE(env, mp); + env->me_dpages = mp; +} + +/** Free a dirty page */ +static void +mdb_dpage_free(MDB_env *env, MDB_page *dp) +{ + if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { + mdb_page_free(env, dp); + } else { + /* large pages just get freed directly */ + VGMEMP_FREE(env, dp); + free(dp); + } +} + +#if MDB_RPAGE_CACHE +/** Free an encrypted dirty page + * We can't check if it's an overflow page, + * caller must tell us how many are being freed. + */ +static void +mdb_dpage_free_n(MDB_env *env, MDB_page *dp, int num) +{ + if (num == 1) + mdb_page_free(env, dp); + else { + /* large pages just get freed directly */ + VGMEMP_FREE(env, dp); + free(dp); + } +} +#endif + +/** Return all dirty pages to dpage list */ +static void +mdb_dlist_free(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned i, n = dl[0].mid; + + if (txn->mt_flags & MDB_TXN_DIRTYNUM) { + int *dl_nump = env->me_dirty_nump; + for (i = 1; i <= n; i++) { + mdb_dpage_free_n(env, dl[i].mptr, dl_nump[i]); + } + txn->mt_flags ^= MDB_TXN_DIRTYNUM; + } else { + for (i = 1; i <= n; i++) { + mdb_dpage_free(env, dl[i].mptr); + } + } + dl[0].mid = 0; +} + +#if MDB_RPAGE_CACHE +#if defined(MDB_VL32) || ((MDB_RPAGE_CACHE) & 2) /* Always remap */ +#define MDB_REMAPPING(flags) 1 +#else +#define MDB_REMAPPING(flags) ((flags) & MDB_REMAP_CHUNKS) +#endif + +static void +mdb_page_unref(MDB_txn *txn, MDB_page *mp) +{ + pgno_t pgno; + MDB_ID3L tl = txn->mt_rpages; + unsigned x, rem; + if (IS_SUBP(mp) || IS_DIRTY_NW(txn, mp)) + return; + rem = mp->mp_pgno & (MDB_RPAGE_CHUNK-1); + pgno = mp->mp_pgno ^ rem; + x = mdb_mid3l_search(tl, pgno); + if (x != tl[0].mid && tl[x+1].mid == mp->mp_pgno) + x++; + if (tl[x].mref) + tl[x].mref--; +} +#define MDB_PAGE_UNREF(txn, mp) \ + (MDB_REMAPPING(txn->mt_env->me_flags) ? mdb_page_unref(txn, mp) : (void)0) + +static void +mdb_cursor_unref(MDB_cursor *mc) +{ + int i; + if (mc->mc_txn->mt_rpages[0].mid) { + if (!mc->mc_snum || !mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0])) + return; + for (i=0; imc_snum; i++) + mdb_page_unref(mc->mc_txn, mc->mc_pg[i]); + if (mc->mc_ovpg) { + mdb_page_unref(mc->mc_txn, mc->mc_ovpg); + mc->mc_ovpg = 0; + } + } + mc->mc_snum = mc->mc_top = 0; + mc->mc_pg[0] = NULL; + mc->mc_flags &= ~C_INITIALIZED; +} +#define MDB_CURSOR_UNREF(mc, force) \ + ((MDB_REMAPPING((mc)->mc_txn->mt_env->me_flags) && \ + ((force) || ((mc)->mc_flags & C_INITIALIZED))) \ + ? mdb_cursor_unref(mc) \ + : (void)0) + +/* Unref ovpage \b omp in \b mc and tracked cursors */ +static void +mdb_ovpage_unref_all(MDB_cursor *mc, MDB_page *omp) +{ + MDB_txn *txn = mc->mc_txn; + MDB_cursor *next = txn->mt_cursors[mc->mc_dbi]; + for (;; mc = next, next = mc->mc_next) { + if (MC_OVPG(mc) == omp) { + mdb_page_unref(mc->mc_txn, omp); + MC_SET_OVPG(mc, NULL); + } + if (next == NULL) + break; + } +} + +#else +#define MDB_REMAPPING(flags) 0 +#define MDB_PAGE_UNREF(txn, mp) +#define MDB_CURSOR_UNREF(mc, force) ((void)0) +#define mdb_ovpage_unref_all(mc, omp, pgno) ((void)0) +#endif /* MDB_RPAGE_CACHE */ + +/** Loosen or free a single page. + * Saves single pages to a list for future reuse + * in this same txn. It has been pulled from the freeDB + * and already resides on the dirty list, but has been + * deleted. Use these pages first before pulling again + * from the freeDB. + * + * If the page wasn't dirtied in this txn, just add it + * to this txn's free list. + */ +static int +mdb_page_loose(MDB_cursor *mc, MDB_page *mp) +{ + int rc = MDB_SUCCESS; + MDB_txn *txn = mc->mc_txn; + + if (IS_WRITABLE(txn, mp) && mc->mc_dbi != FREE_DBI) { + /* Page is dirty in this txn, and is not in freeDB */ + DPRINTF(("loosen db %d page %"Yu, DDBI(mc), mp->mp_pgno)); + NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; + txn->mt_loose_pgs = mp; + txn->mt_loose_count++; + mp->mp_flags |= P_LOOSE; + } else { + rc = mdb_midl_append(&txn->mt_free_pgs, mp->mp_pgno); + } + + return rc; +} + +/** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. + * @param[in] mc A cursor handle for the current operation. + * @param[in] pflags Flags of the pages to update: + * 0 to set P_KEEP, P_KEEP to clear it. + * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush(). + * @return 0 on success, non-zero on failure. + */ +static int +mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) +{ + enum { Mask = P_SUBP|P_LOOSE|P_KEEP }; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m3, *m0 = mc; + MDB_xcursor *mx; + MDB_page *dp, *mp; + MDB_node *leaf; + unsigned i, j, x; + int rc = MDB_SUCCESS; + + /* Mark pages seen by cursors: First m0, then tracked cursors */ + for (i = txn->mt_numdbs;; ) { + if (mc->mc_flags & C_INITIALIZED) { + for (m3 = mc;; m3 = &mx->mx_cursor) { + mp = NULL; + for (j=0; jmc_snum; j++) { + mp = m3->mc_pg[j]; + if ((mp->mp_flags & Mask) == pflags) + if (IS_WRITABLE(txn, mp)) + mp->mp_flags ^= P_KEEP; + } + if (MC_OVPG(m3) && ((MC_OVPG(m3)->mp_flags & Mask) == pflags) && + IS_WRITABLE(txn, MC_OVPG(m3))) + MC_OVPG(m3)->mp_flags ^= P_KEEP; + mx = m3->mc_xcursor; + /* Proceed to mx if it is at a sub-database */ + if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) + break; + if (! (mp && (mp->mp_flags & P_LEAF))) + break; + leaf = NODEPTR(mp, m3->mc_ki[j-1]); + if (!(leaf->mn_flags & F_SUBDATA)) + break; + } + } + mc = mc->mc_next; + for (; !mc || mc == m0; mc = txn->mt_cursors[--i]) + if (i == 0) + goto mark_done; + } + +mark_done: + if (all) { + /* Mark dirty root pages */ + MDB_ID2L dl = txn->mt_u.dirty_list; + for (i=0; imt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + pgno_t pgno = txn->mt_dbs[i].md_root; + if (pgno == P_INVALID) + continue; + x = mdb_mid2l_search(dl, pgno); + if (! (x <= dl[0].mid && dl[x].mid == pgno)) + continue; + dp = dl[x].mptr; + if ((dp->mp_flags & Mask) == pflags) + dp->mp_flags ^= P_KEEP; + } + } + } + + return rc; +} + +static int mdb_page_flush(MDB_txn *txn, int keep); + +/** Spill pages from the dirty list back to disk. + * This is intended to prevent running into #MDB_TXN_FULL situations, + * but note that they may still occur in a few cases: + * 1) our estimate of the txn size could be too small. Currently this + * seems unlikely, except with a large number of #MDB_MULTIPLE items. + * 2) child txns may run out of space if their parents dirtied a + * lot of pages and never spilled them. TODO: we probably should do + * a preemptive spill during #mdb_txn_begin() of a child txn, if + * the parent's dirty_room is below a given threshold. + * + * Otherwise, if not using nested txns, it is expected that apps will + * not run into #MDB_TXN_FULL any more. The pages are flushed to disk + * the same way as for a txn commit. + * If the txn never references them again, they can be left alone. + * If the txn only reads them, they can be used without any fuss. + * If the txn writes them again, they can be dirtied immediately without + * going thru all of the work of #mdb_page_touch(). Such references are + * handled by #mdb_page_unspill(). + * + * Also note, we never spill DB root pages, nor pages of active cursors, + * because we'll need these back again soon anyway. And in nested txns, + * we can't spill a page in a child txn if it was already spilled in a + * parent txn. That would alter the parent txns' data even though + * the child hasn't committed yet, and we'd have no way to undo it if + * the child aborted. + * + * @param[in] m0 cursor A cursor handle identifying the transaction and + * database for which we are checking space. + * @param[in] key For a put operation, the key being stored. + * @param[in] data For a put operation, the data being stored. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) +{ + MDB_txn *txn = m0->mc_txn; + MDB_page *dp; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned int i, j, need; + int rc; + + if (m0->mc_flags & (C_SUB|C_WRITEMAP)) + return MDB_SUCCESS; + + /* Estimate how much space this op will take */ + i = m0->mc_db->md_depth; + /* Named DBs also dirty the main DB */ + if (m0->mc_dbi >= CORE_DBS) + i += txn->mt_dbs[MAIN_DBI].md_depth; + /* For puts, roughly factor in the key+data size */ + if (key) + i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize; + i += i; /* double it for good measure */ + need = i; + + if (txn->mt_dirty_room > i) + return MDB_SUCCESS; + + if (!txn->mt_spill_pgs) { + txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX); + if (!txn->mt_spill_pgs) + return ENOMEM; + } else { + /* purge deleted slots */ + MDB_IDL sl = txn->mt_spill_pgs; + unsigned int num = sl[0]; + j=0; + for (i=1; i<=num; i++) { + if (!(sl[i] & 1)) + sl[++j] = sl[i]; + } + sl[0] = j; + } + + /* Preserve pages which may soon be dirtied again */ + if ((rc = mdb_pages_xkeep(m0, 0, 1)) != MDB_SUCCESS) + goto done; + + /* Less aggressive spill - we originally spilled the entire dirty list, + * with a few exceptions for cursor pages and DB root pages. But this + * turns out to be a lot of wasted effort because in a large txn many + * of those pages will need to be used again. So now we spill only 1/8th + * of the dirty pages. Testing revealed this to be a good tradeoff, + * better than 1/2, 1/4, or 1/10. + */ + if (need < MDB_IDL_UM_MAX / 8) + need = MDB_IDL_UM_MAX / 8; + + /* Save the page IDs of all the pages we're flushing */ + /* flush from the tail forward, this saves a lot of shifting later on. */ + for (i=dl[0].mid; i && need; i--) { + MDB_ID pn = dl[i].mid << 1; + dp = dl[i].mptr; + if (dp->mp_flags & (P_LOOSE|P_KEEP)) + continue; + /* Can't spill twice, make sure it's not already in a parent's + * spill list. + */ + if (txn->mt_parent) { + MDB_txn *tx2; + for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) { + if (tx2->mt_spill_pgs) { + j = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) { + dp->mp_flags |= P_KEEP; + break; + } + } + } + if (tx2) + continue; + } + if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn))) + goto done; + need--; + } + mdb_midl_sort(txn->mt_spill_pgs); + + /* Flush the spilled part of dirty list */ + if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) + goto done; + + /* Reset any dirty pages we kept that page_flush didn't see */ + rc = mdb_pages_xkeep(m0, P_KEEP, i); + +done: + txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS; + return rc; +} + +/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ +static txnid_t +mdb_find_oldest(MDB_txn *txn) +{ + int i; + txnid_t mr, oldest = txn->mt_txnid - 1; + if (txn->mt_env->me_txns) { + MDB_reader *r = txn->mt_env->me_txns->mti_readers; + for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { + if (r[i].mr_pid) { + mr = r[i].mr_txnid; + if (oldest > mr) + oldest = mr; + } + } + } + return oldest; +} + +/** Add a page to the txn's dirty list, if there is one */ +static void +mdb_page_dirty(MDB_txn *txn, MDB_page *mp) +{ + MDB_ID2 mid; + int rc; + + if (txn->mt_flags & MDB_TXN_WRITEMAP) { + txn->mt_flags |= MDB_TXN_DIRTY; + return; + } + mid.mid = mp->mp_pgno; + mid.mptr = mp; + rc = mdb_mid2l_insert(txn->mt_u.dirty_list, &mid); + mdb_tassert(txn, rc == 0); + txn->mt_dirty_room--; +} + +/** Allocate page numbers and memory for writing. Maintain me_pglast, + * me_pghead and mt_next_pgno. Set #MDB_TXN_ERROR on failure. + * + * If there are free pages available from older transactions, they + * are re-used first. Otherwise allocate a new page at mt_next_pgno. + * Do not modify the freedB, just merge freeDB records into me_pghead[] + * and move me_pglast to say which records were consumed. Only this + * function can create me_pghead and move me_pglast/mt_next_pgno. + * When #MDB_DEVEL & 2, it is not affected by #mdb_freelist_save(): it + * then uses the transaction's original snapshot of the freeDB. + * @param[in] mc cursor A cursor handle identifying the transaction and + * database for which we are allocating. + * @param[in] num the number of pages to allocate. + * @param[out] mp Address of the allocated page(s). Requests for multiple pages + * will always be satisfied by a single contiguous chunk of memory. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) +{ +#ifdef MDB_PARANOID /* Seems like we can ignore this now */ + /* Get at most more freeDB records once me_pghead + * has enough pages. If not enough, use new pages from the map. + * If and mc is updating the freeDB, only get new + * records if me_pghead is empty. Then the freelist cannot play + * catch-up with itself by growing while trying to save it. + */ + enum { Paranoid = 1, Max_retries = 500 }; +#else + enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; +#endif + int rc, retry = num * 60; + MDB_txn *txn = mc->mc_txn; + MDB_env *env = txn->mt_env; + pgno_t pgno, *mop = env->me_pghead; + unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; + MDB_page *np; + txnid_t oldest = 0, last; + MDB_cursor_op op; + MDB_cursor m2; + int found_old = 0; + +#if OVERFLOW_NOTYET + MDB_dovpage *dph = NULL; + + if (ov) { + if (!txn->mt_dirty_ovs) { + txn->mt_dirty_ovs = mdb_mid2l_alloc(16); + if (!txn->mt_dirty_ovs) + return ENOMEM; + } else if (mdb_mid2l_need(&txn->mt_dirty_ovs, txn->mt_dirty_ovs[0].mid + 1)) + return ENOMEM; + dph = malloc(sizeof(MDB_dovpage)); + } +#endif + + /* If there are any loose pages, just use them */ + if (num == 1 && txn->mt_loose_pgs) { + np = txn->mt_loose_pgs; + txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); + txn->mt_loose_count--; + DPRINTF(("db %d use loose page %"Yu, DDBI(mc), np->mp_pgno)); +#if OVERFLOW_NOTYET + if (ov) { + dph->mp_hdr = np->mp_hdr; + dph->mp_ptr = np; + np = (MDB_page *)dph; + } +#endif + *mp = np; + np->mp_flags &= P_ADM_FLAGS; + return MDB_SUCCESS; + } + + *mp = NULL; + + /* If our dirty list is already full, we can't do anything */ + if (txn->mt_dirty_room == 0) { + rc = MDB_TXN_FULL; + goto fail; + } + + for (op = MDB_FIRST;; op = MDB_NEXT) { + MDB_val key, data; + MDB_node *leaf; + pgno_t *idl; + + /* Seek a big enough contiguous page range. Prefer + * pages at the tail, just truncating the list. + */ + if (mop_len > n2) { + i = mop_len; + do { + pgno = mop[i]; + if (mop[i-n2] == pgno+n2) + goto search_done; + } while (--i > n2); + if (--retry < 0) + break; + } + + if (op == MDB_FIRST) { /* 1st iteration */ + /* Prepare to fetch more and coalesce */ + last = env->me_pglast; + oldest = env->me_pgoldest; + mdb_cursor_init(&m2, txn, FREE_DBI, NULL); +#if (MDB_DEVEL) & 2 /* "& 2" so MDB_DEVEL=1 won't hide bugs breaking freeDB */ + /* Use original snapshot. TODO: Should need less care in code + * which modifies the database. Maybe we can delete some code? + */ + m2.mc_flags |= C_ORIG_RDONLY; + m2.mc_db = &env->me_metas[(txn->mt_txnid-1) & 1]->mm_dbs[FREE_DBI]; + m2.mc_dbflag = (unsigned char *)""; /* probably unnecessary */ +#endif + if (last) { + op = MDB_SET_RANGE; + key.mv_data = &last; /* will look up last+1 */ + key.mv_size = sizeof(last); + } + if (Paranoid && mc->mc_dbi == FREE_DBI) + retry = -1; + } + if (Paranoid && retry < 0 && mop_len) + break; + + last++; + /* Do not fetch more if the record will be too recent */ + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + rc = mdb_cursor_get(&m2, &key, NULL, op); + if (rc) { + if (rc == MDB_NOTFOUND) + break; + goto fail; + } + last = *(txnid_t*)key.mv_data; + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + np = m2.mc_pg[m2.mc_top]; + leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); + if ((rc = mdb_node_read(&m2, leaf, &data)) != MDB_SUCCESS) + goto fail; + + idl = (MDB_ID *) data.mv_data; + i = idl[0]; + if (!mop) { + if (!(env->me_pghead = mop = mdb_midl_alloc(i))) { + rc = ENOMEM; + goto fail; + } + } else { + if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) + goto fail; + mop = env->me_pghead; + } + env->me_pglast = last; +#if (MDB_DEBUG) > 1 + DPRINTF(("IDL read txn %"Yu" root %"Yu" num %u", + last, txn->mt_dbs[FREE_DBI].md_root, i)); + for (j = i; j; j--) + DPRINTF(("IDL %"Yu, idl[j])); +#endif + /* Merge in descending sorted order */ + mdb_midl_xmerge(mop, idl); + mop_len = mop[0]; + } + + /* Use new pages from the map when nothing suitable in the freeDB */ + i = 0; + pgno = txn->mt_next_pgno; + if (pgno + num >= env->me_maxpg) { + DPUTS("DB size maxed out"); + rc = MDB_MAP_FULL; + goto fail; + } +#if defined(_WIN32) + if (!MDB_REMAPPING(env->me_flags) && !(env->me_flags & MDB_RDONLY)) { + void *p; + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + p = VirtualAlloc(p, env->me_psize * num, MEM_COMMIT, + (env->me_flags & MDB_WRITEMAP) ? PAGE_READWRITE: + PAGE_READONLY); + if (!p) { + DPUTS("VirtualAlloc failed"); + rc = ErrCode(); + goto fail; + } + } +#endif + +search_done: + if (env->me_flags & MDB_WRITEMAP) { + np = (MDB_page *)(env->me_map + env->me_psize * pgno); + } else { + if (!(np = mdb_page_malloc(txn, num, 1))) { + rc = ENOMEM; + goto fail; + } + } + if (i) { + mop[0] = mop_len -= num; + /* Move any stragglers down */ + for (j = i-num; j < mop_len; ) + mop[++j] = mop[++i]; + } else { + txn->mt_next_pgno = pgno + num; + } +#if OVERFLOW_NOTYET + if (ov) { + dph->mp_hdr = np->mp_hdr; + dph->mp_ptr = np; + np = (MDB_page *)dph; + } +#endif + np->mp_pgno = pgno; + SET_PGTXNID(txn, np); + np->mp_flags = 0; +#if OVERFLOW_NOTYET + mdb_page_dirty(txn, np, ov); +#else + mdb_page_dirty(txn, np); +#endif + *mp = np; + + return MDB_SUCCESS; + +fail: +#if OVERFLOW_NOTYET + if (dph) + free(dph); +#endif + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Copy the used portions of a non-overflow page. + * @param[in] dst page to copy into + * @param[in] src page to copy from + * @param[in] psize size of a page + */ +static void +mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) +{ + enum { Align = sizeof(pgno_t) }; + indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; + + /* If page isn't full, just copy the used portion. Adjust + * alignment so memcpy may copy words instead of bytes. + */ + if ((unused &= -Align) && !IS_LEAF2(src)) { + upper = (upper + PAGEBASE) & -Align; + memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align); + memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), + psize - upper); + } else { + memcpy(dst, src, psize - unused); + } +} + +/** Bring back a page which this txn spilled to disk; make it writable again. + * @param[in] txn the transaction handle. + * @param[in] mp the spilled page. + * @param[out] ret the writable page. + */ +static int +mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) +{ + MDB_env *env = txn->mt_env; + unsigned x; + pgno_t pgno = mp->mp_pgno, pn = pgno << 1; + + if (txn->mt_dirty_room == 0) + return MDB_TXN_FULL; + + /* x = position in current spill list, or 0 */ + x = 0; + if (txn->mt_spill_pgs) { + x = mdb_midl_search(txn->mt_spill_pgs, pn); + if (! (x <= txn->mt_spill_pgs[0] && txn->mt_spill_pgs[x] == pn)) + x = 0; + } + if (x == 0 && !txn->mt_parent) + return MDB_PROBLEM; /* should be a spilled page */ + + { + MDB_page *np; + int num; + if (IS_OVERFLOW(mp)) + num = mp->mp_pages; + else + num = 1; + { + np = mdb_page_malloc(txn, num, 1); + if (!np) + return ENOMEM; + if (num > 1) + memcpy(np, mp, num * env->me_psize); + else + mdb_page_copy(np, mp, env->me_psize); + } + if (x) { + /* If in current txn, this page is no longer spilled. + * If it happens to be the last page, truncate the spill list. + * Otherwise mark it as deleted by setting the LSB. + */ + if (x == txn->mt_spill_pgs[0]) + txn->mt_spill_pgs[0]--; + else + txn->mt_spill_pgs[x] |= 1; + } /* otherwise, if belonging to a parent txn, the + * page remains spilled until child commits + */ + + mdb_page_dirty(txn, np); + SET_PGTXNID(txn, np); + *ret = np; + return MDB_SUCCESS; + } +} + +/** Touch a page: make it dirty and re-insert into tree with updated pgno. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc cursor pointing to the page to be touched + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_touch(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top], *np; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m2, *m3; + unsigned np_flags; + pgno_t pgno; + int rc; + + if (IS_SUBP(mp) || IS_WRITABLE(txn, mp)) + return MDB_SUCCESS; + + if (!IS_MUTABLE(txn, mp)) { + /* Page from an older snapshot */ + if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || + (rc = mdb_page_alloc(mc, 1, &np))) + goto fail; + pgno = np->mp_pgno; + DPRINTF(("touched db %d page %"Yu" -> %"Yu, DDBI(mc), + mp->mp_pgno, pgno)); + mdb_cassert(mc, mp->mp_pgno != pgno); + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* Update the parent page, if any, to point to the new page */ + if (mc->mc_top) { + MDB_page *parent = mc->mc_pg[mc->mc_top-1]; + MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); + SETPGNO(node, pgno); + } else { + mc->mc_db->md_root = pgno; + } + } else if (!IS_DIRTY_NW(txn, mp)) { + rc = mdb_page_unspill(txn, mp, &np); + if (rc) + goto fail; + goto done; + } else { + /* Writable in an ancestor txn */ + MDB_ID2 mid, *dl = txn->mt_u.dirty_list; + pgno = mp->mp_pgno; + if (!txn->mt_parent) { + rc = MDB_PROBLEM; + goto fail; + } + mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX); + np = mdb_page_malloc(txn, 1, 1); + if (!np) { + rc = ENOMEM; + goto fail; + } + mid.mid = pgno; + mid.mptr = np; + rc = mdb_mid2l_insert(dl, &mid); + mdb_cassert(mc, rc == 0); + } + + np_flags = np->mp_flags; /* P_ADM_FLAGS */ + mdb_page_copy(np, mp, txn->mt_env->me_psize); + np->mp_flags |= np_flags; + np->mp_pgno = pgno; + SET_PGTXNID(txn, np); + +done: + /* Adjust cursors pointing to mp */ + mc->mc_pg[mc->mc_top] = np; + m2 = txn->mt_cursors[mc->mc_dbi]; + if (mc->mc_flags & C_SUB) { + for (; m2; m2=m2->mc_next) { + m3 = &m2->mc_xcursor->mx_cursor; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[mc->mc_top] == mp) + m3->mc_pg[mc->mc_top] = np; + } + } else { + for (; m2; m2=m2->mc_next) { + if (m2->mc_snum < mc->mc_snum) continue; + if (m2 == mc) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + m2->mc_pg[mc->mc_top] = np; + if (IS_LEAF(np)) + XCURSOR_REFRESH(m2, mc->mc_top, np); + } + } + } + MDB_PAGE_UNREF(mc->mc_txn, mp); + return 0; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs) +{ + int rc = 0; + if (env->me_flags & MDB_RDONLY) + return MDB_IS_READONLY; + if (force || !(env->me_flags & MDB_NOSYNC) +#ifdef _WIN32 /* Sync is normally achieved in Windows by doing WRITE_THROUGH writes */ + && (env->me_flags & MDB_WRITEMAP) +#endif + ) { + if (env->me_flags & MDB_WRITEMAP) { + int flags = ((env->me_flags & MDB_MAPASYNC) && !force) + ? MS_ASYNC : MS_SYNC; + if (MDB_MSYNC(env->me_map, env->me_psize * numpgs, flags)) + rc = ErrCode(); +#if defined(_WIN32) || defined(__APPLE__) + else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); +#endif + } else { +#ifdef BROKEN_FDATASYNC + if (env->me_flags & MDB_FSYNCONLY) { + if (fsync(env->me_fd)) + rc = ErrCode(); + } else +#endif + if (MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); + } + } + return rc; +} + +int +mdb_env_sync(MDB_env *env, int force) +{ + MDB_meta *m = mdb_env_pick_meta(env); + return mdb_env_sync0(env, force, m->mm_last_pg+1); +} + +/** Back up parent txn's cursors, then grab the originals for tracking */ +static int +mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) +{ + MDB_cursor *mc, *bk; + MDB_xcursor *mx; + size_t size; + int i; + + for (i = src->mt_numdbs; --i >= 0; ) { + if ((mc = src->mt_cursors[i]) != NULL) { + size = sizeof(MDB_cursor); + if (mc->mc_xcursor) + size += sizeof(MDB_xcursor); + for (; mc; mc = bk->mc_next) { + bk = malloc(size); + if (!bk) + return ENOMEM; + *bk = *mc; + mc->mc_backup = bk; + mc->mc_db = &dst->mt_dbs[i]; + /* Kill pointers into src to reduce abuse: The + * user may not use mc until dst ends. But we need a valid + * txn pointer here for cursor fixups to keep working. + */ + mc->mc_txn = dst; + mc->mc_dbflag = &dst->mt_dbflags[i]; + if ((mx = mc->mc_xcursor) != NULL) { + *(MDB_xcursor *)(bk+1) = *mx; + mx->mx_cursor.mc_txn = dst; + } + mc->mc_next = dst->mt_cursors[i]; + dst->mt_cursors[i] = mc; + } + } + } + return MDB_SUCCESS; +} + +/** Close this write txn's cursors, give parent txn's cursors back to parent. + * @param[in] txn the transaction handle. + * @param[in] merge true to keep changes to parent cursors, false to revert. + * @return 0 on success, non-zero on failure. + */ +static void +mdb_cursors_close(MDB_txn *txn, unsigned merge) +{ + MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; + MDB_xcursor *mx; + int i; + + for (i = txn->mt_numdbs; --i >= 0; ) { + for (mc = cursors[i]; mc; mc = next) { + next = mc->mc_next; + if ((bk = mc->mc_backup) != NULL) { + if (merge) { + /* Commit changes to parent txn */ + mc->mc_next = bk->mc_next; + mc->mc_backup = bk->mc_backup; + mc->mc_txn = bk->mc_txn; + mc->mc_db = bk->mc_db; + mc->mc_dbflag = bk->mc_dbflag; + if ((mx = mc->mc_xcursor) != NULL) + mx->mx_cursor.mc_txn = bk->mc_txn; + } else { + /* Abort nested txn */ + *mc = *bk; + if ((mx = mc->mc_xcursor) != NULL) + *mx = *(MDB_xcursor *)(bk+1); + } + mc = bk; + } + /* Only malloced cursors are permanently tracked. */ + free(mc); + } + cursors[i] = NULL; + } +} + +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ +enum Pidlock_op { + Pidset, Pidcheck +}; +#else +enum Pidlock_op { + Pidset = F_SETLK, Pidcheck = F_GETLK +}; +#endif + +/** Set or check a pid lock. Set returns 0 on success. + * Check returns 0 if the process is certainly dead, nonzero if it may + * be alive (the lock exists or an error happened so we do not know). + * + * On Windows Pidset is a no-op, we merely check for the existence + * of the process with the given pid. On POSIX we use a single byte + * lock on the lockfile, set at an offset equal to the pid. + */ +static int +mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid) +{ +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ + int ret = 0; + HANDLE h; + if (op == Pidcheck) { + h = OpenProcess(env->me_pidquery, FALSE, pid); + /* No documented "no such process" code, but other program use this: */ + if (!h) + return ErrCode() != ERROR_INVALID_PARAMETER; + /* A process exists until all handles to it close. Has it exited? */ + ret = WaitForSingleObject(h, 0) != 0; + CloseHandle(h); + } + return ret; +#else + for (;;) { + int rc; + struct flock lock_info; + memset(&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = pid; + lock_info.l_len = 1; + if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) { + if (op == F_GETLK && lock_info.l_type != F_UNLCK) + rc = -1; + } else if ((rc = ErrCode()) == EINTR) { + continue; + } + return rc; + } +#endif +} + +/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). + * @param[in] txn the transaction handle to initialize + * @return 0 on success, non-zero on failure. + */ +static int +mdb_txn_renew0(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_txninfo *ti = env->me_txns; + MDB_meta *meta; + unsigned int i, nr, flags = txn->mt_flags; + uint16_t x; + int rc, new_notls = 0; + + if ((flags &= MDB_TXN_RDONLY) != 0) { + if (!ti) { + meta = mdb_env_pick_meta(env); + txn->mt_txnid = meta->mm_txnid; + txn->mt_u.reader = NULL; + } else { + MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : + pthread_getspecific(env->me_txkey); + if (r) { + if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) + return MDB_BAD_RSLOT; + } else { + MDB_PID_T pid = env->me_pid; + MDB_THR_T tid = pthread_self(); + mdb_mutexref_t rmutex = env->me_rmutex; + + if (!env->me_live_reader) { + rc = mdb_reader_pid(env, Pidset, pid); + if (rc) + return rc; + env->me_live_reader = 1; + } + + if (LOCK_MUTEX(rc, env, rmutex)) + return rc; + nr = ti->mti_numreaders; + for (i=0; imti_readers[i].mr_pid == 0) + break; + if (i == env->me_maxreaders) { + UNLOCK_MUTEX(rmutex); + return MDB_READERS_FULL; + } + r = &ti->mti_readers[i]; + /* Claim the reader slot, carefully since other code + * uses the reader table un-mutexed: First reset the + * slot, next publish it in mti_numreaders. After + * that, it is safe for mdb_env_close() to touch it. + * When it will be closed, we can finally claim it. + */ + r->mr_pid = 0; + r->mr_txnid = (txnid_t)-1; + r->mr_tid = tid; + if (i == nr) + ti->mti_numreaders = ++nr; + env->me_close_readers = nr; + r->mr_pid = pid; + UNLOCK_MUTEX(rmutex); + + new_notls = (env->me_flags & MDB_NOTLS); + if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { + r->mr_pid = 0; + return rc; + } + } + do /* LY: Retry on a race, ITS#7970. */ + r->mr_txnid = ti->mti_txnid; + while(r->mr_txnid != ti->mti_txnid); + if (!r->mr_txnid && (env->me_flags & MDB_RDONLY)) { + meta = mdb_env_pick_meta(env); + r->mr_txnid = meta->mm_txnid; + } else { + meta = env->me_metas[r->mr_txnid & 1]; + } + txn->mt_txnid = r->mr_txnid; + txn->mt_u.reader = r; + } + + } else { + /* Not yet touching txn == env->me_txn0, it may be active */ + if (ti) { + if (LOCK_MUTEX(rc, env, env->me_wmutex)) + return rc; + txn->mt_txnid = ti->mti_txnid; + meta = env->me_metas[txn->mt_txnid & 1]; + } else { + meta = mdb_env_pick_meta(env); + txn->mt_txnid = meta->mm_txnid; + } + txn->mt_txnid++; +#if MDB_DEBUG + if (txn->mt_txnid == mdb_debug_start) + mdb_debug = MDB_DBG_INFO; +#endif + txn->mt_child = NULL; + txn->mt_rdonly_child_count = 0; + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + if (env->me_flags & MDB_WRITEMAP) { + txn->mt_workid = txn->mt_txnid; + txn->mt_dirty_room = 1; + } else { + txn->mt_workid = (txn->mt_txnid | MDB_PGTXNID_FLAGMASK) + 1; + txn->mt_dirty_room = MDB_IDL_UM_MAX; + } + txn->mt_last_workid = txn->mt_workid; + txn->mt_u.dirty_list = env->me_dirty_list; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_free_pgs = env->me_free_pgs; + txn->mt_free_pgs[0] = 0; + txn->mt_spill_pgs = NULL; +#if OVERFLOW_NOTYET + txn->mt_dirty_ovs = NULL; +#endif + env->me_txn = txn; + memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); + } + + /* Copy the DB info and flags */ + memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDB_db)); + + /* Moved to here to avoid a data race in read TXNs */ + txn->mt_next_pgno = meta->mm_last_pg+1; +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) + txn->mt_last_pgno = txn->mt_next_pgno - 1; +#endif + + txn->mt_flags = flags; + + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; + for (i=CORE_DBS; imt_numdbs; i++) { + x = env->me_dbflags[i]; + txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_USRVALID|DB_STALE : 0; + } + txn->mt_dbflags[MAIN_DBI] = DB_VALID|DB_USRVALID; + txn->mt_dbflags[FREE_DBI] = DB_VALID; + + if (env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + rc = MDB_PANIC; + } else if (env->me_maxpg < txn->mt_next_pgno) { + rc = MDB_MAP_RESIZED; + } else { + return MDB_SUCCESS; + } + mdb_txn_end(txn, new_notls /*0 or MDB_END_SLOT*/ | MDB_END_FAIL_BEGIN); + return rc; +} + +int +mdb_txn_renew(MDB_txn *txn) +{ + int rc; + + if (!txn || !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY|MDB_TXN_FINISHED)) + return EINVAL; + + rc = mdb_txn_renew0(txn); + if (rc == MDB_SUCCESS) { + DPRINTF(("renew txn %"Yu"%c %p on mdbenv %p, root page %"Yu, + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); + } + return rc; +} + +/** Used up all workids. Rewind it and update dirty pages to match. */ +static txnid_t ESECT +mdb_workid_rewind(MDB_txn *txn) +{ + txnid_t workid, diff; + + while (txn->mt_parent) + txn = txn->mt_parent; + workid = txn->mt_txnid & ~MDB_PGTXNID_FLAGMASK; + do { + workid += MDB_PGTXNID_STEP; + diff = txn->mt_last_workid - workid; + if (diff) { + MDB_ID2L dl = txn->mt_u.dirty_list; + int i; + for (i = dl[0].mid; i; i--) { + if (MDB_PGTXNID_FLAGBITS) + ((MDB_page *)dl[i].mptr)->mp_txnid -= diff; + else + ((MDB_page *)dl[i].mptr)->mp_txnid = workid; + } + txn->mt_workid = txn->mt_last_workid = workid; + } + } while ((txn = txn->mt_child) != NULL); + return workid; +} + +int +mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) +{ + MDB_txn *txn; + MDB_ntxn *ntxn; + int rc, size, tsize; + + flags &= MDB_TXN_BEGIN_FLAGS; + flags |= env->me_flags & MDB_WRITEMAP; + + if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */ + return MDB_IS_READONLY; + + if (parent) { + /* Nested transactions: + * Only write txns may have nested txns; + * if the nested txn is a write txn there may only be 1, no writemap; + * if the nested txn is a read txn there may be arbitrarily many. + */ + if (parent->mt_flags & MDB_TXN_RDONLY) + return EINVAL; + if ((parent->mt_flags & MDB_TXN_WRITEMAP) && !(flags & MDB_RDONLY)) + return EINVAL; + pthread_mutex_lock(&parent->mt_child_mutex); + flags |= parent->mt_flags; + if (parent->mt_child && F_ISSET(parent->mt_child->mt_flags, MDB_RDONLY) && F_ISSET(flags, MDB_RDONLY)) { + flags &= ~MDB_TXN_HAS_CHILD; + } + pthread_mutex_unlock(&parent->mt_child_mutex); + if (flags & MDB_TXN_BLOCKED) + return (flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; + /* Child txns save MDB_pgstate and use own copy of cursors */ + size = env->me_maxdbs * (sizeof(MDB_db)+sizeof(MDB_cursor *)+1); + size += tsize = sizeof(MDB_ntxn); + } else if (flags & MDB_RDONLY) { + size = env->me_maxdbs * (sizeof(MDB_db)+1); + size += tsize = sizeof(MDB_txn); + } else { + /* Reuse preallocated write txn. However, do not touch it until + * mdb_txn_renew0() succeeds, since it currently may be active. + */ + txn = env->me_txn0; + goto renew; + } + if ((txn = calloc(1, size)) == NULL) { + DPRINTF(("calloc: %s", strerror(errno))); + return ENOMEM; + } +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags) && !parent) { + txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3)); + if (!txn->mt_rpages) { + free(txn); + return ENOMEM; + } + txn->mt_rpages[0].mid = 0; + txn->mt_rpcheck = MDB_TRPAGE_SIZE/2; + } +#endif + txn->mt_dbxs = env->me_dbxs; /* static */ + txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); + txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs; + txn->mt_flags = flags; + txn->mt_env = env; + + if (parent) { + unsigned int i; + txnid_t workid = parent->mt_last_workid + MDB_PGTXNID_STEP; + if (!workid) /* wraparound after lots of previous children */ + workid = mdb_workid_rewind(parent) + MDB_PGTXNID_STEP; + txn->mt_workid = txn->mt_last_workid = workid; + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = parent->mt_dbiseqs; + /* share parent dirty and free pages with nested RDONLY txn */ + if (F_ISSET(flags, MDB_RDONLY)) { + txn->mt_u.dirty_list = parent->mt_u.dirty_list; + txn->mt_free_pgs = parent->mt_free_pgs; + } else { + txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); + if (!txn->mt_u.dirty_list || + !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) + { + free(txn->mt_u.dirty_list); + free(txn); + return ENOMEM; + } + txn->mt_u.dirty_list[0].mid = 0; + pthread_mutex_init(&txn->mt_child_mutex, NULL); + } + txn->mt_txnid = parent->mt_txnid; + txn->mt_dirty_room = parent->mt_dirty_room; + txn->mt_spill_pgs = NULL; +#if OVERFLOW_NOTYET + txn->mt_dirty_ovs = NULL; +#endif + txn->mt_next_pgno = parent->mt_next_pgno; + pthread_mutex_lock(&parent->mt_child_mutex); + parent->mt_flags |= MDB_TXN_HAS_CHILD; + parent->mt_child = txn; + if (F_ISSET(flags, MDB_RDONLY)) { + parent->mt_rdonly_child_count++; + } + pthread_mutex_unlock(&parent->mt_child_mutex); + txn->mt_parent = parent; + txn->mt_numdbs = parent->mt_numdbs; +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) + txn->mt_rpages = parent->mt_rpages; +#endif + memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + /* Copy parent's mt_dbflags, but clear DB_NEW */ + for (i=0; imt_numdbs; i++) + txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; + rc = 0; + ntxn = (MDB_ntxn *)txn; + if (!F_ISSET(flags, MDB_RDONLY)) { + ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ + /* Do not copy parent me_pghead when nested and RDONLY */ + if (env->me_pghead) { + size = MDB_IDL_SIZEOF(env->me_pghead); + env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); + if (env->me_pghead) + memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); + else + rc = ENOMEM; + } + if (!rc) + rc = mdb_cursor_shadow(parent, txn); + if (rc) + mdb_txn_end(txn, MDB_END_FAIL_BEGINCHILD); + } + } else { /* MDB_RDONLY */ + txn->mt_dbiseqs = env->me_dbiseqs; +renew: + rc = mdb_txn_renew0(txn); + } + if (rc) { + if (txn != env->me_txn0) { + /* mt_rpages belongs to parent */ + if (!F_ISSET(flags, MDB_RDONLY)) { + free(txn->mt_u.dirty_list); + } + free(txn); + } + } else { + txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ + *ret = txn; + DPRINTF(("begin txn %"Yu"%c %p on mdbenv %p, root page %"Yu, + txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w', + (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); + } + MDB_TRACE(("%p, %p, %u = %p", env, parent, flags, txn)); + + return rc; +} + +MDB_env * +mdb_txn_env(MDB_txn *txn) +{ + if(!txn) return NULL; + return txn->mt_env; +} + +mdb_size_t +mdb_txn_id(MDB_txn *txn) +{ + if(!txn) return 0; + return txn->mt_txnid; +} + +int mdb_txn_flags(MDB_txn *txn, unsigned int *flags) +{ + if(!txn) return EINVAL; + *flags = txn->mt_flags & MDB_RDONLY; + return MDB_SUCCESS; +} + +/** Export or close DBI handles opened in this txn. */ +static void +mdb_dbis_update(MDB_txn *txn, int keep) +{ + int i; + MDB_dbi n = txn->mt_numdbs; + MDB_env *env = txn->mt_env; + unsigned char *tdbflags = txn->mt_dbflags; + + for (i = n; --i >= CORE_DBS;) { + if (tdbflags[i] & DB_NEW) { + if (keep) { + env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; + } else { + char *ptr = env->me_dbxs[i].md_name.mv_data; + if (ptr) { + env->me_dbxs[i].md_name.mv_data = NULL; + env->me_dbxs[i].md_name.mv_size = 0; + env->me_dbflags[i] = 0; + env->me_dbiseqs[i]++; + free(ptr); + } + } + } + } + if (keep && env->me_numdbs < n) + env->me_numdbs = n; +} + +/** End a transaction, except successful commit of a nested transaction. + * May be called twice for readonly txns: First reset it, then abort. + * @param[in] txn the transaction handle to end + * @param[in] mode why and how to end the transaction + */ +static void +mdb_txn_end(MDB_txn *txn, unsigned mode) +{ + MDB_env *env = txn->mt_env; +#if MDB_DEBUG + static const char *const names[] = MDB_END_NAMES; +#endif + + /* Export or close DBI handles opened in this txn */ + mdb_dbis_update(txn, mode & MDB_END_UPDATE); + + DPRINTF(("%s txn %"Yu"%c %p on mdbenv %p, root page %"Yu, + names[mode & MDB_END_OPMASK], + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + if (txn->mt_parent) { + pthread_mutex_lock(&txn->mt_parent->mt_child_mutex); + txn->mt_parent->mt_rdonly_child_count--; + /* mark parent txn has no longer having children if this is the last nested txn */ + if (txn->mt_parent->mt_rdonly_child_count == 0) { + txn->mt_parent->mt_child = NULL; + txn->mt_parent->mt_flags &= ~MDB_TXN_HAS_CHILD; + } + pthread_mutex_unlock(&txn->mt_parent->mt_child_mutex); + } else + if (txn->mt_u.reader) { + txn->mt_u.reader->mr_txnid = (txnid_t)-1; + if (!(env->me_flags & MDB_NOTLS)) { + txn->mt_u.reader = NULL; /* txn does not own reader */ + } else if (mode & MDB_END_SLOT) { + txn->mt_u.reader->mr_pid = 0; + txn->mt_u.reader = NULL; + } /* else txn owns the slot until it does MDB_END_SLOT */ + } + txn->mt_numdbs = 0; /* prevent further DBI activity */ + txn->mt_flags |= MDB_TXN_FINISHED; + + } else if (!F_ISSET(txn->mt_flags, MDB_TXN_FINISHED)) { + pgno_t *pghead = env->me_pghead; + + if (!(mode & MDB_END_UPDATE)) /* !(already closed cursors) */ + mdb_cursors_close(txn, 0); + if (!(env->me_flags & MDB_WRITEMAP)) { + mdb_dlist_free(txn); + } + + txn->mt_numdbs = 0; + txn->mt_flags = MDB_TXN_FINISHED; + + mdb_midl_free(txn->mt_spill_pgs); +#if OVERFLOW_NOTYET + mdb_mid2l_free(txn->mt_dirty_ovs); +#endif + if (!txn->mt_parent) { + mdb_midl_shrink(&txn->mt_free_pgs); + env->me_free_pgs = txn->mt_free_pgs; + /* me_pgstate: */ + env->me_pghead = NULL; + env->me_pglast = 0; + + env->me_txn = NULL; + mode = 0; /* txn == env->me_txn0, do not free() it */ + + /* The writer mutex was locked in mdb_txn_begin. */ + if (env->me_txns) + UNLOCK_MUTEX(env->me_wmutex); + } else { + txn->mt_parent->mt_child = NULL; + txn->mt_parent->mt_flags &= ~MDB_TXN_HAS_CHILD; + env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; + mdb_midl_free(txn->mt_free_pgs); + free(txn->mt_u.dirty_list); + } + + mdb_midl_free(pghead); + } +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags) && !txn->mt_parent) { + MDB_ID3L el = env->me_rpages, tl = txn->mt_rpages; + unsigned i, x, n = tl[0].mid; + pthread_mutex_lock(&env->me_rpmutex); + for (i = 1; i <= n; i++) { + if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) { + /* tmp overflow pages that we didn't share in env */ + munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); + if (tl[i].menc) { + mdb_rpage_dispose(env, &tl[i]); + tl[i].menc = NULL; + } + } else { + x = mdb_mid3l_search(el, tl[i].mid); + if (tl[i].mptr == el[x].mptr) { + el[x].mref--; + if (!el[x].mref) + el[x].muse = 0; + } else { + /* another tmp overflow page */ + munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); + if (tl[i].menc) { + mdb_rpage_dispose(env, &tl[i]); + tl[i].menc = NULL; + } + } + } + } + pthread_mutex_unlock(&env->me_rpmutex); + tl[0].mid = 0; + if (mode & MDB_END_FREE) + free(tl); + } +#endif + if (mode & MDB_END_FREE) { + if (!F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + pthread_mutex_destroy(&txn->mt_child_mutex); + free(txn); + } +} + +void +mdb_txn_reset(MDB_txn *txn) +{ + if (txn == NULL) + return; + + /* This call is only valid for non-nested read-only txns */ + if (!(txn->mt_flags & MDB_TXN_RDONLY) || txn->mt_parent) + return; + + mdb_txn_end(txn, MDB_END_RESET); +} + +static void +_mdb_txn_abort(MDB_txn *txn) +{ + if (txn == NULL) + return; + + if (txn->mt_parent && (txn->mt_flags & MDB_RDONLY)) { + /* You must first abort the child before the parent */ + pthread_mutex_lock(&txn->mt_child_mutex); + int count = txn->mt_rdonly_child_count; + pthread_mutex_unlock(&txn->mt_child_mutex); + mdb_tassert(txn, txn->mt_parent && count == 0); + } + + if (txn->mt_child) + _mdb_txn_abort(txn->mt_child); + + mdb_txn_end(txn, MDB_END_ABORT|MDB_END_SLOT|MDB_END_FREE); +} + +void +mdb_txn_abort(MDB_txn *txn) +{ + MDB_TRACE(("%p", txn)); + _mdb_txn_abort(txn); +} + +/** Save the freelist as of this transaction to the freeDB. + * This changes the freelist. Keep trying until it stabilizes. + * + * When (MDB_DEVEL) & 2, the changes do not affect #mdb_page_alloc(), + * it then uses the transaction's original snapshot of the freeDB. + */ +static int +mdb_freelist_save(MDB_txn *txn) +{ + /* env->me_pghead[] can grow and shrink during this call. + * env->me_pglast and txn->mt_free_pgs[] can only grow. + * Page numbers cannot disappear from txn->mt_free_pgs[]. + */ + MDB_cursor mc; + MDB_env *env = txn->mt_env; + int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1, lost_loose = 0; + txnid_t pglast = 0, head_id = 0; + pgno_t freecnt = 0, *free_pgs, *mop; + ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; + + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + + if (env->me_pghead) { + /* Make sure first page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (!env->me_pghead && txn->mt_loose_pgs) { + /* Put loose page numbers in mt_free_pgs, since + * we may be unable to return them to me_pghead. + */ + MDB_page *mp = txn->mt_loose_pgs; + MDB_ID2 *dl = txn->mt_u.dirty_list; + unsigned x; + if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) + return rc; + for (; mp; mp = NEXT_LOOSE_PAGE(mp)) { + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* must also remove from dirty list */ + if (txn->mt_flags & MDB_TXN_WRITEMAP) { + for (x=1; x<=dl[0].mid; x++) + if (dl[x].mid == mp->mp_pgno) + break; + mdb_tassert(txn, x <= dl[0].mid); + } else { + x = mdb_mid2l_search(dl, mp->mp_pgno); + mdb_tassert(txn, dl[x].mid == mp->mp_pgno); + mdb_dpage_free(env, mp); + } + dl[x].mptr = NULL; + } + { + /* squash freed slots out of the dirty list */ + unsigned y; + for (y=1; dl[y].mptr && y <= dl[0].mid; y++); + if (y <= dl[0].mid) { + for(x=y, y++;;) { + while (!dl[y].mptr && y <= dl[0].mid) y++; + if (y > dl[0].mid) break; + dl[x++] = dl[y++]; + } + dl[0].mid = x-1; + } else { + /* all slots freed */ + dl[0].mid = 0; + } + } + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + } + + /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ + clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) + ? SSIZE_MAX : maxfree_1pg; + + for (;;) { + /* Come back here after each Put() in case freelist changed */ + MDB_val key, data; + pgno_t *pgs; + ssize_t j; + + /* If using records from freeDB which we have not yet + * deleted, delete them and any we reserved for me_pghead. + */ + while (pglast < env->me_pglast) { + rc = mdb_cursor_first(&mc, &key, NULL); + if (rc) + return rc; + pglast = head_id = *(txnid_t *)key.mv_data; + total_room = head_room = 0; + mdb_tassert(txn, pglast <= env->me_pglast); + rc = _mdb_cursor_del(&mc, 0); + if (rc) + return rc; + } + + /* Save the IDL of pages freed by this txn, to a single record */ + if (freecnt < txn->mt_free_pgs[0]) { + if (!freecnt) { + /* Make sure last page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + free_pgs = txn->mt_free_pgs; + /* Write to last page of freeDB */ + key.mv_size = sizeof(txn->mt_txnid); + key.mv_data = &txn->mt_txnid; + do { + freecnt = free_pgs[0]; + data.mv_size = MDB_IDL_SIZEOF(free_pgs); + rc = _mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* Retry if mt_free_pgs[] grew during the Put() */ + free_pgs = txn->mt_free_pgs; + } while (freecnt < free_pgs[0]); + mdb_midl_sort(free_pgs); + memcpy(data.mv_data, free_pgs, data.mv_size); +#if (MDB_DEBUG) > 1 + { + unsigned int i = free_pgs[0]; + DPRINTF(("IDL write txn %"Yu" root %"Yu" num %u", + txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); + for (; i; i--) + DPRINTF(("IDL %"Yu, free_pgs[i])); + } +#endif + continue; + } + + mop = env->me_pghead; + mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; + + /* Reserve records for me_pghead[]. Split it if multi-page, + * to avoid searching freeDB for a page range. Use keys in + * range [1,me_pglast]: Smaller than txnid of oldest reader. + */ + if (total_room >= mop_len) { + if (total_room == mop_len || --more < 0) + break; + } else if (head_room >= maxfree_1pg && head_id > 1) { + /* Keep current record (overflow page), add a new one */ + head_id--; + head_room = 0; + } + /* (Re)write {key = head_id, IDL length = head_room} */ + total_room -= head_room; + head_room = mop_len - total_room; + if (head_room > maxfree_1pg && head_id > 1) { + /* Overflow multi-page for part of me_pghead */ + head_room /= head_id; /* amortize page sizes */ + head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); + } else if (head_room < 0) { + /* Rare case, not bothering to delete this record */ + head_room = 0; + } + key.mv_size = sizeof(head_id); + key.mv_data = &head_id; + data.mv_size = (head_room + 1) * sizeof(pgno_t); + rc = _mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* IDL is initially empty, zero out at least the length */ + pgs = (pgno_t *)data.mv_data; + j = head_room > clean_limit ? head_room : 0; + do { + pgs[j] = 0; + } while (--j >= 0); + total_room += head_room; + } + + /* Return loose page numbers to me_pghead, though usually none are + * left at this point. The pages themselves remain in dirty_list. + */ + if (txn->mt_loose_pgs) { + MDB_page *mp = txn->mt_loose_pgs; + unsigned count = txn->mt_loose_count; + MDB_IDL loose; + /* Room for loose pages + temp IDL with same */ + if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) + return rc; + lost_loose += count; + mop = env->me_pghead; + loose = mop + MDB_IDL_ALLOCLEN(mop) - count; + for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) + loose[ ++count ] = mp->mp_pgno; + loose[0] = count; + mdb_midl_sort(loose); + mdb_midl_xmerge(mop, loose); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + mop_len = mop[0]; + } + + /* Fill in the reserved me_pghead records. Everything is finally + * in place, so this will not allocate or free any DB pages. + */ + rc = MDB_SUCCESS; + if (mop_len) { + MDB_val key, data; + + /* Protect DB env from any (buggy) freelist use when saving mop */ + env->me_pghead = NULL; + txn->mt_dirty_room = 0; + + mop += mop_len; + rc = mdb_cursor_first(&mc, &key, &data); + for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { + txnid_t id = *(txnid_t *)key.mv_data; + ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; + MDB_ID save; + + mdb_tassert(txn, len >= 0 && id <= env->me_pglast); + key.mv_data = &id; + if (len > mop_len) { + len = mop_len; + data.mv_size = (len + 1) * sizeof(MDB_ID); + } + mop_len -= len; + data.mv_data = mop -= len; + save = mop[0]; + mop[0] = len; + rc = _mdb_cursor_put(&mc, &key, &data, MDB_CURRENT); + mop[0] = save; + if (rc || !mop_len) + break; + } + + env->me_pghead = mop - mop_len; + } + + /* Restore this so we can check vs. dirty_list after mdb_page_flush() */ + if (! (txn->mt_flags & MDB_TXN_WRITEMAP)) + txn->mt_loose_count += lost_loose; + + return rc; +} + +#if MDB_RPAGE_CACHE +static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs); +static int mdb_page_encrypt(MDB_env *env, MDB_page *in, MDB_page *out, size_t size); +static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size); +static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size); +#endif + +/** Flush (some) dirty pages to the map, after clearing their dirty flag. + * @param[in] txn the transaction that's being committed + * @param[in] keep number of initial pages in dirty_list to keep dirty. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_flush(MDB_txn *txn, int keep) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned psize = env->me_psize, j; + int i, pagecount = dl[0].mid, rc, *dl_nump, nump = 1; + size_t size = 0; + MDB_OFF_T pos = 0; + pgno_t pgno = 0; + MDB_page *dp = NULL; +#ifdef _WIN32 + OVERLAPPED *ov, *this_ov; + MDB_page *wdp; + int async_i = 0; + HANDLE fd = (env->me_flags & MDB_NOSYNC) ? env->me_fd : env->me_ovfd; +#else + struct iovec iov[MDB_COMMIT_PAGES]; + HANDLE fd = env->me_fd; +#endif + ssize_t wsize = 0, wres; + MDB_OFF_T wpos = 0, next_pos = 1; /* impossible pos, so pos != next_pos */ + int n = 0; + + j = i = keep; + + if (env->me_flags & MDB_WRITEMAP) { + goto done; + } + + /* setup nump list, flag that it's in use */ + dl_nump = env->me_dirty_nump; + for (n=1; n<=pagecount; n++) { + dp = dl[n].mptr; + dl_nump[n] = IS_OVERFLOW(dp) ? dp->mp_pages : 1; + } + n = 0; + txn->mt_flags |= MDB_TXN_DIRTYNUM; + +#ifdef _WIN32 + if (pagecount - keep >= env->me_ovs) { + /* ran out of room in ov array, and re-malloc, copy handles and free previous */ + int ovs = (pagecount - keep) * 1.5; /* provide extra padding to reduce number of re-allocations */ + int new_size = ovs * sizeof(OVERLAPPED); + ov = malloc(new_size); + if (ov == NULL) + return ENOMEM; + int previous_size = env->me_ovs * sizeof(OVERLAPPED); + memcpy(ov, env->me_ov, previous_size); /* Copy previous OVERLAPPED data to retain event handles */ + /* And clear rest of memory */ + memset(&ov[env->me_ovs], 0, new_size - previous_size); + if (env->me_ovs > 0) { + free(env->me_ov); /* release previous allocation */ + } + + env->me_ov = ov; + env->me_ovs = ovs; + } + ov = env->me_ov; +#endif + + /* Write the pages */ + for (;;) { + if (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[i].mid = 0; + continue; + } + pgno = dl[i].mid; + /* Mark the page as clean */ + dp->mp_txnid = txn->mt_txnid; + pos = pgno * psize; + size = psize; + nump = dl_nump[i]; + size *= nump; + } + /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ + if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE +#ifdef _WIN32 + /* Windows actually supports scatter/gather I/O, but only on + * unbuffered file handles. Since we're relying on the OS page + * cache for all our data, that's self-defeating. So we just + * write pages one at a time. We use the ov structure to set + * the write offset, to at least save the overhead of a Seek + * system call. + * If writemap is enabled, consecutive page positions infer + * contiguous (mapped) memory. + * Otherwise force write pages one at a time. + */ + || !(env->me_flags & MDB_WRITEMAP) +#endif + ) { + if (n) { +retry_write: + rc = 0; + /* Write previous page(s) */ +#ifdef _WIN32 + this_ov = &ov[async_i]; + /* Clear status, and keep hEvent, we reuse that */ + this_ov->Internal = 0; + this_ov->Offset = wpos & 0xffffffff; + this_ov->OffsetHigh = wpos >> 16 >> 16; + if (!F_ISSET(env->me_flags, MDB_NOSYNC) && !this_ov->hEvent) { + HANDLE event = CreateEvent(NULL, FALSE, FALSE, NULL); + if (!event) { + rc = ErrCode(); + DPRINTF(("CreateEvent: %s", strerror(rc))); + return rc; + } + this_ov->hEvent = event; + } + if (!WriteFile(fd, wdp, wsize, NULL, this_ov)) { + rc = ErrCode(); + if (rc != ERROR_IO_PENDING) { + DPRINTF(("WriteFile: %d", rc)); + return rc; + } + rc = 0; + } + async_i++; +#else /* _WIN32 */ + if (n == 1) { + while (wsize > MAX_WRITE) { + wsize -= MAX_WRITE; + wres = pwrite(fd, iov[0].iov_base, MAX_WRITE, wpos); + if (wres != MAX_WRITE) + goto bad_write;; + wpos += MAX_WRITE; + iov[0].iov_base += MAX_WRITE; + } + wres = pwrite(fd, iov[0].iov_base, wsize, wpos); + } else { +#ifdef MDB_USE_PWRITEV + wres = pwritev(fd, iov, n, wpos); +#else +retry_seek: + if (lseek(fd, wpos, SEEK_SET) == -1) { + rc = ErrCode(); + if (rc == EINTR) + goto retry_seek; + DPRINTF(("lseek: %s", strerror(rc))); + wres = wsize; + } else { + rc = 0; + wres = writev(fd, iov, n); + } + } +#endif +bad_write: + if (wres != wsize) { + if (wres < 0) { + rc = ErrCode(); + if (rc == EINTR) + goto retry_write; + DPRINTF(("Write error: %s", strerror(rc))); + } else { + rc = MDB_SHORT_WRITE; + DPUTS("short write, filesystem full?"); + } + } +#endif /* _WIN32 */ + if (rc) + return rc; + n = 0; + } + if (i > pagecount) + break; + wpos = pos; + wsize = 0; + } +#if MDB_RPAGE_CACHE + if (env->me_sumfunc) { + mdb_page_set_checksum(env, dp, size); + } + if (env->me_encfunc) { + MDB_page *encp = mdb_page_malloc(txn, nump, 0); + if (!encp) + return ENOMEM; + if (mdb_page_encrypt(env, dp, encp, size)) { + mdb_dpage_free_n(env, encp, nump); + return MDB_CRYPTO_FAIL; + } + mdb_dpage_free_n(env, dp, nump); + dp = encp; + dl[i].mptr = dp; + } +#endif +#ifdef _WIN32 + wdp = dp; +#else + iov[n].iov_len = size; + iov[n].iov_base = (char *)dp; +#endif + DPRINTF(("committing page %"Yu, pgno)); + next_pos = pos + size; + wsize += size; + n++; + } +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags) && pgno > txn->mt_last_pgno) + txn->mt_last_pgno = pgno; +#endif + + /* MIPS has cache coherency issues, this is a no-op everywhere else + * Note: for any size >= on-chip cache size, entire on-chip cache is + * flushed. + */ + CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE); + +#ifdef _WIN32 + if (!F_ISSET(env->me_flags, MDB_NOSYNC)) { + /* Now wait for all the asynchronous/overlapped sync/write-through writes to complete. + * We start with the last one so that all the others should already be complete and + * we reduce thread suspend/resuming (in practice, typically about 99.5% of writes are + * done after the last write is done) */ + rc = 0; + while (--async_i >= 0) { + if (ov[async_i].hEvent) { + DWORD temp_wres; + if (!GetOverlappedResult(fd, &ov[async_i], &temp_wres, TRUE)) { + rc = ErrCode(); /* Continue on so that all the event signals are reset */ + } + wres = temp_wres; + } + } + if (rc) { /* any error on GetOverlappedResult, exit now */ + return rc; + } + } +#endif /* _WIN32 */ + + if (!(env->me_flags & MDB_WRITEMAP)) { + /* Don't free pages when using writemap (can only get here in NOSYNC mode in Windows) + */ + for (i = keep; ++i <= pagecount; ) { + dp = dl[i].mptr; + /* This is a page we skipped above */ + if (!dl[i].mid) { + dl[++j] = dl[i]; + dl[j].mid = dp->mp_pgno; + continue; + } + mdb_dpage_free_n(env, dp, dl_nump[i]); + } + } + txn->mt_flags ^= MDB_TXN_DIRTYNUM; + +done: + i--; + txn->mt_dirty_room += i - j; + dl[0].mid = j; + + return MDB_SUCCESS; +} + +static int ESECT mdb_env_share_locks(MDB_env *env, int *excl); + +static int +_mdb_txn_commit(MDB_txn *txn, int flag) +{ + int rc; + unsigned int i, end_mode; + MDB_env *env; + + if (txn == NULL) + return EINVAL; + + /* mdb_txn_end() mode for a commit which writes nothing */ + end_mode = MDB_END_EMPTY_COMMIT|MDB_END_UPDATE|MDB_END_SLOT|MDB_END_FREE; + + if (txn->mt_child) { + rc = _mdb_txn_commit(txn->mt_child, 0); + if (rc) + goto fail; + } + + env = txn->mt_env; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + goto done; + } + + if (F_ISSET(txn->mt_flags, MDB_TXN_PREPARE)) { + goto prepared; + } + + if (txn->mt_flags & (MDB_TXN_FINISHED|MDB_TXN_ERROR)) { + DPUTS("txn has failed/finished, can't commit"); + if (txn->mt_parent) + txn->mt_parent->mt_flags |= MDB_TXN_ERROR; + rc = MDB_BAD_TXN; + goto fail; + } + + if (txn->mt_parent) { + MDB_txn *parent = txn->mt_parent; + MDB_page **lp; + MDB_ID2L dst, src; + MDB_IDL pspill; + unsigned x, y, len, ps_len; + + /* Append our free list to parent's */ + rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); + if (rc) + goto fail; + mdb_midl_free(txn->mt_free_pgs); + /* Failures after this must either undo the changes + * to the parent or set MDB_TXN_ERROR in the parent. + */ + + parent->mt_next_pgno = txn->mt_next_pgno; + parent->mt_flags = txn->mt_flags; + + /* Merge our cursors into parent's and close them */ + mdb_cursors_close(txn, 1); + + /* Update parent's DB table. */ + memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + parent->mt_numdbs = txn->mt_numdbs; + parent->mt_dbflags[FREE_DBI] = txn->mt_dbflags[FREE_DBI]; + parent->mt_dbflags[MAIN_DBI] = txn->mt_dbflags[MAIN_DBI]; + for (i=CORE_DBS; imt_numdbs; i++) { + /* preserve parent's DB_NEW status */ + x = parent->mt_dbflags[i] & DB_NEW; + parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; + } + + dst = parent->mt_u.dirty_list; + src = txn->mt_u.dirty_list; + /* Remove anything in our dirty list from parent's spill list */ + if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) { + x = y = ps_len; + pspill[0] = (pgno_t)-1; + /* Mark our dirty pages as deleted in parent spill list */ + for (i=0, len=src[0].mid; ++i <= len; ) { + MDB_ID pn = src[i].mid << 1; + while (pn > pspill[x]) + x--; + if (pn == pspill[x]) { + pspill[x] = 1; + y = --x; + } + } + /* Squash deleted pagenums if we deleted any */ + for (x=y; ++x <= ps_len; ) + if (!(pspill[x] & 1)) + pspill[++y] = pspill[x]; + pspill[0] = y; + } + + /* Remove anything in our spill list from parent's dirty list */ + if (txn->mt_spill_pgs && txn->mt_spill_pgs[0]) { + for (i=1; i<=txn->mt_spill_pgs[0]; i++) { + MDB_ID pn = txn->mt_spill_pgs[i]; + if (pn & 1) + continue; /* deleted spillpg */ + pn >>= 1; + y = mdb_mid2l_search(dst, pn); + if (y <= dst[0].mid && dst[y].mid == pn) { + free(dst[y].mptr); + while (y < dst[0].mid) { + dst[y] = dst[y+1]; + y++; + } + dst[0].mid--; + } + } + } + + /* Find len = length of merging our dirty list with parent's */ + x = dst[0].mid; + dst[0].mid = 0; /* simplify loops */ + if (parent->mt_parent) { + len = x + src[0].mid; + y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; + for (i = x; y && i; y--) { + pgno_t yp = src[y].mid; + while (yp < dst[i].mid) + i--; + if (yp == dst[i].mid) { + i--; + len--; + } + } + } else { /* Simplify the above for single-ancestor case */ + len = MDB_IDL_UM_MAX - txn->mt_dirty_room; + } + /* Merge our dirty list with parent's */ + y = src[0].mid; + for (i = len; y; dst[i--] = src[y--]) { + pgno_t yp = src[y].mid; + while (yp < dst[x].mid) + dst[i--] = dst[x--]; + if (yp == dst[x].mid) + free(dst[x--].mptr); + } + mdb_tassert(txn, i == x); + dst[0].mid = len; + free(txn->mt_u.dirty_list); + parent->mt_dirty_room = txn->mt_dirty_room; + if (txn->mt_spill_pgs) { + if (parent->mt_spill_pgs) { + /* TODO: Prevent failure here, so parent does not fail */ + rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs); + if (rc) + parent->mt_flags |= MDB_TXN_ERROR; + mdb_midl_free(txn->mt_spill_pgs); + mdb_midl_sort(parent->mt_spill_pgs); + } else { + parent->mt_spill_pgs = txn->mt_spill_pgs; + } + } + + /* Append our loose page list to parent's */ + for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(*lp)) + ; + *lp = txn->mt_loose_pgs; + parent->mt_loose_count += txn->mt_loose_count; + + parent->mt_last_workid = txn->mt_last_workid; + parent->mt_child = NULL; + mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); + free(txn); + return rc; + } + + if (txn != env->me_txn) { + DPUTS("attempt to commit unknown transaction"); + rc = EINVAL; + goto fail; + } + + mdb_cursors_close(txn, 0); + + if (!txn->mt_u.dirty_list[0].mid && + !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) + goto done; + + DPRINTF(("committing txn %"Yu" %p on mdbenv %p, root page %"Yu, + txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); + + /* Update DB root pointers */ + if (txn->mt_numdbs > CORE_DBS) { + MDB_cursor mc; + MDB_dbi i; + MDB_val data; + data.mv_size = sizeof(MDB_db); + + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + for (i = CORE_DBS; i < txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + if (TXN_DBI_CHANGED(txn, i)) { + rc = MDB_BAD_DBI; + goto fail; + } + data.mv_data = &txn->mt_dbs[i]; + rc = _mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, + F_SUBDATA); + if (rc) + goto fail; + } + } + } + + rc = mdb_freelist_save(txn); + if (rc) + goto fail; + + mdb_midl_free(env->me_pghead); + env->me_pghead = NULL; + mdb_midl_shrink(&txn->mt_free_pgs); + +#if (MDB_DEBUG) > 2 + mdb_audit(txn); +#endif + + if ((rc = mdb_page_flush(txn, 0))) + goto fail; + if ((unsigned)txn->mt_loose_count != txn->mt_u.dirty_list[0].mid) { + rc = MDB_PROBLEM; /* mt_loose_pgs does not match dirty_list */ + goto fail; + } + if (!F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC) && + (rc = mdb_env_sync0(env, 0, txn->mt_next_pgno))) + goto fail; + if (F_ISSET(flag, MDB_TXN_PREPARE)) { + txn->mt_flags |= MDB_TXN_PREPARE; + return MDB_SUCCESS; + } + +prepared: + if ((rc = mdb_env_write_meta(txn))) + goto fail; + end_mode = MDB_END_COMMITTED|MDB_END_UPDATE; + if (env->me_flags & MDB_PREVSNAPSHOT) { + if (!(env->me_flags & MDB_NOLOCK)) { + int excl; + rc = mdb_env_share_locks(env, &excl); + if (rc) + goto fail; + } + env->me_flags ^= MDB_PREVSNAPSHOT; + } + +done: + mdb_txn_end(txn, end_mode); + return MDB_SUCCESS; + +fail: + _mdb_txn_abort(txn); + return rc; +} + +int +mdb_txn_commit(MDB_txn *txn) +{ + MDB_TRACE(("%p", txn)); + return _mdb_txn_commit(txn, 0); +} + +int +mdb_txn_prepare(MDB_txn *txn) +{ + MDB_TRACE(("%p", txn)); + if (F_ISSET(txn->mt_flags, MDB_TXN_PREPARE)) + return MDB_TXN_PENDING; + return _mdb_txn_commit(txn, MDB_TXN_PREPARE); +} + +int +mdb_env_rollback(MDB_env *env, mdb_size_t txnid) +{ + MDB_meta **metas = env->me_metas; + int newest, previous, rc = 0; + + if (env->me_txns && LOCK_MUTEX(rc, env, env->me_wmutex)) + return rc; + newest = metas[0]->mm_txnid < metas[1]->mm_txnid; + previous = newest ^ 1; + if (!metas[previous]->mm_txnid || metas[newest]->mm_txnid != txnid) + rc = MDB_CANT_ROLLBACK; + else { + MDB_txn txn = {0}; + MDB_db dbs[2] = {0}; + txn.mt_env = env; + txn.mt_dbs = dbs; + rc = mdb_env_write_meta(&txn); + } + if (env->me_txns) { + if (rc == MDB_SUCCESS) + env->me_txns->mti_txnid = metas[previous]->mm_txnid; + UNLOCK_MUTEX(env->me_wmutex); + } + return rc; +} + +static int ESECT mdb_env_map(MDB_env *env, void *addr); + +/** Read the environment parameters of a DB environment before + * mapping it into memory. + * @param[in] env the environment handle + * @param[in] prev whether to read the backup meta page + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) +{ + MDB_metabuf pbuf; + MDB_page *p; + MDB_meta *m; + int i, rc, off; + enum { Size = sizeof(pbuf) }; + + if (env->me_flags & MDB_RAWPART) { +#define VM_ALIGN 0x200000 + env->me_mapsize += VM_ALIGN-1; + env->me_mapsize &= ~(VM_ALIGN-1); + env->me_psize = env->me_os_psize; + rc = mdb_env_map(env, NULL); + if (rc) + return rc; + p = (MDB_page *)env->me_map; + for (i=0; imp_flags, P_META)) + return MDB_NO_META; + if (env->me_metas[i]->mm_magic != MDB_MAGIC) + return MDB_INVALID; + if (env->me_metas[i]->mm_version != MDB_DATA_VERSION) + return MDB_VERSION_MISMATCH; + if (i == 0 || env->me_metas[i]->mm_txnid > meta->mm_txnid) + *meta = *env->me_metas[i]; + p = (MDB_page *)((char *)p + env->me_psize); + } + return 0; + } + + /* We don't know the page size yet, so use a minimum value. + * Read both meta pages so we can use the latest one. + */ + + for (i=off=0; imm_psize) { +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1; + if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) + rc = 0; +#else + rc = pread(env->me_fd, &pbuf, Size, off); +#endif + if (rc != Size) { + if (rc == 0 && off == 0) + return MDB_NO_META; + rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; + DPRINTF(("read: %s", mdb_strerror(rc))); + return rc; + } + + p = (MDB_page *)&pbuf; + + if (!F_ISSET(p->mp_flags, P_META)) { + if (env->me_flags & MDB_RAWPART) + return MDB_NO_META; + DPRINTF(("page %"Yu" not a meta page", p->mp_pgno)); + return MDB_INVALID; + } + + m = METADATA(p); + if (m->mm_magic != MDB_MAGIC) { + DPUTS("meta has invalid magic"); + return MDB_INVALID; + } + + if (m->mm_version != MDB_DATA_VERSION) { + DPRINTF(("database is version %u, expected version %u", + m->mm_version, MDB_DATA_VERSION)); + return MDB_VERSION_MISMATCH; + } + + if (off == 0 || (prev ? m->mm_txnid < meta->mm_txnid : m->mm_txnid > meta->mm_txnid)) + *meta = *m; + } + return 0; +} + +/** Fill in most of the zeroed #MDB_meta for an empty database environment */ +static void ESECT +mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) +{ + meta->mm_magic = MDB_MAGIC; + meta->mm_version = MDB_DATA_VERSION; + meta->mm_mapsize = env->me_mapsize; + meta->mm_psize = env->me_psize; + meta->mm_last_pg = NUM_METAS-1; + meta->mm_flags = env->me_flags & 0xffff; + meta->mm_flags |= MDB_INTEGERKEY; /* this is mm_dbs[FREE_DBI].md_flags */ + meta->mm_dbs[FREE_DBI].md_root = P_INVALID; + meta->mm_dbs[MAIN_DBI].md_root = P_INVALID; +} + +/** Write the environment parameters of a freshly created DB environment. + * @param[in] env the environment handle + * @param[in] meta the #MDB_meta to write + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_init_meta(MDB_env *env, MDB_meta *meta) +{ + MDB_page *p, *q; + int rc; + unsigned int psize; +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + ov.Offset = pos; \ + rc = WriteFile(fd, ptr, size, &len, &ov); } while(0) +#else + int len; +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + len = pwrite(fd, ptr, size, pos); \ + if (len == -1 && ErrCode() == EINTR) continue; \ + rc = (len >= 0); break; } while(1) +#endif + DPUTS("writing new meta page"); + + psize = env->me_psize; + + if ((env->me_flags & (MDB_RAWPART|MDB_WRITEMAP)) == (MDB_RAWPART|MDB_WRITEMAP)) { + p = (MDB_page *)env->me_map; + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + return 0; + } + + p = calloc(NUM_METAS, psize); + if (!p) + return ENOMEM; + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + +#if MDB_RPAGE_CACHE + if (env->me_sumsize) { + /* save the checksum size in tail of page 0 */ + char *ptr = (char *)q; + unsigned short *u = (unsigned short *)(ptr-2); + *u = env->me_sumsize; + } +#endif + DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0); + if (!rc) + rc = ErrCode(); + else if ((unsigned) len == psize * NUM_METAS) + rc = MDB_SUCCESS; + else + rc = MDB_SHORT_WRITE; + free(p); + return rc; +} + +/** Update the environment info to commit a transaction. + * @param[in] txn the transaction that's being committed + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_write_meta(MDB_txn *txn) +{ + MDB_env *env; + MDB_meta meta, metab, *mp; + unsigned flags; + mdb_size_t mapsize; + MDB_OFF_T off; + int rc, len, toggle; + char *ptr; + HANDLE mfd; +#ifdef _WIN32 + OVERLAPPED ov; +#else + int r2; +#endif + + toggle = txn->mt_txnid & 1; + DPRINTF(("writing meta page %d for root page %"Yu, + toggle, txn->mt_dbs[MAIN_DBI].md_root)); + + env = txn->mt_env; + flags = txn->mt_flags | env->me_flags; + mp = env->me_metas[toggle]; + mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; + /* Persist any increases of mapsize config */ + if (mapsize < env->me_mapsize) + mapsize = env->me_mapsize; + +#ifndef _WIN32 /* We don't want to ever use MSYNC/FlushViewOfFile in Windows */ + if (flags & MDB_WRITEMAP) { + mp->mm_mapsize = mapsize; + mp->mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI]; + mp->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; + mp->mm_last_pg = txn->mt_next_pgno - 1; +#if (__GNUC__ * 100 + __GNUC_MINOR__ >= 404) && /* TODO: portability */ \ + !(defined(__i386__) || defined(__x86_64__)) + /* LY: issue a memory barrier, if not x86. ITS#7969 */ + __sync_synchronize(); +#endif + mp->mm_txnid = txn->mt_txnid; + if (!(flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { + unsigned meta_size = env->me_psize; + rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; + ptr = (char *)mp - PAGEHDRSZ; + /* POSIX msync() requires ptr = start of OS page */ + r2 = (ptr - env->me_map) & (env->me_os_psize - 1); + ptr -= r2; + meta_size += r2; + if (MDB_MSYNC(ptr, meta_size, rc)) { + rc = ErrCode(); + goto fail; + } +#if defined(__APPLE__) + if (MDB_FDATASYNC(env->me_fd)) { + rc = ErrCode(); + goto fail; + } +#endif + } + goto done; + } +#endif + metab.mm_txnid = mp->mm_txnid; + metab.mm_last_pg = mp->mm_last_pg; + + meta.mm_mapsize = mapsize; + meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI]; + meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; + meta.mm_last_pg = txn->mt_next_pgno - 1; + meta.mm_txnid = txn->mt_txnid; + + off = offsetof(MDB_meta, mm_mapsize); + ptr = (char *)&meta + off; + len = sizeof(MDB_meta) - off; + off += (char *)mp - env->me_map; + + /* Write to the SYNC fd unless MDB_NOSYNC/MDB_NOMETASYNC. + * (me_mfd goes to the same file as me_fd, but writing to it + * also syncs to disk. Avoids a separate fdatasync() call.) + */ + mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd; +#ifdef _WIN32 + { + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) + rc = -1; + } +#else +retry_write: + rc = pwrite(mfd, ptr, len, off); +#endif + if (rc != len) { + rc = rc < 0 ? ErrCode() : MDB_SHORT_WRITE; +#ifndef _WIN32 + if (rc == EINTR) + goto retry_write; +#endif + DPUTS("write failed, disk error?"); + /* On a failure, the pagecache still contains the new data. + * Write some old data back, to prevent it from being used. + * Use the non-SYNC fd; we know it will fail anyway. + */ + meta.mm_last_pg = metab.mm_last_pg; + meta.mm_txnid = metab.mm_txnid; +#ifdef _WIN32 + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + WriteFile(env->me_fd, ptr, len, NULL, &ov); +#else + r2 = pwrite(env->me_fd, ptr, len, off); + (void)r2; /* Silence warnings. We don't care about pwrite's return value */ +#endif +fail: + env->me_flags |= MDB_FATAL_ERROR; + return rc; + } +#if defined(__APPLE__) + if (mfd == env->me_mfd && MDB_FDATASYNC(env->me_mfd)) { + rc = ErrCode(); + return rc; + } +#endif + /* MIPS has cache coherency issues, this is a no-op everywhere else */ + CACHEFLUSH(env->me_map + off, len, DCACHE); +done: + /* Memory ordering issues are irrelevant; since the entire writer + * is wrapped by wmutex, all of these changes will become visible + * after the wmutex is unlocked. Since the DB is multi-version, + * readers will get consistent data regardless of how fresh or + * how stale their view of these values is. + */ + if (env->me_txns) + env->me_txns->mti_txnid = txn->mt_txnid; + + return MDB_SUCCESS; +} + +/** Check both meta pages to see which one is newer. + * @param[in] env the environment handle + * @return newest #MDB_meta. + */ +static MDB_meta * +mdb_env_pick_meta(const MDB_env *env) +{ + MDB_meta *const *metas = env->me_metas; + return metas[ (metas[0]->mm_txnid < metas[1]->mm_txnid) ^ + ((env->me_flags & MDB_PREVSNAPSHOT) != 0) ]; +} + +int ESECT +mdb_env_create(MDB_env **env) +{ + MDB_env *e; + + e = calloc(1, sizeof(MDB_env)); + if (!e) + return ENOMEM; + + e->me_maxreaders = DEFAULT_READERS; + e->me_maxdbs = e->me_numdbs = CORE_DBS; + e->me_fd = INVALID_HANDLE_VALUE; + e->me_lfd = INVALID_HANDLE_VALUE; + e->me_mfd = INVALID_HANDLE_VALUE; +#ifdef MDB_USE_POSIX_SEM + e->me_rmutex = SEM_FAILED; + e->me_wmutex = SEM_FAILED; +#elif defined MDB_USE_SYSV_SEM + e->me_rmutex->semid = -1; + e->me_wmutex->semid = -1; +#endif + e->me_pid = getpid(); + GET_PAGESIZE(e->me_os_psize); + VGMEMP_CREATE(e,0,0); + *env = e; + MDB_TRACE(("%p", e)); + return MDB_SUCCESS; +} + +#ifdef _WIN32 +/** @brief Map a result from an NTAPI call to WIN32. */ +static DWORD +mdb_nt2win32(NTSTATUS st) +{ + OVERLAPPED o = {0}; + DWORD br; + o.Internal = st; + GetOverlappedResult(NULL, &o, &br, FALSE); + return GetLastError(); +} +#endif + +static int ESECT +mdb_env_map(MDB_env *env, void *addr) +{ + MDB_page *p; + unsigned int flags = env->me_flags; +#ifdef _WIN32 + int rc; + int access = SECTION_MAP_READ; + HANDLE mh; + void *map; + SIZE_T msize; + ULONG pageprot = PAGE_READONLY, secprot, alloctype; + + if (flags & MDB_WRITEMAP) { + access |= SECTION_MAP_WRITE; + pageprot = PAGE_READWRITE; + } + if (flags & MDB_RDONLY) { + secprot = PAGE_READONLY; + msize = 0; + alloctype = 0; + } else { + secprot = PAGE_READWRITE; + msize = env->me_mapsize; + alloctype = MEM_RESERVE; + } + + /** Some users are afraid of seeing their disk space getting used + * all at once, so the default is now to do incremental file growth. + * But that has a large performance impact, so give the option of + * allocating the file up front. + */ +#ifdef MDB_FIXEDSIZE + LARGE_INTEGER fsize; + fsize.LowPart = msize & 0xffffffff; + fsize.HighPart = msize >> 16 >> 16; + rc = NtCreateSection(&mh, access, NULL, &fsize, secprot, SEC_RESERVE, env->me_fd); +#else + rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd); +#endif + if (rc) + return mdb_nt2win32(rc); + map = addr; + if (MDB_REMAPPING(env->me_flags)) + msize = NUM_METAS * env->me_psize; + rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, alloctype, pageprot); +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) + env->me_fmh = mh; + else +#endif + NtClose(mh); + if (rc) + return mdb_nt2win32(rc); + env->me_map = map; +#else /* !_WIN32 */ + int mmap_flags = MAP_SHARED; + int prot = PROT_READ; + if (flags & MDB_WRITEMAP) + prot |= PROT_WRITE; +#ifdef MAP_NOSYNC /* Used on FreeBSD */ + if (flags & MDB_NOSYNC) + mmap_flags |= MAP_NOSYNC; +#endif + if (MDB_REMAPPING(env->me_flags)) { + (void) flags; + env->me_map = mmap(addr, NUM_METAS * env->me_psize, prot, mmap_flags, + env->me_fd, 0); + if (env->me_map == MAP_FAILED) { + env->me_map = NULL; + return ErrCode(); + } + } else + { + if (flags & MDB_WRITEMAP) { + if (!(flags & MDB_RAWPART) && ftruncate(env->me_fd, env->me_mapsize) < 0) + return ErrCode(); + } + env->me_map = mmap(addr, env->me_mapsize, prot, mmap_flags, + env->me_fd, 0); + if (env->me_map == MAP_FAILED) { + env->me_map = NULL; + return ErrCode(); + } + + if (flags & MDB_NORDAHEAD) { + /* Turn off readahead. It's harmful when the DB is larger than RAM. */ +#ifdef MADV_RANDOM + madvise(env->me_map, env->me_mapsize, MADV_RANDOM); +#else +#ifdef POSIX_MADV_RANDOM + posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); +#endif /* POSIX_MADV_RANDOM */ +#endif /* MADV_RANDOM */ + } + } +#endif /* _WIN32 */ + + /* Can happen because the address argument to mmap() is just a + * hint. mmap() can pick another, e.g. if the range is in use. + * The MAP_FIXED flag would prevent that, but then mmap could + * instead unmap existing pages to make room for the new map. + */ + if (addr && env->me_map != addr) + return MDB_ADDR_BUSY; + + p = (MDB_page *)env->me_map; + env->me_metas[0] = METADATA(p); + env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize); + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_mapsize(MDB_env *env, mdb_size_t size) +{ + /* If env is already open, caller is responsible for making + * sure there are no active txns. + */ + if (env->me_map) { + MDB_meta *meta; + void *old; + int rc; + + if (env->me_txn) + return EINVAL; + meta = mdb_env_pick_meta(env); + if (!size) + size = meta->mm_mapsize; + { + /* Silently round up to minimum if the size is too small */ + mdb_size_t minsize = (meta->mm_last_pg + 1) * env->me_psize; + if (size < minsize) + size = minsize; + } + if (!(MDB_REMAPPING(env->me_flags))) + { + /* For MDB_REMAP_CHUNKS this bit is a noop since we dynamically remap + * chunks of the DB anyway. + */ + munmap(env->me_map, env->me_mapsize); + env->me_mapsize = size; + old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; + rc = mdb_env_map(env, old); + if (rc) + return rc; + } + } + env->me_mapsize = size; + if (env->me_psize) + env->me_maxpg = env->me_mapsize / env->me_psize; + MDB_TRACE(("%p, %"Yu"", env, size)); + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) +{ + if (env->me_map) + return EINVAL; + env->me_maxdbs = dbs + CORE_DBS; + MDB_TRACE(("%p, %u", env, dbs)); + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) +{ + if (env->me_map || readers < 1) + return EINVAL; + env->me_maxreaders = readers; + MDB_TRACE(("%p, %u", env, readers)); + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) +{ + if (!env || !readers) + return EINVAL; + *readers = env->me_maxreaders; + return MDB_SUCCESS; +} + +static int ESECT +mdb_fsize(HANDLE fd, mdb_size_t *size) +{ +#ifdef _WIN32 + LARGE_INTEGER fsize; + + if (!GetFileSizeEx(fd, &fsize)) + return ErrCode(); + + *size = fsize.QuadPart; +#else + struct stat st; + + if (fstat(fd, &st)) + return ErrCode(); + + *size = st.st_size; +#endif + return MDB_SUCCESS; +} + + +#ifdef _WIN32 +typedef wchar_t mdb_nchar_t; +# define MDB_NAME(str) L##str +# define mdb_name_cpy wcscpy +#else +/** Character type for file names: char on Unix, wchar_t on Windows */ +typedef char mdb_nchar_t; +# define MDB_NAME(str) str /**< #mdb_nchar_t[] string literal */ +# define mdb_name_cpy strcpy /**< Copy name (#mdb_nchar_t string) */ +#endif + +/** Filename - string of #mdb_nchar_t[] */ +typedef struct MDB_name { + int mn_len; /**< Length */ + int mn_alloced; /**< True if #mn_val was malloced */ + mdb_nchar_t *mn_val; /**< Contents */ +} MDB_name; + +/** Filename suffixes [datafile,lockfile][without,with MDB_NOSUBDIR] */ +static const mdb_nchar_t *const mdb_suffixes[2][2] = { + { MDB_NAME("/data.mdb"), MDB_NAME("") }, + { MDB_NAME("/lock.mdb"), MDB_NAME("-lock") } +}; + +#define MDB_SUFFLEN 9 /**< Max string length in #mdb_suffixes[] */ + +/** Set up filename + scratch area for filename suffix, for opening files. + * It should be freed with #mdb_fname_destroy(). + * On Windows, paths are converted from char *UTF-8 to wchar_t *UTF-16. + * + * @param[in] path Pathname for #mdb_env_open(). + * @param[in] envflags Whether a subdir and/or lockfile will be used. + * @param[out] fname Resulting filename, with room for a suffix if necessary. + */ +static int ESECT +mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname) +{ + int no_suffix = F_ISSET(envflags, MDB_NOSUBDIR|MDB_NOLOCK); + fname->mn_alloced = 0; +#ifdef _WIN32 + return utf8_to_utf16(path, fname, no_suffix ? 0 : MDB_SUFFLEN); +#else + fname->mn_len = strlen(path); + if (no_suffix) + fname->mn_val = (char *) path; + else if ((fname->mn_val = malloc(fname->mn_len + MDB_SUFFLEN+1)) != NULL) { + fname->mn_alloced = 1; + strcpy(fname->mn_val, path); + } + else + return ENOMEM; + return MDB_SUCCESS; +#endif +} + +/** Destroy \b fname from #mdb_fname_init() */ +#define mdb_fname_destroy(fname) \ + do { if ((fname).mn_alloced) free((fname).mn_val); } while (0) + +#ifdef O_CLOEXEC /* POSIX.1-2008: Set FD_CLOEXEC atomically at open() */ +# define MDB_CLOEXEC O_CLOEXEC +#else +# define MDB_CLOEXEC 0 +#endif + +/** File type, access mode etc. for #mdb_fopen() */ +enum mdb_fopen_type { +#ifdef _WIN32 + MDB_O_RDONLY, MDB_O_RDWR, MDB_O_OVERLAPPED, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS +#else + /* A comment in mdb_fopen() explains some O_* flag choices. */ + MDB_O_RDONLY= O_RDONLY , /**< for RDONLY me_fd */ + MDB_O_RDWR = O_RDWR |O_CREAT , /**< for me_fd */ + MDB_O_META = O_WRONLY|MDB_DSYNC , /**< for me_mfd */ + MDB_O_COPY = O_WRONLY|O_CREAT|O_EXCL, /**< for #mdb_env_copy() */ + /** Bitmask for open() flags in enum #mdb_fopen_type. The other bits + * distinguish otherwise-equal MDB_O_* constants from each other. + */ + MDB_O_MASK = MDB_O_RDWR| MDB_O_RDONLY|MDB_O_META|MDB_O_COPY, + MDB_O_LOCKS = MDB_O_RDWR| ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */ +#endif +}; + +/** Open an LMDB file. + * @param[in] env The LMDB environment. + * @param[in,out] fname Path from from #mdb_fname_init(). A suffix is + * appended if necessary to create the filename, without changing mn_len. + * @param[in] which Determines file type, access mode, etc. + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[out] res Resulting file handle. + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_fopen(const MDB_env *env, MDB_name *fname, + enum mdb_fopen_type which, mdb_mode_t mode, + HANDLE *res) +{ + int rc = MDB_SUCCESS; + HANDLE fd; +#ifdef _WIN32 + DWORD acc, share, disp, attrs; +#else + int flags; +#endif + + if (fname->mn_alloced) /* modifiable copy */ + mdb_name_cpy(fname->mn_val + fname->mn_len, + mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]); + + /* The directory must already exist. Usually the file need not. + * MDB_O_META requires the file because we already created it using + * MDB_O_RDWR. MDB_O_COPY must not overwrite an existing file. + * + * With MDB_O_COPY we do not want the OS to cache the writes, since + * the source data is already in the OS cache. + * + * All files are opened with MDB_CLOEXEC to prevent leaking across + * an exec(). + */ + +#ifdef _WIN32 + acc = GENERIC_READ|GENERIC_WRITE; + share = FILE_SHARE_READ|FILE_SHARE_WRITE; + disp = OPEN_ALWAYS; + attrs = FILE_ATTRIBUTE_NORMAL; + switch (which) { + case MDB_O_OVERLAPPED: /* for unbuffered asynchronous writes (write-through mode)*/ + acc = GENERIC_WRITE; + disp = OPEN_EXISTING; + attrs = FILE_FLAG_OVERLAPPED|FILE_FLAG_WRITE_THROUGH; + break; + case MDB_O_RDONLY: /* read-only datafile */ + acc = GENERIC_READ; + disp = OPEN_EXISTING; + break; + case MDB_O_META: /* for writing metapages */ + acc = GENERIC_WRITE; + disp = OPEN_EXISTING; + attrs = FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH; + break; + case MDB_O_COPY: /* mdb_env_copy() & co */ + acc = GENERIC_WRITE; + share = 0; + disp = CREATE_NEW; + attrs = FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH; + break; + default: break; /* silence gcc -Wswitch (not all enum values handled) */ + } + fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL); +#else + fd = open(fname->mn_val, (which & MDB_O_MASK)|MDB_CLOEXEC, mode); +#endif + + if (fd == INVALID_HANDLE_VALUE) + rc = ErrCode(); +#ifndef _WIN32 + else { + /* Set CLOEXEC if we could not pass it to open() */ + if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1) { + (void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC); + } + if (which == MDB_O_COPY && env->me_psize >= env->me_os_psize) { + /* This may require buffer alignment. There is no portable + * way to ask how much, so we require OS pagesize alignment. + */ +# ifdef F_NOCACHE /* __APPLE__ */ + (void) fcntl(fd, F_NOCACHE, 1); +# elif defined O_DIRECT + /* open(...O_DIRECT...) would break on filesystems without + * O_DIRECT support (ITS#7682). Try to set it here instead. + */ + if ((flags = fcntl(fd, F_GETFL)) != -1) + (void) fcntl(fd, F_SETFL, flags | O_DIRECT); +# endif + } + } +#endif /* !_WIN32 */ + + *res = fd; + return rc; +} + + +#ifdef BROKEN_FDATASYNC +#include +#include +#endif + +/** Further setup required for opening an LMDB environment + */ +static int ESECT +mdb_env_open2(MDB_env *env, int prev) +{ + unsigned int flags = env->me_flags; + int i, newenv = 0, rc; + MDB_meta meta; + +#ifdef _WIN32 + /* See if we should use QueryLimited */ + rc = GetVersion(); + if ((rc & 0xff) > 5) + env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION; + else + env->me_pidquery = PROCESS_QUERY_INFORMATION; + /* Grab functions we need from NTDLL */ + if (!NtCreateSection) { + HMODULE h = GetModuleHandleW(L"NTDLL.DLL"); + if (!h) + return MDB_PROBLEM; + NtClose = (NtCloseFunc *)GetProcAddress(h, "NtClose"); + if (!NtClose) + return MDB_PROBLEM; + NtMapViewOfSection = (NtMapViewOfSectionFunc *)GetProcAddress(h, "NtMapViewOfSection"); + if (!NtMapViewOfSection) + return MDB_PROBLEM; + NtCreateSection = (NtCreateSectionFunc *)GetProcAddress(h, "NtCreateSection"); + if (!NtCreateSection) + return MDB_PROBLEM; + } + env->me_ovs = 0; +#endif /* _WIN32 */ + +#ifdef BROKEN_FDATASYNC + /* ext3/ext4 fdatasync is broken on some older Linux kernels. + * https://lkml.org/lkml/2012/9/3/83 + * Kernels after 3.6-rc6 are known good. + * https://lkml.org/lkml/2012/9/10/556 + * See if the DB is on ext3/ext4, then check for new enough kernel + * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known + * to be patched. + */ + { + struct statfs st; + fstatfs(env->me_fd, &st); + while (st.f_type == 0xEF53) { + struct utsname uts; + int i; + uname(&uts); + if (uts.release[0] < '3') { + if (!strncmp(uts.release, "2.6.32.", 7)) { + i = atoi(uts.release+7); + if (i >= 60) + break; /* 2.6.32.60 and newer is OK */ + } else if (!strncmp(uts.release, "2.6.34.", 7)) { + i = atoi(uts.release+7); + if (i >= 15) + break; /* 2.6.34.15 and newer is OK */ + } + } else if (uts.release[0] == '3') { + i = atoi(uts.release+2); + if (i > 5) + break; /* 3.6 and newer is OK */ + if (i == 5) { + i = atoi(uts.release+4); + if (i >= 4) + break; /* 3.5.4 and newer is OK */ + } else if (i == 2) { + i = atoi(uts.release+4); + if (i >= 30) + break; /* 3.2.30 and newer is OK */ + } + } else { /* 4.x and newer is OK */ + break; + } + env->me_flags |= MDB_FSYNCONLY; + break; + } + } +#endif + + if ((i = mdb_env_read_header(env, prev, &meta)) != 0) { + if (i != MDB_NO_META) + return i; + DPUTS("new mdbenv"); + newenv = 1; + env->me_psize = env->me_os_psize; + if (env->me_psize > MAX_PAGESIZE) + env->me_psize = MAX_PAGESIZE; + memset(&meta, 0, sizeof(meta)); + mdb_env_init_meta0(env, &meta); + meta.mm_mapsize = DEFAULT_MAPSIZE; + } else { + env->me_psize = meta.mm_psize; + } + env->me_pagespace = env->me_psize - PAGEHDRSZ +#if MDB_RPAGE_CACHE + - env->me_sumsize - env->me_esumsize +#endif + ; + + /* if RAWPART, we did a preliminary mapping in read_header. drop it now. */ + if (flags & MDB_RAWPART) { + munmap(env->me_map, env->me_mapsize); + env->me_map = NULL; + /* if mapsize was defaulted, use mapsize from metapage */ + if (env->me_mapsize == DEFAULT_MAPSIZE) + env->me_mapsize = 0; + } + + /* Verify that root pages are actually inside the file */ + { + mdb_size_t fsize = 0; + if (mdb_fsize(env->me_fd, &fsize) == MDB_SUCCESS && fsize) { + MDB_dbi dbi; + pgno_t maxpgno = fsize / env->me_psize; + pgno_t sum = NUM_METAS; + for (dbi = FREE_DBI; dbi <= MAIN_DBI; dbi++) { + MDB_db *db = &meta.mm_dbs[dbi]; + if (db->md_root != P_INVALID) { + if (db->md_root > maxpgno) + return MDB_PAGE_NOTFOUND; + sum += db->md_branch_pages; + sum += db->md_leaf_pages; + sum += db->md_overflow_pages; + } + } + /* Slack sanity check as sum ignores pages from 2nd meta. */ + /* (2nd meta may be corrupted; thus do not rely on it.) */ + if (sum > maxpgno) + return MDB_CORRUPTED; + } + } + + /* Was a mapsize configured? */ + if (!env->me_mapsize) { + env->me_mapsize = meta.mm_mapsize; + } + { + /* Make sure mapsize >= committed data size. Even when using + * mm_mapsize, which could be broken in old files (ITS#7789). + */ + mdb_size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + if (env->me_mapsize < minsize) + env->me_mapsize = minsize; + } + meta.mm_mapsize = env->me_mapsize; + + if (newenv && !(flags & MDB_FIXEDMAP)) { + /* mdb_env_map() may grow the datafile. Write the metapages + * first, so the file will be valid if initialization fails. + * Except with FIXEDMAP, since we do not yet know mm_address. + * We could fill in mm_address later, but then a different + * program might end up doing that - one with a memory layout + * and map address which does not suit the main program. + */ + rc = mdb_env_init_meta(env, &meta); + if (rc) + return rc; + newenv = 0; + } +#ifdef _WIN32 + /* For FIXEDMAP, make sure the file is non-empty before we attempt to map it */ + if (newenv) { + char dummy = 0; + DWORD len; + rc = WriteFile(env->me_fd, &dummy, 1, &len, NULL); + if (!rc) { + rc = ErrCode(); + return rc; + } + } +#endif + + rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); + if (rc) + return rc; + + if (newenv) { + if (flags & MDB_FIXEDMAP) + meta.mm_address = env->me_map; + i = mdb_env_init_meta(env, &meta); + if (i != MDB_SUCCESS) { + return i; + } + } + if ((env->me_flags ^ env->me_metas[0]->mm_flags) & MDB_ENCRYPT) + return MDB_ENV_ENCRYPTION; + +#if MDB_RPAGE_CACHE + if (!newenv && env->me_sumfunc) { + /* for checksums, check sum size from tail of page 0 */ + char *ptr = env->me_map + env->me_psize; + unsigned short *u = (unsigned short *)(ptr - 2); + if (*u != env->me_sumsize) + return MDB_BAD_CHECKSUM; + } +#endif + + env->me_maxfree_1pg = env->me_pagespace / sizeof(pgno_t) - 1; + env->me_nodemax = ((env->me_pagespace / MDB_MINKEYS) & -2) + - sizeof(indx_t); +#if !(MDB_MAXKEYSIZE) + env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db)); +#endif + env->me_maxpg = env->me_mapsize / env->me_psize; + + if (prev && env->me_txns) + env->me_txns->mti_txnid = meta.mm_txnid; + +#if MDB_DEBUG + { + MDB_meta *meta = mdb_env_pick_meta(env); + MDB_db *db = &meta->mm_dbs[MAIN_DBI]; + + DPRINTF(("opened database version %u, pagesize %u", + meta->mm_version, env->me_psize)); + DPRINTF(("using meta page %d", (int) (meta->mm_txnid & 1))); + DPRINTF(("depth: %u", db->md_depth)); + DPRINTF(("entries: %"Yu, db->md_entries)); + DPRINTF(("branch pages: %"Yu, db->md_branch_pages)); + DPRINTF(("leaf pages: %"Yu, db->md_leaf_pages)); + DPRINTF(("overflow pages: %"Yu, db->md_overflow_pages)); + DPRINTF(("root: %"Yu, db->md_root)); + } +#endif + + return MDB_SUCCESS; +} + + +/** Release a reader thread's slot in the reader lock table. + * This function is called automatically when a thread exits. + * @param[in] ptr This points to the slot in the reader lock table. + */ +static void +mdb_env_reader_dest(void *ptr) +{ + MDB_reader *reader = ptr; + +#ifndef _WIN32 + if (reader->mr_pid == getpid()) /* catch pthread_exit() in child process */ +#endif + /* We omit the mutex, so do this atomically (i.e. skip mr_txnid) */ + reader->mr_pid = 0; +} + +#ifdef _WIN32 +/** Junk for arranging thread-specific callbacks on Windows. This is + * necessarily platform and compiler-specific. Windows supports up + * to 1088 keys. Let's assume nobody opens more than 64 environments + * in a single process, for now. They can override this if needed. + */ +#ifndef MAX_TLS_KEYS +#define MAX_TLS_KEYS 64 +#endif +static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; +static int mdb_tls_nkeys; + +static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) +{ + int i; + switch(reason) { + case DLL_PROCESS_ATTACH: break; + case DLL_THREAD_ATTACH: break; + case DLL_THREAD_DETACH: + for (i=0; ime_txns->mti_txnid = meta->mm_txnid; + +#ifdef _WIN32 + { + OVERLAPPED ov; + /* First acquire a shared lock. The Unlock will + * then release the existing exclusive lock. + */ + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + rc = ErrCode(); + } else { + UnlockFile(env->me_lfd, 0, 0, 1, 0); + *excl = 0; + } + } +#else + { + struct flock lock_info; + /* The shared lock replaces the existing lock */ + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_RDLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + *excl = rc ? -1 : 0; /* error may mean we lost the lock */ + } +#endif + + return rc; +} + +/** Try to get exclusive lock, otherwise shared. + * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. + */ +static int ESECT +mdb_env_excl_lock(MDB_env *env, int *excl) +{ + int rc = 0; +#ifdef _WIN32 + if (LockFile(env->me_lfd, 0, 0, 1, 0)) { + *excl = 1; + } else { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + *excl = 0; + } else { + rc = ErrCode(); + } + } +#else + struct flock lock_info; + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (!rc) { + *excl = 1; + } else +# ifndef MDB_USE_POSIX_MUTEX + if (*excl < 0) /* always true when MDB_USE_POSIX_MUTEX */ +# endif + { + lock_info.l_type = F_RDLCK; + while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (rc == 0) + *excl = 0; + } +#endif + return rc; +} + +#ifdef MDB_USE_HASH +/* + * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code + * + * @(#) $Revision: 5.1 $ + * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ + * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ + * + * http://www.isthe.com/chongo/tech/comp/fnv/index.html + * + *** + * + * Please do not copyright this code. This code is in the public domain. + * + * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO + * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * By: + * chongo /\oo/\ + * http://www.isthe.com/chongo/ + * + * Share and Enjoy! :-) + */ + +/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer + * @param[in] val value to hash + * @param[in] len length of value + * @return 64 bit hash + */ +static mdb_hash_t +mdb_hash(const void *val, size_t len) +{ + const unsigned char *s = (const unsigned char *) val, *end = s + len; + mdb_hash_t hval = 0xcbf29ce484222325ULL; + /* + * FNV-1a hash each octet of the buffer + */ + while (s < end) { + hval = (hval ^ *s++) * 0x100000001b3ULL; + } + /* return our new hash value */ + return hval; +} + +/** Hash the string and output the encoded hash. + * This uses modified RFC1924 Ascii85 encoding to accommodate systems with + * very short name limits. We don't care about the encoding being reversible, + * we just want to preserve as many bits of the input as possible in a + * small printable string. + * @param[in] str string to hash + * @param[out] encbuf an array of 11 chars to hold the hash + */ +static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; + +static void ESECT +mdb_pack85(unsigned long long l, char *out) +{ + int i; + + for (i=0; i<10 && l; i++) { + *out++ = mdb_a85[l % 85]; + l /= 85; + } + *out = '\0'; +} + +/** Init #MDB_env.me_mutexname[] except the char which #MUTEXNAME() will set. + * Changes to this code must be reflected in #MDB_LOCK_FORMAT. + */ +static void ESECT +mdb_env_mname_init(MDB_env *env) +{ + char *nm = env->me_mutexname; + strcpy(nm, MUTEXNAME_PREFIX); + mdb_pack85(env->me_txns->mti_mutexid, nm + sizeof(MUTEXNAME_PREFIX)); +} + +/** Return env->me_mutexname after filling in ch ('r'/'w') for convenience */ +#define MUTEXNAME(env, ch) ( \ + (void) ((env)->me_mutexname[sizeof(MUTEXNAME_PREFIX)-1] = (ch)), \ + (env)->me_mutexname) + +#endif + +/** Open and/or initialize the lock region for the environment. + * @param[in] env The LMDB environment. + * @param[in] fname Filename + scratch area, from #mdb_fname_init(). + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl) +{ +#ifdef _WIN32 +# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT +#else +# define MDB_ERRCODE_ROFS EROFS +#endif +#ifdef MDB_USE_SYSV_SEM + int semid; + union semun semu; +#endif + int rc; + MDB_OFF_T size, rsize; + + rc = mdb_fopen(env, fname, MDB_O_LOCKS, mode, &env->me_lfd); + if (rc) { + /* Omit lockfile if read-only env on read-only filesystem */ + if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { + return MDB_SUCCESS; + } + goto fail; + } + + if (!(env->me_flags & MDB_NOTLS)) { + rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); + if (rc) + goto fail; + env->me_flags |= MDB_ENV_TXKEY; +#ifdef _WIN32 + /* Windows TLS callbacks need help finding their TLS info. */ + if (mdb_tls_nkeys >= MAX_TLS_KEYS) { + rc = MDB_TLS_FULL; + goto fail; + } + mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; +#endif + } + + /* Try to get exclusive lock. If we succeed, then + * nobody is using the lock region and we should initialize it. + */ + if ((rc = mdb_env_excl_lock(env, excl))) goto fail; + +#ifdef _WIN32 + size = GetFileSize(env->me_lfd, NULL); +#else + size = lseek(env->me_lfd, 0, SEEK_END); + if (size == -1) goto fail_errno; +#endif + rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); + if (size < rsize && *excl > 0) { +#ifdef _WIN32 + if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize + || !SetEndOfFile(env->me_lfd)) + goto fail_errno; +#else + if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; +#endif + } else { + rsize = size; + size = rsize - sizeof(MDB_txninfo); + env->me_maxreaders = size/sizeof(MDB_reader) + 1; + } + { +#ifdef _WIN32 + HANDLE mh; + mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, + 0, 0, NULL); + if (!mh) goto fail_errno; + env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); + CloseHandle(mh); + if (!env->me_txns) goto fail_errno; +#else + void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, + env->me_lfd, 0); + if (m == MAP_FAILED) goto fail_errno; + env->me_txns = m; +#endif + } + if (*excl > 0) { +#ifdef _WIN32 + BY_HANDLE_FILE_INFORMATION stbuf; + struct { + DWORD volume; + DWORD nhigh; + DWORD nlow; + } idbuf; + + if (!mdb_sec_inited) { + InitializeSecurityDescriptor(&mdb_null_sd, + SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); + mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); + mdb_all_sa.bInheritHandle = FALSE; + mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; + mdb_sec_inited = 1; + } + if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.volume = stbuf.dwVolumeSerialNumber; + idbuf.nhigh = stbuf.nFileIndexHigh; + idbuf.nlow = stbuf.nFileIndexLow; + env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf)); + mdb_env_mname_init(env); + env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'r')); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'w')); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + struct stat stbuf; + struct { + dev_t dev; + ino_t ino; + } idbuf; + +#if defined(__NetBSD__) +#define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ +#endif + if (fstat(env->me_lfd, &stbuf)) goto fail_errno; + memset(&idbuf, 0, sizeof(idbuf)); + idbuf.dev = stbuf.st_dev; + idbuf.ino = stbuf.st_ino; + env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf)) +#ifdef MDB_SHORT_SEMNAMES + /* Max 9 base85-digits. We truncate here instead of in + * mdb_env_mname_init() to keep the latter portable. + */ + % ((mdb_hash_t)85*85*85*85*85*85*85*85*85) +#endif + ; + mdb_env_mname_init(env); + /* Clean up after a previous run, if needed: Try to + * remove both semaphores before doing anything else. + */ + sem_unlink(MUTEXNAME(env, 'r')); + sem_unlink(MUTEXNAME(env, 'w')); + env->me_rmutex = sem_open(MUTEXNAME(env, 'r'), O_CREAT|O_EXCL, mode, 1); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(MUTEXNAME(env, 'w'), O_CREAT|O_EXCL, mode, 1); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#elif defined(MDB_USE_SYSV_SEM) + unsigned short vals[2] = {1, 1}; + key_t key = ftok(fname->mn_val, 'M'); /* fname is lockfile path now */ + if (key == -1) + goto fail_errno; + semid = semget(key, 2, (mode & 0777) | IPC_CREAT); + if (semid < 0) + goto fail_errno; + semu.array = vals; + if (semctl(semid, 0, SETALL, semu) < 0) + goto fail_errno; + env->me_txns->mti_semid = semid; + env->me_txns->mti_rlocked = 0; + env->me_txns->mti_wlocked = 0; +#else /* MDB_USE_POSIX_MUTEX: */ + pthread_mutexattr_t mattr; + + /* Solaris needs this before initing a robust mutex. Otherwise + * it may skip the init and return EBUSY "seems someone already + * inited" or EINVAL "it was inited differently". + */ + memset(env->me_txns->mti_rmutex, 0, sizeof(*env->me_txns->mti_rmutex)); + memset(env->me_txns->mti_wmutex, 0, sizeof(*env->me_txns->mti_wmutex)); + + if ((rc = pthread_mutexattr_init(&mattr)) != 0) + goto fail; + rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); +#ifdef MDB_ROBUST_SUPPORTED + if (!rc) rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); +#endif + if (!rc) rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr); + if (!rc) rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr); + pthread_mutexattr_destroy(&mattr); + if (rc) + goto fail; +#endif /* _WIN32 || ... */ + + env->me_txns->mti_magic = MDB_MAGIC; + env->me_txns->mti_format = MDB_LOCK_FORMAT; + env->me_txns->mti_txnid = 0; + env->me_txns->mti_numreaders = 0; + + } else { +#ifdef MDB_USE_SYSV_SEM + struct semid_ds buf; +#endif + if (env->me_txns->mti_magic != MDB_MAGIC) { + DPUTS("lock region has invalid magic"); + rc = MDB_INVALID; + goto fail; + } + if (env->me_txns->mti_format != MDB_LOCK_FORMAT) { + DPRINTF(("lock region has format+version 0x%x, expected 0x%x", + env->me_txns->mti_format, MDB_LOCK_FORMAT)); + rc = MDB_VERSION_MISMATCH; + goto fail; + } +#ifdef _WIN32 + mdb_env_mname_init(env); + env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'r')); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'w')); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + mdb_env_mname_init(env); + env->me_rmutex = sem_open(MUTEXNAME(env, 'r'), 0); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(MUTEXNAME(env, 'w'), 0); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#elif defined(MDB_USE_SYSV_SEM) + semid = env->me_txns->mti_semid; + semu.buf = &buf; + /* check for read access */ + if (semctl(semid, 0, IPC_STAT, semu) < 0) + goto fail_errno; + /* check for write access */ + if (semctl(semid, 0, IPC_SET, semu) < 0) + goto fail_errno; +#endif + } +#ifdef MDB_USE_SYSV_SEM + env->me_rmutex->semid = semid; + env->me_wmutex->semid = semid; + env->me_rmutex->semnum = 0; + env->me_wmutex->semnum = 1; + env->me_rmutex->locked = &env->me_txns->mti_rlocked; + env->me_wmutex->locked = &env->me_txns->mti_wlocked; +#endif + + return MDB_SUCCESS; + +fail_errno: + rc = ErrCode(); +fail: + return rc; +} + +#ifdef MDB_TEST +#if MDB_RPAGE_CACHE +/** Trivial encryption for testing */ +static void ESECT +mdb_enctest(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec) +{ + mdb_size_t *sptr = src->mv_data, *dptr = dst->mv_data; + mdb_size_t x=*(mdb_size_t*)key[0].mv_data, v=*(mdb_size_t*)key[1].mv_data; + int i, len = dst->mv_size / sizeof(mdb_size_t); + + for (i = 0; i < len; i++) + x += v += i + sptr[i] + (dptr[i] = sptr[i] ^ x); +} +#endif /* MDB_RPAGE_CACHE */ + +/** Add #mdb_env_open() flags from environment variable $LMDB_FLAGS. + * + * Supports the normal flags plus 'e' = trivial encryption for testing. + */ +static int ESECT +mdb_env_envflags(MDB_env *env) +{ + static const char names[] = "ace" "fhi" "lmn" "rst" "vw"; + static const unsigned f[] = { + /*a*/ MDB_MAPASYNC, /*c*/ MDB_REMAP_CHUNKS, /*e*/ MDB_ENCRYPT, + /*f*/ MDB_FIXEDMAP, /*h*/ MDB_NORDAHEAD, /*i*/ MDB_NOMEMINIT, + /*l*/ MDB_NOLOCK, /*m*/ MDB_NOMETASYNC, /*n*/ MDB_NOSUBDIR, + /*r*/ MDB_RDONLY, /*s*/ MDB_NOSYNC, /*t*/ MDB_NOTLS, + /*v*/ MDB_PREVSNAPSHOT, /*w*/ MDB_WRITEMAP, + }; + unsigned flags = 0; + const char *s, *opts = getenv("LMDB_FLAGS"); + if (opts) { + for (; *opts; opts++) { + if ((s = strchr(names, *opts)) == NULL) + return EINVAL; + flags |= f[s - names]; + } + if (flags & MDB_ENCRYPT) { +#if MDB_RPAGE_CACHE + if (!env->me_encfunc) { + static mdb_size_t k = (MDB_SIZE_MAX/67*73) | 1; + MDB_val key = {sizeof(k), &k}; + int rc; + rc = mdb_env_set_encrypt(env, mdb_enctest, &key, 0); + if (rc) + return rc; + } +#else + return EINVAL; +#endif + } + env->me_flags |= flags; + } + return MDB_SUCCESS; +} +#else +#define mdb_env_envflags(env) MDB_SUCCESS +#endif /* MDB_TEST */ + + /** Only a subset of the @ref mdb_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. + */ +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) +#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \ + MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVSNAPSHOT|MDB_REMAP_CHUNKS) +#define EXPOSED (CHANGEABLE|CHANGELESS | MDB_ENCRYPT) + +#if VALID_FLAGS & PERSISTENT_FLAGS & EXPOSED +# error "Persistent DB flags & env flags overlap, but both go in mm_flags" +#endif + +int ESECT +mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) +{ + int rc, excl = -1; + MDB_name fname; + + if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) + return EINVAL; + + if ((rc = mdb_env_envflags(env)) != MDB_SUCCESS) + return rc; + flags |= env->me_flags; + + if (MDB_REMAPPING(0)) /* if we always remap chunks */ + flags |= MDB_REMAP_CHUNKS; + if (MDB_REMAPPING(flags)) { + /* silently ignore WRITEMAP with REMAP_CHUNKS */ + flags &= ~MDB_WRITEMAP; +#if (MDB_RPAGE_CACHE) & 2 + /* TEST: silently ignore FIXEDMAP, so mtest*.c will work */ + flags &= ~MDB_FIXEDMAP; +#else + /* cannot support FIXEDMAP */ + if (flags & MDB_FIXEDMAP) + return EINVAL; +#endif + } + + rc = mdb_fname_init(path, flags, &fname); + if (rc) + return rc; + +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(flags)) { +#ifdef _WIN32 + env->me_rpmutex = CreateMutex(NULL, FALSE, NULL); + if (!env->me_rpmutex) { + rc = ErrCode(); + goto leave; + } +#else + rc = pthread_mutex_init(&env->me_rpmutex, NULL); + if (rc) + goto leave; +#endif + } +#endif +#ifndef _WIN32 + { + struct stat st; + flags &= ~MDB_RAWPART; + if (!stat(path, &st) && (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))) + flags |= MDB_RAWPART | MDB_NOSUBDIR; + } +#endif + flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close_active() to clean up */ + + if (flags & MDB_RDONLY) { + /* silently ignore WRITEMAP when we're only getting read access */ + flags &= ~MDB_WRITEMAP; + } else { + /* WRITEMAP has a dummy element to match dirty_room = 1 */ + size_t dl_size = (flags & MDB_WRITEMAP) ? 2 : MDB_IDL_UM_SIZE; + if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && + (env->me_dirty_list = calloc(dl_size, sizeof(MDB_ID2))))) + rc = ENOMEM; + if (env->me_dirty_list && !(env->me_dirty_nump = malloc(dl_size * sizeof(int)))) + rc = ENOMEM; + } + + env->me_flags = flags; + if (rc) + goto leave; + +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(flags)) + { + env->me_rpages = malloc(MDB_ERPAGE_SIZE * sizeof(MDB_ID3)); + if (!env->me_rpages) { + rc = ENOMEM; + goto leave; + } + env->me_rpages[0].mid = 0; + env->me_rpcheck = MDB_ERPAGE_SIZE/2; + } +#endif + + env->me_path = strdup(path); + env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); + env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); + env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); + if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { + rc = ENOMEM; + goto leave; + } + env->me_dbxs[FREE_DBI].md_cmp = mdb_cmp_long; /* aligned MDB_INTEGERKEY */ + + /* For RDONLY, get lockfile after we know datafile exists */ + if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { + rc = mdb_env_setup_locks(env, &fname, mode, &excl); + if (rc) + goto leave; + if ((flags & MDB_PREVSNAPSHOT) && !excl) { + rc = MDB_ENV_BUSY; + goto leave; + } + } + + rc = mdb_fopen(env, &fname, + (flags & MDB_RDONLY) ? MDB_O_RDONLY : MDB_O_RDWR, + mode, &env->me_fd); + if (rc) + goto leave; +#ifdef _WIN32 + rc = mdb_fopen(env, &fname, MDB_O_OVERLAPPED, mode, &env->me_ovfd); + if (rc) + goto leave; +#endif + + if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { + rc = mdb_env_setup_locks(env, &fname, mode, &excl); + if (rc) + goto leave; + } + + if ((rc = mdb_env_open2(env, flags & MDB_PREVSNAPSHOT)) == MDB_SUCCESS) { + /* Synchronous fd for meta writes. Needed even with + * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. + */ + if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) { + rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd); + if (rc) + goto leave; + } + DPRINTF(("opened dbenv %p", (void *) env)); + if (excl > 0 && !(flags & MDB_PREVSNAPSHOT)) { + rc = mdb_env_share_locks(env, &excl); + if (rc) + goto leave; + } + if (!(flags & MDB_RDONLY)) { + MDB_txn *txn; + int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * + (sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1); + if ((env->me_pbuf = calloc(1, env->me_psize)) && + (txn = calloc(1, size))) + { + txn->mt_dbs = (MDB_db *)((char *)txn + tsize); + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + txn->mt_env = env; +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3)); + if (!txn->mt_rpages) { + free(txn); + rc = ENOMEM; + goto leave; + } + txn->mt_rpages[0].mid = 0; + txn->mt_rpcheck = MDB_TRPAGE_SIZE/2; + } +#endif + txn->mt_dbxs = env->me_dbxs; + txn->mt_flags = MDB_TXN_FINISHED; + pthread_mutex_init(&txn->mt_child_mutex, NULL); + env->me_txn0 = txn; + } else { + rc = ENOMEM; + } + } + } + +leave: + MDB_TRACE(("%p, %s, %u, %04o", env, path, flags & (CHANGEABLE|CHANGELESS), mode)); + if (rc) { + mdb_env_close_active(env, excl); + } + mdb_fname_destroy(fname); + return rc; +} + +/** When #MDB_ENV_ACTIVE: Clear #mdb_env_open()ed resources, release readers */ +static void ESECT +mdb_env_close_active(MDB_env *env, int excl) +{ + int i; + + if (!(env->me_flags & MDB_ENV_ACTIVE)) + return; + + /* Doing this here since me_dbxs may not exist during mdb_env_close */ + if (env->me_dbxs) { + for (i = env->me_maxdbs; --i >= CORE_DBS; ) + free(env->me_dbxs[i].md_name.mv_data); + free(env->me_dbxs); + } + + free(env->me_pbuf); + free(env->me_dbiseqs); + free(env->me_dbflags); + free(env->me_path); + free(env->me_dirty_list); + free(env->me_dirty_nump); +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + if (env->me_txn0 && env->me_txn0->mt_rpages) + free(env->me_txn0->mt_rpages); + if (env->me_rpages) { + MDB_ID3L el = env->me_rpages; + unsigned int x; + for (x=1; x<=el[0].mid; x++) { + munmap(el[x].mptr, el[x].mcnt * env->me_psize); + free(el[x].menc); + } + free(el); + } + } +#endif + if (env->me_txn0) { + pthread_mutex_destroy(&env->me_txn0->mt_child_mutex); + free(env->me_txn0); + } + mdb_midl_free(env->me_free_pgs); + + if (env->me_flags & MDB_ENV_TXKEY) { + pthread_key_delete(env->me_txkey); +#ifdef _WIN32 + /* Delete our key from the global list */ + for (i=0; ime_txkey) { + mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; + mdb_tls_nkeys--; + break; + } +#endif + } + + if (env->me_map) { + if (MDB_REMAPPING(env->me_flags)) + munmap(env->me_map, NUM_METAS*env->me_psize); + else + munmap(env->me_map, env->me_mapsize); + } + if (env->me_mfd != INVALID_HANDLE_VALUE) + (void) close(env->me_mfd); +#ifdef _WIN32 + if (env->me_ovs > 0) { + for (i = 0; i < env->me_ovs; i++) { + CloseHandle(env->me_ov[i].hEvent); + } + free(env->me_ov); + } + if (env->me_ovfd != INVALID_HANDLE_VALUE) + (void) close(env->me_ovfd); +#endif + if (env->me_fd != INVALID_HANDLE_VALUE) + (void) close(env->me_fd); + if (env->me_txns) { + MDB_PID_T pid = getpid(); + /* Clearing readers is done in this function because + * me_txkey with its destructor must be disabled first. + * + * We skip the the reader mutex, so we touch only + * data owned by this process (me_close_readers and + * our readers), and clear each reader atomically. + */ + for (i = env->me_close_readers; --i >= 0; ) + if (env->me_txns->mti_readers[i].mr_pid == pid) + env->me_txns->mti_readers[i].mr_pid = 0; +#ifdef _WIN32 + if (env->me_rmutex) { + CloseHandle(env->me_rmutex); + if (env->me_wmutex) CloseHandle(env->me_wmutex); + } + /* Windows automatically destroys the mutexes when + * the last handle closes. + */ +#elif defined(MDB_USE_POSIX_SEM) + if (env->me_rmutex != SEM_FAILED) { + sem_close(env->me_rmutex); + if (env->me_wmutex != SEM_FAILED) + sem_close(env->me_wmutex); + /* If we have the filelock: If we are the + * only remaining user, clean up semaphores. + */ + if (excl == 0) + mdb_env_excl_lock(env, &excl); + if (excl > 0) { + sem_unlink(MUTEXNAME(env, 'r')); + sem_unlink(MUTEXNAME(env, 'w')); + } + } +#elif defined(MDB_USE_SYSV_SEM) + if (env->me_rmutex->semid != -1) { + /* If we have the filelock: If we are the + * only remaining user, clean up semaphores. + */ + if (excl == 0) + mdb_env_excl_lock(env, &excl); + if (excl > 0) + semctl(env->me_rmutex->semid, 0, IPC_RMID); + } +#endif + munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); + } + if (env->me_lfd != INVALID_HANDLE_VALUE) { +#ifdef _WIN32 + if (excl >= 0) { + /* Unlock the lockfile. Windows would have unlocked it + * after closing anyway, but not necessarily at once. + */ + UnlockFile(env->me_lfd, 0, 0, 1, 0); + } +#endif + (void) close(env->me_lfd); + } +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) + { +#ifdef _WIN32 + if (env->me_fmh) CloseHandle(env->me_fmh); + if (env->me_rpmutex) CloseHandle(env->me_rpmutex); +#else + pthread_mutex_destroy(&env->me_rpmutex); +#endif + } +#endif + + env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); +} + +void ESECT +mdb_env_close(MDB_env *env) +{ + MDB_page *dp; + + if (env == NULL) + return; + + MDB_TRACE(("%p", env)); + VGMEMP_DESTROY(env); + while ((dp = env->me_dpages) != NULL) { + VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); + env->me_dpages = dp->mp_next; + free(dp); + } + + mdb_env_close_active(env, 0); +#if MDB_RPAGE_CACHE + free(env->me_enckey.mv_data); +#endif + free(env); +} + +/** Compare two items pointing at aligned #mdb_size_t's */ +static int +mdb_cmp_long(const MDB_val *a, const MDB_val *b) +{ + return (*(mdb_size_t *)a->mv_data < *(mdb_size_t *)b->mv_data) ? -1 : + *(mdb_size_t *)a->mv_data > *(mdb_size_t *)b->mv_data; +} + +/** Compare two items pointing at aligned unsigned int's. + * + * This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp, + * but #mdb_cmp_clong() is called instead if the data type is #mdb_size_t. + */ +static int +mdb_cmp_int(const MDB_val *a, const MDB_val *b) +{ + return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : + *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; +} + +/** Compare two items pointing at unsigned ints of unknown alignment. + * Nodes and keys are guaranteed to be 2-byte aligned. + */ +static int +mdb_cmp_cint(const MDB_val *a, const MDB_val *b) +{ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short *u, *c; + int x; + + u = (unsigned short *) ((char *) a->mv_data + a->mv_size); + c = (unsigned short *) ((char *) b->mv_data + a->mv_size); + do { + x = *--u - *--c; + } while(!x && u > (unsigned short *)a->mv_data); + return x; +#else + unsigned short *u, *c, *end; + int x; + + end = (unsigned short *) ((char *) a->mv_data + a->mv_size); + u = (unsigned short *)a->mv_data; + c = (unsigned short *)b->mv_data; + do { + x = *u++ - *c++; + } while(!x && u < end); + return x; +#endif +} + +/** Compare two items lexically */ +static int +mdb_cmp_memn(const MDB_val *a, const MDB_val *b) +{ + int diff; + ssize_t len_diff; + unsigned int len; + + len = a->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + len = b->mv_size; + len_diff = 1; + } + + diff = memcmp(a->mv_data, b->mv_data, len); + return diff ? diff : len_diff<0 ? -1 : len_diff; +} + +/** Compare two items in reverse byte order */ +static int +mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) +{ + const unsigned char *p1, *p2, *p1_lim; + ssize_t len_diff; + int diff; + + p1_lim = (const unsigned char *)a->mv_data; + p1 = (const unsigned char *)a->mv_data + a->mv_size; + p2 = (const unsigned char *)b->mv_data + b->mv_size; + + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + p1_lim += len_diff; + len_diff = 1; + } + + while (p1 > p1_lim) { + diff = *--p1 - *--p2; + if (diff) + return diff; + } + return len_diff<0 ? -1 : len_diff; +} + +/** Search for key within a page, using binary search. + * Returns the smallest entry larger or equal to the key. + * If exactp is non-null, stores whether the found entry was an exact match + * in *exactp (1 or 0). + * Updates the cursor index with the index of the found entry. + * If no entry larger or equal to the key is found, returns NULL. + */ +static MDB_node * +mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) +{ + unsigned int i = 0, nkeys; + int low, high; + int rc = 0; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NULL; + MDB_val nodekey; + MDB_cmp_func *cmp; + DKBUF; + + nkeys = NUMKEYS(mp); + + DPRINTF(("searching %u keys in %s %spage %"Yu, + nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp))); + + low = IS_LEAF(mp) ? 0 : 1; + high = nkeys - 1; + cmp = mc->mc_dbx->md_cmp; + + /* Branch pages have no data, so if using integer keys, + * alignment is guaranteed. Use faster mdb_cmp_int. + */ + if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { + if (NODEPTR(mp, 1)->mn_ksize == sizeof(mdb_size_t)) + cmp = mdb_cmp_long; + else + cmp = mdb_cmp_int; + } + + if (IS_LEAF2(mp)) { + nodekey.mv_size = mc->mc_db->md_pad; + node = NODEPTR(mp, 0); /* fake */ + while (low <= high) { + i = (low + high) >> 1; + nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); + rc = cmp(key, &nodekey); + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } else { + while (low <= high) { + i = (low + high) >> 1; + + node = NODEPTR(mp, i); + nodekey.mv_size = NODEKSZ(node); + nodekey.mv_data = NODEKEY(node); + + rc = cmp(key, &nodekey); +#if MDB_DEBUG + if (IS_LEAF(mp)) + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + else + DPRINTF(("found branch index %u [%s -> %"Yu"], rc = %i", + i, DKEY(&nodekey), NODEPGNO(node), rc)); +#endif + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } + + if (rc > 0) { /* Found entry is less than the key. */ + i++; /* Skip to get the smallest entry larger than key. */ + if (!IS_LEAF2(mp)) + node = NODEPTR(mp, i); + } + if (exactp) + *exactp = (rc == 0 && nkeys > 0); + /* store the key index */ + mc->mc_ki[mc->mc_top] = i; + if (i >= nkeys) + /* There is no entry larger or equal to the key. */ + return NULL; + + /* nodeptr is fake for LEAF2 */ + return node; +} + +#if 0 +static void +mdb_cursor_adjust(MDB_cursor *mc, func) +{ + MDB_cursor *m2; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { + func(mc, m2); + } + } +} +#endif + +/** Pop a page off the top of the cursor's stack. */ +static void +mdb_cursor_pop(MDB_cursor *mc) +{ + if (mc->mc_snum) { + DPRINTF(("popping page %"Yu" off db %d cursor %p", + mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc)); + + mc->mc_snum--; + if (mc->mc_snum) { + mc->mc_top--; + } else { + mc->mc_flags &= ~C_INITIALIZED; + } + } +} + +/** Push a page onto the top of the cursor's stack. + * Set #MDB_TXN_ERROR on failure. + */ +static int +mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) +{ + DPRINTF(("pushing page %"Yu" on db %d cursor %p", mp->mp_pgno, + DDBI(mc), (void *) mc)); + + if (mc->mc_snum >= CURSOR_STACK) { + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CURSOR_FULL; + } + + mc->mc_top = mc->mc_snum++; + mc->mc_pg[mc->mc_top] = mp; + mc->mc_ki[mc->mc_top] = 0; + + return MDB_SUCCESS; +} + +#if MDB_RPAGE_CACHE + +static int +mdb_rpage_encsum(MDB_env *env, MDB_ID3 *id3, unsigned rem, int numpgs) +{ + int rc = 0; + if (env->me_encfunc) { + unsigned short muse = id3->muse; + rc = mdb_rpage_decrypt(env, id3, rem, numpgs); + if (!rc && env->me_sumfunc && muse != id3->muse) { + MDB_page *p = (MDB_page *)((char *)id3->menc + rem * env->me_psize); + rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize); + } + } else { + if (!(id3->muse & (1 << rem))) { + MDB_page *p; + int bit; + /* If this is an overflow page, set all use bits to the end */ + if (rem + numpgs > MDB_RPAGE_CHUNK) + bit = 0xffff; + else + bit = 1; + + id3->muse |= (bit << rem); + p = (MDB_page *)((char *)id3->mptr + rem * env->me_psize); + rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize); + } + } + return rc; +} + +/** Map a read-only page. + * There are two levels of tracking in use, a per-txn list and a per-env list. + * ref'ing and unref'ing the per-txn list is faster since it requires no + * locking. Pages are cached in the per-env list for global reuse, and a lock + * is required. Pages are not immediately unmapped when their refcnt goes to + * zero; they hang around in case they will be reused again soon. + * + * When the per-txn list gets full, all pages with refcnt=0 are purged from the + * list and their refcnts in the per-env list are decremented. + * + * When the per-env list gets full, all pages with refcnt=0 are purged from the + * list and their pages are unmapped. + * + * @note "full" means the list has reached its respective rpcheck threshold. + * This threshold slowly raises if no pages could be purged on a given check, + * and returns to its original value when enough pages were purged. + * + * If purging doesn't free any slots, filling the per-txn list will return + * MDB_TXN_FULL, and filling the per-env list returns MDB_MAP_FULL. + * + * Reference tracking in a txn is imperfect, pages can linger with non-zero + * refcnt even without active references. It was deemed to be too invasive + * to add unrefs in every required location. However, all pages are unref'd + * at the end of the transaction. This guarantees that no stale references + * linger in the per-env list. + * + * Usually we map chunks of 16 pages at a time, but if an overflow page begins + * at the tail of the chunk we extend the chunk to include the entire overflow + * page. Unfortunately, pages can be turned into overflow pages after their + * chunk was already mapped. In that case we must remap the chunk if the + * overflow page is referenced. If the chunk's refcnt is 0 we can just remap + * it, otherwise we temporarily map a new chunk just for the overflow page. + * + * @note this chunk handling means we cannot guarantee that a data item + * returned from the DB will stay alive for the duration of the transaction: + * We unref pages as soon as a cursor moves away from the page + * A subsequent op may cause a purge, which may unmap any unref'd chunks + * The caller must copy the data if it must be used later in the same txn. + * + * Also - our reference counting revolves around cursors, but overflow pages + * aren't pointed to by a cursor's page stack. We have to remember them + * explicitly, in the added mc_ovpg field. A single cursor can only hold a + * reference to one overflow page at a time. + * + * @param[in] txn the transaction for this access. + * @param[in] pg0 the page number for the page to retrieve. + * @param[in] numpgs number of database pages (can be > 1 for overflow pages) + * @param[out] ret address of a pointer where the page's address will be stored. + * @param[out] enc address of a pointer where the encrypted page's address will be stored. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret, MDB_page **enc) +{ + MDB_env *env = txn->mt_env; + MDB_page *p; + MDB_ID3L tl = txn->mt_rpages; + MDB_ID3L el = env->me_rpages; + MDB_ID3 id3; + char *base; + unsigned x, rem; + pgno_t pgno; + int rc, retries = 1; +#ifdef _WIN32 + LARGE_INTEGER off; + SIZE_T len; +#define SET_OFF(off,val) off.QuadPart = val +#define MAP(rc,env,addr,len,off) \ + addr = NULL; \ + rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \ + len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \ + if (rc) rc = mdb_nt2win32(rc) +#else + off_t off; + size_t len; +#define SET_OFF(off,val) off = val +#define MAP(rc,env,addr,len,off) \ + addr = mmap(NULL, len, PROT_READ, MAP_SHARED, env->me_fd, off); \ + rc = (addr == MAP_FAILED) ? errno : 0 +#endif + + /* remember the offset of the actual page number, so we can + * return the correct pointer at the end. + */ + rem = pg0 & (MDB_RPAGE_CHUNK-1); + pgno = pg0 ^ rem; + + id3.mid = 0; + id3.menc = NULL; + id3.muse = 0; + x = mdb_mid3l_search(tl, pgno); + if (x <= tl[0].mid && tl[x].mid == pgno) { + if (x != tl[0].mid && tl[x+1].mid == pg0) + x++; + /* check for overflow size */ + if (numpgs + rem > tl[x].mcnt) { + id3.mcnt = numpgs + rem; + len = id3.mcnt * env->me_psize; + SET_OFF(off, pgno * env->me_psize); + MAP(rc, env, id3.mptr, len, off); + if (rc) + return rc; + /* setup for encryption */ + if (env->me_encfunc) { + id3.menc = malloc(len); + if (!id3.menc) + return ENOMEM; + } + /* check for local-only page */ + if (rem) { + mdb_tassert(txn, tl[x].mid != pg0); + /* hope there's room to insert this locally. + * setting mid here tells later code to just insert + * this id3 instead of searching for a match. + */ + id3.mid = pg0; + goto notlocal; + } else { + /* ignore the mapping we got from env, use new one */ + tl[x].mptr = id3.mptr; + tl[x].mcnt = id3.mcnt; + if (tl[x].menc) + mdb_rpage_dispose(env, &tl[x]); + tl[x].menc = id3.menc; + tl[x].muse = id3.muse; + /* if no active ref, see if we can replace in env */ + if (!tl[x].mref) { + unsigned i; + pthread_mutex_lock(&env->me_rpmutex); + i = mdb_mid3l_search(el, tl[x].mid); + if (el[i].mref == 1) { + /* just us, replace it */ + munmap(el[i].mptr, el[i].mcnt * env->me_psize); + el[i].mptr = tl[x].mptr; + el[i].mcnt = tl[x].mcnt; + if (el[i].menc) + mdb_rpage_dispose(env, &el[i]); + el[i].menc = tl[x].menc; + el[i].muse = tl[x].muse; + } else { + /* there are others, remove ourself */ + el[i].mref--; + } + pthread_mutex_unlock(&env->me_rpmutex); + } + } + } + id3.mptr = tl[x].mptr; + id3.mcnt = tl[x].mcnt; + id3.menc = tl[x].menc; + id3.muse = tl[x].muse; + tl[x].mref++; + if (env->me_encfunc || env->me_sumfunc) { + if (env->me_encfunc) + pthread_mutex_lock(&env->me_rpmutex); + rc = mdb_rpage_encsum(env, &id3, rem, numpgs); + if (env->me_encfunc) + pthread_mutex_unlock(&env->me_rpmutex); + if (rc) return rc; + tl[x].muse = id3.muse; + } + goto ok; + } + +notlocal: + if (tl[0].mid >= MDB_TRPAGE_MAX - txn->mt_rpcheck) { + unsigned i, y; + /* purge unref'd pages from our list and unref in env */ + pthread_mutex_lock(&env->me_rpmutex); +retry: + y = 0; + for (i=1; i<=tl[0].mid; i++) { + if (!tl[i].mref) { + if (!y) y = i; + /* tmp overflow pages don't go to env */ + if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) { + munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); + if (tl[i].menc) + mdb_rpage_dispose(env, &tl[i]); + continue; + } + x = mdb_mid3l_search(el, tl[i].mid); + el[x].mref--; + } + } + pthread_mutex_unlock(&env->me_rpmutex); + if (!y) { + /* we didn't find any unref'd chunks. + * if we're out of room, fail. + */ + if (tl[0].mid >= MDB_TRPAGE_MAX) + return MDB_TXN_FULL; + /* otherwise, raise threshold for next time around + * and let this go. + */ + txn->mt_rpcheck /= 2; + } else { + /* we found some unused; consolidate the list */ + for (i=y+1; i<= tl[0].mid; i++) + if (tl[i].mref) + tl[y++] = tl[i]; + tl[0].mid = y-1; + /* decrease the check threshold toward its original value */ + if (!txn->mt_rpcheck) + txn->mt_rpcheck = 1; + while (txn->mt_rpcheck < tl[0].mid && txn->mt_rpcheck < MDB_TRPAGE_SIZE/2) + txn->mt_rpcheck *= 2; + } + } + if (tl[0].mid < MDB_TRPAGE_SIZE) { + id3.mref = 1; + if (id3.mid) + goto found; + /* don't map past last written page in read-only envs */ + if ((env->me_flags & MDB_RDONLY) && pgno + MDB_RPAGE_CHUNK-1 > txn->mt_last_pgno) + id3.mcnt = txn->mt_last_pgno + 1 - pgno; + else + id3.mcnt = numpgs + rem > MDB_RPAGE_CHUNK ? numpgs + rem : MDB_RPAGE_CHUNK; + len = id3.mcnt * env->me_psize; + id3.mid = pgno; + + /* search for page in env */ + pthread_mutex_lock(&env->me_rpmutex); + x = mdb_mid3l_search(el, pgno); + if (x <= el[0].mid && el[x].mid == pgno) { + id3.mptr = el[x].mptr; + id3.menc = el[x].menc; + id3.muse = el[x].muse; + /* check for overflow size */ + if (id3.mcnt > el[x].mcnt) { + SET_OFF(off, pgno * env->me_psize); + MAP(rc, env, id3.mptr, len, off); + if (rc) + goto fail; + if (env->me_encfunc) { + id3.menc = malloc(len); + if (!id3.menc) { + rc = ENOMEM; + goto fail; + } + id3.muse = 0; + } + if (!el[x].mref) { + munmap(el[x].mptr, env->me_psize * el[x].mcnt); + el[x].mptr = id3.mptr; + el[x].mcnt = id3.mcnt; + if (el[x].menc) + mdb_rpage_dispose(env, &el[x]); + el[x].menc = id3.menc; + el[x].muse = id3.muse; + } else { + id3.mid = pg0; + if (env->me_encfunc || env->me_sumfunc) { + rc = mdb_rpage_encsum(env, &id3, rem, numpgs); + if (rc) { + free(id3.menc); + goto fail; + } + el[x].muse = id3.muse; + } + pthread_mutex_unlock(&env->me_rpmutex); + goto found; + } + } + el[x].mref++; + if (env->me_encfunc || env->me_sumfunc) { + rc = mdb_rpage_encsum(env, &id3, rem, numpgs); + if (rc) + goto fail; + el[x].muse = id3.muse; + } + pthread_mutex_unlock(&env->me_rpmutex); + goto found; + } + if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) { + /* purge unref'd pages */ + unsigned i, y = 0; + for (i=1; i<=el[0].mid; i++) { + if (!el[i].mref) { + if (!y) y = i; + munmap(el[i].mptr, env->me_psize * el[i].mcnt); + if (el[i].menc) + mdb_rpage_dispose(env, &el[i]); + } + } + if (!y) { + if (retries) { + /* see if we can unref some local pages */ + retries--; + id3.mid = 0; + goto retry; + } + if (el[0].mid >= MDB_ERPAGE_MAX) { + pthread_mutex_unlock(&env->me_rpmutex); + return MDB_MAP_FULL; + } + env->me_rpcheck /= 2; + } else { + for (i=y+1; i<= el[0].mid; i++) + if (el[i].mref) + el[y++] = el[i]; + el[0].mid = y-1; + if (!env->me_rpcheck) + env->me_rpcheck = 1; + while (env->me_rpcheck < el[0].mid && env->me_rpcheck < MDB_ERPAGE_SIZE/2) + env->me_rpcheck *= 2; + } + } + SET_OFF(off, pgno * env->me_psize); + MAP(rc, env, id3.mptr, len, off); + if (rc) { +fail: + pthread_mutex_unlock(&env->me_rpmutex); + return rc; + } + if (env->me_encfunc) { + id3.menc = malloc(len); + if (!id3.menc) { + rc = ENOMEM; + goto fail; + } + } + if (env->me_encfunc || env->me_sumfunc) { + rc = mdb_rpage_encsum(env, &id3, rem, numpgs); + if (rc) { + free(id3.menc); + goto fail; + } + } + mdb_mid3l_insert(el, &id3); + pthread_mutex_unlock(&env->me_rpmutex); +found: + mdb_mid3l_insert(tl, &id3); + } else { + return MDB_TXN_FULL; + } +ok: + base = (char *)(env->me_encfunc ? id3.menc : id3.mptr); + p = (MDB_page *)(base + rem * env->me_psize); + rc = MDB_SUCCESS; + if (env->me_encfunc || env->me_sumfunc) { + if (env->me_encfunc) + pthread_mutex_lock(&env->me_rpmutex); + rc = mdb_rpage_encsum(env, &id3, rem, numpgs); + if (env->me_encfunc) + pthread_mutex_unlock(&env->me_rpmutex); + } +#if MDB_DEBUG /* we don't need this check any more */ + if (IS_OVERFLOW(p)) { + mdb_tassert(txn, p->mp_pages + rem <= id3.mcnt); + } +#endif + *ret = p; + if (enc && env->me_encfunc) { + *enc = (MDB_page *)(id3.mptr + rem * env->me_psize); + } + return rc; +} + +static int mdb_page_encrypt(MDB_env *env, MDB_page *dp, MDB_page *encp, size_t size) +{ + MDB_val in, out, enckeys[3]; + int xsize = sizeof(pgno_t) + sizeof(txnid_t); + in.mv_size = size - xsize; + in.mv_data = (char *)dp + xsize; + out.mv_size = in.mv_size; + out.mv_data = (char *)encp + xsize; + if (env->me_esumsize) { + in.mv_size -= env->me_esumsize; + out.mv_size -= env->me_esumsize; + enckeys[2].mv_size = env->me_esumsize; + enckeys[2].mv_data = (char *)out.mv_data + out.mv_size; + } else { + enckeys[2].mv_size = 0; + enckeys[2].mv_data = 0; + } + encp->mp_pgno = dp->mp_pgno; + encp->mp_txnid = dp->mp_txnid; + enckeys[0] = env->me_enckey; + enckeys[1].mv_size = xsize; + enckeys[1].mv_data = dp; + return env->me_encfunc(&in, &out, enckeys, 1); +} + +static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs) +{ + int rc = 0; + if (!(id3->muse & (1 << rem))) { + MDB_val in, out, enckeys[3]; + int bit; + int xsize = sizeof(pgno_t) + sizeof(txnid_t); + + /* If this is an overflow page, set all use bits to the end */ + if (rem + numpgs > MDB_RPAGE_CHUNK) + bit = 0xffff; + else + bit = 1; + + id3->muse |= (bit << rem); + in.mv_size = numpgs * env->me_psize - xsize; + in.mv_data = (char *)id3->mptr + rem * env->me_psize + xsize; + enckeys[0] = env->me_enckey; + enckeys[1].mv_size = xsize; + enckeys[1].mv_data = (char *)in.mv_data - xsize; + if (env->me_esumsize) { + in.mv_size -= env->me_esumsize; + enckeys[2].mv_size = env->me_esumsize; + enckeys[2].mv_data = (char *)in.mv_data + in.mv_size; + } else { + enckeys[2].mv_size = 0; + enckeys[2].mv_data = 0; + } + out.mv_size = in.mv_size; + out.mv_data = (char *)id3->menc + rem * env->me_psize + xsize; + if (env->me_encfunc(&in, &out, enckeys, 0)) + rc = MDB_CRYPTO_FAIL; + else { + MDB_page *penc, *pclr; + penc = (MDB_page *)enckeys[1].mv_data; + pclr = (MDB_page *)((char *)out.mv_data - xsize); + pclr->mp_pgno = penc->mp_pgno; + pclr->mp_txnid = penc->mp_txnid; + } + } + return rc; +} + +/** zero out decrypted pages before freeing them */ +static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3) +{ + char *base = id3->menc; + int i, j; + for (i=0, j=1; i<15; i++) { + if (id3->muse & j) + memset(base, 0, env->me_psize); + j <<= 1; + base += env->me_psize; + } + if (id3->muse & j) { + i = id3->mcnt - (MDB_RPAGE_CHUNK - 1); + memset(base, 0, i * env->me_psize); + } + free(id3->menc); +} + +static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size) +{ + MDB_val src, dst, *key; + src.mv_size = size - env->me_sumsize - env->me_esumsize; + src.mv_data = mp; + dst.mv_size = env->me_sumsize; + dst.mv_data = (char *)src.mv_data + src.mv_size; + if (env->me_encfunc) + key = &env->me_enckey; + else + key = NULL; + env->me_sumfunc(&src, &dst, key); +} + +static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size) +{ + MDB_val src, dst, chk, *key; + char sumbuf[256]; + src.mv_size = size - env->me_sumsize - env->me_esumsize; + src.mv_data = mp; + chk.mv_size = env->me_sumsize; + chk.mv_data = (char *)src.mv_data + src.mv_size; + dst.mv_size = env->me_sumsize; + dst.mv_data = sumbuf; + if (env->me_encfunc) + key = &env->me_enckey; + else + key = NULL; + env->me_sumfunc(&src, &dst, key); + return memcmp(chk.mv_data, dst.mv_data, env->me_sumsize) ? MDB_BAD_CHECKSUM : 0; +} +#endif + +/** Find the address of the page corresponding to a given page number. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc the cursor accessing the page. + * @param[in] pgno the page number for the page to retrieve. + * @param[in] numpgs number of database pages (can be > 1 for overflow pages) + * @param[out] ret address of a pointer where the page's address will be stored. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_get(MDB_cursor *mc, pgno_t pgno, +#if MDB_RPAGE_CACHE + int numpgs, +#endif + MDB_page **ret) +{ + MDB_txn *txn = mc->mc_txn, *tx2; + MDB_page *p = NULL; + + if (! (( mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP) ) && mc->mc_txn->mt_parent == NULL)) { + for (tx2 = txn;; ) { + MDB_ID2L dl = tx2->mt_u.dirty_list; + MDB_IDL sl; + unsigned x; + /* tx2 may have malloced its own "dirty" version of the + * page, with the same page number. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + p = dl[x].mptr; + goto done; + } + } + /* Spilled pages were dirtied in this txn, then cleaned + * and flushed to the map when dirty_list got full. + * Check if tx2 spilled the page before moving on to + * search the parent. (But don't unspill here, leave + * that unless page_touch happens again.) + */ + sl = tx2->mt_spill_pgs; + if ((tx2 = tx2->mt_parent) == NULL) + break; + if (sl) { + MDB_ID pn = pgno << 1; + x = mdb_midl_search(sl, pn); + if (x <= sl[0] && sl[x] == pn) { + goto mapped; + } + } + } + } + + if (pgno >= txn->mt_next_pgno) { + DPRINTF(("page %"Yu" not found", pgno)); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_NOTFOUND; + } + +mapped: +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(txn->mt_env->me_flags)) { + int rc = mdb_rpage_get(txn, pgno, numpgs, &p, NULL); + if (rc) { + txn->mt_flags |= MDB_TXN_ERROR; + return rc; + } + } else +#endif + { + MDB_env *env = txn->mt_env; + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + } + +done: + *ret = p; + return MDB_SUCCESS; +} + +/** Finish #mdb_page_search() / #mdb_page_search_lowest(). + * The cursor is at the root page, set up the rest of it. + */ +static int +mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int rc; + DKBUF; + + while (IS_BRANCH(mp)) { + MDB_node *node; + indx_t i; + + DPRINTF(("branch page %"Yu" has %u keys", mp->mp_pgno, NUMKEYS(mp))); + /* Don't assert on branch pages in the FreeDB. We can get here + * while in the process of rebalancing a FreeDB branch page; we must + * let that proceed. ITS#8336 + */ + mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1); + DPRINTF(("found index 0 to page %"Yu, NODEPGNO(NODEPTR(mp, 0)))); + + if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { + i = 0; + if (flags & MDB_PS_LAST) { + i = NUMKEYS(mp) - 1; + /* if already init'd, see if we're already in right place */ + if (mc->mc_flags & C_INITIALIZED) { + if (mc->mc_ki[mc->mc_top] == i) { + mc->mc_top = mc->mc_snum++; + mp = mc->mc_pg[mc->mc_top]; + goto ready; + } + } + } + } else { + int exact; + node = mdb_node_search(mc, key, &exact); + if (node == NULL) + i = NUMKEYS(mp) - 1; + else { + i = mc->mc_ki[mc->mc_top]; + if (!exact) { + mdb_cassert(mc, i > 0); + i--; + } + } + DPRINTF(("following index %u for key [%s]", i, DKEY(key))); + } + + mdb_cassert(mc, i < NUMKEYS(mp)); + node = NODEPTR(mp, i); + + if ((rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mp)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = i; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + +ready: + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc)) != 0) + return rc; + mp = mc->mc_pg[mc->mc_top]; + } + } + + if (!IS_LEAF(mp)) { + DPRINTF(("internal error, index points to a %02X page!?", + mp->mp_flags)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + + DPRINTF(("found leaf page %"Yu" for key [%s]", mp->mp_pgno, + key ? DKEY(key) : "null")); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + return MDB_SUCCESS; +} + +/** Search for the lowest key under the current branch page. + * This just bypasses a NUMKEYS check in the current page + * before calling mdb_page_search_root(), because the callers + * are all in situations where the current page is known to + * be underfilled. + */ +static int +mdb_page_search_lowest(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NODEPTR(mp, 0); + int rc; + + if ((rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mp)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = 0; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + return mdb_page_search_root(mc, NULL, MDB_PS_FIRST); +} + +/** Search for the page a given key should be in. + * Push it and its parent pages on the cursor stack. + * @param[in,out] mc the cursor for this operation. + * @param[in] key the key to search for, or NULL for first/last page. + * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB + * are touched (updated with new page numbers). + * If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf. + * This is used by #mdb_cursor_first() and #mdb_cursor_last(). + * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) +{ + int rc; + pgno_t root; + + /* Make sure the txn is still viable, then find the root from + * the txn's db table and set it as the root of the cursor's stack. + */ + if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) { + DPUTS("transaction may not be used now"); + return MDB_BAD_TXN; + } else { + /* Make sure we're using an up-to-date root */ + if (*mc->mc_dbflag & DB_STALE) { + MDB_cursor mc2; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); + if (rc) + return rc; + { + MDB_val data; + int exact = 0; + uint16_t flags; + MDB_node *leaf = mdb_node_search(&mc2, + &mc->mc_dbx->md_name, &exact); + if (!exact) + return MDB_BAD_DBI; + if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) + return MDB_INCOMPATIBLE; /* not a named DB */ + rc = mdb_node_read(&mc2, leaf, &data); + if (rc) + return rc; + memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), + sizeof(uint16_t)); + /* The txn may not know this DBI, or another process may + * have dropped and recreated the DB with other flags. + */ + if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) + return MDB_INCOMPATIBLE; + memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); + } + *mc->mc_dbflag &= ~DB_STALE; + } + root = mc->mc_db->md_root; + + if (root == P_INVALID) { /* Tree is empty. */ + DPUTS("tree is empty"); + return MDB_NOTFOUND; + } + } + + mdb_cassert(mc, root > 1); + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) { +#if MDB_RPAGE_CACHE + if (mc->mc_pg[0]) + MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[0]); +#endif + if ((rc = MDB_PAGE_GET(mc, root, 1, &mc->mc_pg[0])) != 0) + return rc; + } + +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(mc->mc_txn->mt_env->me_flags)) + { + int i; + for (i=1; imc_snum; i++) + mdb_page_unref(mc->mc_txn, mc->mc_pg[i]); + } +#endif + mc->mc_snum = 1; + mc->mc_top = 0; + + DPRINTF(("db %d root page %"Yu" has flags 0x%X", + DDBI(mc), root, mc->mc_pg[0]->mp_flags)); + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc))) + return rc; + } + + if (flags & MDB_PS_ROOTONLY) + return MDB_SUCCESS; + + return mdb_page_search_root(mc, key, flags); +} + +static int +mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) +{ + MDB_txn *txn = mc->mc_txn; + pgno_t pg = mp->mp_pgno; + unsigned x = 0, ovpages = mp->mp_pages; + MDB_env *env = txn->mt_env; + MDB_ID pn = pg << 1; + MDB_page *freeme = NULL; + int rc; + + DPRINTF(("free ov page %"Yu" (%d)", pg, ovpages)); + /* If the page is dirty or on the spill list we just acquired it, + * so we should give it back to our current free list, if any. + * Otherwise put it onto the list of pages we freed in this txn. + * + * Won't create me_pghead: me_pglast must be inited along with it. + * Unsupported in nested txns: They would need to hide the page + * range in ancestor txns' dirty and spilled lists. + */ + if (IS_MUTABLE(txn, mp) && env->me_pghead && !txn->mt_parent) { + unsigned i, j; + pgno_t *mop; + MDB_ID2 *dl, ix, iy; + rc = mdb_midl_need(&env->me_pghead, ovpages); + if (rc) + return rc; + if (!IS_DIRTY_NW(txn, mp)) { /* spilled or WRITEMAP */ + MDB_IDL sl = txn->mt_spill_pgs; + if (sl) { + x = mdb_midl_search(sl, pn); + if (! (x <= sl[0] && sl[x] == pn)) + return MDB_PROBLEM; + /* This page is no longer spilled */ + if (x == sl[0]) + sl[0]--; + else + sl[x] |= 1; + } + goto release; + } + /* Remove from dirty list */ + dl = txn->mt_u.dirty_list; + x = dl[0].mid--; + for (ix = dl[x]; ix.mptr != mp; ix = iy) { + if (x > 1) { + x--; + iy = dl[x]; + dl[x] = ix; + } else { + mdb_cassert(mc, x > 1); + j = ++(dl[0].mid); + dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PROBLEM; + } + } + txn->mt_dirty_room++; + freeme = mp; +release: + /* Insert in me_pghead */ + mop = env->me_pghead; + j = mop[0] + ovpages; + for (i = mop[0]; i && mop[i] < pg; i--) + mop[j--] = mop[i]; + while (j>i) + mop[j--] = pg++; + mop[0] += ovpages; + } else { + rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); + if (rc) + return rc; + } + if (MC_OVPG(mc) == mp) + MC_SET_OVPG(mc, NULL); + mc->mc_db->md_overflow_pages -= ovpages; + + if (MDB_REMAPPING(env->me_flags)) + mdb_ovpage_unref_all(mc, mp); + if (freeme) + mdb_dpage_free(env, freeme); + + return 0; +} + +/** Return the data associated with a given node. + * @param[in] mc The cursor for this operation. + * @param[in] leaf The node being read. + * @param[out] data Updated to point to the node's data. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) +{ + MDB_page *omp; /* overflow page */ + MDB_ovpage ovp; + int rc; + + if (MC_OVPG(mc)) { + MDB_PAGE_UNREF(mc->mc_txn, MC_OVPG(mc)); + MC_SET_OVPG(mc, NULL); + } + data->mv_size = NODEDSZ(leaf); + if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { + data->mv_data = NODEDATA(leaf); + return MDB_SUCCESS; + } + + /* Read overflow data. + */ + memcpy(&ovp, NODEDATA(leaf), sizeof(ovp)); + { + if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp)) != 0) { + DPRINTF(("read overflow page %"Yu" failed", ovp.op_pgno)); + return rc; + } + } + data->mv_data = METADATA(omp); + MC_SET_OVPG(mc, omp); + + return MDB_SUCCESS; +} + +int +mdb_get(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + MDB_cursor mc; + MDB_xcursor mx; + int exact = 0, rc; + DKBUF; + + DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); + + if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + mdb_cursor_init(&mc, txn, dbi, &mx); + rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact); + /* unref all the pages when MDB_REMAP_CHUNKS - caller must copy the data + * before doing anything else + */ + MDB_CURSOR_UNREF(&mc, 1); + return rc; +} + +/** Find a sibling for a page. + * Replaces the page at the top of the cursor's stack with the + * specified sibling, if one exists. + * @param[in] mc The cursor for this operation. + * @param[in] move_right Non-zero if the right sibling is requested, + * otherwise the left sibling. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_cursor_sibling(MDB_cursor *mc, int move_right) +{ + int rc; + MDB_node *indx; + MDB_page *mp; +#if MDB_RPAGE_CACHE + MDB_page *op; +#endif + + if (mc->mc_snum < 2) { + return MDB_NOTFOUND; /* root has no siblings */ + } + +#if MDB_RPAGE_CACHE + op = mc->mc_pg[mc->mc_top]; +#endif + mdb_cursor_pop(mc); + DPRINTF(("parent page is page %"Yu", index %u", + mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); + + if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) + : (mc->mc_ki[mc->mc_top] == 0)) { + DPRINTF(("no more keys left, moving to %s sibling", + move_right ? "right" : "left")); + if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { + /* undo cursor_pop before returning */ + mc->mc_top++; + mc->mc_snum++; + return rc; + } + } else { + if (move_right) + mc->mc_ki[mc->mc_top]++; + else + mc->mc_ki[mc->mc_top]--; + DPRINTF(("just moving to %s index key %u", + move_right ? "right" : "left", mc->mc_ki[mc->mc_top])); + } + mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); + + MDB_PAGE_UNREF(mc->mc_txn, op); + + indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((rc = MDB_PAGE_GET(mc, NODEPGNO(indx), 1, &mp)) != 0) { + /* mc will be inconsistent if caller does mc_snum++ as above */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + return rc; + } + + mdb_cursor_push(mc, mp); + if (!move_right) + mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; + + return MDB_SUCCESS; +} + +/** Move the cursor to the next data item. */ +static int +mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + if ((mc->mc_flags & C_DEL && op == MDB_NEXT_DUP)) + return MDB_NOTFOUND; + + if (!(mc->mc_flags & C_INITIALIZED)) + return mdb_cursor_first(mc, key, data); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1) + return MDB_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_NEXT || op == MDB_NEXT_DUP) { + rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); + if (op != MDB_NEXT || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) + MDB_GET_KEY(leaf, key); + return rc; + } + } + else { + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_NEXT_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF(("cursor_next: top page is %"Yu" in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + if (mc->mc_flags & C_DEL) { + mc->mc_flags ^= C_DEL; + goto skip; + } + + if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { + DPUTS("=====> move to next sibling page"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + DPRINTF(("next page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]++; + +skip: + DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } else if (data) { + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the previous data item. */ +static int +mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdb_cursor_last(mc, key, data); + if (rc) + return rc; + mc->mc_ki[mc->mc_top]++; + } + + mp = mc->mc_pg[mc->mc_top]; + + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_PREV || op == MDB_PREV_DUP) { + rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); + if (op != MDB_PREV || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) { + MDB_GET_KEY(leaf, key); + mc->mc_flags &= ~C_EOF; + } + return rc; + } + } + else { + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_PREV_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF(("cursor_prev: top page is %"Yu" in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + + mc->mc_flags &= ~(C_EOF|C_DEL); + + if (mc->mc_ki[mc->mc_top] == 0) { + DPUTS("=====> move to prev sibling page"); + if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; + DPRINTF(("prev page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]--; + + DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (!IS_LEAF(mp)) + return MDB_CORRUPTED; + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } else if (data) { + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Set the cursor on a specific data item. */ +static int +mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op, int *exactp) +{ + int rc; + MDB_page *mp; + MDB_node *leaf = NULL; + DKBUF; + + if (key->mv_size == 0) + return MDB_BAD_VALSIZE; + + if (mc->mc_xcursor) { + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + } + + /* See if we're already on the right page */ + if (mc->mc_flags & C_INITIALIZED) { + MDB_val nodekey; + + mp = mc->mc_pg[mc->mc_top]; + if (!NUMKEYS(mp)) { + mc->mc_ki[mc->mc_top] = 0; + return MDB_NOTFOUND; + } + if (MP_FLAGS(mp) & P_LEAF2) { + nodekey.mv_size = mc->mc_db->md_pad; + nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, 0); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* Probably happens rarely, but first node on the page + * was the one we wanted. + */ + mc->mc_ki[mc->mc_top] = 0; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc > 0) { + unsigned int i; + unsigned int nkeys = NUMKEYS(mp); + if (nkeys > 1) { + if (MP_FLAGS(mp) & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + nkeys-1, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, nkeys-1); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* last node was the one we wanted */ + mc->mc_ki[mc->mc_top] = nkeys-1; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc < 0) { + if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { + /* This is definitely the right page, skip search_page */ + if (MP_FLAGS(mp) & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + mc->mc_ki[mc->mc_top], nodekey.mv_size); + } else { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* current node was the one we wanted */ + if (exactp) + *exactp = 1; + goto set1; + } + } + rc = 0; + mc->mc_flags &= ~C_EOF; + goto set2; + } + } + /* If any parents have right-sibs, search. + * Otherwise, there's nothing further. + */ + for (i=0; imc_top; i++) + if (mc->mc_ki[i] < + NUMKEYS(mc->mc_pg[i])-1) + break; + if (i == mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = nkeys; + return MDB_NOTFOUND; + } + } + if (!mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = 0; + if (op == MDB_SET_RANGE && !exactp) { + rc = 0; + goto set1; + } else + return MDB_NOTFOUND; + } + } else { + mc->mc_pg[0] = 0; + } + + rc = mdb_page_search(mc, key, 0); + if (rc != MDB_SUCCESS) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + +set2: + leaf = mdb_node_search(mc, key, exactp); + if (exactp != NULL && !*exactp) { + /* MDB_SET specified and not an exact match. */ + return MDB_NOTFOUND; + } + + if (leaf == NULL) { + DPUTS("===> inexact leaf not found, goto sibling"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; + return rc; /* no entries matched */ + } + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, 0); + } + +set1: + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + if (IS_LEAF2(mp)) { + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + } else { + int ex2, *ex2p; + if (op == MDB_GET_BOTH) { + ex2p = &ex2; + ex2 = 0; + } else { + ex2p = NULL; + } + rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); + if (rc != MDB_SUCCESS) + return rc; + } + } else if (data) { + if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { + MDB_val olddata; + MDB_cmp_func *dcmp; + if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS) + return rc; + dcmp = mc->mc_dbx->md_dcmp; + if (NEED_CMP_CLONG(dcmp, olddata.mv_size)) + dcmp = mdb_cmp_clong; + rc = dcmp(data, &olddata); + if (rc) { + if (op == MDB_GET_BOTH || rc > 0) + return MDB_NOTFOUND; + rc = 0; + } + *data = olddata; + + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + /* The key already matches in all other cases */ + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) + MDB_GET_KEY(leaf, key); + DPRINTF(("==> cursor placed on key [%s]", DKEY(key))); + + return rc; +} + +/** Move the cursor to the first item in the database. */ +static int +mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) { + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + } + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + mc->mc_ki[mc->mc_top] = 0; + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + if ( key ) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else if (data) { + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the last item in the database. */ +static int +mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) { + MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + } + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_LAST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; + mc->mc_flags |= C_INITIALIZED|C_EOF; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + if (key) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else if (data) { + if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) + return rc; + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +int +mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op) +{ + int rc; + int exact = 0; + int (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data); + + if (mc == NULL) + return EINVAL; + + if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + switch (op) { + case MDB_GET_CURRENT: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + } else { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int nkeys = NUMKEYS(mp); + if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) { + mc->mc_ki[mc->mc_top] = nkeys; + rc = MDB_NOTFOUND; + break; + } + rc = MDB_SUCCESS; + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } else { + MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY(leaf, key); + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); + } else { + rc = mdb_node_read(mc, leaf, data); + } + } + } + } + break; + case MDB_GET_BOTH: + case MDB_GET_BOTH_RANGE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + /* FALLTHRU */ + case MDB_SET: + case MDB_SET_KEY: + case MDB_SET_RANGE: + if (key == NULL) { + rc = EINVAL; + } else { + rc = mdb_cursor_set(mc, key, data, op, + op == MDB_SET_RANGE ? NULL : &exact); + } + break; + case MDB_GET_MULTIPLE: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + rc = MDB_SUCCESS; + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || + (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) + break; + goto fetchm; + case MDB_NEXT_MULTIPLE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); + if (rc == MDB_SUCCESS) { + if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + MDB_cursor *mx; +fetchm: + mx = &mc->mc_xcursor->mx_cursor; + data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * + mx->mc_db->md_pad; + data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); + mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; + } else { + rc = MDB_NOTFOUND; + } + } + break; + case MDB_PREV_MULTIPLE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_last(mc, key, data); + else + rc = MDB_SUCCESS; + if (rc == MDB_SUCCESS) { + MDB_cursor *mx = &mc->mc_xcursor->mx_cursor; + if (mx->mc_flags & C_INITIALIZED) { + rc = mdb_cursor_sibling(mx, 0); + if (rc == MDB_SUCCESS) + goto fetchm; + } else { + rc = MDB_NOTFOUND; + } + } + break; + case MDB_NEXT: + case MDB_NEXT_DUP: + case MDB_NEXT_NODUP: + rc = mdb_cursor_next(mc, key, data, op); + break; + case MDB_PREV: + case MDB_PREV_DUP: + case MDB_PREV_NODUP: + rc = mdb_cursor_prev(mc, key, data, op); + break; + case MDB_FIRST: + rc = mdb_cursor_first(mc, key, data); + break; + case MDB_FIRST_DUP: + mfunc = mdb_cursor_first; + mmove: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) { + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + rc = MDB_NOTFOUND; + break; + } + { + MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + MDB_GET_KEY(leaf, key); + rc = mdb_node_read(mc, leaf, data); + break; + } + } + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); + break; + case MDB_LAST: + rc = mdb_cursor_last(mc, key, data); + break; + case MDB_LAST_DUP: + mfunc = mdb_cursor_last; + goto mmove; + default: + DPRINTF(("unhandled/unimplemented cursor operation %u", op)); + rc = EINVAL; + break; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + return rc; +} + +/** Touch all the pages in the cursor stack. Set mc_top. + * Makes sure all the pages are writable, before attempting a write operation. + * @param[in] mc The cursor to operate on. + */ +static int +mdb_cursor_touch(MDB_cursor *mc) +{ + int rc = MDB_SUCCESS; + + if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & (DB_DIRTY|DB_DUPDATA))) { + /* Touch DB record of named DB */ + MDB_cursor mc2; + MDB_xcursor mcx; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); + if (rc) + return rc; + *mc->mc_dbflag |= DB_DIRTY; + } + mc->mc_top = 0; + if (mc->mc_snum) { + do { + rc = mdb_page_touch(mc); + } while (!rc && ++(mc->mc_top) < mc->mc_snum); + mc->mc_top = mc->mc_snum-1; + } + return rc; +} + +static void +mdb_subdb_adjust(MDB_cursor *mc, MDB_db *old, MDB_db *new) +{ + int delta; + + delta = new->md_branch_pages - old->md_branch_pages; + mc->mc_db->md_branch_pages += delta; + + delta = new->md_leaf_pages - old->md_leaf_pages; + mc->mc_db->md_leaf_pages += delta; +} + +/** Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDB_NOSPILL 0x8000 + +static int +_mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, + unsigned int flags) +{ + MDB_env *env; + MDB_node *leaf = NULL; + MDB_page *fp, *mp, *sub_root = NULL; + uint16_t fp_flags; + MDB_val xdata, *rdata, dkey, olddata; + MDB_db dummy; + int do_sub = 0, insert_key, insert_data; + unsigned int mcount = 0, dcount = 0, nospill; + size_t nsize; + int rc, rc2; + unsigned int nflags; + DKBUF; + + if (mc == NULL || key == NULL) + return EINVAL; + + env = mc->mc_txn->mt_env; + + /* Check this first so counter will always be zero on any + * early failures. + */ + if (flags & MDB_MULTIPLE) { + size_t tmp; + if (!data[1].mv_size) + return EINVAL; + + dcount = data[1].mv_size; + data[1].mv_size = 0; + if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) + return MDB_INCOMPATIBLE; + + /* check for overflow */ + tmp = data[0].mv_size * dcount; + if (tmp/dcount != data[0].mv_size) + return MDB_BAD_VALSIZE; + } + + nospill = flags & MDB_NOSPILL; + flags &= ~MDB_NOSPILL; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? MDB_IS_READONLY : MDB_BAD_TXN; + + if (key->mv_size-1 >= ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; + + if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size-1 >= ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; + +#if SIZE_MAX > MAXDATASIZE + if (data->mv_size > MAXDATASIZE) + return MDB_BAD_VALSIZE; +#endif + + DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", + DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); + + dkey.mv_size = 0; + + if (flags & MDB_CURRENT) { + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + rc = MDB_SUCCESS; + } else if (mc->mc_db->md_root == P_INVALID) { + /* new database, cursor has nothing to point to */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + rc = MDB_NO_ROOT; + } else { + int exact = 0; + MDB_val d2; + if (flags & MDB_APPEND) { + MDB_val k2; + rc = mdb_cursor_last(mc, &k2, &d2); + if (rc == 0) { + rc = mc->mc_dbx->md_cmp(key, &k2); + if (rc > 0) { + rc = MDB_NOTFOUND; + mc->mc_ki[mc->mc_top]++; + } else { + /* new key is <= last key */ + rc = MDB_KEYEXIST; + } + } + } else { + rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); + } + if ((flags & MDB_NOOVERWRITE) && rc == 0) { + DPRINTF(("duplicate key [%s]", DKEY(key))); + *data = d2; + return MDB_KEYEXIST; + } + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + /* Cursor is positioned, check for room in the dirty list */ + if (!nospill) { + if (flags & MDB_MULTIPLE) { + rdata = &xdata; + xdata.mv_size = data->mv_size * dcount; + } else { + rdata = data; + } + if ((rc2 = mdb_page_spill(mc, key, rdata))) + return rc2; + } + + if (rc == MDB_NO_ROOT) { + MDB_page *np; + /* new database, write a root leaf page */ + DPUTS("allocating new root leaf page"); + if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) { + return rc2; + } + mdb_cursor_push(mc, np); + mc->mc_db->md_root = np->mp_pgno; + mc->mc_db->md_depth++; + *mc->mc_dbflag |= DB_DIRTY; + if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) + == MDB_DUPFIXED) + MP_FLAGS(np) |= P_LEAF2; + mc->mc_flags |= C_INITIALIZED; + } else { + /* make sure all cursor pages are writable */ + rc2 = mdb_cursor_touch(mc); + if (rc2) + return rc2; + } + + insert_key = insert_data = rc; + if (insert_key) { + /* The key does not exist */ + DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top])); + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + LEAFSIZE(key, data) > env->me_nodemax) + { + /* Too big for a node, insert in sub-DB. Set up an empty + * "old sub-page" for prep_subDB to expand to a full page. + */ + fp_flags = P_LEAF; + fp = env->me_pbuf; + fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ + MP_LOWER(fp) = MP_UPPER(fp) = (PAGEHDRSZ-PAGEBASE); + olddata.mv_size = PAGEHDRSZ; + goto prep_subDB; + } + } else { + /* there's only a key anyway, so this is a no-op */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + char *ptr; + unsigned int ksize = mc->mc_db->md_pad; + if (key->mv_size != ksize) + return MDB_BAD_VALSIZE; + ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); + memcpy(ptr, key->mv_data, ksize); +fix_parent: + /* if overwriting slot 0 of leaf, need to + * update branch key if there is a parent page + */ + if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + unsigned short dtop = 1; + mc->mc_top--; + /* slot 0 is always an empty key, find real slot */ + while (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + mc->mc_top--; + dtop++; + } + if (mc->mc_ki[mc->mc_top]) + rc2 = mdb_update_key(mc, key); + else + rc2 = MDB_SUCCESS; + mc->mc_top += dtop; + if (rc2) + return rc2; + } + return MDB_SUCCESS; + } + +more: + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + olddata.mv_size = NODEDSZ(leaf); + olddata.mv_data = NODEDATA(leaf); + + /* DB has dups? */ + if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { + /* Prepare (sub-)page/sub-DB to accept the new item, + * if needed. fp: old sub-page or a header faking + * it. mp: new (sub-)page. offset: growth in page + * size. xdata: node data with new page or DB. + */ + unsigned i, offset = 0; + mp = fp = xdata.mv_data = env->me_pbuf; + mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + mp->mp_txnid = mc->mc_txn->mt_txnid; + + /* Was a single item before, must convert now */ + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + MDB_cmp_func *dcmp; + /* Just overwrite the current item */ + if (flags == MDB_CURRENT) + goto current; + dcmp = mc->mc_dbx->md_dcmp; + if (NEED_CMP_CLONG(dcmp, olddata.mv_size)) + dcmp = mdb_cmp_clong; + /* does data match? */ + if (!dcmp(data, &olddata)) { + if (flags & (MDB_NODUPDATA|MDB_APPENDDUP)) + return MDB_KEYEXIST; + /* overwrite it */ + goto current; + } + + /* Back up original data item */ + dkey.mv_size = olddata.mv_size; + dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); + + /* Make sub-page header for the dup items, with dummy body */ + MP_FLAGS(fp) = P_LEAF|P_SUBP; + MP_LOWER(fp) = (PAGEHDRSZ-PAGEBASE); + xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + MP_FLAGS(fp) |= P_LEAF2; + fp->mp_pad = data->mv_size; + xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ + } else { + xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + + (dkey.mv_size & 1) + (data->mv_size & 1); + } + MP_UPPER(fp) = xdata.mv_size - PAGEBASE; + olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ + } else if (leaf->mn_flags & F_SUBDATA) { + /* Data is on sub-DB, just store it */ + flags |= F_DUPDATA|F_SUBDATA; + goto put_sub; + } else { + /* Data is on sub-page */ + fp = olddata.mv_data; + switch (flags) { + default: + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + unsigned int left = SIZELEFT(fp); + offset = EVEN(NODESIZE + sizeof(indx_t) + + data->mv_size); + /* if there's enough space, just use it */ + if (offset < left) + offset = 0; + else + /* else grow by whatever we need */ + offset -= left; + break; + } + offset = fp->mp_pad; + if (SIZELEFT(fp) < offset) { + offset *= 4; /* space for 4 more */ + break; + } + /* FALLTHRU */ /* Big enough MDB_DUPFIXED sub-page */ + case MDB_CURRENT: + COPY_PGNO(MP_PGNO(fp), MP_PGNO(mp)); + mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; + flags |= F_DUPDATA; + goto put_sub; + } + xdata.mv_size = olddata.mv_size + offset; + } + + fp_flags = MP_FLAGS(fp); + if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { + /* Too big for a sub-page, convert to sub-DB */ + fp_flags &= ~P_SUBP; +prep_subDB: + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp_flags |= P_LEAF2; + dummy.md_pad = fp->mp_pad; + dummy.md_flags = MDB_DUPFIXED; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + dummy.md_flags |= MDB_INTEGERKEY; + } else { + dummy.md_pad = 0; + dummy.md_flags = 0; + } + dummy.md_depth = 1; + dummy.md_branch_pages = 0; + dummy.md_leaf_pages = 1; + dummy.md_overflow_pages = 0; + dummy.md_entries = NUMKEYS(fp); + xdata.mv_size = sizeof(MDB_db); + xdata.mv_data = &dummy; + if ((rc = mdb_page_alloc(mc, 1, &mp))) + return rc; + fp_flags |= mp->mp_flags; /* P_ADM_FLAGS */ + offset = env->me_pagespace + PAGEHDRSZ - olddata.mv_size; + flags |= F_DUPDATA|F_SUBDATA; + dummy.md_root = mp->mp_pgno; + sub_root = mp; + mc->mc_db->md_leaf_pages++; + } + if (mp != fp) { + MP_FLAGS(mp) = fp_flags; + MP_PAD(mp) = MP_PAD(fp); + MP_LOWER(mp) = MP_LOWER(fp); + MP_UPPER(mp) = MP_UPPER(fp) + offset; + if (fp_flags & P_LEAF2) { + memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); + } else { + memcpy((char *)mp + MP_UPPER(mp) + PAGEBASE, (char *)fp + MP_UPPER(fp) + PAGEBASE, + olddata.mv_size - MP_UPPER(fp) - PAGEBASE); + memcpy((char *)MP_PTRS(mp), (char *)MP_PTRS(fp), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0])); + for (i=0; imp_ptrs[i] += offset; + } + } + + rdata = &xdata; + flags |= F_DUPDATA; + do_sub = 1; + if (!insert_key) + mdb_node_del(mc, 0); + goto new_sub; + } +current: + /* LMDB passes F_SUBDATA in 'flags' to write a DB record */ + if ((leaf->mn_flags ^ flags) & F_SUBDATA) + return MDB_INCOMPATIBLE; + /* overflow page overwrites need special handling */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + MDB_ovpage ovp; + int ovpages, dpages = OVPAGES(env, data->mv_size); + + memcpy(&ovp, olddata.mv_data, sizeof(ovp)); + if ((rc2 = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp)) != 0) + return rc2; + ovpages = ovp.op_pages; + + /* Is the ov page big enough and from this txn (or a parent)? */ + if (ovpages >= dpages && IS_MUTABLE(mc->mc_txn, omp)) { + /* yes, overwrite it. Note in this case we don't + * bother to try shrinking the page if the new data + * is smaller than the overflow threshold. + */ + if (!IS_WRITABLE(mc->mc_txn, omp)) { + if (!IS_DIRTY_NW(mc->mc_txn, omp)) { + rc = mdb_page_unspill(mc->mc_txn, omp, &omp); + if (rc) + return rc; + } else { + /* It is writable only in a parent txn */ + size_t sz = (size_t) env->me_psize * ovpages, off; + MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages, 1); + MDB_ID2 id2; + if (!np) + return ENOMEM; + id2.mid = ovp.op_pgno; + id2.mptr = np; + /* Note - this page is already counted in parent's dirty_room */ + rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); + mdb_cassert(mc, rc2 == 0); + /* Currently we make the page look as with put() in the + * parent txn, in case the user peeks at MDB_RESERVEd + * or unused parts. Some users treat ovpages specially. + */ + if (!(flags & MDB_RESERVE)) { + /* Skip the part where LMDB will put *data. + * Copy end of page, adjusting alignment so + * compiler may copy words instead of bytes. + */ + off = (PAGEHDRSZ + data->mv_size) & -(int)sizeof(size_t); + memcpy((size_t *)((char *)np + off), + (size_t *)((char *)omp + off), sz - off); + sz = PAGEHDRSZ; + } + memcpy(np, omp, sz); /* Copy beginning of page */ + SET_PGTXNID(mc->mc_txn, np); + omp = np; + } + } + SETDSZ(leaf, data->mv_size); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(omp); + else + memcpy(METADATA(omp), data->mv_data, data->mv_size); + return MDB_SUCCESS; + } + if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) + return rc2; + } else if (data->mv_size == olddata.mv_size) { + /* same size, just replace it. Note that we could + * also reuse this node if the new data is smaller, + * but instead we opt to shrink the node in that case. + */ + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = olddata.mv_data; + else if (!(mc->mc_flags & C_SUB)) + memcpy(olddata.mv_data, data->mv_data, data->mv_size); + else { + memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); + goto fix_parent; + } + return MDB_SUCCESS; + } + mdb_node_del(mc, 0); + } + + rdata = data; + +new_sub: + nflags = flags & NODE_ADD_FLAGS; + nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata); + if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { + if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) + nflags &= ~MDB_APPEND; /* sub-page may need room to grow */ + if (!insert_key) + nflags |= MDB_SPLIT_REPLACE; + rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); + } else { + /* There is room already in this leaf page. */ + rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); + if (rc == 0) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp) continue; + if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) { + m3->mc_ki[i]++; + } + XCURSOR_REFRESH(m3, i, mp); + } + } + } + + if (rc == MDB_SUCCESS) { + /* Now store the actual data in the child DB. Note that we're + * storing the user data in the keys field, so there are strict + * size limits on dupdata. The actual data fields of the child + * DB are all zero size. + */ + if (do_sub) { + int xflags, new_dupdata; + mdb_size_t ecount; +put_sub: + xdata.mv_size = 0; + xdata.mv_data = ""; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((flags & (MDB_CURRENT|MDB_APPENDDUP)) == MDB_CURRENT) { + xflags = MDB_CURRENT|MDB_NOSPILL; + } else { + mdb_xcursor_init1(mc, leaf); + xflags = (flags & MDB_NODUPDATA) ? + MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; + } + if (sub_root) + mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root; + new_dupdata = (int)dkey.mv_size; + /* converted, write the original data first */ + if (dkey.mv_size) { + rc = _mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); + if (rc) + goto bad_sub; + /* we've done our job */ + dkey.mv_size = 0; + } + if (!(leaf->mn_flags & F_SUBDATA) || sub_root) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + MDB_xcursor *mx = mc->mc_xcursor; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[i] == mp) { + if (m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init2(m2, mx, new_dupdata); + } else if (!insert_key) { + XCURSOR_REFRESH(m2, i, mp); + } + } + } + } + ecount = mc->mc_xcursor->mx_db.md_entries; + if (flags & MDB_APPENDDUP) + xflags |= MDB_APPEND; + rc = _mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); + if (flags & F_SUBDATA) { + void *db = NODEDATA(leaf); + mdb_subdb_adjust(mc, db, &mc->mc_xcursor->mx_db); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } + insert_data = mc->mc_xcursor->mx_db.md_entries - ecount; + } + /* Increment count unless we just replaced an existing item. */ + if (insert_data) + mc->mc_db->md_entries++; + if (insert_key) { + /* Invalidate txn if we created an empty sub-DB */ + if (rc) + goto bad_sub; + /* If we succeeded and the key didn't exist before, + * make sure the cursor is marked valid. + */ + mc->mc_flags |= C_INITIALIZED; + } + if (flags & MDB_MULTIPLE) { + if (!rc) { + mcount++; + /* let caller know how many succeeded, if any */ + data[1].mv_size = mcount; + if (mcount < dcount) { + data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; + insert_key = insert_data = 0; + goto more; + } + } + } + return rc; +bad_sub: + if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ + rc = MDB_PROBLEM; + } + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, + unsigned int flags) +{ + DKBUF; + DDBUF; + int rc = _mdb_cursor_put(mc, key, data, flags); + MDB_TRACE(("%p, %"Z"u[%s], %"Z"u%s, %u", + mc, key ? key->mv_size:0, DKEY(key), data ? data->mv_size:0, + data ? mdb_dval(mc->mc_txn, mc->mc_dbi, data, dbuf):"", flags)); + return rc; +} + +static int +_mdb_cursor_del(MDB_cursor *mc, unsigned int flags) +{ + MDB_node *leaf; + MDB_page *mp; + int rc; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? MDB_IS_READONLY : MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + + if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL))) + return rc; + + rc = mdb_cursor_touch(mc); + if (rc) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + if (!IS_LEAF(mp)) + return MDB_CORRUPTED; + if (IS_LEAF2(mp)) + goto del_key; + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (flags & MDB_NODUPDATA) { + /* mdb_cursor_del0() will subtract the final entry */ + mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; + mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + } else { + if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + rc = _mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL); + if (rc) + return rc; + /* If sub-DB still has entries, we're done */ + if (mc->mc_xcursor->mx_db.md_entries) { + if (leaf->mn_flags & F_SUBDATA) { + /* update subDB info */ + void *db = NODEDATA(leaf); + mdb_subdb_adjust(mc, db, &mc->mc_xcursor->mx_db); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } else { + MDB_cursor *m2; + /* shrink fake page */ + mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + /* fix other sub-DB cursors pointed at fake pages on this page */ + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + XCURSOR_REFRESH(m2, mc->mc_top, mp); + } + } + } + mc->mc_db->md_entries--; + return rc; + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + } + /* otherwise fall thru and delete the sub-DB */ + } + + if (leaf->mn_flags & F_SUBDATA) { + /* add all the child DB's pages to the free list */ + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto fail; + } + } + /* LMDB passes F_SUBDATA in 'flags' to delete a DB record */ + else if ((leaf->mn_flags ^ flags) & F_SUBDATA) { + rc = MDB_INCOMPATIBLE; + goto fail; + } + + /* add overflow pages to free list */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + MDB_ovpage ovp; + + memcpy(&ovp, NODEDATA(leaf), sizeof(ovp)); + if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp)) || + (rc = mdb_ovpage_free(mc, omp))) + goto fail; + } + +del_key: + return mdb_cursor_del0(mc); + +fail: + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_cursor_del(MDB_cursor *mc, unsigned int flags) +{ + MDB_TRACE(("%p, %u", + mc, flags)); + return _mdb_cursor_del(mc, flags); +} + +/** Allocate and initialize new pages for a database. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc a cursor on the database being added to. + * @param[in] flags flags defining what type of page is being allocated. + * @param[in] num the number of pages to allocate. This is usually 1, + * unless allocating overflow pages for a large record. + * @param[out] mp Address of a page, or NULL on failure. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) +{ + MDB_page *np; + int rc; + + if ((rc = mdb_page_alloc(mc, num, &np))) + return rc; + DPRINTF(("allocated new mpage %"Yu", page size %u", + np->mp_pgno, mc->mc_txn->mt_env->me_psize)); + np->mp_flags |= flags; + np->mp_lower = (PAGEHDRSZ-PAGEBASE); + np->mp_upper = mc->mc_txn->mt_env->me_pagespace; + + if (IS_BRANCH(np)) + mc->mc_db->md_branch_pages++; + else if (IS_LEAF(np)) + mc->mc_db->md_leaf_pages++; + else if (IS_OVERFLOW(np)) { + mc->mc_db->md_overflow_pages += num; + np->mp_pages = num; + } + *mp = np; + + return 0; +} + +/** Calculate the size of a leaf node. + * The size depends on the environment's page size; if a data item + * is too large it will be put onto an overflow page and the node + * size will only include the key and not the data. Sizes are always + * rounded up to an even number of bytes, to guarantee 2-byte alignment + * of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @param[in] data The data for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) +{ + size_t sz; + + sz = LEAFSIZE(key, data); + if (sz > env->me_nodemax) { + /* put on overflow page */ + sz -= data->mv_size - sizeof(MDB_ovpage); + } + + return EVEN(sz + sizeof(indx_t)); +} + +/** Calculate the size of a branch node. + * The size should depend on the environment's page size but since + * we currently don't support spilling large keys onto overflow + * pages, it's simply the size of the #MDB_node header plus the + * size of the key. Sizes are always rounded up to an even number + * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_branch_size(MDB_env *env, MDB_val *key) +{ + size_t sz; + + sz = INDXSIZE(key); + if (sz > env->me_nodemax) { + /* put on overflow page */ + /* not implemented */ + /* sz -= key->size - sizeof(MDB_ovpage); */ + } + + return sz + sizeof(indx_t); +} + +/** Add a node to the page pointed to by the cursor. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc The cursor for this operation. + * @param[in] indx The index on the page where the new node should be added. + * @param[in] key The key for the new node. + * @param[in] data The data for the new node, if any. + * @param[in] pgno The page number, if adding a branch node. + * @param[in] flags Flags for the node. + * @return 0 on success, non-zero on failure. Possible errors are: + *
    + *
  • ENOMEM - failed to allocate overflow pages for the node. + *
  • MDB_PAGE_FULL - there is insufficient room in the page. This error + * should never happen since all callers already calculate the + * page's free space before calling this function. + *
+ */ +static int +mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) +{ + unsigned int i; + size_t node_size = NODESIZE; + ssize_t room; + indx_t ofs; + MDB_node *node; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_page *ofp = NULL; /* overflow page */ + void *ndata; + DKBUF; + + mdb_cassert(mc, MP_UPPER(mp) >= MP_LOWER(mp)); + + DPRINTF(("add to %s %spage %"Yu" index %i, data size %"Z"u key size %"Z"u [%s]", + IS_LEAF(mp) ? "leaf" : "branch", + IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, + key ? key->mv_size : 0, key ? DKEY(key) : "null")); + + if (IS_LEAF2(mp)) { + /* Move higher keys up one slot. */ + int ksize = mc->mc_db->md_pad, dif; + char *ptr = LEAF2KEY(mp, indx, ksize); + dif = NUMKEYS(mp) - indx; + if (dif > 0) + memmove(ptr+ksize, ptr, dif*ksize); + /* insert new key */ + memcpy(ptr, key->mv_data, ksize); + + /* Just using these for counting */ + MP_LOWER(mp) += sizeof(indx_t); + MP_UPPER(mp) -= ksize - sizeof(indx_t); + return MDB_SUCCESS; + } + + room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); + if (key != NULL) + node_size += EVEN(key->mv_size); + if (IS_LEAF(mp)) { + mdb_cassert(mc, key && data); + if (F_ISSET(flags, F_BIGDATA)) { + /* Data already on overflow page. */ + node_size += sizeof(MDB_ovpage); + } else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) { + int ovpages = OVPAGES(mc->mc_txn->mt_env, data->mv_size); + int rc; + /* Put data on overflow page. */ + DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", + data->mv_size, node_size+data->mv_size)); + node_size = EVEN(node_size + sizeof(MDB_ovpage)); + if ((ssize_t)node_size > room) + goto full; + if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) + return rc; + DPRINTF(("allocated overflow page %"Yu, ofp->mp_pgno)); + flags |= F_BIGDATA; + goto update; + } else { + node_size += EVEN(data->mv_size); + } + } + if ((ssize_t)node_size > room) + goto full; + +update: + /* Move higher pointers up one slot. */ + for (i = NUMKEYS(mp); i > indx; i--) + MP_PTRS(mp)[i] = MP_PTRS(mp)[i - 1]; + + /* Adjust free space offsets. */ + ofs = MP_UPPER(mp) - node_size; + mdb_cassert(mc, ofs >= MP_LOWER(mp) + sizeof(indx_t)); + MP_PTRS(mp)[indx] = ofs; + MP_UPPER(mp) = ofs; + MP_LOWER(mp) += sizeof(indx_t); + + /* Write the node data. */ + node = NODEPTR(mp, indx); + node->mn_ksize = (key == NULL) ? 0 : key->mv_size; + node->mn_flags = flags; + if (IS_LEAF(mp)) + SETDSZ(node,data->mv_size); + else + SETPGNO(node,pgno); + + if (key) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + if (IS_LEAF(mp)) { + ndata = NODEDATA(node); + if (ofp == NULL) { + if (F_ISSET(flags, F_BIGDATA)) + memcpy(ndata, data->mv_data, sizeof(MDB_ovpage)); + else if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = ndata; + else + memcpy(ndata, data->mv_data, data->mv_size); + } else { + MDB_ovpage ovp; + ovp.op_pgno = ofp->mp_pgno; + ovp.op_txnid = mc->mc_txn->mt_txnid; + ovp.op_pages = ofp->mp_pages; + memcpy(ndata, &ovp, sizeof(MDB_ovpage)); + ndata = METADATA(ofp); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = ndata; + else + memcpy(ndata, data->mv_data, data->mv_size); + } + } + + return MDB_SUCCESS; + +full: + DPRINTF(("not enough room in page %"Yu", got %u ptrs", + mdb_dbg_pgno(mp), NUMKEYS(mp))); + DPRINTF(("upper-lower = %u - %u = %"Z"d", MP_UPPER(mp),MP_LOWER(mp),room)); + DPRINTF(("node size = %"Z"u", node_size)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_FULL; +} + +/** Delete the specified node from a page. + * @param[in] mc Cursor pointing to the node to delete. + * @param[in] ksize The size of a node. Only used if the page is + * part of a #MDB_DUPFIXED database. + */ +static void +mdb_node_del(MDB_cursor *mc, int ksize) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + indx_t indx = mc->mc_ki[mc->mc_top]; + unsigned int sz; + indx_t i, j, numkeys, ptr; + MDB_node *node; + char *base; + + DPRINTF(("delete node %u on %s page %"Yu, indx, + IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); + numkeys = NUMKEYS(mp); + mdb_cassert(mc, indx < numkeys); + + if (IS_LEAF2(mp)) { + int x = numkeys - 1 - indx; + base = LEAF2KEY(mp, indx, ksize); + if (x) + memmove(base, base + ksize, x * ksize); + MP_LOWER(mp) -= sizeof(indx_t); + MP_UPPER(mp) += ksize - sizeof(indx_t); + return; + } + + node = NODEPTR(mp, indx); + sz = NODESIZE + EVEN(node->mn_ksize); + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + sz += sizeof(MDB_ovpage); + else + sz += EVEN(NODEDSZ(node)); + } + + ptr = MP_PTRS(mp)[indx]; + for (i = j = 0; i < numkeys; i++) { + if (i != indx) { + MP_PTRS(mp)[j] = MP_PTRS(mp)[i]; + if (MP_PTRS(mp)[i] < ptr) + MP_PTRS(mp)[j] += sz; + j++; + } + } + + base = (char *)mp + MP_UPPER(mp) + PAGEBASE; + memmove(base + sz, base, ptr - MP_UPPER(mp)); + + MP_LOWER(mp) -= sizeof(indx_t); + MP_UPPER(mp) += sz; +} + +/** Compact the main page after deleting a node on a subpage. + * @param[in] mp The main page to operate on. + * @param[in] indx The index of the subpage on the main page. + */ +static void +mdb_node_shrink(MDB_page *mp, indx_t indx) +{ + MDB_node *node; + MDB_page *sp, *xp; + char *base; + indx_t delta, nsize, len, ptr; + int i; + + node = NODEPTR(mp, indx); + sp = (MDB_page *)NODEDATA(node); + delta = SIZELEFT(sp); + nsize = NODEDSZ(node) - delta; + + /* Prepare to shift upward, set len = length(subpage part to shift) */ + if (IS_LEAF2(sp)) { + len = nsize; + if (nsize & 1) + return; /* do not make the node uneven-sized */ + } else { + xp = (MDB_page *)((char *)sp + delta); /* destination subpage */ + for (i = NUMKEYS(sp); --i >= 0; ) + MP_PTRS(xp)[i] = MP_PTRS(sp)[i] - delta; + len = PAGEHDRSZ; + } + MP_UPPER(sp) = MP_LOWER(sp); + COPY_PGNO(MP_PGNO(sp), mp->mp_pgno); + SETDSZ(node, nsize); + + /* Shift upward */ + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + delta, base, (char *)sp + len - base); + + ptr = mp->mp_ptrs[indx]; + for (i = NUMKEYS(mp); --i >= 0; ) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] += delta; + } + mp->mp_upper += delta; +} + +/** Initial setup of a sorted-dups cursor. + * Sorted duplicates are implemented as a sub-database for the given key. + * The duplicate data items are actually keys of the sub-database. + * Operations on the duplicate data items are performed using a sub-cursor + * initialized when the sub-database is first accessed. This function does + * the preliminary setup of the sub-cursor, filling in the fields that + * depend only on the parent DB. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + */ +static void +mdb_xcursor_init0(MDB_cursor *mc) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + mx->mx_cursor.mc_xcursor = NULL; + mx->mx_cursor.mc_txn = mc->mc_txn; + mx->mx_cursor.mc_db = &mx->mx_db; + mx->mx_cursor.mc_dbx = &mx->mx_dbx; + mx->mx_cursor.mc_dbi = mc->mc_dbi; + mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + MC_SET_OVPG(&mx->mx_cursor, NULL); + mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP)); + mx->mx_dbx.md_name.mv_size = 0; + mx->mx_dbx.md_name.mv_data = NULL; + mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; + mx->mx_dbx.md_dcmp = NULL; + mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; +} + +/** Final setup of a sorted-dups cursor. + * Sets up the fields that depend on the data from the main cursor. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + * @param[in] node The data containing the #MDB_db record for the + * sorted-dup database. + */ +static void +mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + mx->mx_cursor.mc_flags &= C_SUB|C_ORIG_RDONLY|C_WRITEMAP; + if (node->mn_flags & F_SUBDATA) { + memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); + mx->mx_cursor.mc_pg[0] = 0; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + } else { + MDB_page *fp = NODEDATA(node); + mx->mx_db.md_pad = 0; + mx->mx_db.md_flags = 0; + mx->mx_db.md_depth = 1; + mx->mx_db.md_branch_pages = 0; + mx->mx_db.md_leaf_pages = 1; + mx->mx_db.md_overflow_pages = 0; + mx->mx_db.md_entries = NUMKEYS(fp); + COPY_PGNO(mx->mx_db.md_root, MP_PGNO(fp)); + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags |= C_INITIALIZED; + mx->mx_cursor.mc_pg[0] = fp; + mx->mx_cursor.mc_ki[0] = 0; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + mx->mx_db.md_flags = MDB_DUPFIXED; + mx->mx_db.md_pad = fp->mp_pad; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + mx->mx_db.md_flags |= MDB_INTEGERKEY; + } + } + DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; + if (NEED_CMP_CLONG(mx->mx_dbx.md_cmp, mx->mx_db.md_pad)) + mx->mx_dbx.md_cmp = mdb_cmp_clong; +} + + +/** Fixup a sorted-dups cursor due to underlying update. + * Sets up some fields that depend on the data from the main cursor. + * Almost the same as init1, but skips initialization steps if the + * xcursor had already been used. + * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up. + * @param[in] src_mx The xcursor of an up-to-date cursor. + * @param[in] new_dupdata True if converting from a non-#F_DUPDATA item. + */ +static void +mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (new_dupdata) { + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags |= C_INITIALIZED; + mx->mx_cursor.mc_ki[0] = 0; + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; +#if UINT_MAX < MDB_SIZE_MAX /* matches mdb_xcursor_init1:NEED_CMP_CLONG() */ + mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; +#endif + } else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { + return; + } + mx->mx_db = src_mx->mx_db; + mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; + DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); +} + +/** Initialize a cursor for a given transaction and database. */ +static void +mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) +{ + mc->mc_next = NULL; + mc->mc_backup = NULL; + mc->mc_dbi = dbi; + mc->mc_txn = txn; + mc->mc_db = &txn->mt_dbs[dbi]; + mc->mc_dbx = &txn->mt_dbxs[dbi]; + mc->mc_dbflag = &txn->mt_dbflags[dbi]; + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_pg[0] = 0; + mc->mc_ki[0] = 0; + MC_SET_OVPG(mc, NULL); + mc->mc_flags = txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP); + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { + mdb_tassert(txn, mx != NULL); + mc->mc_xcursor = mx; + mdb_xcursor_init0(mc); + } else { + mc->mc_xcursor = NULL; + } + if (*mc->mc_dbflag & DB_STALE) { + mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); + } +} + +int +mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) +{ + MDB_cursor *mc; + size_t size = sizeof(MDB_cursor); + + if (!ret || !TXN_DBI_EXIST(txn, dbi, DB_VALID)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + if (dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EINVAL; + + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) + size += sizeof(MDB_xcursor); + + if ((mc = malloc(size)) != NULL) { + mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); + if (txn->mt_cursors) { + mc->mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = mc; + mc->mc_flags |= C_UNTRACK; + } + } else { + return ENOMEM; + } + + MDB_TRACE(("%p, %u = %p", txn, dbi, mc)); + *ret = mc; + + return MDB_SUCCESS; +} + +int +mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) +{ + if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi, DB_VALID)) + return EINVAL; + + if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); + return MDB_SUCCESS; +} + +/* Return the count of duplicate data items for the current key */ +int +mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) +{ + MDB_node *leaf; + + if (mc == NULL || countp == NULL) + return EINVAL; + + if (mc->mc_xcursor == NULL) + return MDB_INCOMPATIBLE; + + if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (!mc->mc_snum) + return MDB_NOTFOUND; + + if (mc->mc_flags & C_EOF) { + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + mc->mc_flags ^= C_EOF; + } + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + *countp = 1; + } else { + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + return EINVAL; + + *countp = mc->mc_xcursor->mx_db.md_entries; + } + return MDB_SUCCESS; +} + +void +mdb_cursor_close(MDB_cursor *mc) +{ + MDB_TRACE(("%p", mc)); + if (mc) { + MDB_CURSOR_UNREF(mc, 0); + } + if (mc && !mc->mc_backup) { + /* Remove from txn, if tracked. + * A read-only txn (!C_UNTRACK) may have been freed already, + * so do not peek inside it. Only write txns track cursors. + */ + if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { + MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + } + free(mc); + } +} + +MDB_txn * +mdb_cursor_txn(MDB_cursor *mc) +{ + if (!mc) return NULL; + return mc->mc_txn; +} + +MDB_dbi +mdb_cursor_dbi(MDB_cursor *mc) +{ + return mc->mc_dbi; +} + +int +mdb_cursor_is_db(MDB_cursor *mc) +{ + + if (mc && (mc->mc_flags & C_INITIALIZED) && mc->mc_snum) { + MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_SUBDATA) + return 1; + } + return 0; +} + +/** Replace the key for a branch node with a new key. + * Set #MDB_TXN_ERROR on failure. + * @param[in] mc Cursor pointing to the node to operate on. + * @param[in] key The new key to use. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_update_key(MDB_cursor *mc, MDB_val *key) +{ + MDB_page *mp; + MDB_node *node; + char *base; + size_t len; + int delta, ksize, oksize; + indx_t ptr, i, numkeys, indx; + DKBUF; + + indx = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + node = NODEPTR(mp, indx); + ptr = mp->mp_ptrs[indx]; +#if MDB_DEBUG + { + MDB_val k2; + char kbuf2[DKBUF_MAXKEYSIZE*2+1]; + k2.mv_data = NODEKEY(node); + k2.mv_size = node->mn_ksize; + DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Yu, + indx, ptr, + mdb_dkey(&k2, kbuf2), + DKEY(key), + mp->mp_pgno)); + } +#endif + + /* Sizes must be 2-byte aligned. */ + ksize = EVEN(key->mv_size); + oksize = EVEN(node->mn_ksize); + delta = ksize - oksize; + + /* Shift node contents if EVEN(key length) changed. */ + if (delta) { + if (delta > 0 && SIZELEFT(mp) < delta) { + pgno_t pgno; + /* not enough space left, do a delete and split */ + DPRINTF(("Not enough room, delta = %d, splitting...", delta)); + pgno = NODEPGNO(node); + mdb_node_del(mc, 0); + return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); + } + + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] -= delta; + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + len = ptr - mp->mp_upper + NODESIZE; + memmove(base - delta, base, len); + mp->mp_upper -= delta; + + node = NODEPTR(mp, indx); + } + + /* But even if no shift was needed, update ksize */ + if (node->mn_ksize != key->mv_size) + node->mn_ksize = key->mv_size; + + if (key->mv_size) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + return MDB_SUCCESS; +} + +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); + +/** Perform \b act while tracking temporary cursor \b mn */ +#define WITH_CURSOR_TRACKING(mn, act) do { \ + MDB_cursor dummy, *tracked, **tp = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \ + if ((mn).mc_flags & C_SUB) { \ + dummy.mc_flags = C_INITIALIZED; \ + dummy.mc_xcursor = (MDB_xcursor *)&(mn); \ + tracked = &dummy; \ + } else { \ + tracked = &(mn); \ + } \ + tracked->mc_next = *tp; \ + *tp = tracked; \ + { act; } \ + *tp = tracked->mc_next; \ +} while (0) + +/** Move a node from csrc to cdst. + */ +static int +mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) +{ + MDB_node *srcnode; + MDB_val key, data; + pgno_t srcpg; + MDB_cursor mn; + int rc; + unsigned short flags; + + DKBUF; + + /* Mark src and dst as dirty. */ + if ((rc = mdb_page_touch(csrc)) || + (rc = mdb_page_touch(cdst))) + return rc; + + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); + data.mv_size = 0; + data.mv_data = NULL; + srcpg = 0; + flags = 0; + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); + mdb_cassert(csrc, !((size_t)srcnode & 1)); + srcpg = NODEPGNO(srcnode); + flags = srcnode->mn_flags; + if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + unsigned int snum = csrc->mc_snum; + MDB_node *s2; + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(csrc); + if (rc) + return rc; + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + csrc->mc_snum = snum--; + csrc->mc_top = snum; + } else { + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + } + mn.mc_xcursor = NULL; + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { + unsigned int snum = cdst->mc_snum; + MDB_node *s2; + MDB_val bkey; + /* must find the lowest key below dst */ + mdb_cursor_copy(cdst, &mn); + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + bkey.mv_size = mn.mc_db->md_pad; + bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + bkey.mv_size = NODEKSZ(s2); + bkey.mv_data = NODEKEY(s2); + } + mn.mc_snum = snum--; + mn.mc_top = snum; + mn.mc_ki[snum] = 0; + rc = mdb_update_key(&mn, &bkey); + if (rc) + return rc; + } + + DPRINTF(("moving %s node %u [%s] on page %"Yu" to node %u on page %"Yu, + IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", + csrc->mc_ki[csrc->mc_top], + DKEY(&key), + csrc->mc_pg[csrc->mc_top]->mp_pgno, + cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno)); + + /* Add the node to the destination page. + */ + rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); + if (rc != MDB_SUCCESS) + return rc; + + /* Delete the node from the source page. + */ + mdb_node_del(csrc, key.mv_size); + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + MDB_page *mpd, *mps; + + mps = csrc->mc_pg[csrc->mc_top]; + /* If we're adding on the left, bump others up */ + if (fromleft) { + mpd = cdst->mc_pg[csrc->mc_top]; + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) + continue; + if (m3 != cdst && + m3->mc_pg[csrc->mc_top] == mpd && + m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) { + m3->mc_ki[csrc->mc_top]++; + } + if (m3 !=csrc && + m3->mc_pg[csrc->mc_top] == mps && + m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + m3->mc_ki[csrc->mc_top-1]++; + } + if (IS_LEAF(mps)) + XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); + } + } else + /* Adding on the right, bump others down */ + { + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) + continue; + if (m3->mc_pg[csrc->mc_top] == mps) { + if (!m3->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + m3->mc_ki[csrc->mc_top-1]--; + } else { + m3->mc_ki[csrc->mc_top]--; + } + if (IS_LEAF(mps)) + XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); + } + } + } + } + + /* Update the parent separators. + */ + if (csrc->mc_ki[csrc->mc_top] == 0) { + if (csrc->mc_ki[csrc->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for source page %"Yu" to [%s]", + csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(csrc, &mn); + mn.mc_snum--; + mn.mc_top--; + /* We want mdb_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, + rc = mdb_update_key(&mn, &key)); + if (rc) + return rc; + } + if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + MDB_val nullkey; + indx_t ix = csrc->mc_ki[csrc->mc_top]; + nullkey.mv_size = 0; + csrc->mc_ki[csrc->mc_top] = 0; + rc = mdb_update_key(csrc, &nullkey); + csrc->mc_ki[csrc->mc_top] = ix; + mdb_cassert(csrc, rc == MDB_SUCCESS); + } + } + + if (cdst->mc_ki[cdst->mc_top] == 0) { + if (cdst->mc_ki[cdst->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for destination page %"Yu" to [%s]", + cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(cdst, &mn); + mn.mc_snum--; + mn.mc_top--; + /* We want mdb_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, + rc = mdb_update_key(&mn, &key)); + if (rc) + return rc; + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { + MDB_val nullkey; + indx_t ix = cdst->mc_ki[cdst->mc_top]; + nullkey.mv_size = 0; + cdst->mc_ki[cdst->mc_top] = 0; + rc = mdb_update_key(cdst, &nullkey); + cdst->mc_ki[cdst->mc_top] = ix; + mdb_cassert(cdst, rc == MDB_SUCCESS); + } + } + + return MDB_SUCCESS; +} + +/** Merge one page into another. + * The nodes from the page pointed to by \b csrc will + * be copied to the page pointed to by \b cdst and then + * the \b csrc page will be freed. + * @param[in] csrc Cursor pointing to the source page. + * @param[in] cdst Cursor pointing to the destination page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_page *psrc, *pdst; + MDB_node *srcnode; + MDB_val key, data; + unsigned nkeys; + int rc; + indx_t i, j; + + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + + DPRINTF(("merging page %"Yu" into %"Yu, psrc->mp_pgno, pdst->mp_pgno)); + + mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ + mdb_cassert(csrc, cdst->mc_snum > 1); + + /* Mark dst as dirty. */ + if ((rc = mdb_page_touch(cdst))) + return rc; + + /* get dst page again now that we've touched it. */ + pdst = cdst->mc_pg[cdst->mc_top]; + + /* Move all nodes from src to dst. + */ + j = nkeys = NUMKEYS(pdst); + if (IS_LEAF2(psrc)) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = METADATA(psrc); + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); + if (rc != MDB_SUCCESS) + return rc; + key.mv_data = (char *)key.mv_data + key.mv_size; + } + } else { + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + srcnode = NODEPTR(psrc, i); + if (i == 0 && IS_BRANCH(psrc)) { + MDB_cursor mn; + MDB_node *s2; + mdb_cursor_copy(csrc, &mn); + mn.mc_xcursor = NULL; + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + key.mv_size = mn.mc_db->md_pad; + key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + } else { + key.mv_size = srcnode->mn_ksize; + key.mv_data = NODEKEY(srcnode); + } + + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); + if (rc != MDB_SUCCESS) + return rc; + } + } + + DPRINTF(("dst page %"Yu" now has %u keys (%.1f%% filled)", + pdst->mp_pgno, NUMKEYS(pdst), + (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); + + /* Unlink the src page from parent and add to free list. + */ + csrc->mc_top--; + mdb_node_del(csrc, 0); + if (csrc->mc_ki[csrc->mc_top] == 0) { + key.mv_size = 0; + rc = mdb_update_key(csrc, &key); + if (rc) { + csrc->mc_top++; + return rc; + } + } + csrc->mc_top++; + + psrc = csrc->mc_pg[csrc->mc_top]; + /* If not operating on FreeDB, allow this page to be reused + * in this txn. Otherwise just add to free list. + */ + rc = mdb_page_loose(csrc, psrc); + if (rc) + return rc; + if (IS_LEAF(psrc)) + csrc->mc_db->md_leaf_pages--; + else + csrc->mc_db->md_branch_pages--; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + unsigned int top = csrc->mc_top; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_snum < csrc->mc_snum) continue; + if (m3->mc_pg[top] == psrc) { + m3->mc_pg[top] = pdst; + m3->mc_ki[top] += nkeys; + m3->mc_ki[top-1] = cdst->mc_ki[top-1]; + } else if (m3->mc_pg[top-1] == csrc->mc_pg[top-1] && + m3->mc_ki[top-1] > csrc->mc_ki[top-1]) { + m3->mc_ki[top-1]--; + } + if (IS_LEAF(psrc)) + XCURSOR_REFRESH(m3, top, m3->mc_pg[top]); + } + } + { + unsigned int snum = cdst->mc_snum; + uint16_t depth = cdst->mc_db->md_depth; + mdb_cursor_pop(cdst); + rc = mdb_rebalance(cdst); + /* Did the tree height change? */ + if (depth != cdst->mc_db->md_depth) + snum += cdst->mc_db->md_depth - depth; + cdst->mc_snum = snum; + cdst->mc_top = snum-1; + } + return rc; +} + +/** Copy the contents of a cursor. + * @param[in] csrc The cursor to copy from. + * @param[out] cdst The cursor to copy to. + */ +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) +{ + unsigned int i; + + cdst->mc_txn = csrc->mc_txn; + cdst->mc_dbi = csrc->mc_dbi; + cdst->mc_db = csrc->mc_db; + cdst->mc_dbx = csrc->mc_dbx; + cdst->mc_snum = csrc->mc_snum; + cdst->mc_top = csrc->mc_top; + cdst->mc_flags = csrc->mc_flags; + MC_SET_OVPG(cdst, MC_OVPG(csrc)); + + for (i=0; imc_snum; i++) { + cdst->mc_pg[i] = csrc->mc_pg[i]; + cdst->mc_ki[i] = csrc->mc_ki[i]; + } +} + +/** Rebalance the tree after a delete operation. + * @param[in] mc Cursor pointing to the page where rebalancing + * should begin. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_rebalance(MDB_cursor *mc) +{ + MDB_node *node; + int rc, fromleft; + unsigned int ptop, minkeys, thresh; + MDB_cursor mn; + indx_t oldki; + + if (IS_BRANCH(mc->mc_pg[mc->mc_top])) { + minkeys = 2; + thresh = 1; + } else { + minkeys = 1; + thresh = FILL_THRESHOLD; + } + DPRINTF(("rebalancing %s page %"Yu" (has %u keys, %.1f%% full)", + IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); + + if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh && + NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { + DPRINTF(("no need to rebalance page %"Yu", above fill threshold", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); + return MDB_SUCCESS; + } + + if (mc->mc_snum < 2) { + MDB_page *mp = mc->mc_pg[0]; + if (IS_SUBP(mp)) { + DPUTS("Can't rebalance a subpage, ignoring"); + return MDB_SUCCESS; + } + if (NUMKEYS(mp) == 0) { + DPUTS("tree is completely empty"); + mc->mc_db->md_root = P_INVALID; + mc->mc_db->md_depth = 0; + mc->mc_db->md_leaf_pages = 0; + rc = mdb_page_loose(mc, mp); + if (rc) + return rc; + /* Adjust cursors pointing to mp */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (!(m3->mc_flags & C_INITIALIZED) || (m3->mc_snum < mc->mc_snum)) + continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum = 0; + m3->mc_top = 0; + m3->mc_flags &= ~C_INITIALIZED; + } + } + } + } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { + int i; + DPUTS("collapsing root page!"); + rc = mdb_page_loose(mc, mp); + if (rc) + return rc; + mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); + rc = MDB_PAGE_GET(mc, mc->mc_db->md_root, 1, &mc->mc_pg[0]); + if (rc) + return rc; + mc->mc_db->md_depth--; + mc->mc_db->md_branch_pages--; + mc->mc_ki[0] = mc->mc_ki[1]; + for (i = 1; imc_db->md_depth; i++) { + mc->mc_pg[i] = mc->mc_pg[i+1]; + mc->mc_ki[i] = mc->mc_ki[i+1]; + } + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc) continue; + if (!(m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_pg[0] == mp) { + for (i=0; imc_db->md_depth; i++) { + m3->mc_pg[i] = m3->mc_pg[i+1]; + m3->mc_ki[i] = m3->mc_ki[i+1]; + } + m3->mc_snum--; + m3->mc_top--; + } + } + } + } else + DPUTS("root page doesn't need rebalancing"); + return MDB_SUCCESS; + } + + /* The parent (branch page) must have at least 2 pointers, + * otherwise the tree is invalid. + */ + ptop = mc->mc_top-1; + mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1); + + /* Leaf page fill factor is below the threshold. + * Try to move keys from left or right neighbor, or + * merge with a neighbor page. + */ + + /* Find neighbors. + */ + mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + + oldki = mc->mc_ki[mc->mc_top]; + if (mc->mc_ki[ptop] == 0) { + /* We're the leftmost leaf in our parent. + */ + DPUTS("reading right neighbor"); + mn.mc_ki[ptop]++; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mn.mc_pg[mn.mc_top]); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = 0; + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + fromleft = 0; + } else { + /* There is at least one neighbor to the left. + */ + DPUTS("reading left neighbor"); + mn.mc_ki[ptop]--; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mn.mc_pg[mn.mc_top]); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; + mc->mc_ki[mc->mc_top] = 0; + fromleft = 1; + } + + DPRINTF(("found neighbor page %"Yu" (%u keys, %.1f%% full)", + mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); + + /* If the neighbor page is above threshold and has enough keys, + * move one key from it. Otherwise we should try to merge them. + * (A branch page must never have less than 2 keys.) + */ + if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { + rc = mdb_node_move(&mn, mc, fromleft); + if (fromleft) { + /* if we inserted on left, bump position up */ + oldki++; + } + } else { + if (!fromleft) { + rc = mdb_page_merge(&mn, mc); + } else { + oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); + mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; + /* We want mdb_rebalance to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, + rc = mdb_page_merge(mc, &mn)); + mdb_cursor_copy(&mn, mc); + } + mc->mc_flags &= ~C_EOF; + } + mc->mc_ki[mc->mc_top] = oldki; + return rc; +} + +/** Complete a delete operation started by #mdb_cursor_del(). */ +static int +mdb_cursor_del0(MDB_cursor *mc) +{ + int rc; + MDB_page *mp; + indx_t ki; + unsigned int nkeys; + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + ki = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + mdb_node_del(mc, mc->mc_db->md_pad); + mc->mc_db->md_entries--; + { + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3 == mc || m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] == ki) { + m3->mc_flags |= C_DEL; + if (mc->mc_db->md_flags & MDB_DUPSORT) { + /* Sub-cursor referred into dataset which is gone */ + m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + } + continue; + } else if (m3->mc_ki[mc->mc_top] > ki) { + m3->mc_ki[mc->mc_top]--; + } + XCURSOR_REFRESH(m3, mc->mc_top, mp); + } + } + } + rc = mdb_rebalance(mc); + if (rc) + goto fail; + + /* DB is totally empty now, just bail out. + * Other cursors adjustments were already done + * by mdb_rebalance and aren't needed here. + */ + if (!mc->mc_snum) { + mc->mc_flags |= C_EOF; + return rc; + } + + mp = mc->mc_pg[mc->mc_top]; + nkeys = NUMKEYS(mp); + + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) { + /* if m3 points past last node in page, find next sibling */ + if (m3->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(m3, 1); + if (rc == MDB_NOTFOUND) { + m3->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + continue; + } + if (rc) + goto fail; + } + if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) { + MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); + /* If this node has dupdata, it may need to be reinited + * because its data has moved. + * If the xcursor was not initd it must be reinited. + * Else if node points to a subDB, nothing is needed. + * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset. + */ + if (node->mn_flags & F_DUPDATA) { + if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + if (!(node->mn_flags & F_SUBDATA)) + m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); + } else { + mdb_xcursor_init1(m3, node); + rc = mdb_cursor_first(&m3->mc_xcursor->mx_cursor, NULL, NULL); + if (rc) + goto fail; + } + } + m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL; + } + } + } + } + mc->mc_flags |= C_DEL; + +fail: + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_del(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + DKBUF; + DDBUF; + if (!key || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) + return (txn->mt_flags & MDB_TXN_RDONLY) ? MDB_IS_READONLY : MDB_BAD_TXN; + + if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) { + /* must ignore any data */ + data = NULL; + } + + MDB_TRACE(("%p, %u, %"Z"u[%s], %"Z"u%s", + txn, dbi, key ? key->mv_size:0, DKEY(key), data ? data->mv_size:0, + data ? mdb_dval(txn, dbi, data, dbuf):"")); + return mdb_del0(txn, dbi, key, data, 0); +} + +static int +mdb_del0(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + MDB_cursor_op op; + MDB_val rdata, *xdata; + int rc, exact = 0; + DKBUF; + + DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key))); + + mdb_cursor_init(&mc, txn, dbi, &mx); + + if (data) { + op = MDB_GET_BOTH; + rdata = *data; + xdata = &rdata; + } else { + op = MDB_SET; + xdata = NULL; + flags |= MDB_NODUPDATA; + } + rc = mdb_cursor_set(&mc, key, xdata, op, &exact); + if (rc == 0) { + /* let mdb_page_split know about this cursor if needed: + * delete will trigger a rebalance; if it needs to move + * a node from one page to another, it will have to + * update the parent's separator key(s). If the new sepkey + * is larger than the current one, the parent page may + * run out of space, triggering a split. We need this + * cursor to be consistent until the end of the rebalance. + */ + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = _mdb_cursor_del(&mc, flags); + txn->mt_cursors[dbi] = mc.mc_next; + } + return rc; +} + +/** Split a page and insert a new node. + * Set #MDB_TXN_ERROR on failure. + * @param[in,out] mc Cursor pointing to the page and desired insertion index. + * The cursor will be updated to point to the actual page and index where + * the node got inserted after the split. + * @param[in] newkey The key for the newly inserted node. + * @param[in] newdata The data for the newly inserted node. + * @param[in] newpgno The page number, if the new node is a branch node. + * @param[in] nflags The #NODE_ADD_FLAGS for the new node. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, + unsigned int nflags) +{ + unsigned int flags; + int rc = MDB_SUCCESS, new_root = 0, did_split = 0; + indx_t newindx; + pgno_t pgno = 0; + int i, j, split_indx, nkeys, pmax; + MDB_env *env = mc->mc_txn->mt_env; + MDB_node *node; + MDB_val sepkey, rkey, xdata, *rdata = &xdata; + MDB_page *copy = NULL; + MDB_page *mp, *rp, *pp; + int ptop; + MDB_cursor mn; + DKBUF; + + mp = mc->mc_pg[mc->mc_top]; + newindx = mc->mc_ki[mc->mc_top]; + nkeys = NUMKEYS(mp); + + DPRINTF(("-----> splitting %s page %"Yu" and adding [%s] at index %i/%i", + IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, + DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); + + /* Create a right sibling. */ + rc = mdb_page_new(mc, mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2), 1, &rp); + if (rc) + return rc; + rp->mp_pad = mp->mp_pad; + DPRINTF(("new right sibling: page %"Yu, rp->mp_pgno)); + + /* Usually when splitting the root page, the cursor + * height is 1. But when called from mdb_update_key, + * the cursor height may be greater because it walks + * up the stack while finding the branch slot to update. + */ + if (mc->mc_top < 1) { + if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) + goto done; + /* shift current top to make room for new parent */ + for (i=mc->mc_snum; i>0; i--) { + mc->mc_pg[i] = mc->mc_pg[i-1]; + mc->mc_ki[i] = mc->mc_ki[i-1]; + } + mc->mc_pg[0] = pp; + mc->mc_ki[0] = 0; + mc->mc_db->md_root = pp->mp_pgno; + DPRINTF(("root split! new root = %"Yu, pp->mp_pgno)); + new_root = mc->mc_db->md_depth++; + + /* Add left (implicit) pointer. */ + if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { + /* undo the pre-push */ + mc->mc_pg[0] = mc->mc_pg[1]; + mc->mc_ki[0] = mc->mc_ki[1]; + mc->mc_db->md_root = mp->mp_pgno; + mc->mc_db->md_depth--; + goto done; + } + mc->mc_snum++; + mc->mc_top++; + ptop = 0; + } else { + ptop = mc->mc_top-1; + DPRINTF(("parent branch page is %"Yu, mc->mc_pg[ptop]->mp_pgno)); + } + + mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + mn.mc_pg[mn.mc_top] = rp; + mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; + + if (nflags & MDB_APPEND) { + mn.mc_ki[mn.mc_top] = 0; + sepkey = *newkey; + split_indx = newindx; + nkeys = 0; + } else { + + split_indx = (nkeys+1) / 2; + + if (IS_LEAF2(rp)) { + char *split, *ins; + int x; + unsigned int lsize, rsize, ksize; + /* Move half of the keys to the right sibling */ + x = mc->mc_ki[mc->mc_top] - split_indx; + ksize = mc->mc_db->md_pad; + split = LEAF2KEY(mp, split_indx, ksize); + rsize = (nkeys - split_indx) * ksize; + lsize = (nkeys - split_indx) * sizeof(indx_t); + mp->mp_lower -= lsize; + rp->mp_lower += lsize; + mp->mp_upper += rsize - lsize; + rp->mp_upper -= rsize - lsize; + sepkey.mv_size = ksize; + if (newindx == split_indx) { + sepkey.mv_data = newkey->mv_data; + } else { + sepkey.mv_data = split; + } + if (x<0) { + ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); + memcpy(rp->mp_ptrs, split, rsize); + sepkey.mv_data = rp->mp_ptrs; + memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); + memcpy(ins, newkey->mv_data, ksize); + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + } else { + if (x) + memcpy(rp->mp_ptrs, split, x * ksize); + ins = LEAF2KEY(rp, x, ksize); + memcpy(ins, newkey->mv_data, ksize); + memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); + rp->mp_lower += sizeof(indx_t); + rp->mp_upper -= ksize - sizeof(indx_t); + mc->mc_ki[mc->mc_top] = x; + } + } else { + int psize, nsize, k, keythresh; + + /* Maximum free space in an empty page */ + pmax = env->me_pagespace; + /* Threshold number of keys considered "small" */ + keythresh = env->me_psize >> 7; + + if (IS_LEAF(mp)) + nsize = mdb_leaf_size(env, newkey, newdata); + else + nsize = mdb_branch_size(env, newkey); + nsize = EVEN(nsize); + + /* grab a page to hold a temporary copy */ + copy = mdb_page_malloc(mc->mc_txn, 1, 1); + if (copy == NULL) { + rc = ENOMEM; + goto done; + } + copy->mp_pgno = mp->mp_pgno; + copy->mp_flags = mp->mp_flags; + copy->mp_lower = (PAGEHDRSZ-PAGEBASE); + copy->mp_upper = env->me_pagespace; + + /* prepare to insert */ + for (i=0, j=0; imp_ptrs[j++] = 0; + } + copy->mp_ptrs[j++] = mp->mp_ptrs[i]; + } + + /* When items are relatively large the split point needs + * to be checked, because being off-by-one will make the + * difference between success or failure in mdb_node_add. + * + * It's also relevant if a page happens to be laid out + * such that one half of its nodes are all "small" and + * the other half of its nodes are "large." If the new + * item is also "large" and falls on the half with + * "large" nodes, it also may not fit. + * + * As a final tweak, if the new item goes on the last + * spot on the page (and thus, onto the new page), bias + * the split so the new page is emptier than the old page. + * This yields better packing during sequential inserts. + */ + if (nkeys < keythresh || nsize > pmax/16 || newindx >= nkeys) { + /* Find split point */ + psize = 0; + if (newindx <= split_indx || newindx >= nkeys) { + i = 0; j = 1; + k = newindx >= nkeys ? nkeys : split_indx+1+IS_LEAF(mp); + } else { + i = nkeys; j = -1; + k = split_indx-1; + } + for (; i!=k; i+=j) { + if (i == newindx) { + psize += nsize; + node = NULL; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + psize += sizeof(MDB_ovpage); + else + psize += NODEDSZ(node); + } + psize = EVEN(psize); + } + if (psize > pmax || i == k-j) { + split_indx = i + (j<0); + break; + } + } + } + if (split_indx == newindx) { + sepkey.mv_size = newkey->mv_size; + sepkey.mv_data = newkey->mv_data; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE); + sepkey.mv_size = node->mn_ksize; + sepkey.mv_data = NODEKEY(node); + } + } + } + + DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey))); + + /* Copy separator key to the parent. + */ + if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { + int snum = mc->mc_snum; + mn.mc_snum--; + mn.mc_top--; + did_split = 1; + /* We want other splits to find mn when doing fixups */ + WITH_CURSOR_TRACKING(mn, + rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0)); + if (rc) + goto done; + + /* root split? */ + if (mc->mc_snum > snum) { + ptop++; + } + /* Right page might now have changed parent. + * Check if left page also changed parent. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; imc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + mc->mc_pg[ptop] = mn.mc_pg[ptop]; + if (mn.mc_ki[ptop]) { + mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; + } else { + /* find right page's left sibling */ + mc->mc_ki[ptop] = mn.mc_ki[ptop]; + rc = mdb_cursor_sibling(mc, 0); + } + } + } else { + mn.mc_top--; + rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); + mn.mc_top++; + } + if (rc != MDB_SUCCESS) { + if (rc == MDB_NOTFOUND) /* improper mdb_cursor_sibling() result */ + rc = MDB_PROBLEM; + goto done; + } + if (nflags & MDB_APPEND) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[mc->mc_top] = 0; + rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); + if (rc) + goto done; + for (i=0; imc_top; i++) + mc->mc_ki[i] = mn.mc_ki[i]; + } else if (!IS_LEAF2(mp)) { + /* Move nodes */ + mc->mc_pg[mc->mc_top] = rp; + i = split_indx; + j = 0; + do { + if (i == newindx) { + rkey.mv_data = newkey->mv_data; + rkey.mv_size = newkey->mv_size; + if (IS_LEAF(mp)) { + rdata = newdata; + } else + pgno = newpgno; + flags = nflags; + /* Update index for the new key. */ + mc->mc_ki[mc->mc_top] = j; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + rkey.mv_data = NODEKEY(node); + rkey.mv_size = node->mn_ksize; + if (IS_LEAF(mp)) { + xdata.mv_data = NODEDATA(node); + xdata.mv_size = NODEDSZ(node); + rdata = &xdata; + } else + pgno = NODEPGNO(node); + flags = node->mn_flags; + } + + if (!IS_LEAF(mp) && j == 0) { + /* First branch index doesn't need key data. */ + rkey.mv_size = 0; + } + + rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); + if (rc) + goto done; + if (i == nkeys) { + i = 0; + j = 0; + mc->mc_pg[mc->mc_top] = copy; + } else { + i++; + j++; + } + } while (i != split_indx); + + nkeys = NUMKEYS(copy); + for (i=0; imp_ptrs[i] = copy->mp_ptrs[i]; + mp->mp_lower = copy->mp_lower; + mp->mp_upper = copy->mp_upper; + memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), + env->me_psize - copy->mp_upper - PAGEBASE); + + /* reset back to original page */ + if (newindx < split_indx) { + mc->mc_pg[mc->mc_top] = mp; + } else { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<=ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + if (nflags & MDB_RESERVE) { + node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!(node->mn_flags & F_BIGDATA)) + newdata->mv_data = NODEDATA(node); + } + } else { + if (newindx >= split_indx) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<=ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + } + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + nkeys = NUMKEYS(mp); + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc) + continue; + if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (new_root) { + int k; + /* sub cursors may be on different DB */ + if (m3->mc_pg[0] != mp) + continue; + /* root split */ + for (k=new_root; k>=0; k--) { + m3->mc_ki[k+1] = m3->mc_ki[k]; + m3->mc_pg[k+1] = m3->mc_pg[k]; + } + if (m3->mc_ki[0] >= nkeys) { + m3->mc_ki[0] = 1; + } else { + m3->mc_ki[0] = 0; + } + m3->mc_pg[0] = mc->mc_pg[0]; + m3->mc_snum++; + m3->mc_top++; + } + if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) + m3->mc_ki[mc->mc_top]++; + if (m3->mc_ki[mc->mc_top] >= nkeys) { + m3->mc_pg[mc->mc_top] = rp; + m3->mc_ki[mc->mc_top] -= nkeys; + for (i=0; imc_top; i++) { + m3->mc_ki[i] = mn.mc_ki[i]; + m3->mc_pg[i] = mn.mc_pg[i]; + } + } + } else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] && + m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { + m3->mc_ki[ptop]++; + } + if (IS_LEAF(mp)) + XCURSOR_REFRESH(m3, mc->mc_top, m3->mc_pg[mc->mc_top]); + } + } + DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); + +done: + if (copy) /* tmp page */ + mdb_page_free(env, copy); + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_put(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned int flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + int rc; + DKBUF; + DDBUF; + + if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + if (flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) + return EINVAL; + + if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) + return (txn->mt_flags & MDB_TXN_RDONLY) ? MDB_IS_READONLY : MDB_BAD_TXN; + + MDB_TRACE(("%p, %u, %"Z"u[%s], %"Z"u%s, %u", + txn, dbi, key ? key->mv_size:0, DKEY(key), data->mv_size, mdb_dval(txn, dbi, data, dbuf), flags)); + mdb_cursor_init(&mc, txn, dbi, &mx); + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = _mdb_cursor_put(&mc, key, data, flags); + txn->mt_cursors[dbi] = mc.mc_next; + return rc; +} + +#ifndef MDB_WBUF +#define MDB_WBUF (1024*1024) +#endif +#define MDB_EOF 0x10 /**< #mdb_env_copyfd1() is done reading */ + + /** State needed for a double-buffering compacting copy. */ +typedef struct mdb_copy { + MDB_env *mc_env; + MDB_txn *mc_txn; + pthread_mutex_t mc_mutex; + pthread_cond_t mc_cond; /**< Condition variable for #mc_new */ + char *mc_wbuf[2]; + char *mc_over[2]; + size_t mc_wlen[2]; + mdb_size_t mc_olen[2]; + pgno_t mc_next_pgno; + HANDLE mc_fd; + int mc_toggle; /**< Buffer number in provider */ + int mc_new; /**< (0-2 buffers to write) | (#MDB_EOF at end) */ + /** Error code. Never cleared if set. Both threads can set nonzero + * to fail the copy. Not mutex-protected, LMDB expects atomic int. + */ + volatile int mc_error; +} mdb_copy; + +#ifdef _WIN32 +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#endif + + /** Dedicated writer thread for compacting copy. */ +static THREAD_RET ESECT CALL_CONV +mdb_env_copythr(void *arg) +{ + mdb_copy *my = arg; + char *ptr; + int toggle = 0, rc; + mdb_size_t wsize; +#if MDB_RPAGE_CACHE + char *pagebuf = NULL; + char *ovpage = NULL; +#endif +#ifdef _WIN32 + DWORD len, w2; +#else + ssize_t len; + size_t w2; +#ifdef F_SETNOSIGPIPE + /* OS X delivers SIGPIPE to the whole process, not the thread that caused it. + * Disable SIGPIPE using platform specific fcntl. + */ + int enabled = 1; + if ((rc = fcntl(my->mc_fd, F_SETNOSIGPIPE, &enabled)) != 0) + my->mc_error = errno; +#elif defined SIGPIPE + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGPIPE); + if ((rc = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0) + my->mc_error = rc; +#endif +#endif + +#if MDB_RPAGE_CACHE + if (my->mc_env->me_encfunc) { + pagebuf = malloc(my->mc_env->me_psize); + if (!pagebuf) { + my->mc_error = ENOMEM; + goto skip; + } + } +#endif + + pthread_mutex_lock(&my->mc_mutex); + for(;;) { + while (!my->mc_new) + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + if (my->mc_new == 0 + MDB_EOF) /* 0 buffers, just EOF */ + break; + wsize = my->mc_wlen[toggle]; + ptr = my->mc_wbuf[toggle]; +#if MDB_RPAGE_CACHE + if (my->mc_env->me_sumfunc || my->mc_env->me_encfunc) { + char *mptr; + for (mptr = ptr; mptr < ptr+wsize; mptr += my->mc_env->me_psize) { + if (((MDB_page *)mptr)->mp_pgno < NUM_METAS) + continue; + if (my->mc_env->me_sumfunc) + mdb_page_set_checksum(my->mc_env, (MDB_page *)mptr, my->mc_env->me_psize); + if (my->mc_env->me_encfunc) { + memcpy(pagebuf, mptr, my->mc_env->me_psize); + mdb_page_encrypt(my->mc_env, (MDB_page *)pagebuf, (MDB_page *)mptr, my->mc_env->me_psize); + } + } + if (my->mc_olen[toggle]) { + if (my->mc_env->me_sumfunc) + mdb_page_set_checksum(my->mc_env, (MDB_page *)my->mc_over[toggle], my->mc_olen[toggle]); + if (my->mc_env->me_encfunc) { + ovpage = malloc(my->mc_olen[toggle]); + if (!ovpage) { + rc = my->mc_error = ENOMEM; + goto skip; + } + mdb_page_encrypt(my->mc_env, (MDB_page *)my->mc_over[toggle], (MDB_page *)ovpage, + my->mc_olen[toggle]); + free(my->mc_over[toggle]); + my->mc_over[toggle] = ovpage; + } + } + } +#endif +again: + rc = MDB_SUCCESS; + while (wsize > 0 && !my->mc_error) { + w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize; + DO_WRITE(rc, my->mc_fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = MDB_SHORT_WRITE; + break; + } + } + if (rc) { + my->mc_error = rc; + } + /* If there's an overflow page tail, write it too */ + if (my->mc_olen[toggle]) { + wsize = my->mc_olen[toggle]; + ptr = my->mc_over[toggle]; + my->mc_olen[toggle] = 0; + goto again; + } +#if MDB_RPAGE_CACHE +skip: + if (ovpage) { + free(ovpage); + ovpage = NULL; + } +#endif + my->mc_wlen[toggle] = 0; + toggle ^= 1; + /* Return the empty buffer to provider */ + my->mc_new--; + pthread_cond_signal(&my->mc_cond); + } + pthread_mutex_unlock(&my->mc_mutex); +#if MDB_RPAGE_CACHE + if (pagebuf) + free(pagebuf); +#endif + return (THREAD_RET)0; +} + + /** Give buffer and/or #MDB_EOF to writer thread, await unused buffer. + * + * @param[in] my control structure. + * @param[in] adjust (1 to hand off 1 buffer) | (MDB_EOF when ending). + */ +static int ESECT +mdb_env_cthr_toggle(mdb_copy *my, int adjust) +{ + pthread_mutex_lock(&my->mc_mutex); + my->mc_new += adjust; + pthread_cond_signal(&my->mc_cond); + while (my->mc_new & 2) /* both buffers in use */ + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + pthread_mutex_unlock(&my->mc_mutex); + + my->mc_toggle ^= (adjust & 1); + /* Both threads reset mc_wlen, to be safe from threading errors */ + my->mc_wlen[my->mc_toggle] = 0; + return my->mc_error; +} + + /** Depth-first tree traversal for compacting copy. + * @param[in] my control structure. + * @param[in,out] pg database root. + * @param[in] flags includes #F_DUPDATA if it is a sorted-duplicate sub-DB. + */ +static int ESECT +mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) +{ + MDB_cursor mc = {0}; + MDB_node *ni; + MDB_page *mo, *mp, *leaf; + char *buf, *ptr; + int rc, toggle; + unsigned int i; + + /* Empty DB, nothing to do */ + if (*pg == P_INVALID) + return MDB_SUCCESS; + + mc.mc_snum = 1; + mc.mc_txn = my->mc_txn; + mc.mc_flags = my->mc_txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP); + + rc = MDB_PAGE_GET(&mc, *pg, 1, &mc.mc_pg[0]); + if (rc) + return rc; + rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); + if (rc) + return rc; + + /* Make cursor pages writable */ + buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum); + if (buf == NULL) + return ENOMEM; + + for (i=0; imc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mc.mc_pg[i]); + mc.mc_pg[i] = (MDB_page *)ptr; + ptr += my->mc_env->me_psize; + } + + /* This is writable space for a leaf page. Usually not needed. */ + leaf = (MDB_page *)ptr; + + toggle = my->mc_toggle; + while (mc.mc_snum > 0) { + unsigned n; + mp = mc.mc_pg[mc.mc_top]; + n = NUMKEYS(mp); + + if (IS_LEAF(mp)) { + if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { + for (i=0; imn_flags & F_BIGDATA) { + MDB_page *omp; + MDB_ovpage ovp; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mp); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&ovp, NODEDATA(ni), sizeof(ovp)); + rc = MDB_PAGE_GET(&mc, ovp.op_pgno, ovp.op_pages, &omp); + if (rc) + goto done; + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + ovp.op_pgno = my->mc_next_pgno; + ovp.op_txnid = 1; + memcpy(NODEDATA(ni), &ovp, sizeof(ovp)); + my->mc_next_pgno += ovp.op_pages; + /* if it's a large overflow page, we want to keep it contiguous + * if we need to do checksum or encryption on it. Otherwise we + * can just modify the leading page and copy the tail separately. + */ + if ((MDB_REMAPPING(my->mc_env->me_flags) || my->mc_env->me_sumfunc) && ovp.op_pages > 1) { + size_t ovlen = ovp.op_pages * my->mc_env->me_psize; + mo = malloc(ovlen); + if (!mo) { + rc = ENOMEM; + goto done; + } + mdb_page_copy(mo, omp, ovlen); + mo->mp_pgno = ovp.op_pgno; + mo->mp_txnid = 1; + my->mc_over[toggle] = (char *)mo; + my->mc_olen[toggle] = ovlen; + MDB_PAGE_UNREF(my->mc_txn, omp); + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } else { + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + memcpy(mo, omp, my->mc_env->me_psize); + mo->mp_pgno = ovp.op_pgno; + mo->mp_txnid = 1; + my->mc_wlen[toggle] += my->mc_env->me_psize; + MDB_PAGE_UNREF(my->mc_txn, omp); + if (ovp.op_pages > 1) { + my->mc_olen[toggle] = my->mc_env->me_psize * (ovp.op_pages - 1); + my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + } + } else if (ni->mn_flags & F_SUBDATA) { + MDB_db db; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mp); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&db, NODEDATA(ni), sizeof(db)); + my->mc_toggle = toggle; + rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA); + if (rc) + goto done; + toggle = my->mc_toggle; + memcpy(NODEDATA(ni), &db, sizeof(db)); + } + } + } + } else { + mc.mc_ki[mc.mc_top]++; + if (mc.mc_ki[mc.mc_top] < n) { + pgno_t pg; +again: + ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]); + pg = NODEPGNO(ni); + rc = MDB_PAGE_GET(&mc, pg, 1, &mp); + if (rc) + goto done; + mc.mc_top++; + mc.mc_snum++; + mc.mc_ki[mc.mc_top] = 0; + if (IS_BRANCH(mp)) { + /* Whenever we advance to a sibling branch page, + * we must proceed all the way down to its first leaf. + */ + mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mp); + goto again; + } else + mc.mc_pg[mc.mc_top] = mp; + continue; + } + } + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + mdb_page_copy(mo, mp, my->mc_env->me_psize); + MDB_PAGE_UNREF(my->mc_txn, mp); + mo->mp_pgno = my->mc_next_pgno++; + mo->mp_txnid = 1; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (mc.mc_top) { + /* Update parent if there is one */ + ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); + SETPGNO(ni, mo->mp_pgno); + mdb_cursor_pop(&mc); + } else { + /* Otherwise we're done */ + *pg = mo->mp_pgno; + break; + } + } +done: + free(buf); + return rc; +} + + /** Copy environment with compaction. */ +static int ESECT +mdb_env_copyfd1(MDB_env *env, HANDLE fd) +{ + MDB_meta *mm; + MDB_page *mp; + mdb_copy my = {0}; + MDB_txn *txn = NULL; + pthread_t thr; + pgno_t root, new_root; + int rc = MDB_SUCCESS; + +#ifdef _WIN32 + if (!(my.mc_mutex = CreateMutex(NULL, FALSE, NULL)) || + !(my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL))) { + rc = ErrCode(); + goto done; + } +#else + if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) != 0) + return rc; + if ((rc = pthread_cond_init(&my.mc_cond, NULL)) != 0) + goto done2; +#endif + MEMALIGN(rc, my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); + if (rc) + goto done; + + memset(my.mc_wbuf[0], 0, MDB_WBUF*2); + my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; + my.mc_next_pgno = NUM_METAS; + my.mc_env = env; + my.mc_fd = fd; + rc = THREAD_CREATE(thr, mdb_env_copythr, &my); + if (rc) + goto done; + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + goto finish; + + mp = (MDB_page *)my.mc_wbuf[0]; + memset(mp, 0, NUM_METAS * env->me_psize); + mp->mp_pgno = 0; + mp->mp_flags = P_META; + mm = (MDB_meta *)METADATA(mp); + mdb_env_init_meta0(env, mm); + mm->mm_address = env->me_metas[0]->mm_address; + + mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize); + mp->mp_pgno = 1; + mp->mp_flags = P_META; + *(MDB_meta *)METADATA(mp) = *mm; + mm = (MDB_meta *)METADATA(mp); + + /* Set metapage 1 with current main DB */ + root = new_root = txn->mt_dbs[MAIN_DBI].md_root; + if (root != P_INVALID) { + /* Count free pages + freeDB pages. Subtract from last_pg + * to find the new last_pg, which also becomes the new root. + */ + MDB_ID freecount = 0; + MDB_cursor mc; + MDB_val key, data; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + if (rc != MDB_NOTFOUND) + goto finish; + freecount += txn->mt_dbs[FREE_DBI].md_branch_pages + + txn->mt_dbs[FREE_DBI].md_leaf_pages + + txn->mt_dbs[FREE_DBI].md_overflow_pages; + + new_root = txn->mt_next_pgno - 1 - freecount; + mm->mm_last_pg = new_root; + mm->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; + mm->mm_dbs[MAIN_DBI].md_root = new_root; + } else { + /* When the DB is empty, handle it specially to + * fix any breakage like page leaks from ITS#8174. + */ + mm->mm_dbs[MAIN_DBI].md_flags = txn->mt_dbs[MAIN_DBI].md_flags; + } + if (root != P_INVALID || mm->mm_dbs[MAIN_DBI].md_flags) { + mm->mm_txnid = 1; /* use metapage 1 */ + } + + my.mc_wlen[0] = env->me_psize * NUM_METAS; + my.mc_txn = txn; + rc = mdb_env_cwalk(&my, &root, 0); + if (rc == MDB_SUCCESS && root != new_root) { + rc = MDB_INCOMPATIBLE; /* page leak or corrupt DB */ + } + +finish: + if (rc) + my.mc_error = rc; + mdb_env_cthr_toggle(&my, 1 | MDB_EOF); + rc = THREAD_FINISH(thr); + _mdb_txn_abort(txn); + +done: +#ifdef _WIN32 + if (my.mc_cond) CloseHandle(my.mc_cond); + if (my.mc_mutex) CloseHandle(my.mc_mutex); +#else + pthread_cond_destroy(&my.mc_cond); +done2: + pthread_mutex_destroy(&my.mc_mutex); +#endif + if (my.mc_wbuf[0]) ALIGNED_FREE(my.mc_wbuf[0]); + return rc ? rc : my.mc_error; +} + + /** Copy environment as-is. */ +static int ESECT +mdb_env_copyfd0(MDB_env *env, HANDLE fd) +{ + MDB_txn *txn = NULL; + mdb_mutexref_t wmutex = NULL; + int rc; + mdb_size_t wsize, w3; + char *ptr; +#ifdef _WIN32 + DWORD len, w2; +#else + ssize_t len; + size_t w2; +#endif +#if MDB_RPAGE_CACHE +#ifdef _WIN32 + LARGE_INTEGER off = {0}; + SIZE_T mlen; +#define OFF_ADD(off,val) off.QuadPart += (val) +#else + off_t off = 0; + size_t mlen; +#define OFF_ADD(off,val) off += (val) +#endif + void *mptr; +#endif + + /* Do the lock/unlock of the reader mutex before starting the + * write txn. Otherwise other read txns could block writers. + */ + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + if (env->me_txns) { + /* We must start the actual read txn after blocking writers */ + mdb_txn_end(txn, MDB_END_RESET_TMP); + + /* Temporarily block writers until we snapshot the meta pages */ + wmutex = env->me_wmutex; + if (LOCK_MUTEX(rc, env, wmutex)) + goto leave; + + rc = mdb_txn_renew0(txn); + if (rc) { + UNLOCK_MUTEX(wmutex); + goto leave; + } + } + + wsize = env->me_psize * NUM_METAS; + ptr = env->me_map; + w2 = wsize; + while (w2 > 0) { + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + w2 -= len; + continue; + } else { + /* Non-blocking or async handles are not supported */ + rc = MDB_SHORT_WRITE; + break; + } + } + if (wmutex) + UNLOCK_MUTEX(wmutex); + + if (rc) + goto leave; + + w3 = txn->mt_next_pgno * env->me_psize; + { + mdb_size_t fsize = 0; + if ((rc = mdb_fsize(env->me_fd, &fsize))) + goto leave; + if (w3 > fsize) + w3 = fsize; + } +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + SET_OFF(off, wsize); + } +#endif + wsize = w3 - wsize; + while (wsize > 0) { + w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize; +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + mlen = w2; + MAP(rc, env, mptr, mlen, off); + ptr = mptr; + } +#endif + DO_WRITE(rc, fd, ptr, w2, len); +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + munmap(ptr, w2); + } +#endif + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + OFF_ADD(off, len); + } +#endif + continue; + } else { + rc = MDB_SHORT_WRITE; + break; + } + } + +leave: + _mdb_txn_abort(txn); + return rc; +} + +int ESECT +mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) +{ + if (flags & MDB_CP_COMPACT) + return mdb_env_copyfd1(env, fd); + else + return mdb_env_copyfd0(env, fd); +} + +int ESECT +mdb_env_copyfd(MDB_env *env, HANDLE fd) +{ + return mdb_env_copyfd2(env, fd, 0); +} + +static int ESECT +mdb_env_copy_open(MDB_env *env, const char *path, HANDLE *retfd) +{ + int rc; + MDB_name fname; + + rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname); + if (rc == MDB_SUCCESS) { + rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, retfd); + mdb_fname_destroy(fname); + } + return rc; +} + +int ESECT +mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) +{ + HANDLE newfd = INVALID_HANDLE_VALUE; + int rc; + + rc = mdb_env_copy_open(env, path, &newfd); + if (rc) + return rc; + rc = mdb_env_copyfd2(env, newfd, flags); + + if (newfd != INVALID_HANDLE_VALUE) + if (close(newfd) < 0 && rc == MDB_SUCCESS) + rc = ErrCode(); + return rc; +} + +int ESECT +mdb_env_copy(MDB_env *env, const char *path) +{ + return mdb_env_copy2(env, path, 0); +} + +int ESECT +mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid) +{ + int rc, npgs; + MDB_page *mp; +#if MDB_RPAGE_CACHE + MDB_page *m2; +#endif + MDB_txn *txn; + mdb_size_t wsize; + char *buf = NULL; + pgno_t pg; + MDB_cursor mc; +#ifdef _WIN32 + DWORD len, w2; +#else + ssize_t len; + size_t w2; +#endif + + MEMALIGN(rc, buf, env->me_psize, 2*env->me_psize); + if (rc) + return rc; + + /* Do the lock/unlock of the reader mutex before starting the + * write txn. Otherwise other read txns could block writers. + */ + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + ALIGNED_FREE(buf); + return rc; + } + + if (env->me_txns) { + /* We must start the actual read txn after blocking writers */ + mdb_txn_reset(txn); + + /* Temporarily block writers until we snapshot the meta pages */ + if (LOCK_MUTEX(rc, env, env->me_wmutex)) + goto leave; + + rc = mdb_txn_renew0(txn); + if (rc) { + UNLOCK_MUTEX(env->me_wmutex); + goto leave; + } + } + + memcpy(buf, env->me_map, env->me_psize*2); + + if (env->me_txns) + UNLOCK_MUTEX(env->me_wmutex); + + { + MDB_meta *mm; + mp = (MDB_page *)buf; + mm = METADATA(mp); + if (mm->mm_txnid > txnid) { + DO_WRITE(rc, fd, mp, env->me_psize, len); + if (!rc) { + rc = ErrCode(); + goto leave; + } + } + mp = (MDB_page *)((char *)mp + env->me_psize); + mm = METADATA(mp); + if (mm->mm_txnid > txnid) { + DO_WRITE(rc, fd, mp, env->me_psize, len); + if (!rc) { + rc = ErrCode(); + goto leave; + } + } + + } + ALIGNED_FREE(buf); + buf = NULL; + + /* dummy cursor for mdb_page_get() */ + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + pg = 2; + while (pg < txn->mt_next_pgno) { + if ((rc = MDB_PAGE_GET(&mc, pg, 1, &mp)) != 0) + goto leave; + wsize = env->me_psize; + npgs = 1; + if (IS_OVERFLOW(mp)) { + npgs = mp->mp_pages; + wsize *= mp->mp_pages; + } +#if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { + mdb_page_unref(txn, mp); + rc = mdb_rpage_get(txn, pg, npgs, &mp, &m2); + if (rc) + goto leave; + } +#endif + if (mp->mp_txnid > txnid) { + mdb_size_t w3 = wsize; + char *ptr = (char *)mp; +#if MDB_RPAGE_CACHE + if (env->me_encfunc) + ptr = (char *)m2; +#endif + while (w3 > 0) { + w2 = (w3 > MAX_WRITE) ? MAX_WRITE : w3; + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + goto leave; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + w3 -= len; + continue; + } else { + rc = MDB_SHORT_WRITE; + goto leave; + } + } + } + MDB_PAGE_UNREF(txn, mp); + pg += npgs; + } + +leave: + mdb_txn_abort(txn); + if (buf != NULL) + ALIGNED_FREE(buf); + return rc; +} + +int ESECT +mdb_env_incr_dump(MDB_env *env, const char *path, size_t txnid) +{ + HANDLE newfd = INVALID_HANDLE_VALUE; + int rc; + + /* Output is just a plain file, not an environment */ + env->me_flags |= MDB_NOSUBDIR; + + rc = mdb_env_copy_open(env, path, &newfd); + if (rc) + return rc; + rc = mdb_env_incr_dumpfd(env, newfd, txnid); + + if (newfd != INVALID_HANDLE_VALUE) + if (close(newfd) < 0 && rc == MDB_SUCCESS) + rc = ErrCode(); + return rc; +} + +int ESECT +mdb_env_incr_loadfd(MDB_env *env, HANDLE fd) +{ +#ifdef _WIN32 + DWORD rsize, rlen, w2; + LARGE_INTEGER off; +#else + size_t rsize, w2; + ssize_t rlen; + off_t off; +#endif + char buf[PAGEHDRSZ], *ptr, *pbuf; + MDB_page *rp; + pgno_t pg, prevpg = 0; + int numpgs, numprev = 0; + int rc = 0; + + pbuf = malloc(env->me_psize); + if (!pbuf) + return ENOMEM; + +#ifdef _WIN32 + { LARGE_INTEGER li0; li0.QuadPart = 0; SetFilePointerEx(env->me_fd, li0, NULL, FILE_BEGIN); } +#else + lseek(env->me_fd, 0, SEEK_SET); +#endif + + for (;;) { +#ifdef _WIN32 + rc = ReadFile(fd, buf, sizeof(buf), &rlen, NULL) ? (int)rlen : -1; + if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) + rc = 0; +#else + rlen = read(fd, buf, sizeof(buf)); +#endif + if (rlen != sizeof(buf)) + break; + rsize = env->me_psize; + rp = (MDB_page *)buf; + pg = rp->mp_pgno; +#if MDB_RPAGE_CACHE + /* decrypt the header if needed */ + if (env->me_encfunc && pg > 1) { + MDB_val in, out, enckeys[3]; + int xsize = sizeof(pgno_t) + sizeof(txnid_t); + in.mv_size = PAGEHDRSZ - xsize; + in.mv_data = buf + xsize; + out.mv_size = in.mv_size; + out.mv_data = pbuf + xsize; + enckeys[0] = env->me_enckey; + enckeys[1].mv_size = xsize; + enckeys[1].mv_data = buf; + enckeys[2].mv_size = 0; + enckeys[2].mv_data = pbuf + xsize; /* dummy */ + env->me_encfunc(&in, &out, enckeys, 0); + rp = (MDB_page *)pbuf; + } +#endif + numpgs = 1; + if (IS_OVERFLOW(rp)) { + numpgs = rp->mp_pages; + rsize *= rp->mp_pages; + if (rp->mp_pages > 1) { + ptr = realloc(pbuf, rsize); + if (!ptr) { + free(pbuf); + return ENOMEM; + } + pbuf = ptr; + } + } + memcpy(pbuf, buf, sizeof(buf)); + rsize -= rlen; + ptr = pbuf + sizeof(buf); + while (rsize > 0) { +#ifdef _WIN32 + rc = ReadFile(fd, ptr, rsize, &rlen, NULL) ? (int)rlen : -1; + if (rc == -1) + rlen = -1; +#else + rlen = read(fd, ptr, rsize); +#endif + if (rlen == -1) + return ErrCode(); + ptr += rlen; + rsize -= rlen; + } +#ifdef _WIN32 + off.QuadPart = (pg-prevpg-numprev) * env->me_psize; + rsize = numpgs * env->me_psize; + if (off.QuadPart) { + SetFilePointerEx(env->me_fd, off, NULL, FILE_CURRENT); + } +#else + off = (pg-prevpg-numprev) * env->me_psize; + rsize = numpgs * env->me_psize; + if (off) { + lseek(env->me_fd, off, SEEK_CUR); + } +#endif + ptr = pbuf; + while (rsize > 0) { + w2 = (rsize > MAX_WRITE) ? MAX_WRITE: rsize; + DO_WRITE(rc, env->me_fd, ptr, w2, rlen); + if (!rc) { + rc = ErrCode(); + break; + } else if (rlen > 0) { + rc = MDB_SUCCESS; + ptr += rlen; + rsize -= rlen; + continue; + } else { + rc = MDB_SHORT_WRITE; + break; + } + } + if (rc) + break; + numprev = numpgs; + prevpg = pg; + } + free(pbuf); + return rc; +} + +int ESECT +mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) +{ + if (flag & ~CHANGEABLE) + return EINVAL; + if (onoff) + env->me_flags |= flag; + else + env->me_flags &= ~flag; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_flags(MDB_env *env, unsigned int *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_flags & EXPOSED; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_userctx(MDB_env *env, void *ctx) +{ + if (!env) + return EINVAL; + env->me_userctx = ctx; + return MDB_SUCCESS; +} + +void * ESECT +mdb_env_get_userctx(MDB_env *env) +{ + return env ? env->me_userctx : NULL; +} + +int ESECT +mdb_env_set_assert(MDB_env *env, MDB_assert_func *func) +{ + if (!env) + return EINVAL; +#ifndef NDEBUG + env->me_assert_func = func; +#endif + return MDB_SUCCESS; +} + +#if MDB_RPAGE_CACHE +int ESECT +mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size) +{ + char *kdata; + + if (!env || !func || !key) + return EINVAL; + if (env->me_flags & MDB_ENV_ACTIVE) + return EINVAL; + if (! (kdata = malloc(key[0].mv_size))) + return ENOMEM; + + memcpy(kdata, key->mv_data, key->mv_size); + free(env->me_enckey.mv_data); + env->me_enckey.mv_data = kdata; + env->me_enckey.mv_size = key->mv_size; + env->me_encfunc = func; + if (size) + env->me_esumsize = size; + env->me_flags |= MDB_REMAP_CHUNKS | MDB_ENCRYPT; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size) +{ + if (!env || !func || !size) + return EINVAL; + if (env->me_flags & MDB_ENV_ACTIVE) + return EINVAL; + env->me_sumfunc = func; + env->me_sumsize = size; + return MDB_SUCCESS; +} +#endif + +int ESECT +mdb_env_get_path(MDB_env *env, const char **arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_path; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_fd; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_pagesize(MDB_env *env, int size) +{ + if (!env || env->me_map) + return EINVAL; + if (size > MAX_PAGESIZE || size < 256) + return EINVAL; + if (size & (size-1)) + return EINVAL; + env->me_os_psize = size; + return MDB_SUCCESS; +} + +/** Common code for #mdb_stat() and #mdb_env_stat(). + * @param[in] env the environment to operate in. + * @param[in] db the #MDB_db record containing the stats to return. + * @param[out] arg the address of an #MDB_stat structure to receive the stats. + * @return 0, this function always succeeds. + */ +static int ESECT +mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) +{ + arg->ms_psize = env->me_psize; + arg->ms_depth = db->md_depth; + arg->ms_branch_pages = db->md_branch_pages; + arg->ms_leaf_pages = db->md_leaf_pages; + arg->ms_overflow_pages = db->md_overflow_pages; + arg->ms_entries = db->md_entries; + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_stat(MDB_env *env, MDB_stat *arg) +{ + MDB_meta *meta; + + if (env == NULL || arg == NULL) + return EINVAL; + + meta = mdb_env_pick_meta(env); + + return mdb_stat0(env, &meta->mm_dbs[MAIN_DBI], arg); +} + +int ESECT +mdb_env_info(MDB_env *env, MDB_envinfo *arg) +{ + MDB_meta *meta; + + if (env == NULL || arg == NULL) + return EINVAL; + + meta = mdb_env_pick_meta(env); + arg->me_mapaddr = meta->mm_address ? meta->mm_address : env->me_map; + arg->me_last_pgno = meta->mm_last_pg; + arg->me_last_txnid = meta->mm_txnid; + + arg->me_mapsize = env->me_mapsize; + arg->me_maxreaders = env->me_maxreaders; + arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : 0; + return MDB_SUCCESS; +} + +/** Set the default comparison functions for a database. + * Called immediately after a database is opened to set the defaults. + * The user can then override them with #mdb_set_compare() or + * #mdb_set_dupsort(). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +static void +mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) +{ + uint16_t f = txn->mt_dbs[dbi].md_flags; + + txn->mt_dbxs[dbi].md_cmp = + (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : + (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; + + txn->mt_dbxs[dbi].md_dcmp = + !(f & MDB_DUPSORT) ? 0 : + ((f & MDB_INTEGERDUP) + ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) + : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); +} + +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) +{ + MDB_val key, data; + MDB_dbi i; + MDB_cursor mc; + MDB_db dummy; + int rc, dbflag, exact; + unsigned int unused = 0, seq; + char *namedup; + size_t size; + + if (flags & ~VALID_FLAGS) + return EINVAL; + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + /* main DB? */ + if (!name) { + *dbi = MAIN_DBI; + if (flags & PERSISTENT_FLAGS) { + uint16_t f2 = flags & PERSISTENT_FLAGS; + /* make sure flag changes get committed */ + if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { + txn->mt_dbs[MAIN_DBI].md_flags |= f2; + txn->mt_flags |= MDB_TXN_DIRTY; + } + } + mdb_default_cmp(txn, MAIN_DBI); + MDB_TRACE(("%p, (null), %u = %u", txn, flags, MAIN_DBI)); + return MDB_SUCCESS; + } + + if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, MAIN_DBI); + } + + /* Is the DB already open? */ + size = strlen(name) + 1; + for (i=CORE_DBS; imt_numdbs; i++) { + if (!txn->mt_dbxs[i].md_name.mv_size) { + /* Remember this free slot */ + if (!unused) unused = i; + continue; + } + if (size == txn->mt_dbxs[i].md_name.mv_size && + !strcmp(name, txn->mt_dbxs[i].md_name.mv_data)) { + *dbi = i; + return MDB_SUCCESS; + } + } + + /* If no free slot and max hit, fail */ + if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) + return MDB_DBS_FULL; + + /* Cannot mix named databases with some mainDB flags */ + if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) + return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; + + /* Find the DB info */ + dbflag = DB_NEW|DB_VALID|DB_USRVALID; + exact = 0; + key.mv_size = size; + key.mv_data = (void *)name; + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); + if (rc == MDB_SUCCESS) { + /* make sure this is actually a DB */ + MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); + if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) + return MDB_INCOMPATIBLE; + } else { + if (rc != MDB_NOTFOUND || !(flags & MDB_CREATE)) + return rc; + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return MDB_IS_READONLY; + } + + /* Done here so we cannot fail after creating a new DB */ + if ((namedup = strdup(name)) == NULL) + return ENOMEM; + + if (rc) { + /* MDB_NOTFOUND and MDB_CREATE: Create new DB */ + data.mv_size = sizeof(MDB_db); + data.mv_data = &dummy; + memset(&dummy, 0, sizeof(dummy)); + dummy.md_root = P_INVALID; + dummy.md_flags = flags & PERSISTENT_FLAGS; + WITH_CURSOR_TRACKING(mc, + rc = _mdb_cursor_put(&mc, &key, &data, F_SUBDATA)); + dbflag |= DB_DIRTY; + } + + if (rc) { + free(namedup); + } else { + /* Got info, register DBI in this txn */ + unsigned int slot = unused ? unused : txn->mt_numdbs; + txn->mt_dbxs[slot].md_name.mv_data = namedup; + txn->mt_dbxs[slot].md_name.mv_size = size; + txn->mt_dbxs[slot].md_rel = NULL; + txn->mt_dbflags[slot] = dbflag; + /* txn-> and env-> are the same in read txns, use + * tmp variable to avoid undefined assignment + */ + seq = ++txn->mt_env->me_dbiseqs[slot]; + txn->mt_dbiseqs[slot] = seq; + + memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); + *dbi = slot; + mdb_default_cmp(txn, slot); + if (!unused) { + txn->mt_numdbs++; + } + MDB_TRACE(("%p, %s, %u = %u", txn, name, flags, slot)); + } + + return rc; +} + +int ESECT +mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) +{ + if (!arg || !TXN_DBI_EXIST(txn, dbi, DB_VALID)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_BLOCKED) + return MDB_BAD_TXN; + + if (txn->mt_dbflags[dbi] & DB_STALE) { + MDB_cursor mc; + MDB_xcursor mx; + /* Stale, must read the DB's root. cursor_init does it for us. */ + mdb_cursor_init(&mc, txn, dbi, &mx); + } + return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); +} + +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) +{ + char *ptr; + if (dbi < CORE_DBS || dbi >= env->me_maxdbs) + return; + ptr = env->me_dbxs[dbi].md_name.mv_data; + /* If there was no name, this was already closed */ + if (ptr) { + MDB_TRACE(("%p, %u", env, dbi)); + env->me_dbxs[dbi].md_name.mv_data = NULL; + env->me_dbxs[dbi].md_name.mv_size = 0; + env->me_dbflags[dbi] = 0; + env->me_dbiseqs[dbi]++; + free(ptr); + } +} + +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) +{ + /* We could return the flags for the FREE_DBI too but what's the point? */ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; + return MDB_SUCCESS; +} + +/** Add all the DB's pages to the free list. + * @param[in] mc Cursor on the DB to free. + * @param[in] subs non-Zero to check for sub-DBs in this DB. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_drop0(MDB_cursor *mc, int subs) +{ + int rc; + + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc == MDB_SUCCESS) { + MDB_txn *txn = mc->mc_txn; + MDB_node *ni; + MDB_cursor mx; + unsigned int i; + + /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves. + * This also avoids any P_LEAF2 pages, which have no nodes. + * Also if the DB doesn't have sub-DBs and has no overflow + * pages, omit scanning leaves. + */ + if ((mc->mc_flags & C_SUB) || + (!subs && !mc->mc_db->md_overflow_pages)) + mdb_cursor_pop(mc); + + mdb_cursor_copy(mc, &mx); + if (MDB_REMAPPING(mc->mc_txn->mt_env->me_flags)) { + /* bump refcount for mx's pages */ + for (i=0; imc_snum; i++) + MDB_PAGE_GET(&mx, mc->mc_pg[i]->mp_pgno, 1, &mx.mc_pg[i]); + } + + while (mc->mc_snum > 0) { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + unsigned n = NUMKEYS(mp); + if (IS_LEAF(mp)) { + for (i=0; imn_flags & F_BIGDATA) { + MDB_ovpage ovp; + memcpy(&ovp, NODEDATA(ni), sizeof(ovp)); + rc = mdb_midl_append_range(&txn->mt_free_pgs, + ovp.op_pgno, ovp.op_pages); + if (rc) + goto done; + mc->mc_db->md_overflow_pages -= ovp.op_pages; + if (!mc->mc_db->md_overflow_pages && !subs) + break; + } else if (subs && (ni->mn_flags & F_SUBDATA)) { + mdb_xcursor_init1(mc, ni); + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto done; + } + } + if (!subs && !mc->mc_db->md_overflow_pages) + goto pop; + } else { + if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) + goto done; + for (i=0; imt_free_pgs, pg); + } + } + if (!mc->mc_top) + break; + mc->mc_ki[mc->mc_top] = i; + rc = mdb_cursor_sibling(mc, 1); + if (rc) { + if (rc != MDB_NOTFOUND) + goto done; + /* no more siblings, go back to beginning + * of previous level. + */ +pop: + mdb_cursor_pop(mc); + mc->mc_ki[0] = 0; + for (i=1; imc_snum; i++) { + mc->mc_ki[i] = 0; + mc->mc_pg[i] = mx.mc_pg[i]; + } + } + } + /* free it */ + rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); +done: + if (rc) + txn->mt_flags |= MDB_TXN_ERROR; + /* drop refcount for mx's pages */ + MDB_CURSOR_UNREF(&mx, 0); + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + mc->mc_flags &= ~C_INITIALIZED; + return rc; +} + +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) +{ + MDB_cursor *mc, *m2; + int rc; + + if ((unsigned)del > 1 || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return MDB_IS_READONLY; + + if (TXN_DBI_CHANGED(txn, dbi)) + return MDB_BAD_DBI; + + MDB_TRACE(("%u, %d", dbi, del)); + + /* Dropping the main DBI empties/resets the entire environment. + * Can't do it if any other DBIs are still open. Can only commit + * or abort after this. + */ + if (dbi == MAIN_DBI) { + MDB_dbi i; + for (i = CORE_DBS; imt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_VALID) + return MDB_DBIS_BUSY; + } + for (i = 0; imt_dbflags[i] |= DB_DIRTY; + txn->mt_dbs[i].md_depth = 0; + txn->mt_dbs[i].md_branch_pages = 0; + txn->mt_dbs[i].md_leaf_pages = 0; + txn->mt_dbs[i].md_overflow_pages = 0; + txn->mt_dbs[i].md_entries = 0; + txn->mt_dbs[i].md_root = P_INVALID; + } + txn->mt_flags |= MDB_TXN_DIRTY|MDB_TXN_DROPPED; + return MDB_SUCCESS; + } + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) + return rc; + + rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); + /* Invalidate the dropped DB's cursors */ + for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) + m2->mc_flags &= ~(C_INITIALIZED|C_EOF); + if (rc) + goto leave; + + /* Can't delete the main DB */ + if (del && dbi >= CORE_DBS) { + rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA); + if (!rc) { + txn->mt_dbflags[dbi] = DB_STALE; + mdb_dbi_close(txn->mt_env, dbi); + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + } else { + /* reset the DB record, mark it dirty */ + txn->mt_dbflags[dbi] |= DB_DIRTY; + txn->mt_dbs[dbi].md_depth = 0; + txn->mt_dbs[dbi].md_branch_pages = 0; + txn->mt_dbs[dbi].md_leaf_pages = 0; + txn->mt_dbs[dbi].md_overflow_pages = 0; + txn->mt_dbs[dbi].md_entries = 0; + txn->mt_dbs[dbi].md_root = P_INVALID; + + txn->mt_flags |= MDB_TXN_DIRTY; + } +leave: + mdb_cursor_close(mc); + return rc; +} + +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_cmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_dcmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) +{ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_rel = rel; + return MDB_SUCCESS; +} + +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) +{ + if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) + return EINVAL; + + txn->mt_dbxs[dbi].md_relctx = ctx; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxkeysize(MDB_env *env) +{ + return ENV_MAXKEY(env); +} + +int ESECT +mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) +{ + unsigned int i, rdrs; + MDB_reader *mr; + char buf[64]; + int rc = 0, first = 1; + + if (!env || !func) + return -1; + if (!env->me_txns) { + return func("(no reader locks)\n", ctx); + } + rdrs = env->me_txns->mti_numreaders; + mr = env->me_txns->mti_readers; + for (i=0; i> 1; + cursor = base + pivot + 1; + val = pid - ids[cursor]; + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + /* found, so it's a duplicate */ + return -1; + } + } + + if( val > 0 ) { + ++cursor; + } + ids[0]++; + for (n = ids[0]; n > cursor; n--) + ids[n] = ids[n-1]; + ids[n] = pid; + return 0; +} + +int ESECT +mdb_reader_check(MDB_env *env, int *dead) +{ + if (!env) + return EINVAL; + if (dead) + *dead = 0; + return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS; +} + +/** As #mdb_reader_check(). \b rlocked is set if caller locked #me_rmutex. */ +static int ESECT +mdb_reader_check0(MDB_env *env, int rlocked, int *dead) +{ + mdb_mutexref_t rmutex = rlocked ? NULL : env->me_rmutex; + unsigned int i, j, rdrs; + MDB_reader *mr; + MDB_PID_T *pids, pid; + int rc = MDB_SUCCESS, count = 0; + + rdrs = env->me_txns->mti_numreaders; + pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); + if (!pids) + return ENOMEM; + pids[0] = 0; + mr = env->me_txns->mti_readers; + for (i=0; ime_pid) { + if (mdb_pid_insert(pids, pid) == 0) { + if (!mdb_reader_pid(env, Pidcheck, pid)) { + /* Stale reader found */ + j = i; + if (rmutex) { + if ((rc = LOCK_MUTEX0(rmutex)) != 0) { + if ((rc = mdb_mutex_failed(env, rmutex, rc))) + break; + rdrs = 0; /* the above checked all readers */ + } else { + /* Recheck, a new process may have reused pid */ + if (mdb_reader_pid(env, Pidcheck, pid)) + j = rdrs; + } + } + for (; jme_rmutex); + if (!rlocked) { + /* Keep mti_txnid updated, otherwise next writer can + * overwrite data which latest meta page refers to. + */ + meta = mdb_env_pick_meta(env); + env->me_txns->mti_txnid = meta->mm_txnid; + /* env is hosed if the dead thread was ours */ + if (env->me_txn) { + env->me_flags |= MDB_FATAL_ERROR; + env->me_txn = NULL; + rc = MDB_PANIC; + } + } + DPRINTF(("%cmutex owner died, %s", (rlocked ? 'r' : 'w'), + (rc ? "this process' env is hosed" : "recovering"))); + rc2 = mdb_reader_check0(env, rlocked, NULL); + if (rc2 == 0) + rc2 = mdb_mutex_consistent(mutex); + if (rc || (rc = rc2)) { + DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc))); + UNLOCK_MUTEX(mutex); + } + } else { +#ifdef _WIN32 + rc = ErrCode(); +#endif + DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc))); + } + + return rc; +} +#endif /* MDB_ROBUST_SUPPORTED */ + +#if defined(_WIN32) +/** Convert \b src to new wchar_t[] string with room for \b xtra extra chars */ +static int ESECT +utf8_to_utf16(const char *src, MDB_name *dst, int xtra) +{ + int rc, need = 0; + wchar_t *result = NULL; + for (;;) { /* malloc result, then fill it in */ + need = MultiByteToWideChar(CP_UTF8, 0, src, -1, result, need); + if (!need) { + rc = ErrCode(); + free(result); + return rc; + } + if (!result) { + result = malloc(sizeof(wchar_t) * (need + xtra)); + if (!result) + return ENOMEM; + continue; + } + dst->mn_alloced = 1; + dst->mn_len = need - 1; + dst->mn_val = result; + return MDB_SUCCESS; + } +} +#endif /* defined(_WIN32) */ +/** @} */ diff --git a/src/LightningDB/wasm/midl.c b/src/LightningDB/wasm/midl.c new file mode 100644 index 0000000..8a8fda7 --- /dev/null +++ b/src/LightningDB/wasm/midl.c @@ -0,0 +1,452 @@ +/** @file midl.c + * @brief ldap bdb back-end ID List functions */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 2000-2026 The OpenLDAP Foundation. + * Portions Copyright 2001-2026 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include +#include +#include +#include +#include +#include "midl.h" + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup idls ID List Management + * @{ + */ +#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) + +unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( ids[cursor], id ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +#if 0 /* superseded by append/sort */ +int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) +{ + unsigned x, i; + + x = mdb_midl_search( ids, id ); + assert( x > 0 ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0] && ids[x] == id ) { + /* duplicate */ + assert(0); + return -1; + } + + if ( ++ids[0] >= MDB_IDL_DB_MAX ) { + /* no room */ + --ids[0]; + return -2; + + } else { + /* insert id */ + for (i=ids[0]; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = id; + } + + return 0; +} +#endif + +MDB_IDL mdb_midl_alloc(int num) +{ + MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); + if (ids) { + *ids++ = num; + *ids = 0; + } + return ids; +} + +void mdb_midl_free(MDB_IDL ids) +{ + if (ids) + free(ids-1); +} + +void mdb_midl_shrink( MDB_IDL *idp ) +{ + MDB_IDL ids = *idp; + if (*(--ids) > MDB_IDL_UM_MAX && + (ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID)))) + { + *ids++ = MDB_IDL_UM_MAX; + *idp = ids; + } +} + +static int mdb_midl_grow( MDB_IDL *idp, int num ) +{ + MDB_IDL idn = *idp-1; + /* grow it */ + idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); + if (!idn) + return ENOMEM; + *idn++ += num; + *idp = idn; + return 0; +} + +int mdb_midl_need( MDB_IDL *idp, unsigned num ) +{ + MDB_IDL ids = *idp; + num += ids[0]; + if (num > ids[-1]) { + num = (num + num/4 + (256 + 2)) & -256; + if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) + return ENOMEM; + *ids++ = num - 2; + *idp = ids; + } + return 0; +} + +int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] >= ids[-1]) { + if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0]++; + ids[ids[0]] = id; + return 0; +} + +int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] + app[0] >= ids[-1]) { + if (mdb_midl_grow(idp, app[0])) + return ENOMEM; + ids = *idp; + } + memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); + ids[0] += app[0]; + return 0; +} + +int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) +{ + MDB_ID *ids = *idp, len = ids[0]; + /* Too big? */ + if (len + n > ids[-1]) { + if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0] = len + n; + ids += len; + while (n) + ids[n--] = id++; + return 0; +} + +void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) +{ + MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; + idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ + old_id = idl[j]; + while (i) { + merge_id = merge[i--]; + for (; old_id < merge_id; old_id = idl[--j]) + idl[k--] = old_id; + idl[k--] = merge_id; + } + idl[0] = total; +} + +/* Quicksort + Insertion sort for small arrays */ + +#define SMALL 8 +#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } + +void +mdb_midl_sort( MDB_IDL ids ) +{ + /* Max possible depth of int-indexed tree * 2 items/level */ + int istack[sizeof(int)*CHAR_BIT * 2]; + int i,j,k,l,ir,jstack; + MDB_ID a, itmp; + + ir = (int)ids[0]; + l = 1; + jstack = 0; + for(;;) { + if (ir - l < SMALL) { /* Insertion sort */ + for (j=l+1;j<=ir;j++) { + a = ids[j]; + for (i=j-1;i>=1;i--) { + if (ids[i] >= a) break; + ids[i+1] = ids[i]; + } + ids[i+1] = a; + } + if (jstack == 0) break; + ir = istack[jstack--]; + l = istack[jstack--]; + } else { + k = (l + ir) >> 1; /* Choose median of left, center, right */ + MIDL_SWAP(ids[k], ids[l+1]); + if (ids[l] < ids[ir]) { + MIDL_SWAP(ids[l], ids[ir]); + } + if (ids[l+1] < ids[ir]) { + MIDL_SWAP(ids[l+1], ids[ir]); + } + if (ids[l] < ids[l+1]) { + MIDL_SWAP(ids[l], ids[l+1]); + } + i = l+1; + j = ir; + a = ids[l+1]; + for(;;) { + do i++; while(ids[i] > a); + do j--; while(ids[j] < a); + if (j < i) break; + MIDL_SWAP(ids[i],ids[j]); + } + ids[l+1] = ids[j]; + ids[j] = a; + jstack += 2; + if (ir-i+1 >= j-l) { + istack[jstack] = ir; + istack[jstack-1] = i; + ir = j-1; + } else { + istack[jstack] = j-1; + istack[jstack-1] = l; + l = i; + } + } + } +} + +unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = (unsigned)ids[0].mid; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( id, ids[cursor].mid ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) +{ + unsigned x, i; + + x = mdb_mid2l_search( ids, id->mid ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0].mid && ids[x].mid == id->mid ) { + /* duplicate */ + return -1; + } + + if ( ids[0].mid >= MDB_IDL_UM_MAX ) { + /* too big */ + return -2; + + } else { + /* insert id */ + ids[0].mid++; + for (i=(unsigned)ids[0].mid; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = *id; + } + + return 0; +} + +int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) +{ + /* Too big? */ + if (ids[0].mid >= MDB_IDL_UM_MAX) { + return -2; + } + ids[0].mid++; + ids[ids[0].mid] = *id; + return 0; +} + +MDB_ID2L mdb_mid2l_alloc(int num) +{ + MDB_ID2L ids = malloc((num+2) * sizeof(MDB_ID2)); + if (ids) { + ids->mid = num; + ids++; + ids->mid = 0; + } + return ids; +} + +void mdb_mid2l_free(MDB_ID2L ids) +{ + if (ids) + free(ids-1); +} + +int mdb_mid2l_need( MDB_ID2L *idp, unsigned num ) +{ + MDB_ID2L ids = *idp; + num += ids[0].mid; + if (num > ids[-1].mid) { + num = (num + num/4 + (256 + 2)) & -256; + if (!(ids = realloc(ids-1, num * sizeof(MDB_ID2)))) + return ENOMEM; + ids[0].mid = num - 2; + *idp = ids+1; + } + return 0; +} + +#if MDB_RPAGE_CACHE +unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = (unsigned)ids[0].mid; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( id, ids[cursor].mid ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id ) +{ + unsigned x, i; + + x = mdb_mid3l_search( ids, id->mid ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0].mid && ids[x].mid == id->mid ) { + /* duplicate */ + return -1; + } + + /* insert id */ + ids[0].mid++; + for (i=(unsigned)ids[0].mid; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = *id; + + return 0; +} +#endif /* MDB_RPAGE_CACHE */ + +/** @} */ +/** @} */ diff --git a/src/LightningDB/wasm/midl.h b/src/LightningDB/wasm/midl.h new file mode 100644 index 0000000..1cccb92 --- /dev/null +++ b/src/LightningDB/wasm/midl.h @@ -0,0 +1,210 @@ +/** @file midl.h + * @brief LMDB ID List header file. + * + * This file was originally part of back-bdb but has been + * modified for use in libmdb. Most of the macros defined + * in this file are unused, just left over from the original. + * + * This file is only used internally in libmdb and its definitions + * are not exposed publicly. + */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 2000-2026 The OpenLDAP Foundation. + * Portions Copyright 2001-2026 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#ifndef _MDB_MIDL_H_ +#define _MDB_MIDL_H_ + +#include "lmdb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ + +/** @defgroup idls ID List Management + * @{ + */ + /** A generic unsigned ID number. These were entryIDs in back-bdb. + * Preferably it should have the same size as a pointer. + */ +typedef mdb_size_t MDB_ID; + + /** An IDL is an ID List, a sorted array of IDs. The first + * element of the array is a counter for how many actual + * IDs are in the list. In the original back-bdb code, IDLs are + * sorted in ascending order. For libmdb IDLs are sorted in + * descending order. + */ +typedef MDB_ID *MDB_IDL; + +/* IDL sizes - likely should be even bigger + * limiting factors: sizeof(ID), thread stack size + */ +#ifndef MDB_IDL_LOGN +#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ +#endif +#define MDB_IDL_DB_SIZE (1< + +#include "monocypher.h" + +#ifdef MONOCYPHER_CPP_NAMESPACE +namespace MONOCYPHER_CPP_NAMESPACE { +#endif + +///////////////// +/// Utilities /// +///////////////// +#define FOR_T(type, i, start, end) for (type i = (start); i < (end); i++) +#define FOR(i, start, end) FOR_T(size_t, i, start, end) +#define COPY(dst, src, size) FOR(_i_, 0, size) (dst)[_i_] = (src)[_i_] +#define ZERO(buf, size) FOR(_i_, 0, size) (buf)[_i_] = 0 +#define WIPE_CTX(ctx) crypto_wipe(ctx , sizeof(*(ctx))) +#define WIPE_BUFFER(buffer) crypto_wipe(buffer, sizeof(buffer)) +#define MIN(a, b) ((a) <= (b) ? (a) : (b)) +#define MAX(a, b) ((a) >= (b) ? (a) : (b)) + +typedef int8_t i8; +typedef uint8_t u8; +typedef int16_t i16; +typedef uint32_t u32; +typedef int32_t i32; +typedef int64_t i64; +typedef uint64_t u64; + +static const u8 zero[128] = {0}; + +// returns the smallest positive integer y such that +// (x + y) % pow_2 == 0 +// Basically, y is the "gap" missing to align x. +// Only works when pow_2 is a power of 2. +// Note: we use ~x+1 instead of -x to avoid compiler warnings +static size_t gap(size_t x, size_t pow_2) +{ + return (~x + 1) & (pow_2 - 1); +} + +static u32 load24_le(const u8 s[3]) +{ + return + ((u32)s[0] << 0) | + ((u32)s[1] << 8) | + ((u32)s[2] << 16); +} + +static u32 load32_le(const u8 s[4]) +{ + return + ((u32)s[0] << 0) | + ((u32)s[1] << 8) | + ((u32)s[2] << 16) | + ((u32)s[3] << 24); +} + +static u64 load64_le(const u8 s[8]) +{ + return load32_le(s) | ((u64)load32_le(s+4) << 32); +} + +static void store32_le(u8 out[4], u32 in) +{ + out[0] = in & 0xff; + out[1] = (in >> 8) & 0xff; + out[2] = (in >> 16) & 0xff; + out[3] = (in >> 24) & 0xff; +} + +static void store64_le(u8 out[8], u64 in) +{ + store32_le(out , (u32)in ); + store32_le(out + 4, in >> 32); +} + +static void load32_le_buf (u32 *dst, const u8 *src, size_t size) { + FOR(i, 0, size) { dst[i] = load32_le(src + i*4); } +} +static void load64_le_buf (u64 *dst, const u8 *src, size_t size) { + FOR(i, 0, size) { dst[i] = load64_le(src + i*8); } +} +static void store32_le_buf(u8 *dst, const u32 *src, size_t size) { + FOR(i, 0, size) { store32_le(dst + i*4, src[i]); } +} +static void store64_le_buf(u8 *dst, const u64 *src, size_t size) { + FOR(i, 0, size) { store64_le(dst + i*8, src[i]); } +} + +static u64 rotr64(u64 x, u64 n) { return (x >> n) ^ (x << (64 - n)); } +static u32 rotl32(u32 x, u32 n) { return (x << n) ^ (x >> (32 - n)); } + +static int neq0(u64 diff) +{ + // constant time comparison to zero + // return diff != 0 ? -1 : 0 + u64 half = (diff >> 32) | ((u32)diff); + return (1 & ((half - 1) >> 32)) - 1; +} + +static u64 x16(const u8 a[16], const u8 b[16]) +{ + return (load64_le(a + 0) ^ load64_le(b + 0)) + | (load64_le(a + 8) ^ load64_le(b + 8)); +} +static u64 x32(const u8 a[32],const u8 b[32]){return x16(a,b)| x16(a+16, b+16);} +static u64 x64(const u8 a[64],const u8 b[64]){return x32(a,b)| x32(a+32, b+32);} +int crypto_verify16(const u8 a[16], const u8 b[16]){ return neq0(x16(a, b)); } +int crypto_verify32(const u8 a[32], const u8 b[32]){ return neq0(x32(a, b)); } +int crypto_verify64(const u8 a[64], const u8 b[64]){ return neq0(x64(a, b)); } + +void crypto_wipe(void *secret, size_t size) +{ + volatile u8 *v_secret = (u8*)secret; + ZERO(v_secret, size); +} + +///////////////// +/// Chacha 20 /// +///////////////// +#define QUARTERROUND(a, b, c, d) \ + a += b; d = rotl32(d ^ a, 16); \ + c += d; b = rotl32(b ^ c, 12); \ + a += b; d = rotl32(d ^ a, 8); \ + c += d; b = rotl32(b ^ c, 7) + +static void chacha20_rounds(u32 out[16], const u32 in[16]) +{ + // The temporary variables make Chacha20 10% faster. + u32 t0 = in[ 0]; u32 t1 = in[ 1]; u32 t2 = in[ 2]; u32 t3 = in[ 3]; + u32 t4 = in[ 4]; u32 t5 = in[ 5]; u32 t6 = in[ 6]; u32 t7 = in[ 7]; + u32 t8 = in[ 8]; u32 t9 = in[ 9]; u32 t10 = in[10]; u32 t11 = in[11]; + u32 t12 = in[12]; u32 t13 = in[13]; u32 t14 = in[14]; u32 t15 = in[15]; + + FOR (i, 0, 10) { // 20 rounds, 2 rounds per loop. + QUARTERROUND(t0, t4, t8 , t12); // column 0 + QUARTERROUND(t1, t5, t9 , t13); // column 1 + QUARTERROUND(t2, t6, t10, t14); // column 2 + QUARTERROUND(t3, t7, t11, t15); // column 3 + QUARTERROUND(t0, t5, t10, t15); // diagonal 0 + QUARTERROUND(t1, t6, t11, t12); // diagonal 1 + QUARTERROUND(t2, t7, t8 , t13); // diagonal 2 + QUARTERROUND(t3, t4, t9 , t14); // diagonal 3 + } + out[ 0] = t0; out[ 1] = t1; out[ 2] = t2; out[ 3] = t3; + out[ 4] = t4; out[ 5] = t5; out[ 6] = t6; out[ 7] = t7; + out[ 8] = t8; out[ 9] = t9; out[10] = t10; out[11] = t11; + out[12] = t12; out[13] = t13; out[14] = t14; out[15] = t15; +} + +static const u8 *chacha20_constant = (const u8*)"expand 32-byte k"; // 16 bytes + +void crypto_chacha20_h(u8 out[32], const u8 key[32], const u8 in [16]) +{ + u32 block[16]; + load32_le_buf(block , chacha20_constant, 4); + load32_le_buf(block + 4, key , 8); + load32_le_buf(block + 12, in , 4); + + chacha20_rounds(block, block); + + // prevent reversal of the rounds by revealing only half of the buffer. + store32_le_buf(out , block , 4); // constant + store32_le_buf(out+16, block+12, 4); // counter and nonce + WIPE_BUFFER(block); +} + +u64 crypto_chacha20_djb(u8 *cipher_text, const u8 *plain_text, + size_t text_size, const u8 key[32], const u8 nonce[8], + u64 ctr) +{ + u32 input[16]; + load32_le_buf(input , chacha20_constant, 4); + load32_le_buf(input + 4, key , 8); + load32_le_buf(input + 14, nonce , 2); + input[12] = (u32) ctr; + input[13] = (u32)(ctr >> 32); + + // Whole blocks + u32 pool[16]; + size_t nb_blocks = text_size >> 6; + FOR (i, 0, nb_blocks) { + chacha20_rounds(pool, input); + if (plain_text != 0) { + FOR (j, 0, 16) { + u32 p = pool[j] + input[j]; + store32_le(cipher_text, p ^ load32_le(plain_text)); + cipher_text += 4; + plain_text += 4; + } + } else { + FOR (j, 0, 16) { + u32 p = pool[j] + input[j]; + store32_le(cipher_text, p); + cipher_text += 4; + } + } + input[12]++; + if (input[12] == 0) { + input[13]++; + } + } + text_size &= 63; + + // Last (incomplete) block + if (text_size > 0) { + if (plain_text == 0) { + plain_text = zero; + } + chacha20_rounds(pool, input); + u8 tmp[64]; + FOR (i, 0, 16) { + store32_le(tmp + i*4, pool[i] + input[i]); + } + FOR (i, 0, text_size) { + cipher_text[i] = tmp[i] ^ plain_text[i]; + } + WIPE_BUFFER(tmp); + } + ctr = input[12] + ((u64)input[13] << 32) + (text_size > 0); + + WIPE_BUFFER(pool); + WIPE_BUFFER(input); + return ctr; +} + +u32 crypto_chacha20_ietf(u8 *cipher_text, const u8 *plain_text, + size_t text_size, + const u8 key[32], const u8 nonce[12], u32 ctr) +{ + u64 big_ctr = ctr + ((u64)load32_le(nonce) << 32); + return (u32)crypto_chacha20_djb(cipher_text, plain_text, text_size, + key, nonce + 4, big_ctr); +} + +u64 crypto_chacha20_x(u8 *cipher_text, const u8 *plain_text, + size_t text_size, + const u8 key[32], const u8 nonce[24], u64 ctr) +{ + u8 sub_key[32]; + crypto_chacha20_h(sub_key, key, nonce); + ctr = crypto_chacha20_djb(cipher_text, plain_text, text_size, + sub_key, nonce + 16, ctr); + WIPE_BUFFER(sub_key); + return ctr; +} + +///////////////// +/// Poly 1305 /// +///////////////// + +// h = (h + c) * r +// preconditions: +// ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff +// ctx->r <= 0ffffffc_0ffffffc_0ffffffc_0fffffff +// end <= 1 +// Postcondition: +// ctx->h <= 4_ffffffff_ffffffff_ffffffff_ffffffff +static void poly_blocks(crypto_poly1305_ctx *ctx, const u8 *in, + size_t nb_blocks, unsigned end) +{ + // Local all the things! + const u32 r0 = ctx->r[0]; + const u32 r1 = ctx->r[1]; + const u32 r2 = ctx->r[2]; + const u32 r3 = ctx->r[3]; + const u32 rr0 = (r0 >> 2) * 5; // lose 2 bits... + const u32 rr1 = (r1 >> 2) + r1; // rr1 == (r1 >> 2) * 5 + const u32 rr2 = (r2 >> 2) + r2; // rr1 == (r2 >> 2) * 5 + const u32 rr3 = (r3 >> 2) + r3; // rr1 == (r3 >> 2) * 5 + const u32 rr4 = r0 & 3; // ...recover 2 bits + u32 h0 = ctx->h[0]; + u32 h1 = ctx->h[1]; + u32 h2 = ctx->h[2]; + u32 h3 = ctx->h[3]; + u32 h4 = ctx->h[4]; + + FOR (i, 0, nb_blocks) { + // h + c, without carry propagation + const u64 s0 = (u64)h0 + load32_le(in); in += 4; + const u64 s1 = (u64)h1 + load32_le(in); in += 4; + const u64 s2 = (u64)h2 + load32_le(in); in += 4; + const u64 s3 = (u64)h3 + load32_le(in); in += 4; + const u32 s4 = h4 + end; + + // (h + c) * r, without carry propagation + const u64 x0 = s0*r0+ s1*rr3+ s2*rr2+ s3*rr1+ s4*rr0; + const u64 x1 = s0*r1+ s1*r0 + s2*rr3+ s3*rr2+ s4*rr1; + const u64 x2 = s0*r2+ s1*r1 + s2*r0 + s3*rr3+ s4*rr2; + const u64 x3 = s0*r3+ s1*r2 + s2*r1 + s3*r0 + s4*rr3; + const u32 x4 = s4*rr4; + + // partial reduction modulo 2^130 - 5 + const u32 u5 = x4 + (x3 >> 32); // u5 <= 7ffffff5 + const u64 u0 = (u5 >> 2) * 5 + (x0 & 0xffffffff); + const u64 u1 = (u0 >> 32) + (x1 & 0xffffffff) + (x0 >> 32); + const u64 u2 = (u1 >> 32) + (x2 & 0xffffffff) + (x1 >> 32); + const u64 u3 = (u2 >> 32) + (x3 & 0xffffffff) + (x2 >> 32); + const u32 u4 = (u3 >> 32) + (u5 & 3); // u4 <= 4 + + // Update the hash + h0 = u0 & 0xffffffff; + h1 = u1 & 0xffffffff; + h2 = u2 & 0xffffffff; + h3 = u3 & 0xffffffff; + h4 = u4; + } + ctx->h[0] = h0; + ctx->h[1] = h1; + ctx->h[2] = h2; + ctx->h[3] = h3; + ctx->h[4] = h4; +} + +void crypto_poly1305_init(crypto_poly1305_ctx *ctx, const u8 key[32]) +{ + ZERO(ctx->h, 5); // Initial hash is zero + ctx->c_idx = 0; + // load r and pad (r has some of its bits cleared) + load32_le_buf(ctx->r , key , 4); + load32_le_buf(ctx->pad, key+16, 4); + FOR (i, 0, 1) { ctx->r[i] &= 0x0fffffff; } + FOR (i, 1, 4) { ctx->r[i] &= 0x0ffffffc; } +} + +void crypto_poly1305_update(crypto_poly1305_ctx *ctx, + const u8 *message, size_t message_size) +{ + // Avoid undefined NULL pointer increments with empty messages + if (message_size == 0) { + return; + } + + // Align ourselves with block boundaries + size_t aligned = MIN(gap(ctx->c_idx, 16), message_size); + FOR (i, 0, aligned) { + ctx->c[ctx->c_idx] = *message; + ctx->c_idx++; + message++; + message_size--; + } + + // If block is complete, process it + if (ctx->c_idx == 16) { + poly_blocks(ctx, ctx->c, 1, 1); + ctx->c_idx = 0; + } + + // Process the message block by block + size_t nb_blocks = message_size >> 4; + poly_blocks(ctx, message, nb_blocks, 1); + message += nb_blocks << 4; + message_size &= 15; + + // remaining bytes (we never complete a block here) + FOR (i, 0, message_size) { + ctx->c[ctx->c_idx] = message[i]; + ctx->c_idx++; + } +} + +void crypto_poly1305_final(crypto_poly1305_ctx *ctx, u8 mac[16]) +{ + // Process the last block (if any) + // We move the final 1 according to remaining input length + // (this will add less than 2^130 to the last input block) + if (ctx->c_idx != 0) { + ZERO(ctx->c + ctx->c_idx, 16 - ctx->c_idx); + ctx->c[ctx->c_idx] = 1; + poly_blocks(ctx, ctx->c, 1, 0); + } + + // check if we should subtract 2^130-5 by performing the + // corresponding carry propagation. + u64 c = 5; + FOR (i, 0, 4) { + c += ctx->h[i]; + c >>= 32; + } + c += ctx->h[4]; + c = (c >> 2) * 5; // shift the carry back to the beginning + // c now indicates how many times we should subtract 2^130-5 (0 or 1) + FOR (i, 0, 4) { + c += (u64)ctx->h[i] + ctx->pad[i]; + store32_le(mac + i*4, (u32)c); + c = c >> 32; + } + WIPE_CTX(ctx); +} + +void crypto_poly1305(u8 mac[16], const u8 *message, + size_t message_size, const u8 key[32]) +{ + crypto_poly1305_ctx ctx; + crypto_poly1305_init (&ctx, key); + crypto_poly1305_update(&ctx, message, message_size); + crypto_poly1305_final (&ctx, mac); +} + +//////////////// +/// BLAKE2 b /// +//////////////// +static const u64 iv[8] = { + 0x6a09e667f3bcc908, 0xbb67ae8584caa73b, + 0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1, + 0x510e527fade682d1, 0x9b05688c2b3e6c1f, + 0x1f83d9abfb41bd6b, 0x5be0cd19137e2179, +}; + +static void blake2b_compress(crypto_blake2b_ctx *ctx, int is_last_block) +{ + static const u8 sigma[12][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + }; + + // increment input offset + u64 *x = ctx->input_offset; + size_t y = ctx->input_idx; + x[0] += y; + if (x[0] < y) { + x[1]++; + } + + // init work vector + u64 v0 = ctx->hash[0]; u64 v8 = iv[0]; + u64 v1 = ctx->hash[1]; u64 v9 = iv[1]; + u64 v2 = ctx->hash[2]; u64 v10 = iv[2]; + u64 v3 = ctx->hash[3]; u64 v11 = iv[3]; + u64 v4 = ctx->hash[4]; u64 v12 = iv[4] ^ ctx->input_offset[0]; + u64 v5 = ctx->hash[5]; u64 v13 = iv[5] ^ ctx->input_offset[1]; + u64 v6 = ctx->hash[6]; u64 v14 = iv[6] ^ (u64)~(is_last_block - 1); + u64 v7 = ctx->hash[7]; u64 v15 = iv[7]; + + // mangle work vector + u64 *input = ctx->input; +#define BLAKE2_G(a, b, c, d, x, y) \ + a += b + x; d = rotr64(d ^ a, 32); \ + c += d; b = rotr64(b ^ c, 24); \ + a += b + y; d = rotr64(d ^ a, 16); \ + c += d; b = rotr64(b ^ c, 63) +#define BLAKE2_ROUND(i) \ + BLAKE2_G(v0, v4, v8 , v12, input[sigma[i][ 0]], input[sigma[i][ 1]]); \ + BLAKE2_G(v1, v5, v9 , v13, input[sigma[i][ 2]], input[sigma[i][ 3]]); \ + BLAKE2_G(v2, v6, v10, v14, input[sigma[i][ 4]], input[sigma[i][ 5]]); \ + BLAKE2_G(v3, v7, v11, v15, input[sigma[i][ 6]], input[sigma[i][ 7]]); \ + BLAKE2_G(v0, v5, v10, v15, input[sigma[i][ 8]], input[sigma[i][ 9]]); \ + BLAKE2_G(v1, v6, v11, v12, input[sigma[i][10]], input[sigma[i][11]]); \ + BLAKE2_G(v2, v7, v8 , v13, input[sigma[i][12]], input[sigma[i][13]]); \ + BLAKE2_G(v3, v4, v9 , v14, input[sigma[i][14]], input[sigma[i][15]]) + +#ifdef BLAKE2_NO_UNROLLING + FOR (i, 0, 12) { + BLAKE2_ROUND(i); + } +#else + BLAKE2_ROUND(0); BLAKE2_ROUND(1); BLAKE2_ROUND(2); BLAKE2_ROUND(3); + BLAKE2_ROUND(4); BLAKE2_ROUND(5); BLAKE2_ROUND(6); BLAKE2_ROUND(7); + BLAKE2_ROUND(8); BLAKE2_ROUND(9); BLAKE2_ROUND(10); BLAKE2_ROUND(11); +#endif + + // update hash + ctx->hash[0] ^= v0 ^ v8; ctx->hash[1] ^= v1 ^ v9; + ctx->hash[2] ^= v2 ^ v10; ctx->hash[3] ^= v3 ^ v11; + ctx->hash[4] ^= v4 ^ v12; ctx->hash[5] ^= v5 ^ v13; + ctx->hash[6] ^= v6 ^ v14; ctx->hash[7] ^= v7 ^ v15; +} + +void crypto_blake2b_keyed_init(crypto_blake2b_ctx *ctx, size_t hash_size, + const u8 *key, size_t key_size) +{ + // initial hash + COPY(ctx->hash, iv, 8); + ctx->hash[0] ^= 0x01010000 ^ (key_size << 8) ^ hash_size; + + ctx->input_offset[0] = 0; // beginning of the input, no offset + ctx->input_offset[1] = 0; // beginning of the input, no offset + ctx->hash_size = hash_size; + ctx->input_idx = 0; + ZERO(ctx->input, 16); + + // if there is a key, the first block is that key (padded with zeroes) + if (key_size > 0) { + u8 key_block[128] = {0}; + COPY(key_block, key, key_size); + // same as calling crypto_blake2b_update(ctx, key_block , 128) + load64_le_buf(ctx->input, key_block, 16); + ctx->input_idx = 128; + } +} + +void crypto_blake2b_init(crypto_blake2b_ctx *ctx, size_t hash_size) +{ + crypto_blake2b_keyed_init(ctx, hash_size, 0, 0); +} + +void crypto_blake2b_update(crypto_blake2b_ctx *ctx, + const u8 *message, size_t message_size) +{ + // Avoid undefined NULL pointer increments with empty messages + if (message_size == 0) { + return; + } + + // Align with word boundaries + if ((ctx->input_idx & 7) != 0) { + size_t nb_bytes = MIN(gap(ctx->input_idx, 8), message_size); + size_t word = ctx->input_idx >> 3; + size_t byte = ctx->input_idx & 7; + FOR (i, 0, nb_bytes) { + ctx->input[word] |= (u64)message[i] << ((byte + i) << 3); + } + ctx->input_idx += nb_bytes; + message += nb_bytes; + message_size -= nb_bytes; + } + + // Align with block boundaries (faster than byte by byte) + if ((ctx->input_idx & 127) != 0) { + size_t nb_words = MIN(gap(ctx->input_idx, 128), message_size) >> 3; + load64_le_buf(ctx->input + (ctx->input_idx >> 3), message, nb_words); + ctx->input_idx += nb_words << 3; + message += nb_words << 3; + message_size -= nb_words << 3; + } + + // Process block by block + size_t nb_blocks = message_size >> 7; + FOR (i, 0, nb_blocks) { + if (ctx->input_idx == 128) { + blake2b_compress(ctx, 0); + } + load64_le_buf(ctx->input, message, 16); + message += 128; + ctx->input_idx = 128; + } + message_size &= 127; + + if (message_size != 0) { + // Compress block & flush input buffer as needed + if (ctx->input_idx == 128) { + blake2b_compress(ctx, 0); + ctx->input_idx = 0; + } + if (ctx->input_idx == 0) { + ZERO(ctx->input, 16); + } + // Fill remaining words (faster than byte by byte) + size_t nb_words = message_size >> 3; + load64_le_buf(ctx->input, message, nb_words); + ctx->input_idx += nb_words << 3; + message += nb_words << 3; + message_size -= nb_words << 3; + + // Fill remaining bytes + FOR (i, 0, message_size) { + size_t word = ctx->input_idx >> 3; + size_t byte = ctx->input_idx & 7; + ctx->input[word] |= (u64)message[i] << (byte << 3); + ctx->input_idx++; + } + } +} + +void crypto_blake2b_final(crypto_blake2b_ctx *ctx, u8 *hash) +{ + blake2b_compress(ctx, 1); // compress the last block + size_t hash_size = MIN(ctx->hash_size, 64); + size_t nb_words = hash_size >> 3; + store64_le_buf(hash, ctx->hash, nb_words); + FOR (i, nb_words << 3, hash_size) { + hash[i] = (ctx->hash[i >> 3] >> (8 * (i & 7))) & 0xff; + } + WIPE_CTX(ctx); +} + +void crypto_blake2b_keyed(u8 *hash, size_t hash_size, + const u8 *key, size_t key_size, + const u8 *message, size_t message_size) +{ + crypto_blake2b_ctx ctx; + crypto_blake2b_keyed_init(&ctx, hash_size, key, key_size); + crypto_blake2b_update (&ctx, message, message_size); + crypto_blake2b_final (&ctx, hash); +} + +void crypto_blake2b(u8 *hash, size_t hash_size, const u8 *msg, size_t msg_size) +{ + crypto_blake2b_keyed(hash, hash_size, 0, 0, msg, msg_size); +} + +////////////// +/// Argon2 /// +////////////// +// references to R, Z, Q etc. come from the spec + +// Argon2 operates on 1024 byte blocks. +typedef struct { u64 a[128]; } blk; + +// updates a BLAKE2 hash with a 32 bit word, little endian. +static void blake_update_32(crypto_blake2b_ctx *ctx, u32 input) +{ + u8 buf[4]; + store32_le(buf, input); + crypto_blake2b_update(ctx, buf, 4); + WIPE_BUFFER(buf); +} + +static void blake_update_32_buf(crypto_blake2b_ctx *ctx, + const u8 *buf, u32 size) +{ + blake_update_32(ctx, size); + crypto_blake2b_update(ctx, buf, size); +} + + +static void copy_block(blk *o,const blk*in){FOR(i, 0, 128) o->a[i] = in->a[i];} +static void xor_block(blk *o,const blk*in){FOR(i, 0, 128) o->a[i] ^= in->a[i];} + +// Hash with a virtually unlimited digest size. +// Doesn't extract more entropy than the base hash function. +// Mainly used for filling a whole kilobyte block with pseudo-random bytes. +// (One could use a stream cipher with a seed hash as the key, but +// this would introduce another dependency —and point of failure.) +static void extended_hash(u8 *digest, u32 digest_size, + const u8 *input , u32 input_size) +{ + crypto_blake2b_ctx ctx; + crypto_blake2b_init (&ctx, MIN(digest_size, 64)); + blake_update_32 (&ctx, digest_size); + crypto_blake2b_update(&ctx, input, input_size); + crypto_blake2b_final (&ctx, digest); + + if (digest_size > 64) { + // the conversion to u64 avoids integer overflow on + // ludicrously big hash sizes. + u32 r = (u32)(((u64)digest_size + 31) >> 5) - 2; + u32 i = 1; + u32 in = 0; + u32 out = 32; + while (i < r) { + // Input and output overlap. This is intentional + crypto_blake2b(digest + out, 64, digest + in, 64); + i += 1; + in += 32; + out += 32; + } + crypto_blake2b(digest + out, digest_size - (32 * r), digest + in , 64); + } +} + +#define LSB(x) ((u64)(u32)x) +#define G(a, b, c, d) \ + a += b + ((LSB(a) * LSB(b)) << 1); d ^= a; d = rotr64(d, 32); \ + c += d + ((LSB(c) * LSB(d)) << 1); b ^= c; b = rotr64(b, 24); \ + a += b + ((LSB(a) * LSB(b)) << 1); d ^= a; d = rotr64(d, 16); \ + c += d + ((LSB(c) * LSB(d)) << 1); b ^= c; b = rotr64(b, 63) +#define ROUND(v0, v1, v2, v3, v4, v5, v6, v7, \ + v8, v9, v10, v11, v12, v13, v14, v15) \ + G(v0, v4, v8, v12); G(v1, v5, v9, v13); \ + G(v2, v6, v10, v14); G(v3, v7, v11, v15); \ + G(v0, v5, v10, v15); G(v1, v6, v11, v12); \ + G(v2, v7, v8, v13); G(v3, v4, v9, v14) + +// Core of the compression function G. Computes Z from R in place. +static void g_rounds(blk *b) +{ + // column rounds (work_block = Q) + for (int i = 0; i < 128; i += 16) { + ROUND(b->a[i ], b->a[i+ 1], b->a[i+ 2], b->a[i+ 3], + b->a[i+ 4], b->a[i+ 5], b->a[i+ 6], b->a[i+ 7], + b->a[i+ 8], b->a[i+ 9], b->a[i+10], b->a[i+11], + b->a[i+12], b->a[i+13], b->a[i+14], b->a[i+15]); + } + // row rounds (b = Z) + for (int i = 0; i < 16; i += 2) { + ROUND(b->a[i ], b->a[i+ 1], b->a[i+ 16], b->a[i+ 17], + b->a[i+32], b->a[i+33], b->a[i+ 48], b->a[i+ 49], + b->a[i+64], b->a[i+65], b->a[i+ 80], b->a[i+ 81], + b->a[i+96], b->a[i+97], b->a[i+112], b->a[i+113]); + } +} + +const crypto_argon2_extras crypto_argon2_no_extras = { 0, 0, 0, 0 }; + +void crypto_argon2(u8 *hash, u32 hash_size, void *work_area, + crypto_argon2_config config, + crypto_argon2_inputs inputs, + crypto_argon2_extras extras) +{ + const u32 segment_size = config.nb_blocks / config.nb_lanes / 4; + const u32 lane_size = segment_size * 4; + const u32 nb_blocks = lane_size * config.nb_lanes; // rounding down + + // work area seen as blocks (must be suitably aligned) + blk *blocks = (blk*)work_area; + { + u8 initial_hash[72]; // 64 bytes plus 2 words for future hashes + crypto_blake2b_ctx ctx; + crypto_blake2b_init (&ctx, 64); + blake_update_32 (&ctx, config.nb_lanes ); // p: number of "threads" + blake_update_32 (&ctx, hash_size); + blake_update_32 (&ctx, config.nb_blocks); + blake_update_32 (&ctx, config.nb_passes); + blake_update_32 (&ctx, 0x13); // v: version number + blake_update_32 (&ctx, config.algorithm); // y: Argon2i, Argon2d... + blake_update_32_buf (&ctx, inputs.pass, inputs.pass_size); + blake_update_32_buf (&ctx, inputs.salt, inputs.salt_size); + blake_update_32_buf (&ctx, extras.key, extras.key_size); + blake_update_32_buf (&ctx, extras.ad, extras.ad_size); + crypto_blake2b_final(&ctx, initial_hash); // fill 64 first bytes only + + // fill first 2 blocks of each lane + u8 hash_area[1024]; + FOR_T(u32, l, 0, config.nb_lanes) { + FOR_T(u32, i, 0, 2) { + store32_le(initial_hash + 64, i); // first additional word + store32_le(initial_hash + 68, l); // second additional word + extended_hash(hash_area, 1024, initial_hash, 72); + load64_le_buf(blocks[l * lane_size + i].a, hash_area, 128); + } + } + + WIPE_BUFFER(initial_hash); + WIPE_BUFFER(hash_area); + } + + // Argon2i and Argon2id start with constant time indexing + int constant_time = config.algorithm != CRYPTO_ARGON2_D; + + // Fill (and re-fill) the rest of the blocks + // + // Note: even though each segment within the same slice can be + // computed in parallel, (one thread per lane), we are computing + // them sequentially, because Monocypher doesn't support threads. + // + // Yet optimal performance (and therefore security) requires one + // thread per lane. The only reason Monocypher supports multiple + // lanes is compatibility. + blk tmp; + FOR_T(u32, pass, 0, config.nb_passes) { + FOR_T(u32, slice, 0, 4) { + // On the first slice of the first pass, + // blocks 0 and 1 are already filled, hence pass_offset. + u32 pass_offset = pass == 0 && slice == 0 ? 2 : 0; + u32 slice_offset = slice * segment_size; + + // Argon2id switches back to non-constant time indexing + // after the first two slices of the first pass + if (slice == 2 && config.algorithm == CRYPTO_ARGON2_ID) { + constant_time = 0; + } + + // Each iteration of the following loop may be performed in + // a separate thread. All segments must be fully completed + // before we start filling the next slice. + FOR_T(u32, segment, 0, config.nb_lanes) { + blk index_block; + u32 index_ctr = 1; + FOR_T (u32, block, pass_offset, segment_size) { + // Current and previous blocks + u32 lane_offset = segment * lane_size; + blk *segment_start = blocks + lane_offset + slice_offset; + blk *current = segment_start + block; + blk *previous = + block == 0 && slice_offset == 0 + ? segment_start + lane_size - 1 + : segment_start + block - 1; + + u64 index_seed; + if (constant_time) { + if (block == pass_offset || (block % 128) == 0) { + // Fill or refresh deterministic indices block + + // seed the beginning of the block... + ZERO(index_block.a, 128); + index_block.a[0] = pass; + index_block.a[1] = segment; + index_block.a[2] = slice; + index_block.a[3] = nb_blocks; + index_block.a[4] = config.nb_passes; + index_block.a[5] = config.algorithm; + index_block.a[6] = index_ctr; + index_ctr++; + + // ... then shuffle it + copy_block(&tmp, &index_block); + g_rounds (&index_block); + xor_block (&index_block, &tmp); + copy_block(&tmp, &index_block); + g_rounds (&index_block); + xor_block (&index_block, &tmp); + } + index_seed = index_block.a[block % 128]; + } else { + index_seed = previous->a[0]; + } + + // Establish the reference set. *Approximately* comprises: + // - The last 3 slices (if they exist yet) + // - The already constructed blocks in the current segment + u32 next_slice = ((slice + 1) % 4) * segment_size; + u32 window_start = pass == 0 ? 0 : next_slice; + u32 nb_segments = pass == 0 ? slice : 3; + u64 lane = + pass == 0 && slice == 0 + ? segment + : (index_seed >> 32) % config.nb_lanes; + u32 window_size = + nb_segments * segment_size + + (lane == segment ? block-1 : + block == 0 ? (u32)-1 : 0); + + // Find reference block + u64 j1 = index_seed & 0xffffffff; // block selector + u64 x = (j1 * j1) >> 32; + u64 y = (window_size * x) >> 32; + u64 z = (window_size - 1) - y; + u64 ref = (window_start + z) % lane_size; + u32 index = lane * lane_size + (u32)ref; + blk *reference = blocks + index; + + // Shuffle the previous & reference block + // into the current block + copy_block(&tmp, previous); + xor_block (&tmp, reference); + if (pass == 0) { copy_block(current, &tmp); } + else { xor_block (current, &tmp); } + g_rounds (&tmp); + xor_block (current, &tmp); + } + } + } + } + + // Wipe temporary block + volatile u64* p = tmp.a; + ZERO(p, 128); + + // XOR last blocks of each lane + blk *last_block = blocks + lane_size - 1; + FOR_T (u32, lane, 1, config.nb_lanes) { + blk *next_block = last_block + lane_size; + xor_block(next_block, last_block); + last_block = next_block; + } + + // Serialize last block + u8 final_block[1024]; + store64_le_buf(final_block, last_block->a, 128); + + // Wipe work area + p = (u64*)work_area; + ZERO(p, 128 * nb_blocks); + + // Hash the very last block with H' into the output hash + extended_hash(hash, hash_size, final_block, 1024); + WIPE_BUFFER(final_block); +} + +//////////////////////////////////// +/// Arithmetic modulo 2^255 - 19 /// +//////////////////////////////////// +// Originally taken from SUPERCOP's ref10 implementation. +// A bit bigger than TweetNaCl, over 4 times faster. + +// field element +typedef i32 fe[10]; + +// field constants +// +// fe_one : 1 +// sqrtm1 : sqrt(-1) +// d : -121665 / 121666 +// D2 : 2 * -121665 / 121666 +// lop_x, lop_y: low order point in Edwards coordinates +// ufactor : -sqrt(-1) * 2 +// A2 : 486662^2 (A squared) +static const fe fe_one = {1}; +static const fe sqrtm1 = { + -32595792, -7943725, 9377950, 3500415, 12389472, + -272473, -25146209, -2005654, 326686, 11406482, +}; +static const fe d = { + -10913610, 13857413, -15372611, 6949391, 114729, + -8787816, -6275908, -3247719, -18696448, -12055116, +}; +static const fe D2 = { + -21827239, -5839606, -30745221, 13898782, 229458, + 15978800, -12551817, -6495438, 29715968, 9444199, +}; +static const fe lop_x = { + 21352778, 5345713, 4660180, -8347857, 24143090, + 14568123, 30185756, -12247770, -33528939, 8345319, +}; +static const fe lop_y = { + -6952922, -1265500, 6862341, -7057498, -4037696, + -5447722, 31680899, -15325402, -19365852, 1569102, +}; +static const fe ufactor = { + -1917299, 15887451, -18755900, -7000830, -24778944, + 544946, -16816446, 4011309, -653372, 10741468, +}; +static const fe A2 = { + 12721188, 3529, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +static void fe_0(fe h) { ZERO(h , 10); } +static void fe_1(fe h) { h[0] = 1; ZERO(h+1, 9); } + +static void fe_copy(fe h,const fe f ){FOR(i,0,10) h[i] = f[i]; } +static void fe_neg (fe h,const fe f ){FOR(i,0,10) h[i] = -f[i]; } +static void fe_add (fe h,const fe f,const fe g){FOR(i,0,10) h[i] = f[i] + g[i];} +static void fe_sub (fe h,const fe f,const fe g){FOR(i,0,10) h[i] = f[i] - g[i];} + +static void fe_cswap(fe f, fe g, int b) +{ + i32 mask = -b; // -1 = 0xffffffff + FOR (i, 0, 10) { + i32 x = (f[i] ^ g[i]) & mask; + f[i] = f[i] ^ x; + g[i] = g[i] ^ x; + } +} + +static void fe_ccopy(fe f, const fe g, int b) +{ + i32 mask = -b; // -1 = 0xffffffff + FOR (i, 0, 10) { + i32 x = (f[i] ^ g[i]) & mask; + f[i] = f[i] ^ x; + } +} + + +// Signed carry propagation +// ------------------------ +// +// Let t be a number. It can be uniquely decomposed thus: +// +// t = h*2^26 + l +// such that -2^25 <= l < 2^25 +// +// Let c = (t + 2^25) / 2^26 (rounded down) +// c = (h*2^26 + l + 2^25) / 2^26 (rounded down) +// c = h + (l + 2^25) / 2^26 (rounded down) +// c = h (exactly) +// Because 0 <= l + 2^25 < 2^26 +// +// Let u = t - c*2^26 +// u = h*2^26 + l - h*2^26 +// u = l +// Therefore, -2^25 <= u < 2^25 +// +// Additionally, if |t| < x, then |h| < x/2^26 (rounded down) +// +// Notations: +// - In C, 1<<25 means 2^25. +// - In C, x>>25 means floor(x / (2^25)). +// - All of the above applies with 25 & 24 as well as 26 & 25. +// +// +// Note on negative right shifts +// ----------------------------- +// +// In C, x >> n, where x is a negative integer, is implementation +// defined. In practice, all platforms do arithmetic shift, which is +// equivalent to division by 2^26, rounded down. Some compilers, like +// GCC, even guarantee it. +// +// If we ever stumble upon a platform that does not propagate the sign +// bit (we won't), visible failures will show at the slightest test, and +// the signed shifts can be replaced by the following: +// +// typedef struct { i64 x:39; } s25; +// typedef struct { i64 x:38; } s26; +// i64 shift25(i64 x) { s25 s; s.x = ((u64)x)>>25; return s.x; } +// i64 shift26(i64 x) { s26 s; s.x = ((u64)x)>>26; return s.x; } +// +// Current compilers cannot optimise this, causing a 30% drop in +// performance. Fairly expensive for something that never happens. +// +// +// Precondition +// ------------ +// +// |t0| < 2^63 +// |t1|..|t9| < 2^62 +// +// Algorithm +// --------- +// c = t0 + 2^25 / 2^26 -- |c| <= 2^36 +// t0 -= c * 2^26 -- |t0| <= 2^25 +// t1 += c -- |t1| <= 2^63 +// +// c = t4 + 2^25 / 2^26 -- |c| <= 2^36 +// t4 -= c * 2^26 -- |t4| <= 2^25 +// t5 += c -- |t5| <= 2^63 +// +// c = t1 + 2^24 / 2^25 -- |c| <= 2^38 +// t1 -= c * 2^25 -- |t1| <= 2^24 +// t2 += c -- |t2| <= 2^63 +// +// c = t5 + 2^24 / 2^25 -- |c| <= 2^38 +// t5 -= c * 2^25 -- |t5| <= 2^24 +// t6 += c -- |t6| <= 2^63 +// +// c = t2 + 2^25 / 2^26 -- |c| <= 2^37 +// t2 -= c * 2^26 -- |t2| <= 2^25 < 1.1 * 2^25 (final t2) +// t3 += c -- |t3| <= 2^63 +// +// c = t6 + 2^25 / 2^26 -- |c| <= 2^37 +// t6 -= c * 2^26 -- |t6| <= 2^25 < 1.1 * 2^25 (final t6) +// t7 += c -- |t7| <= 2^63 +// +// c = t3 + 2^24 / 2^25 -- |c| <= 2^38 +// t3 -= c * 2^25 -- |t3| <= 2^24 < 1.1 * 2^24 (final t3) +// t4 += c -- |t4| <= 2^25 + 2^38 < 2^39 +// +// c = t7 + 2^24 / 2^25 -- |c| <= 2^38 +// t7 -= c * 2^25 -- |t7| <= 2^24 < 1.1 * 2^24 (final t7) +// t8 += c -- |t8| <= 2^63 +// +// c = t4 + 2^25 / 2^26 -- |c| <= 2^13 +// t4 -= c * 2^26 -- |t4| <= 2^25 < 1.1 * 2^25 (final t4) +// t5 += c -- |t5| <= 2^24 + 2^13 < 1.1 * 2^24 (final t5) +// +// c = t8 + 2^25 / 2^26 -- |c| <= 2^37 +// t8 -= c * 2^26 -- |t8| <= 2^25 < 1.1 * 2^25 (final t8) +// t9 += c -- |t9| <= 2^63 +// +// c = t9 + 2^24 / 2^25 -- |c| <= 2^38 +// t9 -= c * 2^25 -- |t9| <= 2^24 < 1.1 * 2^24 (final t9) +// t0 += c * 19 -- |t0| <= 2^25 + 2^38*19 < 2^44 +// +// c = t0 + 2^25 / 2^26 -- |c| <= 2^18 +// t0 -= c * 2^26 -- |t0| <= 2^25 < 1.1 * 2^25 (final t0) +// t1 += c -- |t1| <= 2^24 + 2^18 < 1.1 * 2^24 (final t1) +// +// Postcondition +// ------------- +// |t0|, |t2|, |t4|, |t6|, |t8| < 1.1 * 2^25 +// |t1|, |t3|, |t5|, |t7|, |t9| < 1.1 * 2^24 +#define FE_CARRY \ + i64 c; \ + c = (t0 + ((i64)1<<25)) >> 26; t0 -= c * ((i64)1 << 26); t1 += c; \ + c = (t4 + ((i64)1<<25)) >> 26; t4 -= c * ((i64)1 << 26); t5 += c; \ + c = (t1 + ((i64)1<<24)) >> 25; t1 -= c * ((i64)1 << 25); t2 += c; \ + c = (t5 + ((i64)1<<24)) >> 25; t5 -= c * ((i64)1 << 25); t6 += c; \ + c = (t2 + ((i64)1<<25)) >> 26; t2 -= c * ((i64)1 << 26); t3 += c; \ + c = (t6 + ((i64)1<<25)) >> 26; t6 -= c * ((i64)1 << 26); t7 += c; \ + c = (t3 + ((i64)1<<24)) >> 25; t3 -= c * ((i64)1 << 25); t4 += c; \ + c = (t7 + ((i64)1<<24)) >> 25; t7 -= c * ((i64)1 << 25); t8 += c; \ + c = (t4 + ((i64)1<<25)) >> 26; t4 -= c * ((i64)1 << 26); t5 += c; \ + c = (t8 + ((i64)1<<25)) >> 26; t8 -= c * ((i64)1 << 26); t9 += c; \ + c = (t9 + ((i64)1<<24)) >> 25; t9 -= c * ((i64)1 << 25); t0 += c * 19; \ + c = (t0 + ((i64)1<<25)) >> 26; t0 -= c * ((i64)1 << 26); t1 += c; \ + h[0]=(i32)t0; h[1]=(i32)t1; h[2]=(i32)t2; h[3]=(i32)t3; h[4]=(i32)t4; \ + h[5]=(i32)t5; h[6]=(i32)t6; h[7]=(i32)t7; h[8]=(i32)t8; h[9]=(i32)t9 + +// Decodes a field element from a byte buffer. +// mask specifies how many bits we ignore. +// Traditionally we ignore 1. It's useful for EdDSA, +// which uses that bit to denote the sign of x. +// Elligator however uses positive representatives, +// which means ignoring 2 bits instead. +static void fe_frombytes_mask(fe h, const u8 s[32], unsigned nb_mask) +{ + u32 mask = 0xffffff >> nb_mask; + i64 t0 = load32_le(s); // t0 < 2^32 + i64 t1 = load24_le(s + 4) << 6; // t1 < 2^30 + i64 t2 = load24_le(s + 7) << 5; // t2 < 2^29 + i64 t3 = load24_le(s + 10) << 3; // t3 < 2^27 + i64 t4 = load24_le(s + 13) << 2; // t4 < 2^26 + i64 t5 = load32_le(s + 16); // t5 < 2^32 + i64 t6 = load24_le(s + 20) << 7; // t6 < 2^31 + i64 t7 = load24_le(s + 23) << 5; // t7 < 2^29 + i64 t8 = load24_le(s + 26) << 4; // t8 < 2^28 + i64 t9 = (load24_le(s + 29) & mask) << 2; // t9 < 2^25 + FE_CARRY; // Carry precondition OK +} + +static void fe_frombytes(fe h, const u8 s[32]) +{ + fe_frombytes_mask(h, s, 1); +} + + +// Precondition +// |h[0]|, |h[2]|, |h[4]|, |h[6]|, |h[8]| < 1.1 * 2^25 +// |h[1]|, |h[3]|, |h[5]|, |h[7]|, |h[9]| < 1.1 * 2^24 +// +// Therefore, |h| < 2^255-19 +// There are two possibilities: +// +// - If h is positive, all we need to do is reduce its individual +// limbs down to their tight positive range. +// - If h is negative, we also need to add 2^255-19 to it. +// Or just remove 19 and chop off any excess bit. +static void fe_tobytes(u8 s[32], const fe h) +{ + i32 t[10]; + COPY(t, h, 10); + i32 q = (19 * t[9] + (((i32) 1) << 24)) >> 25; + // |t9| < 1.1 * 2^24 + // -1.1 * 2^24 < t9 < 1.1 * 2^24 + // -21 * 2^24 < 19 * t9 < 21 * 2^24 + // -2^29 < 19 * t9 + 2^24 < 2^29 + // -2^29 / 2^25 < (19 * t9 + 2^24) / 2^25 < 2^29 / 2^25 + // -16 < (19 * t9 + 2^24) / 2^25 < 16 + FOR (i, 0, 5) { + q += t[2*i ]; q >>= 26; // q = 0 or -1 + q += t[2*i+1]; q >>= 25; // q = 0 or -1 + } + // q = 0 iff h >= 0 + // q = -1 iff h < 0 + // Adding q * 19 to h reduces h to its proper range. + q *= 19; // Shift carry back to the beginning + FOR (i, 0, 5) { + t[i*2 ] += q; q = t[i*2 ] >> 26; t[i*2 ] -= q * ((i32)1 << 26); + t[i*2+1] += q; q = t[i*2+1] >> 25; t[i*2+1] -= q * ((i32)1 << 25); + } + // h is now fully reduced, and q represents the excess bit. + + store32_le(s + 0, ((u32)t[0] >> 0) | ((u32)t[1] << 26)); + store32_le(s + 4, ((u32)t[1] >> 6) | ((u32)t[2] << 19)); + store32_le(s + 8, ((u32)t[2] >> 13) | ((u32)t[3] << 13)); + store32_le(s + 12, ((u32)t[3] >> 19) | ((u32)t[4] << 6)); + store32_le(s + 16, ((u32)t[5] >> 0) | ((u32)t[6] << 25)); + store32_le(s + 20, ((u32)t[6] >> 7) | ((u32)t[7] << 19)); + store32_le(s + 24, ((u32)t[7] >> 13) | ((u32)t[8] << 12)); + store32_le(s + 28, ((u32)t[8] >> 20) | ((u32)t[9] << 6)); + + WIPE_BUFFER(t); +} + +// Precondition +// ------------- +// |f0|, |f2|, |f4|, |f6|, |f8| < 1.65 * 2^26 +// |f1|, |f3|, |f5|, |f7|, |f9| < 1.65 * 2^25 +// +// |g0|, |g2|, |g4|, |g6|, |g8| < 1.65 * 2^26 +// |g1|, |g3|, |g5|, |g7|, |g9| < 1.65 * 2^25 +static void fe_mul_small(fe h, const fe f, i32 g) +{ + i64 t0 = f[0] * (i64) g; i64 t1 = f[1] * (i64) g; + i64 t2 = f[2] * (i64) g; i64 t3 = f[3] * (i64) g; + i64 t4 = f[4] * (i64) g; i64 t5 = f[5] * (i64) g; + i64 t6 = f[6] * (i64) g; i64 t7 = f[7] * (i64) g; + i64 t8 = f[8] * (i64) g; i64 t9 = f[9] * (i64) g; + // |t0|, |t2|, |t4|, |t6|, |t8| < 1.65 * 2^26 * 2^31 < 2^58 + // |t1|, |t3|, |t5|, |t7|, |t9| < 1.65 * 2^25 * 2^31 < 2^57 + + FE_CARRY; // Carry precondition OK +} + +// Precondition +// ------------- +// |f0|, |f2|, |f4|, |f6|, |f8| < 1.65 * 2^26 +// |f1|, |f3|, |f5|, |f7|, |f9| < 1.65 * 2^25 +// +// |g0|, |g2|, |g4|, |g6|, |g8| < 1.65 * 2^26 +// |g1|, |g3|, |g5|, |g7|, |g9| < 1.65 * 2^25 +static void fe_mul(fe h, const fe f, const fe g) +{ + // Everything is unrolled and put in temporary variables. + // We could roll the loop, but that would make curve25519 twice as slow. + i32 f0 = f[0]; i32 f1 = f[1]; i32 f2 = f[2]; i32 f3 = f[3]; i32 f4 = f[4]; + i32 f5 = f[5]; i32 f6 = f[6]; i32 f7 = f[7]; i32 f8 = f[8]; i32 f9 = f[9]; + i32 g0 = g[0]; i32 g1 = g[1]; i32 g2 = g[2]; i32 g3 = g[3]; i32 g4 = g[4]; + i32 g5 = g[5]; i32 g6 = g[6]; i32 g7 = g[7]; i32 g8 = g[8]; i32 g9 = g[9]; + i32 F1 = f1*2; i32 F3 = f3*2; i32 F5 = f5*2; i32 F7 = f7*2; i32 F9 = f9*2; + i32 G1 = g1*19; i32 G2 = g2*19; i32 G3 = g3*19; + i32 G4 = g4*19; i32 G5 = g5*19; i32 G6 = g6*19; + i32 G7 = g7*19; i32 G8 = g8*19; i32 G9 = g9*19; + // |F1|, |F3|, |F5|, |F7|, |F9| < 1.65 * 2^26 + // |G0|, |G2|, |G4|, |G6|, |G8| < 2^31 + // |G1|, |G3|, |G5|, |G7|, |G9| < 2^30 + + i64 t0 = f0*(i64)g0 + F1*(i64)G9 + f2*(i64)G8 + F3*(i64)G7 + f4*(i64)G6 + + F5*(i64)G5 + f6*(i64)G4 + F7*(i64)G3 + f8*(i64)G2 + F9*(i64)G1; + i64 t1 = f0*(i64)g1 + f1*(i64)g0 + f2*(i64)G9 + f3*(i64)G8 + f4*(i64)G7 + + f5*(i64)G6 + f6*(i64)G5 + f7*(i64)G4 + f8*(i64)G3 + f9*(i64)G2; + i64 t2 = f0*(i64)g2 + F1*(i64)g1 + f2*(i64)g0 + F3*(i64)G9 + f4*(i64)G8 + + F5*(i64)G7 + f6*(i64)G6 + F7*(i64)G5 + f8*(i64)G4 + F9*(i64)G3; + i64 t3 = f0*(i64)g3 + f1*(i64)g2 + f2*(i64)g1 + f3*(i64)g0 + f4*(i64)G9 + + f5*(i64)G8 + f6*(i64)G7 + f7*(i64)G6 + f8*(i64)G5 + f9*(i64)G4; + i64 t4 = f0*(i64)g4 + F1*(i64)g3 + f2*(i64)g2 + F3*(i64)g1 + f4*(i64)g0 + + F5*(i64)G9 + f6*(i64)G8 + F7*(i64)G7 + f8*(i64)G6 + F9*(i64)G5; + i64 t5 = f0*(i64)g5 + f1*(i64)g4 + f2*(i64)g3 + f3*(i64)g2 + f4*(i64)g1 + + f5*(i64)g0 + f6*(i64)G9 + f7*(i64)G8 + f8*(i64)G7 + f9*(i64)G6; + i64 t6 = f0*(i64)g6 + F1*(i64)g5 + f2*(i64)g4 + F3*(i64)g3 + f4*(i64)g2 + + F5*(i64)g1 + f6*(i64)g0 + F7*(i64)G9 + f8*(i64)G8 + F9*(i64)G7; + i64 t7 = f0*(i64)g7 + f1*(i64)g6 + f2*(i64)g5 + f3*(i64)g4 + f4*(i64)g3 + + f5*(i64)g2 + f6*(i64)g1 + f7*(i64)g0 + f8*(i64)G9 + f9*(i64)G8; + i64 t8 = f0*(i64)g8 + F1*(i64)g7 + f2*(i64)g6 + F3*(i64)g5 + f4*(i64)g4 + + F5*(i64)g3 + f6*(i64)g2 + F7*(i64)g1 + f8*(i64)g0 + F9*(i64)G9; + i64 t9 = f0*(i64)g9 + f1*(i64)g8 + f2*(i64)g7 + f3*(i64)g6 + f4*(i64)g5 + + f5*(i64)g4 + f6*(i64)g3 + f7*(i64)g2 + f8*(i64)g1 + f9*(i64)g0; + // t0 < 0.67 * 2^61 + // t1 < 0.41 * 2^61 + // t2 < 0.52 * 2^61 + // t3 < 0.32 * 2^61 + // t4 < 0.38 * 2^61 + // t5 < 0.22 * 2^61 + // t6 < 0.23 * 2^61 + // t7 < 0.13 * 2^61 + // t8 < 0.09 * 2^61 + // t9 < 0.03 * 2^61 + + FE_CARRY; // Everything below 2^62, Carry precondition OK +} + +// Precondition +// ------------- +// |f0|, |f2|, |f4|, |f6|, |f8| < 1.65 * 2^26 +// |f1|, |f3|, |f5|, |f7|, |f9| < 1.65 * 2^25 +// +// Note: we could use fe_mul() for this, but this is significantly faster +static void fe_sq(fe h, const fe f) +{ + i32 f0 = f[0]; i32 f1 = f[1]; i32 f2 = f[2]; i32 f3 = f[3]; i32 f4 = f[4]; + i32 f5 = f[5]; i32 f6 = f[6]; i32 f7 = f[7]; i32 f8 = f[8]; i32 f9 = f[9]; + i32 f0_2 = f0*2; i32 f1_2 = f1*2; i32 f2_2 = f2*2; i32 f3_2 = f3*2; + i32 f4_2 = f4*2; i32 f5_2 = f5*2; i32 f6_2 = f6*2; i32 f7_2 = f7*2; + i32 f5_38 = f5*38; i32 f6_19 = f6*19; i32 f7_38 = f7*38; + i32 f8_19 = f8*19; i32 f9_38 = f9*38; + // |f0_2| , |f2_2| , |f4_2| , |f6_2| , |f8_2| < 1.65 * 2^27 + // |f1_2| , |f3_2| , |f5_2| , |f7_2| , |f9_2| < 1.65 * 2^26 + // |f5_38|, |f6_19|, |f7_38|, |f8_19|, |f9_38| < 2^31 + + i64 t0 = f0 *(i64)f0 + f1_2*(i64)f9_38 + f2_2*(i64)f8_19 + + f3_2*(i64)f7_38 + f4_2*(i64)f6_19 + f5 *(i64)f5_38; + i64 t1 = f0_2*(i64)f1 + f2 *(i64)f9_38 + f3_2*(i64)f8_19 + + f4 *(i64)f7_38 + f5_2*(i64)f6_19; + i64 t2 = f0_2*(i64)f2 + f1_2*(i64)f1 + f3_2*(i64)f9_38 + + f4_2*(i64)f8_19 + f5_2*(i64)f7_38 + f6 *(i64)f6_19; + i64 t3 = f0_2*(i64)f3 + f1_2*(i64)f2 + f4 *(i64)f9_38 + + f5_2*(i64)f8_19 + f6 *(i64)f7_38; + i64 t4 = f0_2*(i64)f4 + f1_2*(i64)f3_2 + f2 *(i64)f2 + + f5_2*(i64)f9_38 + f6_2*(i64)f8_19 + f7 *(i64)f7_38; + i64 t5 = f0_2*(i64)f5 + f1_2*(i64)f4 + f2_2*(i64)f3 + + f6 *(i64)f9_38 + f7_2*(i64)f8_19; + i64 t6 = f0_2*(i64)f6 + f1_2*(i64)f5_2 + f2_2*(i64)f4 + + f3_2*(i64)f3 + f7_2*(i64)f9_38 + f8 *(i64)f8_19; + i64 t7 = f0_2*(i64)f7 + f1_2*(i64)f6 + f2_2*(i64)f5 + + f3_2*(i64)f4 + f8 *(i64)f9_38; + i64 t8 = f0_2*(i64)f8 + f1_2*(i64)f7_2 + f2_2*(i64)f6 + + f3_2*(i64)f5_2 + f4 *(i64)f4 + f9 *(i64)f9_38; + i64 t9 = f0_2*(i64)f9 + f1_2*(i64)f8 + f2_2*(i64)f7 + + f3_2*(i64)f6 + f4 *(i64)f5_2; + // t0 < 0.67 * 2^61 + // t1 < 0.41 * 2^61 + // t2 < 0.52 * 2^61 + // t3 < 0.32 * 2^61 + // t4 < 0.38 * 2^61 + // t5 < 0.22 * 2^61 + // t6 < 0.23 * 2^61 + // t7 < 0.13 * 2^61 + // t8 < 0.09 * 2^61 + // t9 < 0.03 * 2^61 + + FE_CARRY; +} + +// Parity check. Returns 0 if even, 1 if odd +static int fe_isodd(const fe f) +{ + u8 s[32]; + fe_tobytes(s, f); + u8 isodd = s[0] & 1; + WIPE_BUFFER(s); + return isodd; +} + +// Returns 1 if equal, 0 if not equal +static int fe_isequal(const fe f, const fe g) +{ + u8 fs[32]; + u8 gs[32]; + fe_tobytes(fs, f); + fe_tobytes(gs, g); + int isdifferent = crypto_verify32(fs, gs); + WIPE_BUFFER(fs); + WIPE_BUFFER(gs); + return 1 + isdifferent; +} + +// Inverse square root. +// Returns true if x is a square, false otherwise. +// After the call: +// isr = sqrt(1/x) if x is a non-zero square. +// isr = sqrt(sqrt(-1)/x) if x is not a square. +// isr = 0 if x is zero. +// We do not guarantee the sign of the square root. +// +// Notes: +// Let quartic = x^((p-1)/4) +// +// x^((p-1)/2) = chi(x) +// quartic^2 = chi(x) +// quartic = sqrt(chi(x)) +// quartic = 1 or -1 or sqrt(-1) or -sqrt(-1) +// +// Note that x is a square if quartic is 1 or -1 +// There are 4 cases to consider: +// +// if quartic = 1 (x is a square) +// then x^((p-1)/4) = 1 +// x^((p-5)/4) * x = 1 +// x^((p-5)/4) = 1/x +// x^((p-5)/8) = sqrt(1/x) or -sqrt(1/x) +// +// if quartic = -1 (x is a square) +// then x^((p-1)/4) = -1 +// x^((p-5)/4) * x = -1 +// x^((p-5)/4) = -1/x +// x^((p-5)/8) = sqrt(-1) / sqrt(x) +// x^((p-5)/8) * sqrt(-1) = sqrt(-1)^2 / sqrt(x) +// x^((p-5)/8) * sqrt(-1) = -1/sqrt(x) +// x^((p-5)/8) * sqrt(-1) = -sqrt(1/x) or sqrt(1/x) +// +// if quartic = sqrt(-1) (x is not a square) +// then x^((p-1)/4) = sqrt(-1) +// x^((p-5)/4) * x = sqrt(-1) +// x^((p-5)/4) = sqrt(-1)/x +// x^((p-5)/8) = sqrt(sqrt(-1)/x) or -sqrt(sqrt(-1)/x) +// +// Note that the product of two non-squares is always a square: +// For any non-squares a and b, chi(a) = -1 and chi(b) = -1. +// Since chi(x) = x^((p-1)/2), chi(a)*chi(b) = chi(a*b) = 1. +// Therefore a*b is a square. +// +// Since sqrt(-1) and x are both non-squares, their product is a +// square, and we can compute their square root. +// +// if quartic = -sqrt(-1) (x is not a square) +// then x^((p-1)/4) = -sqrt(-1) +// x^((p-5)/4) * x = -sqrt(-1) +// x^((p-5)/4) = -sqrt(-1)/x +// x^((p-5)/8) = sqrt(-sqrt(-1)/x) +// x^((p-5)/8) = sqrt( sqrt(-1)/x) * sqrt(-1) +// x^((p-5)/8) * sqrt(-1) = sqrt( sqrt(-1)/x) * sqrt(-1)^2 +// x^((p-5)/8) * sqrt(-1) = sqrt( sqrt(-1)/x) * -1 +// x^((p-5)/8) * sqrt(-1) = -sqrt(sqrt(-1)/x) or sqrt(sqrt(-1)/x) +static int invsqrt(fe isr, const fe x) +{ + fe t0, t1, t2; + + // t0 = x^((p-5)/8) + // Can be achieved with a simple double & add ladder, + // but it would be slower. + fe_sq(t0, x); + fe_sq(t1,t0); fe_sq(t1, t1); fe_mul(t1, x, t1); + fe_mul(t0, t0, t1); + fe_sq(t0, t0); fe_mul(t0, t1, t0); + fe_sq(t1, t0); FOR (i, 1, 5) { fe_sq(t1, t1); } fe_mul(t0, t1, t0); + fe_sq(t1, t0); FOR (i, 1, 10) { fe_sq(t1, t1); } fe_mul(t1, t1, t0); + fe_sq(t2, t1); FOR (i, 1, 20) { fe_sq(t2, t2); } fe_mul(t1, t2, t1); + fe_sq(t1, t1); FOR (i, 1, 10) { fe_sq(t1, t1); } fe_mul(t0, t1, t0); + fe_sq(t1, t0); FOR (i, 1, 50) { fe_sq(t1, t1); } fe_mul(t1, t1, t0); + fe_sq(t2, t1); FOR (i, 1, 100) { fe_sq(t2, t2); } fe_mul(t1, t2, t1); + fe_sq(t1, t1); FOR (i, 1, 50) { fe_sq(t1, t1); } fe_mul(t0, t1, t0); + fe_sq(t0, t0); FOR (i, 1, 2) { fe_sq(t0, t0); } fe_mul(t0, t0, x); + + // quartic = x^((p-1)/4) + i32 *quartic = t1; + fe_sq (quartic, t0); + fe_mul(quartic, quartic, x); + + i32 *check = t2; + fe_0 (check); int z0 = fe_isequal(x , check); + fe_1 (check); int p1 = fe_isequal(quartic, check); + fe_neg(check, check ); int m1 = fe_isequal(quartic, check); + fe_neg(check, sqrtm1); int ms = fe_isequal(quartic, check); + + // if quartic == -1 or sqrt(-1) + // then isr = x^((p-1)/4) * sqrt(-1) + // else isr = x^((p-1)/4) + fe_mul(isr, t0, sqrtm1); + fe_ccopy(isr, t0, 1 - (m1 | ms)); + + WIPE_BUFFER(t0); + WIPE_BUFFER(t1); + WIPE_BUFFER(t2); + return p1 | m1 | z0; +} + +// Inverse in terms of inverse square root. +// Requires two additional squarings to get rid of the sign. +// +// 1/x = x * (+invsqrt(x^2))^2 +// = x * (-invsqrt(x^2))^2 +// +// A fully optimised exponentiation by p-1 would save 6 field +// multiplications, but it would require more code. +static void fe_invert(fe out, const fe x) +{ + fe tmp; + fe_sq(tmp, x); + invsqrt(tmp, tmp); + fe_sq(tmp, tmp); + fe_mul(out, tmp, x); + WIPE_BUFFER(tmp); +} + +// trim a scalar for scalar multiplication +void crypto_eddsa_trim_scalar(u8 out[32], const u8 in[32]) +{ + COPY(out, in, 32); + out[ 0] &= 248; + out[31] &= 127; + out[31] |= 64; +} + +// get bit from scalar at position i +static int scalar_bit(const u8 s[32], int i) +{ + if (i < 0) { return 0; } // handle -1 for sliding windows + return (s[i>>3] >> (i&7)) & 1; +} + +/////////////// +/// X-25519 /// Taken from SUPERCOP's ref10 implementation. +/////////////// +static void scalarmult(u8 q[32], const u8 scalar[32], const u8 p[32], + int nb_bits) +{ + // computes the scalar product + fe x1; + fe_frombytes(x1, p); + + // computes the actual scalar product (the result is in x2 and z2) + fe x2, z2, x3, z3, t0, t1; + // Montgomery ladder + // In projective coordinates, to avoid divisions: x = X / Z + // We don't care about the y coordinate, it's only 1 bit of information + fe_1(x2); fe_0(z2); // "zero" point + fe_copy(x3, x1); fe_1(z3); // "one" point + int swap = 0; + for (int pos = nb_bits-1; pos >= 0; --pos) { + // constant time conditional swap before ladder step + int b = scalar_bit(scalar, pos); + swap ^= b; // xor trick avoids swapping at the end of the loop + fe_cswap(x2, x3, swap); + fe_cswap(z2, z3, swap); + swap = b; // anticipates one last swap after the loop + + // Montgomery ladder step: replaces (P2, P3) by (P2*2, P2+P3) + // with differential addition + fe_sub(t0, x3, z3); + fe_sub(t1, x2, z2); + fe_add(x2, x2, z2); + fe_add(z2, x3, z3); + fe_mul(z3, t0, x2); + fe_mul(z2, z2, t1); + fe_sq (t0, t1 ); + fe_sq (t1, x2 ); + fe_add(x3, z3, z2); + fe_sub(z2, z3, z2); + fe_mul(x2, t1, t0); + fe_sub(t1, t1, t0); + fe_sq (z2, z2 ); + fe_mul_small(z3, t1, 121666); + fe_sq (x3, x3 ); + fe_add(t0, t0, z3); + fe_mul(z3, x1, z2); + fe_mul(z2, t1, t0); + } + // last swap is necessary to compensate for the xor trick + // Note: after this swap, P3 == P2 + P1. + fe_cswap(x2, x3, swap); + fe_cswap(z2, z3, swap); + + // normalises the coordinates: x == X / Z + fe_invert(z2, z2); + fe_mul(x2, x2, z2); + fe_tobytes(q, x2); + + WIPE_BUFFER(x1); + WIPE_BUFFER(x2); WIPE_BUFFER(z2); WIPE_BUFFER(t0); + WIPE_BUFFER(x3); WIPE_BUFFER(z3); WIPE_BUFFER(t1); +} + +void crypto_x25519(u8 raw_shared_secret[32], + const u8 your_secret_key [32], + const u8 their_public_key [32]) +{ + // restrict the possible scalar values + u8 e[32]; + crypto_eddsa_trim_scalar(e, your_secret_key); + scalarmult(raw_shared_secret, e, their_public_key, 255); + WIPE_BUFFER(e); +} + +void crypto_x25519_public_key(u8 public_key[32], + const u8 secret_key[32]) +{ + static const u8 base_point[32] = {9}; + crypto_x25519(public_key, secret_key, base_point); +} + +/////////////////////////// +/// Arithmetic modulo L /// +/////////////////////////// +static const u32 L[8] = { + 0x5cf5d3ed, 0x5812631a, 0xa2f79cd6, 0x14def9de, + 0x00000000, 0x00000000, 0x00000000, 0x10000000, +}; + +// p = a*b + p +static void multiply(u32 p[16], const u32 a[8], const u32 b[8]) +{ + FOR (i, 0, 8) { + u64 carry = 0; + FOR (j, 0, 8) { + carry += p[i+j] + (u64)a[i] * b[j]; + p[i+j] = (u32)carry; + carry >>= 32; + } + p[i+8] = (u32)carry; + } +} + +static int is_above_l(const u32 x[8]) +{ + // We work with L directly, in a 2's complement encoding + // (-L == ~L + 1) + u64 carry = 1; + FOR (i, 0, 8) { + carry += (u64)x[i] + (~L[i] & 0xffffffff); + carry >>= 32; + } + return (int)carry; // carry is either 0 or 1 +} + +// Final reduction modulo L, by conditionally removing L. +// if x < l , then r = x +// if l <= x 2*l, then r = x-l +// otherwise the result will be wrong +static void remove_l(u32 r[8], const u32 x[8]) +{ + u64 carry = (u64)is_above_l(x); + u32 mask = ~(u32)carry + 1; // carry == 0 or 1 + FOR (i, 0, 8) { + carry += (u64)x[i] + (~L[i] & mask); + r[i] = (u32)carry; + carry >>= 32; + } +} + +// Full reduction modulo L (Barrett reduction) +static void mod_l(u8 reduced[32], const u32 x[16]) +{ + static const u32 r[9] = { + 0x0a2c131b,0xed9ce5a3,0x086329a7,0x2106215d, + 0xffffffeb,0xffffffff,0xffffffff,0xffffffff,0xf, + }; + // xr = x * r + u32 xr[25] = {0}; + FOR (i, 0, 9) { + u64 carry = 0; + FOR (j, 0, 16) { + carry += xr[i+j] + (u64)r[i] * x[j]; + xr[i+j] = (u32)carry; + carry >>= 32; + } + xr[i+16] = (u32)carry; + } + // xr = floor(xr / 2^512) * L + // Since the result is guaranteed to be below 2*L, + // it is enough to only compute the first 256 bits. + // The division is performed by saying xr[i+16]. (16 * 32 = 512) + ZERO(xr, 8); + FOR (i, 0, 8) { + u64 carry = 0; + FOR (j, 0, 8-i) { + carry += xr[i+j] + (u64)xr[i+16] * L[j]; + xr[i+j] = (u32)carry; + carry >>= 32; + } + } + // xr = x - xr + u64 carry = 1; + FOR (i, 0, 8) { + carry += (u64)x[i] + (~xr[i] & 0xffffffff); + xr[i] = (u32)carry; + carry >>= 32; + } + // Final reduction modulo L (conditional subtraction) + remove_l(xr, xr); + store32_le_buf(reduced, xr, 8); + + WIPE_BUFFER(xr); +} + +void crypto_eddsa_reduce(u8 reduced[32], const u8 expanded[64]) +{ + u32 x[16]; + load32_le_buf(x, expanded, 16); + mod_l(reduced, x); + WIPE_BUFFER(x); +} + +// r = (a * b) + c +void crypto_eddsa_mul_add(u8 r[32], + const u8 a[32], const u8 b[32], const u8 c[32]) +{ + u32 A[8]; load32_le_buf(A, a, 8); + u32 B[8]; load32_le_buf(B, b, 8); + u32 p[16]; load32_le_buf(p, c, 8); ZERO(p + 8, 8); + multiply(p, A, B); + mod_l(r, p); + WIPE_BUFFER(p); + WIPE_BUFFER(A); + WIPE_BUFFER(B); +} + +/////////////// +/// Ed25519 /// +/////////////// + +// Point (group element, ge) in a twisted Edwards curve, +// in extended projective coordinates. +// ge : x = X/Z, y = Y/Z, T = XY/Z +// ge_cached : Yp = X+Y, Ym = X-Y, T2 = T*D2 +// ge_precomp: Z = 1 +typedef struct { fe X; fe Y; fe Z; fe T; } ge; +typedef struct { fe Yp; fe Ym; fe Z; fe T2; } ge_cached; +typedef struct { fe Yp; fe Ym; fe T2; } ge_precomp; + +static void ge_zero(ge *p) +{ + fe_0(p->X); + fe_1(p->Y); + fe_1(p->Z); + fe_0(p->T); +} + +static void ge_tobytes(u8 s[32], const ge *h) +{ + fe recip, x, y; + fe_invert(recip, h->Z); + fe_mul(x, h->X, recip); + fe_mul(y, h->Y, recip); + fe_tobytes(s, y); + s[31] ^= fe_isodd(x) << 7; + + WIPE_BUFFER(recip); + WIPE_BUFFER(x); + WIPE_BUFFER(y); +} + +// h = -s, where s is a point encoded in 32 bytes +// +// Variable time! Inputs must not be secret! +// => Use only to *check* signatures. +// +// From the specifications: +// The encoding of s contains y and the sign of x +// x = sqrt((y^2 - 1) / (d*y^2 + 1)) +// In extended coordinates: +// X = x, Y = y, Z = 1, T = x*y +// +// Note that num * den is a square iff num / den is a square +// If num * den is not a square, the point was not on the curve. +// From the above: +// Let num = y^2 - 1 +// Let den = d*y^2 + 1 +// x = sqrt((y^2 - 1) / (d*y^2 + 1)) +// x = sqrt(num / den) +// x = sqrt(num^2 / (num * den)) +// x = num * sqrt(1 / (num * den)) +// +// Therefore, we can just compute: +// num = y^2 - 1 +// den = d*y^2 + 1 +// isr = invsqrt(num * den) // abort if not square +// x = num * isr +// Finally, negate x if its sign is not as specified. +static int ge_frombytes_neg_vartime(ge *h, const u8 s[32]) +{ + fe_frombytes(h->Y, s); + fe_1(h->Z); + fe_sq (h->T, h->Y); // t = y^2 + fe_mul(h->X, h->T, d ); // x = d*y^2 + fe_sub(h->T, h->T, h->Z); // t = y^2 - 1 + fe_add(h->X, h->X, h->Z); // x = d*y^2 + 1 + fe_mul(h->X, h->T, h->X); // x = (y^2 - 1) * (d*y^2 + 1) + int is_square = invsqrt(h->X, h->X); + if (!is_square) { + return -1; // Not on the curve, abort + } + fe_mul(h->X, h->T, h->X); // x = sqrt((y^2 - 1) / (d*y^2 + 1)) + if (fe_isodd(h->X) == (s[31] >> 7)) { + fe_neg(h->X, h->X); + } + fe_mul(h->T, h->X, h->Y); + return 0; +} + +static void ge_cache(ge_cached *c, const ge *p) +{ + fe_add (c->Yp, p->Y, p->X); + fe_sub (c->Ym, p->Y, p->X); + fe_copy(c->Z , p->Z ); + fe_mul (c->T2, p->T, D2 ); +} + +// Internal buffers are not wiped! Inputs must not be secret! +// => Use only to *check* signatures. +static void ge_add(ge *s, const ge *p, const ge_cached *q) +{ + fe a, b; + fe_add(a , p->Y, p->X ); + fe_sub(b , p->Y, p->X ); + fe_mul(a , a , q->Yp); + fe_mul(b , b , q->Ym); + fe_add(s->Y, a , b ); + fe_sub(s->X, a , b ); + + fe_add(s->Z, p->Z, p->Z ); + fe_mul(s->Z, s->Z, q->Z ); + fe_mul(s->T, p->T, q->T2); + fe_add(a , s->Z, s->T ); + fe_sub(b , s->Z, s->T ); + + fe_mul(s->T, s->X, s->Y); + fe_mul(s->X, s->X, b ); + fe_mul(s->Y, s->Y, a ); + fe_mul(s->Z, a , b ); +} + +// Internal buffers are not wiped! Inputs must not be secret! +// => Use only to *check* signatures. +static void ge_sub(ge *s, const ge *p, const ge_cached *q) +{ + ge_cached neg; + fe_copy(neg.Ym, q->Yp); + fe_copy(neg.Yp, q->Ym); + fe_copy(neg.Z , q->Z ); + fe_neg (neg.T2, q->T2); + ge_add(s, p, &neg); +} + +static void ge_madd(ge *s, const ge *p, const ge_precomp *q, fe a, fe b) +{ + fe_add(a , p->Y, p->X ); + fe_sub(b , p->Y, p->X ); + fe_mul(a , a , q->Yp); + fe_mul(b , b , q->Ym); + fe_add(s->Y, a , b ); + fe_sub(s->X, a , b ); + + fe_add(s->Z, p->Z, p->Z ); + fe_mul(s->T, p->T, q->T2); + fe_add(a , s->Z, s->T ); + fe_sub(b , s->Z, s->T ); + + fe_mul(s->T, s->X, s->Y); + fe_mul(s->X, s->X, b ); + fe_mul(s->Y, s->Y, a ); + fe_mul(s->Z, a , b ); +} + +// Internal buffers are not wiped! Inputs must not be secret! +// => Use only to *check* signatures. +static void ge_msub(ge *s, const ge *p, const ge_precomp *q, fe a, fe b) +{ + ge_precomp neg; + fe_copy(neg.Ym, q->Yp); + fe_copy(neg.Yp, q->Ym); + fe_neg (neg.T2, q->T2); + ge_madd(s, p, &neg, a, b); +} + +static void ge_double(ge *s, const ge *p, ge *q) +{ + fe_sq (q->X, p->X); + fe_sq (q->Y, p->Y); + fe_sq (q->Z, p->Z); // qZ = pZ^2 + fe_mul_small(q->Z, q->Z, 2); // qZ = pZ^2 * 2 + fe_add(q->T, p->X, p->Y); + fe_sq (s->T, q->T); + fe_add(q->T, q->Y, q->X); + fe_sub(q->Y, q->Y, q->X); + fe_sub(q->X, s->T, q->T); + fe_sub(q->Z, q->Z, q->Y); + + fe_mul(s->X, q->X , q->Z); + fe_mul(s->Y, q->T , q->Y); + fe_mul(s->Z, q->Y , q->Z); + fe_mul(s->T, q->X , q->T); +} + +// 5-bit signed window in cached format (Niels coordinates, Z=1) +static const ge_precomp b_window[8] = { + {{25967493,-14356035,29566456,3660896,-12694345, + 4014787,27544626,-11754271,-6079156,2047605,}, + {-12545711,934262,-2722910,3049990,-727428, + 9406986,12720692,5043384,19500929,-15469378,}, + {-8738181,4489570,9688441,-14785194,10184609, + -12363380,29287919,11864899,-24514362,-4438546,},}, + {{15636291,-9688557,24204773,-7912398,616977, + -16685262,27787600,-14772189,28944400,-1550024,}, + {16568933,4717097,-11556148,-1102322,15682896, + -11807043,16354577,-11775962,7689662,11199574,}, + {30464156,-5976125,-11779434,-15670865,23220365, + 15915852,7512774,10017326,-17749093,-9920357,},}, + {{10861363,11473154,27284546,1981175,-30064349, + 12577861,32867885,14515107,-15438304,10819380,}, + {4708026,6336745,20377586,9066809,-11272109, + 6594696,-25653668,12483688,-12668491,5581306,}, + {19563160,16186464,-29386857,4097519,10237984, + -4348115,28542350,13850243,-23678021,-15815942,},}, + {{5153746,9909285,1723747,-2777874,30523605, + 5516873,19480852,5230134,-23952439,-15175766,}, + {-30269007,-3463509,7665486,10083793,28475525, + 1649722,20654025,16520125,30598449,7715701,}, + {28881845,14381568,9657904,3680757,-20181635, + 7843316,-31400660,1370708,29794553,-1409300,},}, + {{-22518993,-6692182,14201702,-8745502,-23510406, + 8844726,18474211,-1361450,-13062696,13821877,}, + {-6455177,-7839871,3374702,-4740862,-27098617, + -10571707,31655028,-7212327,18853322,-14220951,}, + {4566830,-12963868,-28974889,-12240689,-7602672, + -2830569,-8514358,-10431137,2207753,-3209784,},}, + {{-25154831,-4185821,29681144,7868801,-6854661, + -9423865,-12437364,-663000,-31111463,-16132436,}, + {25576264,-2703214,7349804,-11814844,16472782, + 9300885,3844789,15725684,171356,6466918,}, + {23103977,13316479,9739013,-16149481,817875, + -15038942,8965339,-14088058,-30714912,16193877,},}, + {{-33521811,3180713,-2394130,14003687,-16903474, + -16270840,17238398,4729455,-18074513,9256800,}, + {-25182317,-4174131,32336398,5036987,-21236817, + 11360617,22616405,9761698,-19827198,630305,}, + {-13720693,2639453,-24237460,-7406481,9494427, + -5774029,-6554551,-15960994,-2449256,-14291300,},}, + {{-3151181,-5046075,9282714,6866145,-31907062, + -863023,-18940575,15033784,25105118,-7894876,}, + {-24326370,15950226,-31801215,-14592823,-11662737, + -5090925,1573892,-2625887,2198790,-15804619,}, + {-3099351,10324967,-2241613,7453183,-5446979, + -2735503,-13812022,-16236442,-32461234,-12290683,},}, +}; + +// Incremental sliding windows (left to right) +// Based on Roberto Maria Avanzi[2005] +typedef struct { + i16 next_index; // position of the next signed digit + i8 next_digit; // next signed digit (odd number below 2^window_width) + u8 next_check; // point at which we must check for a new window +} slide_ctx; + +static void slide_init(slide_ctx *ctx, const u8 scalar[32]) +{ + // scalar is guaranteed to be below L, either because we checked (s), + // or because we reduced it modulo L (h_ram). L is under 2^253, so + // so bits 253 to 255 are guaranteed to be zero. No need to test them. + // + // Note however that L is very close to 2^252, so bit 252 is almost + // always zero. If we were to start at bit 251, the tests wouldn't + // catch the off-by-one error (constructing one that does would be + // prohibitively expensive). + // + // We should still check bit 252, though. + int i = 252; + while (i > 0 && scalar_bit(scalar, i) == 0) { + i--; + } + ctx->next_check = (u8)(i + 1); + ctx->next_index = -1; + ctx->next_digit = -1; +} + +static int slide_step(slide_ctx *ctx, int width, int i, const u8 scalar[32]) +{ + if (i == ctx->next_check) { + if (scalar_bit(scalar, i) == scalar_bit(scalar, i - 1)) { + ctx->next_check--; + } else { + // compute digit of next window + int w = MIN(width, i + 1); + int v = -(scalar_bit(scalar, i) << (w-1)); + FOR_T (int, j, 0, w-1) { + v += scalar_bit(scalar, i-(w-1)+j) << j; + } + v += scalar_bit(scalar, i-w); + int lsb = v & (~v + 1); // smallest bit of v + int s = // log2(lsb) + (((lsb & 0xAA) != 0) << 0) | + (((lsb & 0xCC) != 0) << 1) | + (((lsb & 0xF0) != 0) << 2); + ctx->next_index = (i16)(i-(w-1)+s); + ctx->next_digit = (i8) (v >> s ); + ctx->next_check -= (u8) w; + } + } + return i == ctx->next_index ? ctx->next_digit: 0; +} + +#define P_W_WIDTH 3 // Affects the size of the stack +#define B_W_WIDTH 5 // Affects the size of the binary +#define P_W_SIZE (1<<(P_W_WIDTH-2)) + +int crypto_eddsa_check_equation(const u8 signature[64], const u8 public_key[32], + const u8 h[32]) +{ + ge minus_A; // -public_key + ge minus_R; // -first_half_of_signature + const u8 *s = signature + 32; + + // Check that A and R are on the curve + // Check that 0 <= S < L (prevents malleability) + // *Allow* non-cannonical encoding for A and R + { + u32 s32[8]; + load32_le_buf(s32, s, 8); + if (ge_frombytes_neg_vartime(&minus_A, public_key) || + ge_frombytes_neg_vartime(&minus_R, signature) || + is_above_l(s32)) { + return -1; + } + } + + // look-up table for minus_A + ge_cached lutA[P_W_SIZE]; + { + ge minus_A2, tmp; + ge_double(&minus_A2, &minus_A, &tmp); + ge_cache(&lutA[0], &minus_A); + FOR (i, 1, P_W_SIZE) { + ge_add(&tmp, &minus_A2, &lutA[i-1]); + ge_cache(&lutA[i], &tmp); + } + } + + // sum = [s]B - [h]A + // Merged double and add ladder, fused with sliding + slide_ctx h_slide; slide_init(&h_slide, h); + slide_ctx s_slide; slide_init(&s_slide, s); + int i = MAX(h_slide.next_check, s_slide.next_check); + ge *sum = &minus_A; // reuse minus_A for the sum + ge_zero(sum); + while (i >= 0) { + ge tmp; + ge_double(sum, sum, &tmp); + int h_digit = slide_step(&h_slide, P_W_WIDTH, i, h); + int s_digit = slide_step(&s_slide, B_W_WIDTH, i, s); + if (h_digit > 0) { ge_add(sum, sum, &lutA[ h_digit / 2]); } + if (h_digit < 0) { ge_sub(sum, sum, &lutA[-h_digit / 2]); } + fe t1, t2; + if (s_digit > 0) { ge_madd(sum, sum, b_window + s_digit/2, t1, t2); } + if (s_digit < 0) { ge_msub(sum, sum, b_window + -s_digit/2, t1, t2); } + i--; + } + + // Compare [8](sum-R) and the zero point + // The multiplication by 8 eliminates any low-order component + // and ensures consistency with batched verification. + ge_cached cached; + u8 check[32]; + static const u8 zero_point[32] = {1}; // Point of order 1 + ge_cache(&cached, &minus_R); + ge_add(sum, sum, &cached); + ge_double(sum, sum, &minus_R); // reuse minus_R as temporary + ge_double(sum, sum, &minus_R); // reuse minus_R as temporary + ge_double(sum, sum, &minus_R); // reuse minus_R as temporary + ge_tobytes(check, sum); + return crypto_verify32(check, zero_point); +} + +// 5-bit signed comb in cached format (Niels coordinates, Z=1) +static const ge_precomp b_comb_low[8] = { + {{-6816601,-2324159,-22559413,124364,18015490, + 8373481,19993724,1979872,-18549925,9085059,}, + {10306321,403248,14839893,9633706,8463310, + -8354981,-14305673,14668847,26301366,2818560,}, + {-22701500,-3210264,-13831292,-2927732,-16326337, + -14016360,12940910,177905,12165515,-2397893,},}, + {{-12282262,-7022066,9920413,-3064358,-32147467, + 2927790,22392436,-14852487,2719975,16402117,}, + {-7236961,-4729776,2685954,-6525055,-24242706, + -15940211,-6238521,14082855,10047669,12228189,}, + {-30495588,-12893761,-11161261,3539405,-11502464, + 16491580,-27286798,-15030530,-7272871,-15934455,},}, + {{17650926,582297,-860412,-187745,-12072900, + -10683391,-20352381,15557840,-31072141,-5019061,}, + {-6283632,-2259834,-4674247,-4598977,-4089240, + 12435688,-31278303,1060251,6256175,10480726,}, + {-13871026,2026300,-21928428,-2741605,-2406664, + -8034988,7355518,15733500,-23379862,7489131,},}, + {{6883359,695140,23196907,9644202,-33430614, + 11354760,-20134606,6388313,-8263585,-8491918,}, + {-7716174,-13605463,-13646110,14757414,-19430591, + -14967316,10359532,-11059670,-21935259,12082603,}, + {-11253345,-15943946,10046784,5414629,24840771, + 8086951,-6694742,9868723,15842692,-16224787,},}, + {{9639399,11810955,-24007778,-9320054,3912937, + -9856959,996125,-8727907,-8919186,-14097242,}, + {7248867,14468564,25228636,-8795035,14346339, + 8224790,6388427,-7181107,6468218,-8720783,}, + {15513115,15439095,7342322,-10157390,18005294, + -7265713,2186239,4884640,10826567,7135781,},}, + {{-14204238,5297536,-5862318,-6004934,28095835, + 4236101,-14203318,1958636,-16816875,3837147,}, + {-5511166,-13176782,-29588215,12339465,15325758, + -15945770,-8813185,11075932,-19608050,-3776283,}, + {11728032,9603156,-4637821,-5304487,-7827751, + 2724948,31236191,-16760175,-7268616,14799772,},}, + {{-28842672,4840636,-12047946,-9101456,-1445464, + 381905,-30977094,-16523389,1290540,12798615,}, + {27246947,-10320914,14792098,-14518944,5302070, + -8746152,-3403974,-4149637,-27061213,10749585,}, + {25572375,-6270368,-15353037,16037944,1146292, + 32198,23487090,9585613,24714571,-1418265,},}, + {{19844825,282124,-17583147,11004019,-32004269, + -2716035,6105106,-1711007,-21010044,14338445,}, + {8027505,8191102,-18504907,-12335737,25173494, + -5923905,15446145,7483684,-30440441,10009108,}, + {-14134701,-4174411,10246585,-14677495,33553567, + -14012935,23366126,15080531,-7969992,7663473,},}, +}; + +static const ge_precomp b_comb_high[8] = { + {{33055887,-4431773,-521787,6654165,951411, + -6266464,-5158124,6995613,-5397442,-6985227,}, + {4014062,6967095,-11977872,3960002,8001989, + 5130302,-2154812,-1899602,-31954493,-16173976,}, + {16271757,-9212948,23792794,731486,-25808309, + -3546396,6964344,-4767590,10976593,10050757,},}, + {{2533007,-4288439,-24467768,-12387405,-13450051, + 14542280,12876301,13893535,15067764,8594792,}, + {20073501,-11623621,3165391,-13119866,13188608, + -11540496,-10751437,-13482671,29588810,2197295,}, + {-1084082,11831693,6031797,14062724,14748428, + -8159962,-20721760,11742548,31368706,13161200,},}, + {{2050412,-6457589,15321215,5273360,25484180, + 124590,-18187548,-7097255,-6691621,-14604792,}, + {9938196,2162889,-6158074,-1711248,4278932, + -2598531,-22865792,-7168500,-24323168,11746309,}, + {-22691768,-14268164,5965485,9383325,20443693, + 5854192,28250679,-1381811,-10837134,13717818,},}, + {{-8495530,16382250,9548884,-4971523,-4491811, + -3902147,6182256,-12832479,26628081,10395408,}, + {27329048,-15853735,7715764,8717446,-9215518, + -14633480,28982250,-5668414,4227628,242148,}, + {-13279943,-7986904,-7100016,8764468,-27276630, + 3096719,29678419,-9141299,3906709,11265498,},}, + {{11918285,15686328,-17757323,-11217300,-27548967, + 4853165,-27168827,6807359,6871949,-1075745,}, + {-29002610,13984323,-27111812,-2713442,28107359, + -13266203,6155126,15104658,3538727,-7513788,}, + {14103158,11233913,-33165269,9279850,31014152, + 4335090,-1827936,4590951,13960841,12787712,},}, + {{1469134,-16738009,33411928,13942824,8092558, + -8778224,-11165065,1437842,22521552,-2792954,}, + {31352705,-4807352,-25327300,3962447,12541566, + -9399651,-27425693,7964818,-23829869,5541287,}, + {-25732021,-6864887,23848984,3039395,-9147354, + 6022816,-27421653,10590137,25309915,-1584678,},}, + {{-22951376,5048948,31139401,-190316,-19542447, + -626310,-17486305,-16511925,-18851313,-12985140,}, + {-9684890,14681754,30487568,7717771,-10829709, + 9630497,30290549,-10531496,-27798994,-13812825,}, + {5827835,16097107,-24501327,12094619,7413972, + 11447087,28057551,-1793987,-14056981,4359312,},}, + {{26323183,2342588,-21887793,-1623758,-6062284, + 2107090,-28724907,9036464,-19618351,-13055189,}, + {-29697200,14829398,-4596333,14220089,-30022969, + 2955645,12094100,-13693652,-5941445,7047569,}, + {-3201977,14413268,-12058324,-16417589,-9035655, + -7224648,9258160,1399236,30397584,-5684634,},}, +}; + +static void lookup_add(ge *p, ge_precomp *tmp_c, fe tmp_a, fe tmp_b, + const ge_precomp comb[8], const u8 scalar[32], int i) +{ + u8 teeth = (u8)((scalar_bit(scalar, i) ) + + (scalar_bit(scalar, i + 32) << 1) + + (scalar_bit(scalar, i + 64) << 2) + + (scalar_bit(scalar, i + 96) << 3)); + u8 high = teeth >> 3; + u8 index = (teeth ^ (high - 1)) & 7; + FOR (j, 0, 8) { + i32 select = 1 & (((j ^ index) - 1) >> 8); + fe_ccopy(tmp_c->Yp, comb[j].Yp, select); + fe_ccopy(tmp_c->Ym, comb[j].Ym, select); + fe_ccopy(tmp_c->T2, comb[j].T2, select); + } + fe_neg(tmp_a, tmp_c->T2); + fe_cswap(tmp_c->T2, tmp_a , high ^ 1); + fe_cswap(tmp_c->Yp, tmp_c->Ym, high ^ 1); + ge_madd(p, p, tmp_c, tmp_a, tmp_b); +} + +// p = [scalar]B, where B is the base point +static void ge_scalarmult_base(ge *p, const u8 scalar[32]) +{ + // twin 4-bits signed combs, from Mike Hamburg's + // Fast and compact elliptic-curve cryptography (2012) + // 1 / 2 modulo L + static const u8 half_mod_L[32] = { + 247,233,122,46,141,49,9,44,107,206,123,81,239,124,111,10, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8, + }; + // (2^256 - 1) / 2 modulo L + static const u8 half_ones[32] = { + 142,74,204,70,186,24,118,107,184,231,190,57,250,173,119,99, + 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,7, + }; + + // All bits set form: 1 means 1, 0 means -1 + u8 s_scalar[32]; + crypto_eddsa_mul_add(s_scalar, scalar, half_mod_L, half_ones); + + // Double and add ladder + fe tmp_a, tmp_b; // temporaries for addition + ge_precomp tmp_c; // temporary for comb lookup + ge tmp_d; // temporary for doubling + fe_1(tmp_c.Yp); + fe_1(tmp_c.Ym); + fe_0(tmp_c.T2); + + // Save a double on the first iteration + ge_zero(p); + lookup_add(p, &tmp_c, tmp_a, tmp_b, b_comb_low , s_scalar, 31); + lookup_add(p, &tmp_c, tmp_a, tmp_b, b_comb_high, s_scalar, 31+128); + // Regular double & add for the rest + for (int i = 30; i >= 0; i--) { + ge_double(p, p, &tmp_d); + lookup_add(p, &tmp_c, tmp_a, tmp_b, b_comb_low , s_scalar, i); + lookup_add(p, &tmp_c, tmp_a, tmp_b, b_comb_high, s_scalar, i+128); + } + // Note: we could save one addition at the end if we assumed the + // scalar fit in 252 bits. Which it does in practice if it is + // selected at random. However, non-random, non-hashed scalars + // *can* overflow 252 bits in practice. Better account for that + // than leaving that kind of subtle corner case. + + WIPE_BUFFER(tmp_a); WIPE_CTX(&tmp_d); + WIPE_BUFFER(tmp_b); WIPE_CTX(&tmp_c); + WIPE_BUFFER(s_scalar); +} + +void crypto_eddsa_scalarbase(u8 point[32], const u8 scalar[32]) +{ + ge P; + ge_scalarmult_base(&P, scalar); + ge_tobytes(point, &P); + WIPE_CTX(&P); +} + +void crypto_eddsa_key_pair(u8 secret_key[64], u8 public_key[32], u8 seed[32]) +{ + // To allow overlaps, observable writes happen in this order: + // 1. seed + // 2. secret_key + // 3. public_key + u8 a[64]; + COPY(a, seed, 32); + crypto_wipe(seed, 32); + COPY(secret_key, a, 32); + crypto_blake2b(a, 64, a, 32); + crypto_eddsa_trim_scalar(a, a); + crypto_eddsa_scalarbase(secret_key + 32, a); + COPY(public_key, secret_key + 32, 32); + WIPE_BUFFER(a); +} + +static void hash_reduce(u8 h[32], + const u8 *a, size_t a_size, + const u8 *b, size_t b_size, + const u8 *c, size_t c_size) +{ + u8 hash[64]; + crypto_blake2b_ctx ctx; + crypto_blake2b_init (&ctx, 64); + crypto_blake2b_update(&ctx, a, a_size); + crypto_blake2b_update(&ctx, b, b_size); + crypto_blake2b_update(&ctx, c, c_size); + crypto_blake2b_final (&ctx, hash); + crypto_eddsa_reduce(h, hash); +} + +// Digital signature of a message with from a secret key. +// +// The secret key comprises two parts: +// - The seed that generates the key (secret_key[ 0..31]) +// - The public key (secret_key[32..63]) +// +// The seed and the public key are bundled together to make sure users +// don't use mismatched seeds and public keys, which would instantly +// leak the secret scalar and allow forgeries (allowing this to happen +// has resulted in critical vulnerabilities in the wild). +// +// The seed is hashed to derive the secret scalar and a secret prefix. +// The sole purpose of the prefix is to generate a secret random nonce. +// The properties of that nonce must be as follows: +// - Unique: we need a different one for each message. +// - Secret: third parties must not be able to predict it. +// - Random: any detectable bias would break all security. +// +// There are two ways to achieve these properties. The obvious one is +// to simply generate a random number. Here that would be a parameter +// (Monocypher doesn't have an RNG). It works, but then users may reuse +// the nonce by accident, which _also_ leaks the secret scalar and +// allows forgeries. This has happened in the wild too. +// +// This is no good, so instead we generate that nonce deterministically +// by reducing modulo L a hash of the secret prefix and the message. +// The secret prefix makes the nonce unpredictable, the message makes it +// unique, and the hash/reduce removes all bias. +// +// The cost of that safety is hashing the message twice. If that cost +// is unacceptable, there are two alternatives: +// +// - Signing a hash of the message instead of the message itself. This +// is fine as long as the hash is collision resistant. It is not +// compatible with existing "pure" signatures, but at least it's safe. +// +// - Using a random nonce. Please exercise **EXTREME CAUTION** if you +// ever do that. It is absolutely **critical** that the nonce is +// really an unbiased random number between 0 and L-1, never reused, +// and wiped immediately. +// +// To lower the likelihood of complete catastrophe if the RNG is +// either flawed or misused, you can hash the RNG output together with +// the secret prefix and the beginning of the message, and use the +// reduction of that hash instead of the RNG output itself. It's not +// foolproof (you'd need to hash the whole message) but it helps. +// +// Signing a message involves the following operations: +// +// scalar, prefix = HASH(secret_key) +// r = HASH(prefix || message) % L +// R = [r]B +// h = HASH(R || public_key || message) % L +// S = ((h * a) + r) % L +// signature = R || S +void crypto_eddsa_sign(u8 signature [64], const u8 secret_key[64], + const u8 *message, size_t message_size) +{ + u8 a[64]; // secret scalar and prefix + u8 r[32]; // secret deterministic "random" nonce + u8 h[32]; // publically verifiable hash of the message (not wiped) + u8 R[32]; // first half of the signature (allows overlapping inputs) + + crypto_blake2b(a, 64, secret_key, 32); + crypto_eddsa_trim_scalar(a, a); + hash_reduce(r, a + 32, 32, message, message_size, 0, 0); + crypto_eddsa_scalarbase(R, r); + hash_reduce(h, R, 32, secret_key + 32, 32, message, message_size); + COPY(signature, R, 32); + crypto_eddsa_mul_add(signature + 32, h, a, r); + + WIPE_BUFFER(a); + WIPE_BUFFER(r); +} + +// To check the signature R, S of the message M with the public key A, +// there are 3 steps: +// +// compute h = HASH(R || A || message) % L +// check that A is on the curve. +// check that R == [s]B - [h]A +// +// The last two steps are done in crypto_eddsa_check_equation() +int crypto_eddsa_check(const u8 signature[64], const u8 public_key[32], + const u8 *message, size_t message_size) +{ + u8 h[32]; + hash_reduce(h, signature, 32, public_key, 32, message, message_size); + return crypto_eddsa_check_equation(signature, public_key, h); +} + +///////////////////////// +/// EdDSA <--> X25519 /// +///////////////////////// +void crypto_eddsa_to_x25519(u8 x25519[32], const u8 eddsa[32]) +{ + // (u, v) = ((1+y)/(1-y), sqrt(-486664)*u/x) + // Only converting y to u, the sign of x is ignored. + fe t1, t2; + fe_frombytes(t2, eddsa); + fe_add(t1, fe_one, t2); + fe_sub(t2, fe_one, t2); + fe_invert(t2, t2); + fe_mul(t1, t1, t2); + fe_tobytes(x25519, t1); + WIPE_BUFFER(t1); + WIPE_BUFFER(t2); +} + +void crypto_x25519_to_eddsa(u8 eddsa[32], const u8 x25519[32]) +{ + // (x, y) = (sqrt(-486664)*u/v, (u-1)/(u+1)) + // Only converting u to y, x is assumed positive. + fe t1, t2; + fe_frombytes(t2, x25519); + fe_sub(t1, t2, fe_one); + fe_add(t2, t2, fe_one); + fe_invert(t2, t2); + fe_mul(t1, t1, t2); + fe_tobytes(eddsa, t1); + WIPE_BUFFER(t1); + WIPE_BUFFER(t2); +} + +///////////////////////////////////////////// +/// Dirty ephemeral public key generation /// +///////////////////////////////////////////// + +// Those functions generates a public key, *without* clearing the +// cofactor. Sending that key over the network leaks 3 bits of the +// private key. Use only to generate ephemeral keys that will be hidden +// with crypto_curve_to_hidden(). +// +// The public key is otherwise compatible with crypto_x25519(), which +// properly clears the cofactor. +// +// Note that the distribution of the resulting public keys is almost +// uniform. Flipping the sign of the v coordinate (not provided by this +// function), covers the entire key space almost perfectly, where +// "almost" means a 2^-128 bias (undetectable). This uniformity is +// needed to ensure the proper randomness of the resulting +// representatives (once we apply crypto_curve_to_hidden()). +// +// Recall that Curve25519 has order C = 2^255 + e, with e < 2^128 (not +// to be confused with the prime order of the main subgroup, L, which is +// 8 times less than that). +// +// Generating all points would require us to multiply a point of order C +// (the base point plus any point of order 8) by all scalars from 0 to +// C-1. Clamping limits us to scalars between 2^254 and 2^255 - 1. But +// by negating the resulting point at random, we also cover scalars from +// -2^255 + 1 to -2^254 (which modulo C is congruent to e+1 to 2^254 + e). +// +// In practice: +// - Scalars from 0 to e + 1 are never generated +// - Scalars from 2^255 to 2^255 + e are never generated +// - Scalars from 2^254 + 1 to 2^254 + e are generated twice +// +// Since e < 2^128, detecting this bias requires observing over 2^100 +// representatives from a given source (this will never happen), *and* +// recovering enough of the private key to determine that they do, or do +// not, belong to the biased set (this practically requires solving +// discrete logarithm, which is conjecturally intractable). +// +// In practice, this means the bias is impossible to detect. + +// s + (x*L) % 8*L +// Guaranteed to fit in 256 bits iff s fits in 255 bits. +// L < 2^253 +// x%8 < 2^3 +// L * (x%8) < 2^255 +// s < 2^255 +// s + L * (x%8) < 2^256 +static void add_xl(u8 s[32], u8 x) +{ + u64 mod8 = x & 7; + u64 carry = 0; + FOR (i , 0, 8) { + carry = carry + load32_le(s + 4*i) + L[i] * mod8; + store32_le(s + 4*i, (u32)carry); + carry >>= 32; + } +} + +// "Small" dirty ephemeral key. +// Use if you need to shrink the size of the binary, and can afford to +// slow down by a factor of two (compared to the fast version) +// +// This version works by decoupling the cofactor from the main factor. +// +// - The trimmed scalar determines the main factor +// - The clamped bits of the scalar determine the cofactor. +// +// Cofactor and main factor are combined into a single scalar, which is +// then multiplied by a point of order 8*L (unlike the base point, which +// has prime order). That "dirty" base point is the addition of the +// regular base point (9), and a point of order 8. +void crypto_x25519_dirty_small(u8 public_key[32], const u8 secret_key[32]) +{ + // Base point of order 8*L + // Raw scalar multiplication with it does not clear the cofactor, + // and the resulting public key will reveal 3 bits of the scalar. + // + // The low order component of this base point has been chosen + // to yield the same results as crypto_x25519_dirty_fast(). + static const u8 dirty_base_point[32] = { + 0xd8, 0x86, 0x1a, 0xa2, 0x78, 0x7a, 0xd9, 0x26, + 0x8b, 0x74, 0x74, 0xb6, 0x82, 0xe3, 0xbe, 0xc3, + 0xce, 0x36, 0x9a, 0x1e, 0x5e, 0x31, 0x47, 0xa2, + 0x6d, 0x37, 0x7c, 0xfd, 0x20, 0xb5, 0xdf, 0x75, + }; + // separate the main factor & the cofactor of the scalar + u8 scalar[32]; + crypto_eddsa_trim_scalar(scalar, secret_key); + + // Separate the main factor and the cofactor + // + // The scalar is trimmed, so its cofactor is cleared. The three + // least significant bits however still have a main factor. We must + // remove it for X25519 compatibility. + // + // cofactor = lsb * L (modulo 8*L) + // combined = scalar + cofactor (modulo 8*L) + add_xl(scalar, secret_key[0]); + scalarmult(public_key, scalar, dirty_base_point, 256); + WIPE_BUFFER(scalar); +} + +// Select low order point +// We're computing the [cofactor]lop scalar multiplication, where: +// +// cofactor = tweak & 7. +// lop = (lop_x, lop_y) +// lop_x = sqrt((sqrt(d + 1) + 1) / d) +// lop_y = -lop_x * sqrtm1 +// +// The low order point has order 8. There are 4 such points. We've +// chosen the one whose both coordinates are positive (below p/2). +// The 8 low order points are as follows: +// +// [0]lop = ( 0 , 1 ) +// [1]lop = ( lop_x , lop_y) +// [2]lop = ( sqrt(-1), -0 ) +// [3]lop = ( lop_x , -lop_y) +// [4]lop = (-0 , -1 ) +// [5]lop = (-lop_x , -lop_y) +// [6]lop = (-sqrt(-1), 0 ) +// [7]lop = (-lop_x , lop_y) +// +// The x coordinate is either 0, sqrt(-1), lop_x, or their opposite. +// The y coordinate is either 0, -1 , lop_y, or their opposite. +// The pattern for both is the same, except for a rotation of 2 (modulo 8) +// +// This helper function captures the pattern, and we can use it thus: +// +// select_lop(x, lop_x, sqrtm1, cofactor); +// select_lop(y, lop_y, fe_one, cofactor + 2); +// +// This is faster than an actual scalar multiplication, +// and requires less code than naive constant time look up. +static void select_lop(fe out, const fe x, const fe k, u8 cofactor) +{ + fe tmp; + fe_0(out); + fe_ccopy(out, k , (cofactor >> 1) & 1); // bit 1 + fe_ccopy(out, x , (cofactor >> 0) & 1); // bit 0 + fe_neg (tmp, out); + fe_ccopy(out, tmp, (cofactor >> 2) & 1); // bit 2 + WIPE_BUFFER(tmp); +} + +// "Fast" dirty ephemeral key +// We use this one by default. +// +// This version works by performing a regular scalar multiplication, +// then add a low order point. The scalar multiplication is done in +// Edwards space for more speed (*2 compared to the "small" version). +// The cost is a bigger binary for programs that don't also sign messages. +void crypto_x25519_dirty_fast(u8 public_key[32], const u8 secret_key[32]) +{ + // Compute clean scalar multiplication + u8 scalar[32]; + ge pk; + crypto_eddsa_trim_scalar(scalar, secret_key); + ge_scalarmult_base(&pk, scalar); + + // Compute low order point + fe t1, t2; + select_lop(t1, lop_x, sqrtm1, secret_key[0]); + select_lop(t2, lop_y, fe_one, secret_key[0] + 2); + ge_precomp low_order_point; + fe_add(low_order_point.Yp, t2, t1); + fe_sub(low_order_point.Ym, t2, t1); + fe_mul(low_order_point.T2, t2, t1); + fe_mul(low_order_point.T2, low_order_point.T2, D2); + + // Add low order point to the public key + ge_madd(&pk, &pk, &low_order_point, t1, t2); + + // Convert to Montgomery u coordinate (we ignore the sign) + fe_add(t1, pk.Z, pk.Y); + fe_sub(t2, pk.Z, pk.Y); + fe_invert(t2, t2); + fe_mul(t1, t1, t2); + + fe_tobytes(public_key, t1); + + WIPE_BUFFER(t1); WIPE_CTX(&pk); + WIPE_BUFFER(t2); WIPE_CTX(&low_order_point); + WIPE_BUFFER(scalar); +} + +/////////////////// +/// Elligator 2 /// +/////////////////// +static const fe A = {486662}; + +// Elligator direct map +// +// Computes the point corresponding to a representative, encoded in 32 +// bytes (little Endian). Since positive representatives fits in 254 +// bits, The two most significant bits are ignored. +// +// From the paper: +// w = -A / (fe(1) + non_square * r^2) +// e = chi(w^3 + A*w^2 + w) +// u = e*w - (fe(1)-e)*(A//2) +// v = -e * sqrt(u^3 + A*u^2 + u) +// +// We ignore v because we don't need it for X25519 (the Montgomery +// ladder only uses u). +// +// Note that e is either 0, 1 or -1 +// if e = 0 u = 0 and v = 0 +// if e = 1 u = w +// if e = -1 u = -w - A = w * non_square * r^2 +// +// Let r1 = non_square * r^2 +// Let r2 = 1 + r1 +// Note that r2 cannot be zero, -1/non_square is not a square. +// We can (tediously) verify that: +// w^3 + A*w^2 + w = (A^2*r1 - r2^2) * A / r2^3 +// Therefore: +// chi(w^3 + A*w^2 + w) = chi((A^2*r1 - r2^2) * (A / r2^3)) +// chi(w^3 + A*w^2 + w) = chi((A^2*r1 - r2^2) * (A / r2^3)) * 1 +// chi(w^3 + A*w^2 + w) = chi((A^2*r1 - r2^2) * (A / r2^3)) * chi(r2^6) +// chi(w^3 + A*w^2 + w) = chi((A^2*r1 - r2^2) * (A / r2^3) * r2^6) +// chi(w^3 + A*w^2 + w) = chi((A^2*r1 - r2^2) * A * r2^3) +// Corollary: +// e = 1 if (A^2*r1 - r2^2) * A * r2^3) is a non-zero square +// e = -1 if (A^2*r1 - r2^2) * A * r2^3) is not a square +// Note that w^3 + A*w^2 + w (and therefore e) can never be zero: +// w^3 + A*w^2 + w = w * (w^2 + A*w + 1) +// w^3 + A*w^2 + w = w * (w^2 + A*w + A^2/4 - A^2/4 + 1) +// w^3 + A*w^2 + w = w * (w + A/2)^2 - A^2/4 + 1) +// which is zero only if: +// w = 0 (impossible) +// (w + A/2)^2 = A^2/4 - 1 (impossible, because A^2/4-1 is not a square) +// +// Let isr = invsqrt((A^2*r1 - r2^2) * A * r2^3) +// isr = sqrt(1 / ((A^2*r1 - r2^2) * A * r2^3)) if e = 1 +// isr = sqrt(sqrt(-1) / ((A^2*r1 - r2^2) * A * r2^3)) if e = -1 +// +// if e = 1 +// let u1 = -A * (A^2*r1 - r2^2) * A * r2^2 * isr^2 +// u1 = w +// u1 = u +// +// if e = -1 +// let ufactor = -non_square * sqrt(-1) * r^2 +// let vfactor = sqrt(ufactor) +// let u2 = -A * (A^2*r1 - r2^2) * A * r2^2 * isr^2 * ufactor +// u2 = w * -1 * -non_square * r^2 +// u2 = w * non_square * r^2 +// u2 = u +void crypto_elligator_map(u8 curve[32], const u8 hidden[32]) +{ + fe r, u, t1, t2, t3; + fe_frombytes_mask(r, hidden, 2); // r is encoded in 254 bits. + fe_sq(r, r); + fe_add(t1, r, r); + fe_add(u, t1, fe_one); + fe_sq (t2, u); + fe_mul(t3, A2, t1); + fe_sub(t3, t3, t2); + fe_mul(t3, t3, A); + fe_mul(t1, t2, u); + fe_mul(t1, t3, t1); + int is_square = invsqrt(t1, t1); + fe_mul(u, r, ufactor); + fe_ccopy(u, fe_one, is_square); + fe_sq (t1, t1); + fe_mul(u, u, A); + fe_mul(u, u, t3); + fe_mul(u, u, t2); + fe_mul(u, u, t1); + fe_neg(u, u); + fe_tobytes(curve, u); + + WIPE_BUFFER(t1); WIPE_BUFFER(r); + WIPE_BUFFER(t2); WIPE_BUFFER(u); + WIPE_BUFFER(t3); +} + +// Elligator inverse map +// +// Computes the representative of a point, if possible. If not, it does +// nothing and returns -1. Note that the success of the operation +// depends only on the point (more precisely its u coordinate). The +// tweak parameter is used only upon success +// +// The tweak should be a random byte. Beyond that, its contents are an +// implementation detail. Currently, the tweak comprises: +// - Bit 1 : sign of the v coordinate (0 if positive, 1 if negative) +// - Bit 2-5: not used +// - Bits 6-7: random padding +// +// From the paper: +// Let sq = -non_square * u * (u+A) +// if sq is not a square, or u = -A, there is no mapping +// Assuming there is a mapping: +// if v is positive: r = sqrt(-u / (non_square * (u+A))) +// if v is negative: r = sqrt(-(u+A) / (non_square * u )) +// +// We compute isr = invsqrt(-non_square * u * (u+A)) +// if it wasn't a square, abort. +// else, isr = sqrt(-1 / (non_square * u * (u+A)) +// +// If v is positive, we return isr * u: +// isr * u = sqrt(-1 / (non_square * u * (u+A)) * u +// isr * u = sqrt(-u / (non_square * (u+A)) +// +// If v is negative, we return isr * (u+A): +// isr * (u+A) = sqrt(-1 / (non_square * u * (u+A)) * (u+A) +// isr * (u+A) = sqrt(-(u+A) / (non_square * u) +int crypto_elligator_rev(u8 hidden[32], const u8 public_key[32], u8 tweak) +{ + fe t1, t2, t3; + fe_frombytes(t1, public_key); // t1 = u + + fe_add(t2, t1, A); // t2 = u + A + fe_mul(t3, t1, t2); + fe_mul_small(t3, t3, -2); + int is_square = invsqrt(t3, t3); // t3 = sqrt(-1 / non_square * u * (u+A)) + if (is_square) { + // The only variable time bit. This ultimately reveals how many + // tries it took us to find a representable key. + // This does not affect security as long as we try keys at random. + + fe_ccopy (t1, t2, tweak & 1); // multiply by u if v is positive, + fe_mul (t3, t1, t3); // multiply by u+A otherwise + fe_mul_small(t1, t3, 2); + fe_neg (t2, t3); + fe_ccopy (t3, t2, fe_isodd(t1)); + fe_tobytes(hidden, t3); + + // Pad with two random bits + hidden[31] |= tweak & 0xc0; + } + + WIPE_BUFFER(t1); + WIPE_BUFFER(t2); + WIPE_BUFFER(t3); + return is_square - 1; +} + +void crypto_elligator_key_pair(u8 hidden[32], u8 secret_key[32], u8 seed[32]) +{ + u8 pk [32]; // public key + u8 buf[64]; // seed + representative + COPY(buf + 32, seed, 32); + do { + crypto_chacha20_djb(buf, 0, 64, buf+32, zero, 0); + crypto_x25519_dirty_fast(pk, buf); // or the "small" version + } while(crypto_elligator_rev(buf+32, pk, buf[32])); + // Note that the return value of crypto_elligator_rev() is + // independent from its tweak parameter. + // Therefore, buf[32] is not actually reused. Either we loop one + // more time and buf[32] is used for the new seed, or we succeeded, + // and buf[32] becomes the tweak parameter. + + crypto_wipe(seed, 32); + COPY(hidden , buf + 32, 32); + COPY(secret_key, buf , 32); + WIPE_BUFFER(buf); + WIPE_BUFFER(pk); +} + +/////////////////////// +/// Scalar division /// +/////////////////////// + +// Montgomery reduction. +// Divides x by (2^256), and reduces the result modulo L +// +// Precondition: +// x < L * 2^256 +// Constants: +// r = 2^256 (makes division by r trivial) +// k = (r * (1/r) - 1) // L (1/r is computed modulo L ) +// Algorithm: +// s = (x * k) % r +// t = x + s*L (t is always a multiple of r) +// u = (t/r) % L (u is always below 2*L, conditional subtraction is enough) +static void redc(u32 u[8], u32 x[16]) +{ + static const u32 k[8] = { + 0x12547e1b, 0xd2b51da3, 0xfdba84ff, 0xb1a206f2, + 0xffa36bea, 0x14e75438, 0x6fe91836, 0x9db6c6f2, + }; + + // s = x * k (modulo 2^256) + // This is cheaper than the full multiplication. + u32 s[8] = {0}; + FOR (i, 0, 8) { + u64 carry = 0; + FOR (j, 0, 8-i) { + carry += s[i+j] + (u64)x[i] * k[j]; + s[i+j] = (u32)carry; + carry >>= 32; + } + } + u32 t[16] = {0}; + multiply(t, s, L); + + // t = t + x + u64 carry = 0; + FOR (i, 0, 16) { + carry += (u64)t[i] + x[i]; + t[i] = (u32)carry; + carry >>= 32; + } + + // u = (t / 2^256) % L + // Note that t / 2^256 is always below 2*L, + // So a constant time conditional subtraction is enough + remove_l(u, t+8); + + WIPE_BUFFER(s); + WIPE_BUFFER(t); +} + +void crypto_x25519_inverse(u8 blind_salt [32], const u8 private_key[32], + const u8 curve_point[32]) +{ + static const u8 Lm2[32] = { // L - 2 + 0xeb, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, + 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, + }; + // 1 in Montgomery form + u32 m_inv [8] = { + 0x8d98951d, 0xd6ec3174, 0x737dcf70, 0xc6ef5bf4, + 0xfffffffe, 0xffffffff, 0xffffffff, 0x0fffffff, + }; + + u8 scalar[32]; + crypto_eddsa_trim_scalar(scalar, private_key); + + // Convert the scalar in Montgomery form + // m_scl = scalar * 2^256 (modulo L) + u32 m_scl[8]; + { + u32 tmp[16]; + ZERO(tmp, 8); + load32_le_buf(tmp+8, scalar, 8); + mod_l(scalar, tmp); + load32_le_buf(m_scl, scalar, 8); + WIPE_BUFFER(tmp); // Wipe ASAP to save stack space + } + + // Compute the inverse + u32 product[16]; + for (int i = 252; i >= 0; i--) { + ZERO(product, 16); + multiply(product, m_inv, m_inv); + redc(m_inv, product); + if (scalar_bit(Lm2, i)) { + ZERO(product, 16); + multiply(product, m_inv, m_scl); + redc(m_inv, product); + } + } + // Convert the inverse *out* of Montgomery form + // scalar = m_inv / 2^256 (modulo L) + COPY(product, m_inv, 8); + ZERO(product + 8, 8); + redc(m_inv, product); + store32_le_buf(scalar, m_inv, 8); // the *inverse* of the scalar + + // Clear the cofactor of scalar: + // cleared = scalar * (3*L + 1) (modulo 8*L) + // cleared = scalar + scalar * 3 * L (modulo 8*L) + // Note that (scalar * 3) is reduced modulo 8, so we only need the + // first byte. + add_xl(scalar, scalar[0] * 3); + + // Recall that 8*L < 2^256. However it is also very close to + // 2^255. If we spanned the ladder over 255 bits, random tests + // wouldn't catch the off-by-one error. + scalarmult(blind_salt, scalar, curve_point, 256); + + WIPE_BUFFER(scalar); WIPE_BUFFER(m_scl); + WIPE_BUFFER(product); WIPE_BUFFER(m_inv); +} + +//////////////////////////////// +/// Authenticated encryption /// +//////////////////////////////// +static void lock_auth(u8 mac[16], const u8 auth_key[32], + const u8 *ad , size_t ad_size, + const u8 *cipher_text, size_t text_size) +{ + u8 sizes[16]; // Not secret, not wiped + store64_le(sizes + 0, ad_size); + store64_le(sizes + 8, text_size); + crypto_poly1305_ctx poly_ctx; // auto wiped... + crypto_poly1305_init (&poly_ctx, auth_key); + crypto_poly1305_update(&poly_ctx, ad , ad_size); + crypto_poly1305_update(&poly_ctx, zero , gap(ad_size, 16)); + crypto_poly1305_update(&poly_ctx, cipher_text, text_size); + crypto_poly1305_update(&poly_ctx, zero , gap(text_size, 16)); + crypto_poly1305_update(&poly_ctx, sizes , 16); + crypto_poly1305_final (&poly_ctx, mac); // ...here +} + +void crypto_aead_init_x(crypto_aead_ctx *ctx, + u8 const key[32], const u8 nonce[24]) +{ + crypto_chacha20_h(ctx->key, key, nonce); + COPY(ctx->nonce, nonce + 16, 8); + ctx->counter = 0; +} + +void crypto_aead_init_djb(crypto_aead_ctx *ctx, + const u8 key[32], const u8 nonce[8]) +{ + COPY(ctx->key , key , 32); + COPY(ctx->nonce, nonce, 8); + ctx->counter = 0; +} + +void crypto_aead_init_ietf(crypto_aead_ctx *ctx, + const u8 key[32], const u8 nonce[12]) +{ + COPY(ctx->key , key , 32); + COPY(ctx->nonce, nonce + 4, 8); + ctx->counter = (u64)load32_le(nonce) << 32; +} + +void crypto_aead_write(crypto_aead_ctx *ctx, u8 *cipher_text, u8 mac[16], + const u8 *ad, size_t ad_size, + const u8 *plain_text, size_t text_size) +{ + u8 auth_key[64]; // the last 32 bytes are used for rekeying. + crypto_chacha20_djb(auth_key, 0, 64, ctx->key, ctx->nonce, ctx->counter); + crypto_chacha20_djb(cipher_text, plain_text, text_size, + ctx->key, ctx->nonce, ctx->counter + 1); + lock_auth(mac, auth_key, ad, ad_size, cipher_text, text_size); + COPY(ctx->key, auth_key + 32, 32); + WIPE_BUFFER(auth_key); +} + +int crypto_aead_read(crypto_aead_ctx *ctx, u8 *plain_text, const u8 mac[16], + const u8 *ad, size_t ad_size, + const u8 *cipher_text, size_t text_size) +{ + u8 auth_key[64]; // the last 32 bytes are used for rekeying. + u8 real_mac[16]; + crypto_chacha20_djb(auth_key, 0, 64, ctx->key, ctx->nonce, ctx->counter); + lock_auth(real_mac, auth_key, ad, ad_size, cipher_text, text_size); + int mismatch = crypto_verify16(mac, real_mac); + if (!mismatch) { + crypto_chacha20_djb(plain_text, cipher_text, text_size, + ctx->key, ctx->nonce, ctx->counter + 1); + COPY(ctx->key, auth_key + 32, 32); + } + WIPE_BUFFER(auth_key); + WIPE_BUFFER(real_mac); + return mismatch; +} + +void crypto_aead_lock(u8 *cipher_text, u8 mac[16], const u8 key[32], + const u8 nonce[24], const u8 *ad, size_t ad_size, + const u8 *plain_text, size_t text_size) +{ + crypto_aead_ctx ctx; + crypto_aead_init_x(&ctx, key, nonce); + crypto_aead_write(&ctx, cipher_text, mac, ad, ad_size, + plain_text, text_size); + crypto_wipe(&ctx, sizeof(ctx)); +} + +int crypto_aead_unlock(u8 *plain_text, const u8 mac[16], const u8 key[32], + const u8 nonce[24], const u8 *ad, size_t ad_size, + const u8 *cipher_text, size_t text_size) +{ + crypto_aead_ctx ctx; + crypto_aead_init_x(&ctx, key, nonce); + int mismatch = crypto_aead_read(&ctx, plain_text, mac, ad, ad_size, + cipher_text, text_size); + crypto_wipe(&ctx, sizeof(ctx)); + return mismatch; +} + +#ifdef MONOCYPHER_CPP_NAMESPACE +} +#endif diff --git a/src/LightningDB/wasm/monocypher.h b/src/LightningDB/wasm/monocypher.h new file mode 100644 index 0000000..765a07f --- /dev/null +++ b/src/LightningDB/wasm/monocypher.h @@ -0,0 +1,321 @@ +// Monocypher version 4.0.2 +// +// This file is dual-licensed. Choose whichever licence you want from +// the two licences listed below. +// +// The first licence is a regular 2-clause BSD licence. The second licence +// is the CC-0 from Creative Commons. It is intended to release Monocypher +// to the public domain. The BSD licence serves as a fallback option. +// +// SPDX-License-Identifier: BSD-2-Clause OR CC0-1.0 +// +// ------------------------------------------------------------------------ +// +// Copyright (c) 2017-2019, Loup Vaillant +// All rights reserved. +// +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the +// distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// ------------------------------------------------------------------------ +// +// Written in 2017-2019 by Loup Vaillant +// +// To the extent possible under law, the author(s) have dedicated all copyright +// and related neighboring rights to this software to the public domain +// worldwide. This software is distributed without any warranty. +// +// You should have received a copy of the CC0 Public Domain Dedication along +// with this software. If not, see +// + +#ifndef MONOCYPHER_H +#define MONOCYPHER_H + +#include +#include + +#ifdef MONOCYPHER_CPP_NAMESPACE +namespace MONOCYPHER_CPP_NAMESPACE { +#elif defined(__cplusplus) +extern "C" { +#endif + +// Constant time comparisons +// ------------------------- + +// Return 0 if a and b are equal, -1 otherwise +int crypto_verify16(const uint8_t a[16], const uint8_t b[16]); +int crypto_verify32(const uint8_t a[32], const uint8_t b[32]); +int crypto_verify64(const uint8_t a[64], const uint8_t b[64]); + + +// Erase sensitive data +// -------------------- +void crypto_wipe(void *secret, size_t size); + + +// Authenticated encryption +// ------------------------ +void crypto_aead_lock(uint8_t *cipher_text, + uint8_t mac [16], + const uint8_t key [32], + const uint8_t nonce[24], + const uint8_t *ad, size_t ad_size, + const uint8_t *plain_text, size_t text_size); +int crypto_aead_unlock(uint8_t *plain_text, + const uint8_t mac [16], + const uint8_t key [32], + const uint8_t nonce[24], + const uint8_t *ad, size_t ad_size, + const uint8_t *cipher_text, size_t text_size); + +// Authenticated stream +// -------------------- +typedef struct { + uint64_t counter; + uint8_t key[32]; + uint8_t nonce[8]; +} crypto_aead_ctx; + +void crypto_aead_init_x(crypto_aead_ctx *ctx, + const uint8_t key[32], const uint8_t nonce[24]); +void crypto_aead_init_djb(crypto_aead_ctx *ctx, + const uint8_t key[32], const uint8_t nonce[8]); +void crypto_aead_init_ietf(crypto_aead_ctx *ctx, + const uint8_t key[32], const uint8_t nonce[12]); + +void crypto_aead_write(crypto_aead_ctx *ctx, + uint8_t *cipher_text, + uint8_t mac[16], + const uint8_t *ad , size_t ad_size, + const uint8_t *plain_text, size_t text_size); +int crypto_aead_read(crypto_aead_ctx *ctx, + uint8_t *plain_text, + const uint8_t mac[16], + const uint8_t *ad , size_t ad_size, + const uint8_t *cipher_text, size_t text_size); + + +// General purpose hash (BLAKE2b) +// ------------------------------ + +// Direct interface +void crypto_blake2b(uint8_t *hash, size_t hash_size, + const uint8_t *message, size_t message_size); + +void crypto_blake2b_keyed(uint8_t *hash, size_t hash_size, + const uint8_t *key, size_t key_size, + const uint8_t *message, size_t message_size); + +// Incremental interface +typedef struct { + // Do not rely on the size or contents of this type, + // for they may change without notice. + uint64_t hash[8]; + uint64_t input_offset[2]; + uint64_t input[16]; + size_t input_idx; + size_t hash_size; +} crypto_blake2b_ctx; + +void crypto_blake2b_init(crypto_blake2b_ctx *ctx, size_t hash_size); +void crypto_blake2b_keyed_init(crypto_blake2b_ctx *ctx, size_t hash_size, + const uint8_t *key, size_t key_size); +void crypto_blake2b_update(crypto_blake2b_ctx *ctx, + const uint8_t *message, size_t message_size); +void crypto_blake2b_final(crypto_blake2b_ctx *ctx, uint8_t *hash); + + +// Password key derivation (Argon2) +// -------------------------------- +#define CRYPTO_ARGON2_D 0 +#define CRYPTO_ARGON2_I 1 +#define CRYPTO_ARGON2_ID 2 + +typedef struct { + uint32_t algorithm; // Argon2d, Argon2i, Argon2id + uint32_t nb_blocks; // memory hardness, >= 8 * nb_lanes + uint32_t nb_passes; // CPU hardness, >= 1 (>= 3 recommended for Argon2i) + uint32_t nb_lanes; // parallelism level (single threaded anyway) +} crypto_argon2_config; + +typedef struct { + const uint8_t *pass; + const uint8_t *salt; + uint32_t pass_size; + uint32_t salt_size; // 16 bytes recommended +} crypto_argon2_inputs; + +typedef struct { + const uint8_t *key; // may be NULL if no key + const uint8_t *ad; // may be NULL if no additional data + uint32_t key_size; // 0 if no key (32 bytes recommended otherwise) + uint32_t ad_size; // 0 if no additional data +} crypto_argon2_extras; + +extern const crypto_argon2_extras crypto_argon2_no_extras; + +void crypto_argon2(uint8_t *hash, uint32_t hash_size, void *work_area, + crypto_argon2_config config, + crypto_argon2_inputs inputs, + crypto_argon2_extras extras); + + +// Key exchange (X-25519) +// ---------------------- + +// Shared secrets are not quite random. +// Hash them to derive an actual shared key. +void crypto_x25519_public_key(uint8_t public_key[32], + const uint8_t secret_key[32]); +void crypto_x25519(uint8_t raw_shared_secret[32], + const uint8_t your_secret_key [32], + const uint8_t their_public_key [32]); + +// Conversion to EdDSA +void crypto_x25519_to_eddsa(uint8_t eddsa[32], const uint8_t x25519[32]); + +// scalar "division" +// Used for OPRF. Be aware that exponential blinding is less secure +// than Diffie-Hellman key exchange. +void crypto_x25519_inverse(uint8_t blind_salt [32], + const uint8_t private_key[32], + const uint8_t curve_point[32]); + +// "Dirty" versions of x25519_public_key(). +// Use with crypto_elligator_rev(). +// Leaks 3 bits of the private key. +void crypto_x25519_dirty_small(uint8_t pk[32], const uint8_t sk[32]); +void crypto_x25519_dirty_fast (uint8_t pk[32], const uint8_t sk[32]); + + +// Signatures +// ---------- + +// EdDSA with curve25519 + BLAKE2b +void crypto_eddsa_key_pair(uint8_t secret_key[64], + uint8_t public_key[32], + uint8_t seed[32]); +void crypto_eddsa_sign(uint8_t signature [64], + const uint8_t secret_key[64], + const uint8_t *message, size_t message_size); +int crypto_eddsa_check(const uint8_t signature [64], + const uint8_t public_key[32], + const uint8_t *message, size_t message_size); + +// Conversion to X25519 +void crypto_eddsa_to_x25519(uint8_t x25519[32], const uint8_t eddsa[32]); + +// EdDSA building blocks +void crypto_eddsa_trim_scalar(uint8_t out[32], const uint8_t in[32]); +void crypto_eddsa_reduce(uint8_t reduced[32], const uint8_t expanded[64]); +void crypto_eddsa_mul_add(uint8_t r[32], + const uint8_t a[32], + const uint8_t b[32], + const uint8_t c[32]); +void crypto_eddsa_scalarbase(uint8_t point[32], const uint8_t scalar[32]); +int crypto_eddsa_check_equation(const uint8_t signature[64], + const uint8_t public_key[32], + const uint8_t h_ram[32]); + + +// Chacha20 +// -------- + +// Specialised hash. +// Used to hash X25519 shared secrets. +void crypto_chacha20_h(uint8_t out[32], + const uint8_t key[32], + const uint8_t in [16]); + +// Unauthenticated stream cipher. +// Don't forget to add authentication. +uint64_t crypto_chacha20_djb(uint8_t *cipher_text, + const uint8_t *plain_text, + size_t text_size, + const uint8_t key[32], + const uint8_t nonce[8], + uint64_t ctr); +uint32_t crypto_chacha20_ietf(uint8_t *cipher_text, + const uint8_t *plain_text, + size_t text_size, + const uint8_t key[32], + const uint8_t nonce[12], + uint32_t ctr); +uint64_t crypto_chacha20_x(uint8_t *cipher_text, + const uint8_t *plain_text, + size_t text_size, + const uint8_t key[32], + const uint8_t nonce[24], + uint64_t ctr); + + +// Poly 1305 +// --------- + +// This is a *one time* authenticator. +// Disclosing the mac reveals the key. +// See crypto_lock() on how to use it properly. + +// Direct interface +void crypto_poly1305(uint8_t mac[16], + const uint8_t *message, size_t message_size, + const uint8_t key[32]); + +// Incremental interface +typedef struct { + // Do not rely on the size or contents of this type, + // for they may change without notice. + uint8_t c[16]; // chunk of the message + size_t c_idx; // How many bytes are there in the chunk. + uint32_t r [4]; // constant multiplier (from the secret key) + uint32_t pad[4]; // random number added at the end (from the secret key) + uint32_t h [5]; // accumulated hash +} crypto_poly1305_ctx; + +void crypto_poly1305_init (crypto_poly1305_ctx *ctx, const uint8_t key[32]); +void crypto_poly1305_update(crypto_poly1305_ctx *ctx, + const uint8_t *message, size_t message_size); +void crypto_poly1305_final (crypto_poly1305_ctx *ctx, uint8_t mac[16]); + + +// Elligator 2 +// ----------- + +// Elligator mappings proper +void crypto_elligator_map(uint8_t curve [32], const uint8_t hidden[32]); +int crypto_elligator_rev(uint8_t hidden[32], const uint8_t curve [32], + uint8_t tweak); + +// Easy to use key pair generation +void crypto_elligator_key_pair(uint8_t hidden[32], uint8_t secret_key[32], + uint8_t seed[32]); + +#ifdef __cplusplus +} +#endif + +#endif // MONOCYPHER_H