{ "format": "v8c", "createdUtc": "2026-10-08T13:15:06Z", "commandLine": "consolidate --session v7=20261006-130619-eshoponweb,20261006-142724-eshoponweb,20261006-154837-eshoponweb --session v8=20261007-205106-eshoponweb,20261007-221429-eshoponweb,20261007-233551-eshoponweb --basis-session v5=20261005-023459-eshoponweb,20261005-031556-eshoponweb,20261005-035711-eshoponweb --basis-session v6=20261005-073329-eshoponweb,20261005-085448-eshoponweb,20261005-101815-eshoponweb --out ~/ForClaude/GenericVectorBuilder/bench-results/candidate-v8 --facts ~/gvb-work/v8-run/freeze-report/bin/engine-facts.json --exclusions ~/gvb-work/v8-run/freeze-report/bin/basis-exclusions.json --repo ~/ForClaude/GenericVectorBuilder --observer ~/gvb-work/v8-run/consolidate-work-report-f5/observer/summary.json --observer ~/gvb-work/v8-run/consolidate-work-report-f5/observer/summary-v7.json", "mode": "two-session", "status": "complete", "stopReasons": [], "sessions": [ { "name": "v7", "runs": [ { "folder": "20261006-130619-eshoponweb", "seed": 701, "startedUtc": "2026-10-06T13:06:19Z", "resultsSha256": "db2b2cacc7bec3f8d787b21bbef5af189da99b98f03504fc66f553d342f7a316", "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] }, { "folder": "20261006-142724-eshoponweb", "seed": 702, "startedUtc": "2026-10-06T14:27:24Z", "resultsSha256": "10bafe1e141cdfa2eadd1019d03ca8ef907a030e4595b847ada7348270b361ef", "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] }, { "folder": "20261006-154837-eshoponweb", "seed": 703, "startedUtc": "2026-10-06T15:48:37Z", "resultsSha256": "11e136639abf6ada17d07c3029345a86efdd42aeb749cb62efa31e4920d9689a", "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] } ] }, { "name": "v8", "runs": [ { "folder": "20261007-205106-eshoponweb", "seed": 801, "startedUtc": "2026-10-07T20:51:06Z", "resultsSha256": "f723e0233ced1f1f14403328c199b92bab39b5ba26b075ef13a18cce48eb21b0", "staleLines": [] }, { "folder": "20261007-221429-eshoponweb", "seed": 802, "startedUtc": "2026-10-07T22:14:29Z", "resultsSha256": "235eb74d0ddb51c0914bf382ff7db3a5abcc025d887fedb947ba2cd6d2b0498a", "staleLines": [] }, { "folder": "20261007-233551-eshoponweb", "seed": 803, "startedUtc": "2026-10-07T23:35:51Z", "resultsSha256": "64fd3d69b2dd8334c3c70bec05a11b7e856a586137ccab900b24a58651f04c37", "staleLines": [] } ] } ], "targets": [ "clickhouse", "vespa", "oracle", "elasticsearch", "sql", "pgvector", "sqlitevec", "qdrant", "opensearch", "qdrant-hnsw", "redis", "milvus", "mariadb", "weaviate", "mongodb", "chroma", "typesense", "sql-diskann", "duckdb" ], "runsPerSession": 3, "runsShown": 6, "claimRunCount": 6, "imageTargets": 17, "headlineNamedCount": 1, "queries": { "recorded": "golden: 20 labelled questions from ~/ForClaude/evalkit/questions_golden.json, embedded with qwen3-emb-0.6b (cached in ~/gvb-data/bench-cache/golden-eshoponweb-68df42ca69efa079.json, no embedding calls)", "copyPath": "design/bench-inputs/questions_golden.json", "copySha256": "e3a917ec8c5e687d2a2d6b83bc8ffd7eb901f0ffac8d8b277995b86898e9a712", "matchingRuns": [ "20261007-205106-eshoponweb", "20261007-221429-eshoponweb", "20261007-233551-eshoponweb" ], "unrecordedRuns": [ "20261006-130619-eshoponweb", "20261006-142724-eshoponweb", "20261006-154837-eshoponweb" ] }, "reuse": [ { "session": "v7", "verdict": "design/verdicts/v7-verdict.md", "blockedFolders": [ "blocked-2026-10-06-v7" ], "runs": [ "20261006-130619-eshoponweb", "20261006-142724-eshoponweb", "20261006-154837-eshoponweb" ] }, { "session": "v5", "verdict": "design/verdicts/v5-verdict.txt", "blockedFolders": [ "blocked-2026-10-05-v5" ], "runs": [ "20261005-023459-eshoponweb", "20261005-031556-eshoponweb", "20261005-035711-eshoponweb" ] }, { "session": "v6", "verdict": "design/verdicts/v6-verdict.md", "blockedFolders": [ "blocked-2026-10-05-v6" ], "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ] } ], "notInReport": [], "basis": { "runs": [ { "folder": "20261005-023459-eshoponweb", "seed": 501, "session": "v5", "startedUtc": "2026-10-05T02:34:59Z", "resultsSha256": "7514dac7c2d83fc5c22af773ff64fb4205fdfe8c88500382c8362650d41fc928", "conditions": { "turbo": "turbo not pinned (no no_turbo change recorded)", "uncore": "uncore not pinned", "warmup": "20-search warm-up", "rehearsal": "rehearsal 5 s", "build": "~/gvb-work/lanes/v5-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." } ] }, { "folder": "20261005-031556-eshoponweb", "seed": 502, "session": "v5", "startedUtc": "2026-10-05T03:15:56Z", "resultsSha256": "6e6671a431dc570145a8d410c276257fe1efe508b1f59bc76b1dee987c2a8fe9", "conditions": { "turbo": "turbo not pinned (no no_turbo change recorded)", "uncore": "uncore not pinned", "warmup": "20-search warm-up", "rehearsal": "rehearsal 5 s", "build": "~/gvb-work/lanes/v5-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." } ] }, { "folder": "20261005-035711-eshoponweb", "seed": 503, "session": "v5", "startedUtc": "2026-10-05T03:57:11Z", "resultsSha256": "00914a96391a61303f8a9c513af1934a6c4d40a5dac5089ea4b68329b528565d", "conditions": { "turbo": "turbo not pinned (no no_turbo change recorded)", "uncore": "uncore not pinned", "warmup": "20-search warm-up", "rehearsal": "rehearsal 5 s", "build": "~/gvb-work/lanes/v5-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." } ] }, { "folder": "20261005-073329-eshoponweb", "seed": 601, "session": "v6", "startedUtc": "2026-10-05T07:33:29Z", "resultsSha256": "0e91f3990213e577bb98089235db9f521d700977d0b105531e6e87e4e8411549", "conditions": { "turbo": "turbo not pinned (no no_turbo change recorded)", "uncore": "uncore not pinned", "warmup": "settle check", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/lanes/v6-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] }, { "folder": "20261005-085448-eshoponweb", "seed": 602, "session": "v6", "startedUtc": "2026-10-05T08:54:48Z", "resultsSha256": "044e47aa35749255a2fe00604a9bacefb1dc1b1d2dca7643658446d508e0b189", "conditions": { "turbo": "turbo not pinned (no no_turbo change recorded)", "uncore": "uncore not pinned", "warmup": "settle check", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/lanes/v6-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] }, { "folder": "20261005-101815-eshoponweb", "seed": 603, "session": "v6", "startedUtc": "2026-10-05T10:18:15Z", "resultsSha256": "0e93a162a36d88d83f9fe9b9d060d00995eacfd798ca865f21517bb9000add43", "conditions": { "turbo": "turbo not pinned (no no_turbo change recorded)", "uncore": "uncore not pinned", "warmup": "settle check", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/lanes/v6-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] }, { "folder": "20261006-130619-eshoponweb", "seed": 701, "session": "v7", "startedUtc": "2026-10-06T13:06:19Z", "resultsSha256": "db2b2cacc7bec3f8d787b21bbef5af189da99b98f03504fc66f553d342f7a316", "conditions": { "turbo": "turbo off (no_turbo 0 -> 1)", "uncore": "uncore pinned (MSR 0x620 = 0x1e1e)", "warmup": "settle check with level test", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/lanes/v7-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] }, { "folder": "20261006-142724-eshoponweb", "seed": 702, "session": "v7", "startedUtc": "2026-10-06T14:27:24Z", "resultsSha256": "10bafe1e141cdfa2eadd1019d03ca8ef907a030e4595b847ada7348270b361ef", "conditions": { "turbo": "turbo off (no_turbo 0 -> 1)", "uncore": "uncore pinned (MSR 0x620 = 0x1e1e)", "warmup": "settle check with level test", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/lanes/v7-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] }, { "folder": "20261006-154837-eshoponweb", "seed": 703, "session": "v7", "startedUtc": "2026-10-06T15:48:37Z", "resultsSha256": "11e136639abf6ada17d07c3029345a86efdd42aeb749cb62efa31e4920d9689a", "conditions": { "turbo": "turbo off (no_turbo 0 -> 1)", "uncore": "uncore pinned (MSR 0x620 = 0x1e1e)", "warmup": "settle check with level test", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/lanes/v7-final", "machineControl": "on" }, "staleLines": [ { "line": 13, "retired": "Where it is close to the latency, the client library is a large part of what is measured." }, { "line": 13, "retired": "not client overhead" } ] }, { "folder": "20261007-205106-eshoponweb", "seed": 801, "session": "v8", "startedUtc": "2026-10-07T20:51:06Z", "resultsSha256": "f723e0233ced1f1f14403328c199b92bab39b5ba26b075ef13a18cce48eb21b0", "conditions": { "turbo": "turbo off (no_turbo 0 -> 1)", "uncore": "uncore pinned (MSR 0x620 = 0x1e1e)", "warmup": "settle check with level test", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "machineControl": "on" }, "staleLines": [] }, { "folder": "20261007-221429-eshoponweb", "seed": 802, "session": "v8", "startedUtc": "2026-10-07T22:14:29Z", "resultsSha256": "235eb74d0ddb51c0914bf382ff7db3a5abcc025d887fedb947ba2cd6d2b0498a", "conditions": { "turbo": "turbo off (no_turbo 0 -> 1)", "uncore": "uncore pinned (MSR 0x620 = 0x1e1e)", "warmup": "settle check with level test", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "machineControl": "on" }, "staleLines": [] }, { "folder": "20261007-233551-eshoponweb", "seed": 803, "session": "v8", "startedUtc": "2026-10-07T23:35:51Z", "resultsSha256": "64fd3d69b2dd8334c3c70bec05a11b7e856a586137ccab900b24a58651f04c37", "conditions": { "turbo": "turbo off (no_turbo 0 -> 1)", "uncore": "uncore pinned (MSR 0x620 = 0x1e1e)", "warmup": "settle check with level test", "rehearsal": "rehearsal 30 s", "build": "~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "machineControl": "on" }, "staleLines": [] } ], "leftOut": [ { "folder": "20261003-183955-eshoponweb", "reason": "machineControl not recorded" }, { "folder": "20261003-184207-eshoponweb", "reason": "machineControl not recorded" }, { "folder": "20261003-184445-eshoponweb", "reason": "machineControl not recorded" }, { "folder": "20261003-234838-eshoponweb", "reason": "machineControl not recorded" }, { "folder": "20261004-130858-eshoponweb", "reason": "machineControl not recorded" }, { "folder": "20261004-132735-eshoponweb", "reason": "machineControl not recorded" } ], "exclusions": [ { "target": "vespa", "seeds": [ 502, 503 ], "metric": "qps8", "kind": "method defect", "why": "QPS@8 ran first in runs 502 and 503 (914.6 and 977.3 QPS) against 1550.0 in run 501, where it ran after the exact pass; cause inferred, not proven (JVM warm-up).", "source": "design/verdicts/v5-verdict.txt", "item": 2, "evidence": [ "cold", "The @8 pass ran first in runs 502 and 503" ], "seedEvidence": "runs 502 and 503" }, { "target": "elasticsearch", "seeds": [ 503 ], "metric": "qps8", "kind": "method defect", "why": "Elasticsearch QPS@8 shows the same effect: run 503 read 2335, the one run where it was the first pass; cause inferred, not proven.", "source": "design/verdicts/v5-verdict.txt", "item": 2, "evidence": [ "Elasticsearch QPS@8 shows the same effect", "run 503, the one run where @8 was its first pass", "cold" ], "seedEvidence": "run 503" }, { "target": "clickhouse", "seeds": [ 501, 502, 503 ], "metric": "*", "kind": "unrecorded config change", "why": "21 of ClickHouse's built-in log tables were switched off after the v5 runs ended; the v5 results do not say so.", "source": "design/verdicts/v6-verdict.md", "item": 1, "evidence": [ "21 of ClickHouse's built-in log tables", "after the v5 runs ended at 04:38", "Nothing in results.json, results.md or consolidated.md mentions it" ], "seedEvidence": "after the v5 runs ended" } ], "perMetric": { "p50Ms": { "oneEngine": { "metric": "p50Ms", "moveBp": 1528, "move": 0.152733584032398, "moveBpExact": 1527.3358403239802, "target": "redis", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 503, 703 ], "values": [ 0.3457, 0.3985 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "pair": { "metric": "p50Ms", "moveBp": 2634, "move": 0.26334851036070495, "moveBpExact": 2633.4851036070495, "target": null, "pair": "mongodb/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 503, 703 ], "values": [ 4.000867804454729, 3.166875784190715 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "oneEngineSameSettings": { "metric": "p50Ms", "moveBp": 1202, "move": 0.12012678288431067, "moveBpExact": 1201.2678288431066, "target": "mongodb", "pair": null, "runs": [ "20261006-130619-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 701, 703 ], "values": [ 1.4136, 1.262 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pairSameSettings": { "metric": "p50Ms", "moveBp": 1888, "move": 0.1887557465222991, "moveBpExact": 1887.557465222991, "target": null, "pair": "mongodb/redis", "runs": [ "20261006-154837-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 703, 801 ], "values": [ 3.166875784190715, 3.7646417869790247 ], "runsWithBoth": 12, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "oneEngineExclusionsKept": [ { "kind": "method defect", "oneEngine": { "metric": "p50Ms", "moveBp": 1528, "move": 0.152733584032398, "moveBpExact": 1527.3358403239802, "target": "redis", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 503, 703 ], "values": [ 0.3457, 0.3985 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "pair": null, "tBpIfKept": null }, { "kind": "unrecorded config change", "oneEngine": { "metric": "p50Ms", "moveBp": 1561, "move": 0.15606101031661312, "moveBpExact": 1560.6101031661312, "target": "clickhouse", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261005-101815-eshoponweb" ], "seeds": [ 503, 603 ], "values": [ 5.1995, 4.4976 ], "runsWithBoth": null, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "clickhouse pass median MHz engine/client: (3592, 3591) vs (3590, 3590)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": null, "tBpIfKept": null } ], "pairExclusionsKept": [ { "kind": "method defect", "oneEngine": null, "pair": { "metric": "p50Ms", "moveBp": 2634, "move": 0.26334851036070495, "moveBpExact": 2633.4851036070495, "target": null, "pair": "mongodb/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 503, 703 ], "values": [ 4.000867804454729, 3.166875784190715 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "tBpIfKept": null }, { "kind": "unrecorded config change", "oneEngine": null, "pair": { "metric": "p50Ms", "moveBp": 2634, "move": 0.26334851036070495, "moveBpExact": 2633.4851036070495, "target": null, "pair": "mongodb/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 503, 703 ], "values": [ 4.000867804454729, 3.166875784190715 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "tBpIfKept": null } ], "oneEngineNoMachineControl": { "metric": "p50Ms", "moveBp": 11282, "move": 1.1281633529975204, "moveBpExact": 11281.633529975203, "target": "typesense", "pair": null, "runs": [ "20261004-130858-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 11.88675, 5.58545 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pairNoMachineControl": { "metric": "p50Ms", "moveBp": 14882, "move": 1.4881283482972751, "moveBpExact": 14881.28348297275, "target": null, "pair": "mariadb/typesense", "runs": [ "20261004-130858-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 0.14237281006162325, 0.35424182474106836 ], "runsWithBoth": 3, "differences": [], "differenceNames": [] } }, "qps1": { "oneEngine": { "metric": "qps1", "moveBp": 1149, "move": 0.11486340473089629, "moveBpExact": 1148.6340473089629, "target": "mongodb", "pair": null, "runs": [ "20261006-130619-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 701, 703 ], "values": [ 691.0434489081044, 770.4190522666705 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pair": { "metric": "qps1", "moveBp": 1933, "move": 0.19321604061740616, "moveBpExact": 1932.1604061740616, "target": null, "pair": "mongodb/pgvector", "runs": [ "20261005-085448-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 602, 703 ], "values": [ 0.5778515777477949, 0.6895017716647451 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/lanes/v7-final" ], "differenceNames": [ "turbo", "uncore", "warmup", "build" ] }, "oneEngineSameSettings": { "metric": "qps1", "moveBp": 1149, "move": 0.11486340473089629, "moveBpExact": 1148.6340473089629, "target": "mongodb", "pair": null, "runs": [ "20261006-130619-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 701, 703 ], "values": [ 691.0434489081044, 770.4190522666705 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pairSameSettings": { "metric": "qps1", "moveBp": 1401, "move": 0.1400438239384485, "moveBpExact": 1400.4382393844849, "target": null, "pair": "mongodb/sqlitevec", "runs": [ "20261006-154837-eshoponweb", "20261007-233551-eshoponweb" ], "seeds": [ 703, 803 ], "values": [ 1.521234670431564, 1.3343650818406576 ], "runsWithBoth": 9, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "sqlitevec pass median MHz engine/client: (3492, 3492) vs (3499, 3492)" ], "differenceNames": [ "build", "median MHz" ] }, "oneEngineExclusionsKept": [ { "kind": "method defect", "oneEngine": { "metric": "qps1", "moveBp": 1149, "move": 0.11486340473089629, "moveBpExact": 1148.6340473089629, "target": "mongodb", "pair": null, "runs": [ "20261006-130619-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 701, 703 ], "values": [ 691.0434489081044, 770.4190522666705 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pair": null, "tBpIfKept": null }, { "kind": "unrecorded config change", "oneEngine": { "metric": "qps1", "moveBp": 2181, "move": 0.2180970564230289, "moveBpExact": 2180.970564230289, "target": "clickhouse", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261005-073329-eshoponweb" ], "seeds": [ 503, 601 ], "values": [ 176.63207979234465, 215.15501646493257 ], "runsWithBoth": null, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "clickhouse pass median MHz engine/client: (3592, 3591) vs (3574, 3585)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": null, "tBpIfKept": null } ], "pairExclusionsKept": [ { "kind": "method defect", "oneEngine": null, "pair": { "metric": "qps1", "moveBp": 1933, "move": 0.19321604061740616, "moveBpExact": 1932.1604061740616, "target": null, "pair": "mongodb/pgvector", "runs": [ "20261005-085448-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 602, 703 ], "values": [ 0.5778515777477949, 0.6895017716647451 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/lanes/v7-final" ], "differenceNames": [ "turbo", "uncore", "warmup", "build" ] }, "tBpIfKept": null }, { "kind": "unrecorded config change", "oneEngine": null, "pair": { "metric": "qps1", "moveBp": 2815, "move": 0.2814002802629627, "moveBpExact": 2814.0028026296272, "target": null, "pair": "clickhouse/redis", "runs": [ "20261005-035711-eshoponweb", "20261005-101815-eshoponweb" ], "seeds": [ 503, 603 ], "values": [ 0.06456748115091682, 0.08273678844265837 ], "runsWithBoth": 6, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "clickhouse pass median MHz engine/client: (3592, 3591) vs (3590, 3590)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "tBpIfKept": null } ], "oneEngineNoMachineControl": { "metric": "qps1", "moveBp": 4245, "move": 0.4244622394864437, "moveBpExact": 4244.6223948644365, "target": "typesense", "pair": null, "runs": [ "20261003-184445-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 95.42371267563212, 135.92747545804187 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pairNoMachineControl": { "metric": "qps1", "moveBp": 9455, "move": 0.9454970962224176, "moveBpExact": 9454.970962224175, "target": null, "pair": "mongodb/typesense", "runs": [ "20261003-184445-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 3.224594073842532, 1.657465374841085 ], "runsWithBoth": 3, "differences": [], "differenceNames": [] } }, "qps8": { "oneEngine": { "metric": "qps8", "moveBp": 2736, "move": 0.27350002280617347, "moveBpExact": 2735.0002280617346, "target": "clickhouse", "pair": null, "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 428.4725710440155, 336.45273919969844 ], "runsWithBoth": null, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "pair": { "metric": "qps8", "moveBp": 3582, "move": 0.3581503086609965, "moveBpExact": 3581.503086609965, "target": null, "pair": "clickhouse/oracle", "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 0.16136394796348733, 0.1188115534300297 ], "runsWithBoth": 6, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "oneEngineSameSettings": { "metric": "qps8", "moveBp": 2736, "move": 0.27350002280617347, "moveBpExact": 2735.0002280617346, "target": "clickhouse", "pair": null, "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 428.4725710440155, 336.45273919969844 ], "runsWithBoth": null, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "pairSameSettings": { "metric": "qps8", "moveBp": 3582, "move": 0.3581503086609965, "moveBpExact": 3581.503086609965, "target": null, "pair": "clickhouse/oracle", "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 0.16136394796348733, 0.1188115534300297 ], "runsWithBoth": 6, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "oneEngineExclusionsKept": [ { "kind": "method defect", "oneEngine": { "metric": "qps8", "moveBp": 7442, "move": 0.7441162292802652, "moveBpExact": 7441.162292802652, "target": "vespa", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261005-101815-eshoponweb" ], "seeds": [ 502, 603 ], "values": [ 914.6355971415784, 1595.2307888520734 ], "runsWithBoth": null, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "vespa pass median MHz engine/client: (3501, 3497) vs (3492, 3493)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": null, "tBpIfKept": null }, { "kind": "unrecorded config change", "oneEngine": { "metric": "qps8", "moveBp": 2736, "move": 0.27350002280617347, "moveBpExact": 2735.0002280617346, "target": "clickhouse", "pair": null, "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 428.4725710440155, 336.45273919969844 ], "runsWithBoth": null, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "pair": null, "tBpIfKept": null } ], "pairExclusionsKept": [ { "kind": "method defect", "oneEngine": null, "pair": { "metric": "qps8", "moveBp": 9564, "move": 0.956391788564052, "moveBpExact": 9563.91788564052, "target": null, "pair": "oracle/vespa", "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 3.450090093312333, 1.7634965110156295 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only", "vespa pass median MHz engine/client: (3501, 3497) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded", "median MHz" ] }, "tBpIfKept": null }, { "kind": "unrecorded config change", "oneEngine": null, "pair": { "metric": "qps8", "moveBp": 3582, "move": 0.3581503086609965, "moveBpExact": 3581.503086609965, "target": null, "pair": "clickhouse/oracle", "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 0.16136394796348733, 0.1188115534300297 ], "runsWithBoth": 6, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "tBpIfKept": null } ], "oneEngineNoMachineControl": { "metric": "qps8", "moveBp": 8428, "move": 0.8427470842697431, "moveBpExact": 8427.47084269743, "target": "chroma", "pair": null, "runs": [ "20261003-184445-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 330.89762574959, 609.7606350418376 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pairNoMachineControl": { "metric": "qps8", "moveBp": 14788, "move": 1.478782695143662, "moveBpExact": 14787.82695143662, "target": null, "pair": "chroma/oracle", "runs": [ "20261003-184445-eshoponweb", "20261004-130858-eshoponweb" ], "seeds": [ null, null ], "values": [ 0.47605388055294745, 1.180034121070634 ], "runsWithBoth": 3, "differences": [], "differenceNames": [] } }, "exactP50Ms": { "oneEngine": { "metric": "exactP50Ms", "moveBp": 2023, "move": 0.2022047244094487, "moveBpExact": 2022.047244094487, "target": "redis", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 0.3175, 0.3817 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "pair": { "metric": "exactP50Ms", "moveBp": 1966, "move": 0.19654729041222785, "moveBpExact": 1965.4729041222786, "target": null, "pair": "qdrant-hnsw/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 2.797795275590551, 2.3382237359182603 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "oneEngineSameSettings": { "metric": "exactP50Ms", "moveBp": 774, "move": 0.07733594515181186, "moveBpExact": 773.3594515181186, "target": "opensearch", "pair": null, "runs": [ "20261006-130619-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 701, 801 ], "values": [ 2.7499, 2.5525 ], "runsWithBoth": null, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "pairSameSettings": { "metric": "exactP50Ms", "moveBp": 841, "move": 0.08400710602124462, "moveBpExact": 840.0710602124461, "target": null, "pair": "opensearch/sqlitevec", "runs": [ "20261006-130619-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 701, 801 ], "values": [ 1.44306255247691, 1.331229790341087 ], "runsWithBoth": 9, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "sqlitevec pass median MHz engine/client: (3492, 3492) vs (3493, 3492)" ], "differenceNames": [ "build", "median MHz" ] }, "oneEngineExclusionsKept": [ { "kind": "method defect", "oneEngine": { "metric": "exactP50Ms", "moveBp": 2023, "move": 0.2022047244094487, "moveBpExact": 2022.047244094487, "target": "redis", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 0.3175, 0.3817 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "pair": null, "tBpIfKept": null }, { "kind": "unrecorded config change", "oneEngine": { "metric": "exactP50Ms", "moveBp": 2023, "move": 0.2022047244094487, "moveBpExact": 2022.047244094487, "target": "redis", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 0.3175, 0.3817 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "pair": null, "tBpIfKept": null } ], "pairExclusionsKept": [ { "kind": "method defect", "oneEngine": null, "pair": { "metric": "exactP50Ms", "moveBp": 1966, "move": 0.19654729041222785, "moveBpExact": 1965.4729041222786, "target": null, "pair": "qdrant-hnsw/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 2.797795275590551, 2.3382237359182603 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "tBpIfKept": null }, { "kind": "unrecorded config change", "oneEngine": null, "pair": { "metric": "exactP50Ms", "moveBp": 1966, "move": 0.19654729041222785, "moveBpExact": 1965.4729041222786, "target": null, "pair": "qdrant-hnsw/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 2.797795275590551, 2.3382237359182603 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "tBpIfKept": null } ], "oneEngineNoMachineControl": { "metric": "exactP50Ms", "moveBp": 6968, "move": 0.6967868077855244, "moveBpExact": 6967.868077855243, "target": "typesense", "pair": null, "runs": [ "20261004-130858-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 12.496749999999999, 7.36495 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pairNoMachineControl": { "metric": "exactP50Ms", "moveBp": 12391, "move": 1.2390442215698303, "moveBpExact": 12390.442215698304, "target": null, "pair": "duckdb/typesense", "runs": [ "20261004-130858-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 0.3580891031668234, 0.8017773372527988 ], "runsWithBoth": 3, "differences": [], "differenceNames": [] } } }, "noMachineControl": { "oneEngine": { "metric": "p50Ms", "moveBp": 11282, "move": 1.1281633529975204, "moveBpExact": 11281.633529975203, "target": "typesense", "pair": null, "runs": [ "20261004-130858-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 11.88675, 5.58545 ], "runsWithBoth": null, "differences": [], "differenceNames": [] }, "pair": { "metric": "p50Ms", "moveBp": 14882, "move": 1.4881283482972751, "moveBpExact": 14881.28348297275, "target": null, "pair": "mariadb/typesense", "runs": [ "20261004-130858-eshoponweb", "20261004-132735-eshoponweb" ], "seeds": [ null, null ], "values": [ 0.14237281006162325, 0.35424182474106836 ], "runsWithBoth": 3, "differences": [], "differenceNames": [] } }, "exclusionsKept": [ { "kind": "method defect", "oneEngine": { "metric": "qps8", "moveBp": 7442, "move": 0.7441162292802652, "moveBpExact": 7441.162292802652, "target": "vespa", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261005-101815-eshoponweb" ], "seeds": [ 502, 603 ], "values": [ 914.6355971415784, 1595.2307888520734 ], "runsWithBoth": null, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "vespa pass median MHz engine/client: (3501, 3497) vs (3492, 3493)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps8", "moveBp": 9564, "move": 0.956391788564052, "moveBpExact": 9563.91788564052, "target": null, "pair": "oracle/vespa", "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 3.450090093312333, 1.7634965110156295 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only", "vespa pass median MHz engine/client: (3501, 3497) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded", "median MHz" ] }, "tBpIfKept": 10500 }, { "kind": "unrecorded config change", "oneEngine": { "metric": "qps8", "moveBp": 2736, "move": 0.27350002280617347, "moveBpExact": 2735.0002280617346, "target": "clickhouse", "pair": null, "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 428.4725710440155, 336.45273919969844 ], "runsWithBoth": null, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "pair": { "metric": "qps8", "moveBp": 3582, "move": 0.3581503086609965, "moveBpExact": 3581.503086609965, "target": null, "pair": "clickhouse/oracle", "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 0.16136394796348733, 0.1188115534300297 ], "runsWithBoth": 6, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "tBpIfKept": 4500 } ], "setupSplits": [ { "target": "clickhouse", "fields": [ "engine" ], "changes": [ { "field": "engine", "before": "ClickHouse 26.3.39.7 (MergeTree + vector_similarity index)", "after": "ClickHouse 26.3.39.7 (MergeTree + vector_similarity index; image-default system logging, each log table with a TTL of 1 hour)" } ], "before": { "sessions": [ "v6" ], "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ] }, "after": { "sessions": [ "v7", "v8" ], "runs": [ "20261006-130619-eshoponweb", "20261006-142724-eshoponweb", "20261006-154837-eshoponweb", "20261007-205106-eshoponweb", "20261007-221429-eshoponweb", "20261007-233551-eshoponweb" ] }, "oneEngine": { "metric": "qps8", "moveBp": 4736, "move": 0.4735507730412365, "moveBpExact": 4735.507730412365, "target": "clickhouse", "pair": null, "runs": [ "20261005-073329-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 601, 801 ], "values": [ 495.78019393955714, 336.45273919969844 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "clickhouse pass median MHz engine/client: (3542, 3552) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] }, "pair": { "metric": "qps8", "moveBp": 5208, "move": 0.5207143280504289, "moveBpExact": 5207.14328050429, "target": null, "pair": "clickhouse/redis", "runs": [ "20261005-073329-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 601, 801 ], "values": [ 0.09769935188923334, 0.06424569696432393 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "clickhouse pass median MHz engine/client: (3542, 3552) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] }, "perMetric": { "p50Ms": { "oneEngine": { "metric": "p50Ms", "moveBp": 1058, "move": 0.10571193525435785, "moveBpExact": 1057.1193525435785, "target": "clickhouse", "pair": null, "runs": [ "20261005-101815-eshoponweb", "20261006-130619-eshoponweb" ], "seeds": [ 603, 701 ], "values": [ 4.4976, 4.97305 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/lanes/v7-final", "clickhouse pass median MHz engine/client: (3590, 3590) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] }, "pair": { "metric": "p50Ms", "moveBp": 2305, "move": 0.23041678563767087, "moveBpExact": 2304.1678563767086, "target": null, "pair": "clickhouse/mongodb", "runs": [ "20261005-085448-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 602, 703 ], "values": [ 3.192133399419387, 3.9276545166402532 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/lanes/v7-final", "clickhouse pass median MHz engine/client: (3591, 3589) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] } }, "qps1": { "oneEngine": { "metric": "qps1", "moveBp": 1232, "move": 0.12312641182352735, "moveBpExact": 1231.2641182352736, "target": "clickhouse", "pair": null, "runs": [ "20261005-073329-eshoponweb", "20261006-130619-eshoponweb" ], "seeds": [ 601, 701 ], "values": [ 215.15501646493257, 191.56794302041496 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/lanes/v7-final", "clickhouse pass median MHz engine/client: (3574, 3585) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] }, "pair": { "metric": "qps1", "moveBp": 2349, "move": 0.23487916683192012, "moveBpExact": 2348.791668319201, "target": null, "pair": "clickhouse/mongodb", "runs": [ "20261005-085448-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 602, 703 ], "values": [ 0.3094923392845588, 0.25062560580608123 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/lanes/v7-final", "clickhouse pass median MHz engine/client: (3591, 3589) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] } }, "qps8": { "oneEngine": { "metric": "qps8", "moveBp": 4736, "move": 0.4735507730412365, "moveBpExact": 4735.507730412365, "target": "clickhouse", "pair": null, "runs": [ "20261005-073329-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 601, 801 ], "values": [ 495.78019393955714, 336.45273919969844 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "clickhouse pass median MHz engine/client: (3542, 3552) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] }, "pair": { "metric": "qps8", "moveBp": 5208, "move": 0.5207143280504289, "moveBpExact": 5207.14328050429, "target": null, "pair": "clickhouse/redis", "runs": [ "20261005-073329-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 601, 801 ], "values": [ 0.09769935188923334, 0.06424569696432393 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "clickhouse pass median MHz engine/client: (3542, 3552) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] } }, "exactP50Ms": { "oneEngine": { "metric": "exactP50Ms", "moveBp": 1026, "move": 0.10253616009510602, "moveBpExact": 1025.3616009510602, "target": "clickhouse", "pair": null, "runs": [ "20261005-101815-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 603, 703 ], "values": [ 6.0564, 6.6774 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/lanes/v7-final", "clickhouse pass median MHz engine/client: (3592, 3592) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] }, "pair": { "metric": "exactP50Ms", "moveBp": 1300, "move": 0.12994201227365765, "moveBpExact": 1299.4201227365766, "target": null, "pair": "clickhouse/opensearch", "runs": [ "20261005-101815-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 603, 801 ], "values": [ 2.298749359497467, 2.5974534769833495 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: settle check vs settle check with level test", "build: ~/gvb-work/lanes/v6-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)", "clickhouse pass median MHz engine/client: (3592, 3592) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "build", "median MHz" ] } } }, "tBpIfCounted": 6000 }, { "target": "duckdb", "fields": [ "index" ], "changes": [ { "field": "index", "before": "HNSW (vss extension) FLOAT[n] metric=cosine m=16 ef_construction=128, ef_search=100 per connection, persistent (hnsw_enable_experimental_persistence=true, checkpoint_threshold=256MB); exact mode = array_cosine_similarity sequential scan; score = 1 - cosine distance", "after": "HNSW (vss extension) FLOAT[n] metric=cosine m=16 ef_construction=128, ef_search=100 per connection, persistent (hnsw_enable_experimental_persistence=true, checkpoint_threshold=256MB); exact mode = array_cosine_similarity sequential scan; score = 1 - cosine distance; searches run concurrently, one connection per searcher (opened as searchers arrive, at most 32), writes run one at a time and never overlap a search" } ], "before": { "sessions": [ "v5" ], "runs": [ "20261005-023459-eshoponweb", "20261005-031556-eshoponweb", "20261005-035711-eshoponweb" ] }, "after": { "sessions": [ "v6", "v7", "v8" ], "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb", "20261006-130619-eshoponweb", "20261006-142724-eshoponweb", "20261006-154837-eshoponweb", "20261007-205106-eshoponweb", "20261007-221429-eshoponweb", "20261007-233551-eshoponweb" ] }, "oneEngine": { "metric": "qps8", "moveBp": 11556, "move": 1.1555948111177292, "moveBpExact": 11555.948111177293, "target": "duckdb", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261005-101815-eshoponweb" ], "seeds": [ 503, 603 ], "values": [ 279.21221193127604, 601.8683952397623 ], "runsWithBoth": null, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "duckdb pass median MHz engine/client: (3597, 3592) vs (3593, 3592)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps8", "moveBp": 14495, "move": 1.4494080321881153, "moveBpExact": 14494.080321881152, "target": null, "pair": "duckdb/oracle", "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 0.08855562475783227, 0.21690885857727107 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "duckdb pass median MHz engine/client: (3597, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "perMetric": { "p50Ms": { "oneEngine": { "metric": "p50Ms", "moveBp": 537, "move": 0.053629748467312055, "moveBpExact": 536.2974846731206, "target": "duckdb", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 3.4906, 3.6778 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "duckdb pass median MHz engine/client: (3593, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "p50Ms", "moveBp": 1483, "move": 0.1482267703262452, "moveBpExact": 1482.267703262452, "target": null, "pair": "duckdb/mongodb", "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 2.530153667729777, 2.905190174326466 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "duckdb pass median MHz engine/client: (3593, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] } }, "qps1": { "oneEngine": { "metric": "qps1", "moveBp": 549, "move": 0.05484231587998489, "moveBpExact": 548.4231587998489, "target": "duckdb", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 283.8571446675838, 269.09912542784235 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "duckdb pass median MHz engine/client: (3593, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps1", "moveBp": 1580, "move": 0.15792448991257357, "moveBpExact": 1579.2448991257356, "target": null, "pair": "duckdb/mongodb", "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 0.4060443803734322, 0.35066568149369537 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "duckdb pass median MHz engine/client: (3593, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] } }, "qps8": { "oneEngine": { "metric": "qps8", "moveBp": 11556, "move": 1.1555948111177292, "moveBpExact": 11555.948111177293, "target": "duckdb", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261005-101815-eshoponweb" ], "seeds": [ 503, 603 ], "values": [ 279.21221193127604, 601.8683952397623 ], "runsWithBoth": null, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "duckdb pass median MHz engine/client: (3597, 3592) vs (3593, 3592)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps8", "moveBp": 14495, "move": 1.4494080321881153, "moveBpExact": 14494.080321881152, "target": null, "pair": "duckdb/oracle", "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 0.08855562475783227, 0.21690885857727107 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "duckdb pass median MHz engine/client: (3597, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] } }, "exactP50Ms": { "oneEngine": { "metric": "exactP50Ms", "moveBp": 504, "move": 0.05031156530408776, "moveBpExact": 503.1156530408776, "target": "duckdb", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-130619-eshoponweb" ], "seeds": [ 502, 701 ], "values": [ 4.0120000000000005, 4.213850000000001 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "duckdb pass median MHz engine/client: (3593, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "exactP50Ms", "moveBp": 1581, "move": 0.15803328009365814, "moveBpExact": 1580.3328009365814, "target": null, "pair": "duckdb/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 12.740944881889764, 11.002226879748495 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "duckdb pass median MHz engine/client: (3594, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] } } }, "tBpIfCounted": 15000 }, { "target": "mariadb", "fields": [ "durability" ], "changes": [ { "field": "durability", "before": "innodb_flush_log_at_trx_commit=2 (mariadb.compose.yaml): the InnoDB redo log is written to the operating system at every commit but fsynced about once a second, so a crash of the mariadbd process loses nothing, while an operating-system crash or power cut can lose the last second of commits; innodb_doublewrite is on (default), the binary log is off, and the vector graph is an InnoDB table under the same log (settings read from the running server with SHOW VARIABLES; the crash behaviour is InnoDB's documented behaviour for this setting and was not tested, because gvb-mariadb is shared and stays up)", "after": "innodb_flush_log_at_trx_commit=2 (set in mariadb-bench.compose.yaml for the benchmark's own container gvbbench-mariadb, and in mariadb.compose.yaml for the daily gvb-mariadb, which has the same server settings): the InnoDB redo log is written to the operating system at every commit but fsynced about once a second, so a crash of the mariadbd process loses nothing, while an operating-system crash or power cut can lose the last second of commits; innodb_doublewrite is on (default), the binary log is off, and the vector graph is an InnoDB table under the same log (settings read from the running server with SHOW VARIABLES; the crash behaviour is InnoDB's documented behaviour for this setting and was not tested on either container)" } ], "before": { "sessions": [ "v5" ], "runs": [ "20261005-023459-eshoponweb", "20261005-031556-eshoponweb", "20261005-035711-eshoponweb" ] }, "after": { "sessions": [ "v6", "v7", "v8" ], "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb", "20261006-130619-eshoponweb", "20261006-142724-eshoponweb", "20261006-154837-eshoponweb", "20261007-205106-eshoponweb", "20261007-221429-eshoponweb", "20261007-233551-eshoponweb" ] }, "oneEngine": { "metric": "exactP50Ms", "moveBp": 409, "move": 0.04082278481012658, "moveBpExact": 408.22784810126575, "target": "mariadb", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 1.58, 1.6445 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "mariadb pass median MHz engine/client: (3592, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps8", "moveBp": 1785, "move": 0.1784601458879258, "moveBpExact": 1784.601458879258, "target": null, "pair": "mariadb/oracle", "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 1.8917692092447151, 2.229374618312813 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "perMetric": { "p50Ms": { "oneEngine": { "metric": "p50Ms", "moveBp": 240, "move": 0.023974255832662772, "moveBpExact": 239.74255832662772, "target": "mariadb", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261006-130619-eshoponweb" ], "seeds": [ 503, 701 ], "values": [ 0.6215, 0.6364 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "pair": { "metric": "p50Ms", "moveBp": 1338, "move": 0.1337227437177011, "moveBpExact": 1337.2274371770109, "target": null, "pair": "mariadb/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 1.7978015620480186, 1.5857506361323155 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] } }, "qps1": { "oneEngine": { "metric": "qps1", "moveBp": 218, "move": 0.02171033995365046, "moveBpExact": 217.10339953650458, "target": "mariadb", "pair": null, "runs": [ "20261005-035711-eshoponweb", "20261006-130619-eshoponweb" ], "seeds": [ 503, 701 ], "values": [ 1579.9450310728773, 1546.3727529120936 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps1", "moveBp": 1169, "move": 0.11689889903648787, "moveBpExact": 1168.9889903648786, "target": null, "pair": "mariadb/mongodb", "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 2.2497472822483653, 2.0142801503243923 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] } }, "qps8": { "oneEngine": { "metric": "qps8", "moveBp": 99, "move": 0.009838273446269419, "moveBpExact": 98.38273446269419, "target": "mariadb", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-130619-eshoponweb" ], "seeds": [ 502, 701 ], "values": [ 5969.620024820835, 5911.461450602725 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps8", "moveBp": 1785, "move": 0.1784601458879258, "moveBpExact": 1784.601458879258, "target": null, "pair": "mariadb/oracle", "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 1.8917692092447151, 2.229374618312813 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] } }, "exactP50Ms": { "oneEngine": { "metric": "exactP50Ms", "moveBp": 409, "move": 0.04082278481012658, "moveBpExact": 408.22784810126575, "target": "mariadb", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 1.58, 1.6445 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "mariadb pass median MHz engine/client: (3592, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "exactP50Ms", "moveBp": 1660, "move": 0.16594810673440996, "moveBpExact": 1659.4810673440995, "target": null, "pair": "mariadb/redis", "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 4.993385826771653, 4.282682735132303 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "mariadb pass median MHz engine/client: (3592, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] } } }, "tBpIfCounted": 4500 }, { "target": "sqlitevec", "fields": [ "index" ], "changes": [ { "field": "index", "before": "vec0 brute-force scan, no ANN index (exact), float32, cosine distance, default chunk_size=1024; score = 1 - cosine distance", "after": "vec0 brute-force scan, no ANN index (exact), float32, cosine distance, default chunk_size=1024; score = 1 - cosine distance; searches run concurrently, one WAL reader connection per searcher (opened as searchers arrive, at most 32), writes run one at a time and may overlap searches" } ], "before": { "sessions": [ "v5" ], "runs": [ "20261005-023459-eshoponweb", "20261005-031556-eshoponweb", "20261005-035711-eshoponweb" ] }, "after": { "sessions": [ "v6", "v7", "v8" ], "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb", "20261006-130619-eshoponweb", "20261006-142724-eshoponweb", "20261006-154837-eshoponweb", "20261007-205106-eshoponweb", "20261007-221429-eshoponweb", "20261007-233551-eshoponweb" ] }, "oneEngine": { "metric": "qps8", "moveBp": 11936, "move": 1.1935642586795336, "moveBpExact": 11935.642586795337, "target": "sqlitevec", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261005-073329-eshoponweb" ], "seeds": [ 502, 601 ], "values": [ 529.4626901002513, 1161.4104333082294 ], "runsWithBoth": null, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "sqlitevec pass median MHz engine/client: (3598, 3592) vs (3592, 3592)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps8", "moveBp": 15508, "move": 1.5507809613594987, "moveBpExact": 15507.809613594987, "target": null, "pair": "oracle/sqlitevec", "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 5.9599576546016415, 2.336522713978993 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only", "sqlitevec pass median MHz engine/client: (3598, 3592) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded", "median MHz" ] }, "perMetric": { "p50Ms": { "oneEngine": { "metric": "p50Ms", "moveBp": 689, "move": 0.06888925747774644, "moveBpExact": 688.8925747774643, "target": "sqlitevec", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 1.8087, 1.9333 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "sqlitevec pass median MHz engine/client: (3598, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "p50Ms", "moveBp": 1685, "move": 0.16849415183541905, "moveBpExact": 1684.9415183541905, "target": null, "pair": "mongodb/sqlitevec", "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 0.7627577818322552, 0.6527698753426783 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only", "sqlitevec pass median MHz engine/client: (3598, 3592) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded", "median MHz" ] } }, "qps1": { "oneEngine": { "metric": "qps1", "moveBp": 767, "move": 0.07664969699871649, "moveBpExact": 766.4969699871649, "target": "sqlitevec", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 545.2619870605721, 506.4432643045848 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "sqlitevec pass median MHz engine/client: (3598, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps1", "moveBp": 1866, "move": 0.18652013129960232, "moveBpExact": 1865.2013129960233, "target": null, "pair": "mongodb/sqlitevec", "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 1.282097648663868, 1.521234670431564 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only", "sqlitevec pass median MHz engine/client: (3598, 3592) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded", "median MHz" ] } }, "qps8": { "oneEngine": { "metric": "qps8", "moveBp": 11936, "move": 1.1935642586795336, "moveBpExact": 11935.642586795337, "target": "sqlitevec", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261005-073329-eshoponweb" ], "seeds": [ 502, 601 ], "values": [ 529.4626901002513, 1161.4104333082294 ], "runsWithBoth": null, "differences": [ "warmup: 20-search warm-up vs settle check", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v6-final", "sqlitevec pass median MHz engine/client: (3598, 3592) vs (3592, 3592)", "searchSettings recorded in one run only" ], "differenceNames": [ "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "qps8", "moveBp": 15508, "move": 1.5507809613594987, "moveBpExact": 15507.809613594987, "target": null, "pair": "oracle/sqlitevec", "runs": [ "20261005-031556-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 502, 702 ], "values": [ 5.9599576546016415, 2.336522713978993 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only", "sqlitevec pass median MHz engine/client: (3598, 3592) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded", "median MHz" ] } }, "exactP50Ms": { "oneEngine": { "metric": "exactP50Ms", "moveBp": 686, "move": 0.06853926383753639, "moveBpExact": 685.392638375364, "target": "sqlitevec", "pair": null, "runs": [ "20261005-031556-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 502, 703 ], "values": [ 1.8121, 1.9363 ], "runsWithBoth": null, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "sqlitevec pass median MHz engine/client: (3600, 3592) vs (3492, 3492)", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "median MHz", "whether searchSettings was recorded" ] }, "pair": { "metric": "exactP50Ms", "moveBp": 1484, "move": 0.14831278848764606, "moveBpExact": 1483.1278848764607, "target": null, "pair": "redis/sqlitevec", "runs": [ "20261005-035711-eshoponweb", "20261006-142724-eshoponweb" ], "seeds": [ 503, 702 ], "values": [ 0.17366808883054372, 0.19942528735632184 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only", "sqlitevec pass median MHz engine/client: (3597, 3592) vs (3492, 3492)" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded", "median MHz" ] } } }, "tBpIfCounted": 16500 } ], "setupSplitCount": 4, "clockHeld": [ { "session": "v5", "runs": 3, "held": 0 }, { "session": "v6", "runs": 3, "held": 0 }, { "session": "v7", "runs": 3, "held": 3 }, { "session": "v8", "runs": 3, "held": 3 } ], "maxBp": 3582, "maxMove": { "metric": "qps8", "moveBp": 3582, "move": 0.3581503086609965, "moveBpExact": 3581.503086609965, "target": null, "pair": "clickhouse/oracle", "runs": [ "20261006-142724-eshoponweb", "20261007-205106-eshoponweb" ], "seeds": [ 702, 801 ], "values": [ 0.16136394796348733, 0.1188115534300297 ], "runsWithBoth": 6, "differences": [ "build: ~/gvb-work/lanes/v7-final vs ~/gvb-work/v8-run/bin/GenericVectorBuilder.Bench.dll (commit 9b924200abc3f2c3e41c2a9520e5c7bb16748e6c)" ], "differenceNames": [ "build" ] }, "maxMoveNotHeld": [ { "run": "20261007-205106-eshoponweb", "seed": 801, "target": "clickhouse", "pass": "default@8", "metric": "qps8", "token": "the timed pass 336 QPS, 24% from the settled 418 QPS (limit 10%), NOT HELD", "timed": "336 QPS", "offPercent": "24", "settled": "418 QPS", "limitPercent": "10" } ], "maxMoveWithout": { "targets": [ "clickhouse" ], "maxBp": 2908, "maxMove": { "metric": "qps8", "moveBp": 2908, "move": 0.2907866621687567, "moveBpExact": 2907.866621687567, "target": null, "pair": "mongodb/oracle", "runs": [ "20261005-035711-eshoponweb", "20261006-154837-eshoponweb" ], "seeds": [ 503, 703 ], "values": [ 0.64994356185512, 0.8389384808050432 ], "runsWithBoth": 12, "differences": [ "turbo: turbo not pinned (no no_turbo change recorded) vs turbo off (no_turbo 0 -> 1)", "uncore: uncore not pinned vs uncore pinned (MSR 0x620 = 0x1e1e)", "warmup: 20-search warm-up vs settle check with level test", "rehearsal: rehearsal 5 s vs rehearsal 30 s", "build: ~/gvb-work/lanes/v5-final vs ~/gvb-work/lanes/v7-final", "searchSettings recorded in one run only" ], "differenceNames": [ "turbo", "uncore", "warmup", "rehearsal", "build", "whether searchSettings was recorded" ] }, "tBp": 3500 }, "tBp": 4500, "floorBp": 3500, "stepBp": 500, "marginBp": 500, "tBpRule": "TBp = max(FLOOR_BP 3500, the smallest multiple of STEP_BP 500 that is at least MaxBp + MARGIN_BP 500)", "runCount": 12, "noMachineControlCount": 6, "leftOutCount": 6, "moveRule": "moveBp = ceiling(10000 x hi / lo) - 10000, with hi and lo the two values (one engine) or the two same-run ratios (pair)" }, "threshold": { "tBp": 4500, "ratio": 1.45, "rule": "ordered when floor(10000 x B_best / A_worst) >= 10000 + TBp for latency, or floor(10000 x A_worst / B_best) >= 10000 + TBp for searches per second, in every claim session separately" }, "metrics": [ { "metric": "p50Ms", "lowerIsBetter": true, "rows": [ { "target": "redis", "display": "redis", "status": "ranked", "perSession": { "v7": { "min": 0.393, "max": 0.3985, "runs": [ 0.3978, 0.393, 0.3985 ] }, "v8": { "min": 0.3671, "max": 0.3917, "runs": [ 0.3671, 0.3869, 0.3917 ] } }, "median": 0.39235, "notSeparatedFrom": [], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, documented", "note": "redis holds its data in memory: its saved docs page says 'Redis is an in-memory but persistent on disk database', and its compose file sets save \"300 1\" and appendonly no.", "noteSources": [ { "kind": "doc", "ref": "doc:design/engine-docs/redis-faq-2026-10-07.html#Redis is an in-memory but persistent on disk database", "value": "Redis is an in-memory but persistent on disk database" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--save \"300 1\"", "value": "--save \"300 1\"" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--appendonly no", "value": "--appendonly no" } ] }, { "target": "mariadb", "display": "mariadb", "status": "ranked", "perSession": { "v7": { "min": 0.6232, "max": 0.6364, "runs": [ 0.6364, 0.6232, 0.6334 ] }, "v8": { "min": 0.6232, "max": 0.6348, "runs": [ 0.6348, 0.6275, 0.6232 ] } }, "median": 0.63045, "notSeparatedFrom": [ "pgvector", "qdrant", "qdrant-hnsw", "oracle" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "pgvector", "display": "pgvector", "status": "ranked", "perSession": { "v7": { "min": 0.8695, "max": 0.8846, "runs": [ 0.8846, 0.8695, 0.8768 ] }, "v8": { "min": 0.8446, "max": 0.8689, "runs": [ 0.8446, 0.8689, 0.8523 ] } }, "median": 0.8692, "notSeparatedFrom": [ "mariadb", "qdrant", "qdrant-hnsw", "oracle", "mongodb" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "qdrant", "display": "qdrant", "status": "ranked", "perSession": { "v7": { "min": 0.8812, "max": 0.8858, "runs": [ 0.8858, 0.8834, 0.8812 ] }, "v8": { "min": 0.887, "max": 0.89245, "runs": [ 0.89245, 0.887, 0.8899 ] } }, "median": 0.8864000000000001, "notSeparatedFrom": [ "mariadb", "pgvector", "qdrant-hnsw", "oracle", "mongodb" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "qdrant-hnsw", "display": "qdrant-hnsw", "status": "ranked", "perSession": { "v7": { "min": 0.916, "max": 0.92, "runs": [ 0.92, 0.916, 0.9163 ] }, "v8": { "min": 0.9196, "max": 0.9218, "runs": [ 0.92, 0.9196, 0.9218 ] } }, "median": 0.9198, "notSeparatedFrom": [ "mariadb", "pgvector", "qdrant", "oracle", "elasticsearch", "mongodb" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "oracle", "display": "oracle", "status": "ranked", "perSession": { "v7": { "min": 0.92345, "max": 0.9342, "runs": [ 0.9269, 0.9342, 0.92345 ] }, "v8": { "min": 0.9191, "max": 0.9443, "runs": [ 0.9191, 0.9443, 0.9279 ] } }, "median": 0.9274, "notSeparatedFrom": [ "mariadb", "pgvector", "qdrant", "qdrant-hnsw", "elasticsearch", "mongodb" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "elasticsearch", "display": "elasticsearch", "status": "ranked", "perSession": { "v7": { "min": 1.3102, "max": 1.3287, "runs": [ 1.3287, 1.3284500000000001, 1.3102 ] }, "v8": { "min": 1.3119, "max": 1.3248, "runs": [ 1.3127, 1.3119, 1.3248 ] } }, "median": 1.31875, "notSeparatedFrom": [ "qdrant-hnsw", "oracle", "mongodb", "sqlitevec", "vespa" ], "flags": [ { "code": "index-not-ready", "text": "Run 20261006-130619-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-130619-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-142724-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-142724-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-154837-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-154837-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-205106-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-205106-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-221429-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-221429-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-233551-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-233551-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "mongodb", "display": "mongodb", "status": "ranked", "perSession": { "v7": { "min": 1.262, "max": 1.4136, "runs": [ 1.4136, 1.40755, 1.262 ] }, "v8": { "min": 1.3677, "max": 1.3894, "runs": [ 1.382, 1.3677, 1.3894 ] } }, "median": 1.3857, "notSeparatedFrom": [ "pgvector", "qdrant", "qdrant-hnsw", "oracle", "elasticsearch", "sqlitevec", "vespa", "opensearch" ], "flags": [ { "code": "busy-box", "text": "Run 20261006-154837-eshoponweb: busy box during the one-searcher pass, with 0.350 CPUs of outside load.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[47].outsideLoadDuring", "value": "0.350" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[47].target#mongodb", "value": "mongodb" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[47].pass#default@1", "value": "default@1" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-130619-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-142724-eshoponweb: mongodb's index state after the searches reads 2 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[mongodb].indexState.afterSearch.detail#2 segment(s)", "value": "2 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-154837-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-205106-eshoponweb: mongodb's index state after the searches reads 3 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].indexState.afterSearch.detail#3 segment(s)", "value": "3 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-221429-eshoponweb: mongodb's index state after the searches reads 2 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[mongodb].indexState.afterSearch.detail#2 segment(s)", "value": "2 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-233551-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": "approximate", "searchConfidence": "recorded (engine's own report)", "note": null, "noteSources": [] }, { "target": "sqlitevec", "display": "sqlitevec", "status": "ranked", "perSession": { "v7": { "min": 1.91025, "max": 1.9333, "runs": [ 1.9247, 1.91025, 1.9333 ] }, "v8": { "min": 1.8633, "max": 1.9154, "runs": [ 1.9154, 1.8944, 1.8633 ] } }, "median": 1.912825, "notSeparatedFrom": [ "elasticsearch", "mongodb", "vespa", "opensearch", "chroma", "milvus" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "vespa", "display": "vespa", "status": "ranked", "perSession": { "v7": { "min": 1.9468, "max": 1.9701, "runs": [ 1.9701, 1.9468, 1.9484 ] }, "v8": { "min": 1.8711, "max": 1.92185, "runs": [ 1.9088, 1.92185, 1.8711 ] } }, "median": 1.934325, "notSeparatedFrom": [ "elasticsearch", "mongodb", "sqlitevec", "opensearch", "chroma", "milvus" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "opensearch", "display": "opensearch", "status": "ranked", "perSession": { "v7": { "min": 1.9948, "max": 2.1568, "runs": [ 2.1568, 1.9948, 2.027 ] }, "v8": { "min": 1.9826, "max": 2.0648, "runs": [ 1.9826, 2.0648, 1.9826 ] } }, "median": 2.0109, "notSeparatedFrom": [ "mongodb", "sqlitevec", "vespa", "chroma", "milvus" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "chroma", "display": "chroma", "status": "ranked", "perSession": { "v7": { "min": 2.105, "max": 2.106, "runs": [ 2.1054, 2.105, 2.106 ] }, "v8": { "min": 2.0665, "max": 2.1166, "runs": [ 2.0792, 2.1166, 2.0665 ] } }, "median": 2.1052, "notSeparatedFrom": [ "sqlitevec", "vespa", "opensearch", "milvus" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] }, { "target": "milvus", "display": "milvus", "status": "ranked", "perSession": { "v7": { "min": 2.3158, "max": 2.3394500000000003, "runs": [ 2.3394500000000003, 2.3158, 2.3231 ] }, "v8": { "min": 2.22985, "max": 2.2509, "runs": [ 2.2509, 2.22985, 2.2433 ] } }, "median": 2.28335, "notSeparatedFrom": [ "sqlitevec", "vespa", "opensearch", "chroma" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] }, { "target": "sql-diskann", "display": "sql-diskann", "status": "ranked", "perSession": { "v7": { "min": 3.5957, "max": 3.694, "runs": [ 3.5957, 3.694, 3.6845 ] }, "v8": { "min": 3.5638, "max": 3.5827, "runs": [ 3.5827, 3.5638, 3.5683499999999997 ] } }, "median": 3.5892, "notSeparatedFrom": [ "duckdb", "sql", "typesense", "clickhouse" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "duckdb", "display": "duckdb", "status": "ranked", "perSession": { "v7": { "min": 3.66635, "max": 3.6778, "runs": [ 3.6669, 3.6778, 3.66635 ] }, "v8": { "min": 3.6162, "max": 3.641, "runs": [ 3.641, 3.6162, 3.6359 ] } }, "median": 3.653675, "notSeparatedFrom": [ "sql-diskann", "sql", "typesense", "clickhouse" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "sql", "display": "sql", "status": "ranked", "perSession": { "v7": { "min": 4.01585, "max": 4.06025, "runs": [ 4.06025, 4.01585, 4.0501 ] }, "v8": { "min": 3.92575, "max": 3.9423, "runs": [ 3.9423, 3.92575, 3.933 ] } }, "median": 3.979075, "notSeparatedFrom": [ "sql-diskann", "duckdb", "typesense", "clickhouse", "weaviate" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "typesense", "display": "typesense", "status": "ranked", "perSession": { "v7": { "min": 4.1819, "max": 4.203, "runs": [ 4.1819, 4.203, 4.1859 ] }, "v8": { "min": 4.1599, "max": 4.1765, "runs": [ 4.1765, 4.1631, 4.1599 ] } }, "median": 4.1792, "notSeparatedFrom": [ "sql-diskann", "duckdb", "sql", "clickhouse", "weaviate" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "clickhouse", "display": "clickhouse", "status": "ranked", "perSession": { "v7": { "min": 4.9413, "max": 4.97305, "runs": [ 4.97305, 4.9413, 4.9567 ] }, "v8": { "min": 4.839700000000001, "max": 4.86925, "runs": [ 4.839700000000001, 4.86925, 4.8588 ] } }, "median": 4.905275, "notSeparatedFrom": [ "sql-diskann", "duckdb", "sql", "typesense", "weaviate" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "weaviate", "display": "weaviate", "status": "ranked", "perSession": { "v7": { "min": 5.3595, "max": 5.3722, "runs": [ 5.3612, 5.3595, 5.3722 ] }, "v8": { "min": 5.31125, "max": 5.3273, "runs": [ 5.3251, 5.31125, 5.3273 ] } }, "median": 5.3434, "notSeparatedFrom": [ "sql", "typesense", "clickhouse" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] } ], "noExactPass": [], "orderedPairs": 126, "comparedPairs": 171 }, { "metric": "qps1", "lowerIsBetter": false, "rows": [ { "target": "redis", "display": "redis", "status": "ranked", "perSession": { "v7": { "min": 2518.8969647291574, "max": 2536.5766254463965, "runs": [ 2526.9181229278793, 2536.5766254463965, 2518.8969647291574 ] }, "v8": { "min": 2559.3416309528666, "max": 2617.7536134059706, "runs": [ 2617.7536134059706, 2587.0392767221983, 2559.3416309528666 ] } }, "median": 2547.959128199632, "notSeparatedFrom": [], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, documented", "note": "redis holds its data in memory: its saved docs page says 'Redis is an in-memory but persistent on disk database', and its compose file sets save \"300 1\" and appendonly no.", "noteSources": [ { "kind": "doc", "ref": "doc:design/engine-docs/redis-faq-2026-10-07.html#Redis is an in-memory but persistent on disk database", "value": "Redis is an in-memory but persistent on disk database" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--save \"300 1\"", "value": "--save \"300 1\"" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--appendonly no", "value": "--appendonly no" } ] }, { "target": "mariadb", "display": "mariadb", "status": "ranked", "perSession": { "v7": { "min": 1546.3727529120936, "max": 1577.9512097485945, "runs": [ 1546.3727529120936, 1577.9512097485945, 1551.839804412485 ] }, "v8": { "min": 1549.1505743509254, "max": 1575.9984003616237, "runs": [ 1549.1505743509254, 1563.2255042563484, 1575.9984003616237 ] } }, "median": 1557.5326543344167, "notSeparatedFrom": [ "pgvector", "qdrant", "qdrant-hnsw" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "pgvector", "display": "pgvector", "status": "ranked", "perSession": { "v7": { "min": 1107.11890103007, "max": 1125.6652394574055, "runs": [ 1107.11890103007, 1125.6652394574055, 1117.3561605310415 ] }, "v8": { "min": 1126.741650844367, "max": 1158.3633204254586, "runs": [ 1158.3633204254586, 1126.741650844367, 1146.5767244924928 ] } }, "median": 1126.2034451508862, "notSeparatedFrom": [ "mariadb", "qdrant", "qdrant-hnsw", "oracle", "mongodb" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "qdrant", "display": "qdrant", "status": "ranked", "perSession": { "v7": { "min": 1111.1778042233607, "max": 1118.3726781554726, "runs": [ 1111.1778042233607, 1113.792673091116, 1118.3726781554726 ] }, "v8": { "min": 1102.981679474304, "max": 1113.4915207620695, "runs": [ 1102.981679474304, 1113.4915207620695, 1109.2483860435984 ] } }, "median": 1112.334662492715, "notSeparatedFrom": [ "mariadb", "pgvector", "qdrant-hnsw", "oracle", "mongodb" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "qdrant-hnsw", "display": "qdrant-hnsw", "status": "ranked", "perSession": { "v7": { "min": 1071.4659166691908, "max": 1077.2869648277256, "runs": [ 1071.4659166691908, 1076.6271324397069, 1077.2869648277256 ] }, "v8": { "min": 1072.1899374974364, "max": 1074.8833661799083, "runs": [ 1074.129532461759, 1074.8833661799083, 1072.1899374974364 ] } }, "median": 1074.5064493208338, "notSeparatedFrom": [ "mariadb", "pgvector", "qdrant", "oracle", "elasticsearch", "mongodb" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "oracle", "display": "oracle", "status": "ranked", "perSession": { "v7": { "min": 1042.0507943614903, "max": 1055.3737634082415, "runs": [ 1050.894051939666, 1042.0507943614903, 1055.3737634082415 ] }, "v8": { "min": 1028.675687250132, "max": 1059.8820297001864, "runs": [ 1059.8820297001864, 1028.675687250132, 1049.8651024839935 ] } }, "median": 1050.3795772118297, "notSeparatedFrom": [ "pgvector", "qdrant", "qdrant-hnsw", "elasticsearch", "mongodb" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "elasticsearch", "display": "elasticsearch", "status": "ranked", "perSession": { "v7": { "min": 739.3797594790843, "max": 748.0569381023581, "runs": [ 739.4510557346209, 739.3797594790843, 748.0569381023581 ] }, "v8": { "min": 742.1834493090804, "max": 749.4569212161684, "runs": [ 749.4569212161684, 747.4549621012586, 742.1834493090804 ] } }, "median": 744.8192057051695, "notSeparatedFrom": [ "qdrant-hnsw", "oracle", "mongodb", "sqlitevec", "vespa" ], "flags": [ { "code": "index-not-ready", "text": "Run 20261006-130619-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-130619-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-142724-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-142724-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-154837-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-154837-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-205106-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-205106-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-221429-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-221429-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-233551-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-233551-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "mongodb", "display": "mongodb", "status": "ranked", "perSession": { "v7": { "min": 691.0434489081044, "max": 770.4190522666705, "runs": [ 691.0434489081044, 693.3759745224828, 770.4190522666705 ] }, "v8": { "min": 705.2958422810098, "max": 717.458326433109, "runs": [ 708.5293676248148, 717.458326433109, 705.2958422810098 ] } }, "median": 706.9126049529123, "notSeparatedFrom": [ "pgvector", "qdrant", "qdrant-hnsw", "oracle", "elasticsearch", "sqlitevec", "vespa", "opensearch" ], "flags": [ { "code": "busy-box", "text": "Run 20261006-154837-eshoponweb: busy box during the one-searcher pass, with 0.350 CPUs of outside load.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[47].outsideLoadDuring", "value": "0.350" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[47].target#mongodb", "value": "mongodb" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[47].pass#default@1", "value": "default@1" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-130619-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-142724-eshoponweb: mongodb's index state after the searches reads 2 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[mongodb].indexState.afterSearch.detail#2 segment(s)", "value": "2 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-154837-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-205106-eshoponweb: mongodb's index state after the searches reads 3 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].indexState.afterSearch.detail#3 segment(s)", "value": "3 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-221429-eshoponweb: mongodb's index state after the searches reads 2 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[mongodb].indexState.afterSearch.detail#2 segment(s)", "value": "2 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-233551-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": "approximate", "searchConfidence": "recorded (engine's own report)", "note": null, "noteSources": [] }, { "target": "sqlitevec", "display": "sqlitevec", "status": "ranked", "perSession": { "v7": { "min": 506.4432643045848, "max": 514.2720673126639, "runs": [ 509.88742362669626, 514.2720673126639, 506.4432643045848 ] }, "v8": { "min": 514.1936779887292, "max": 528.5628737437482, "runs": [ 514.1936779887292, 520.5732919872546, 528.5628737437482 ] } }, "median": 514.2328726506965, "notSeparatedFrom": [ "elasticsearch", "mongodb", "vespa", "opensearch", "chroma", "milvus" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "vespa", "display": "vespa", "status": "ranked", "perSession": { "v7": { "min": 491.5462200095681, "max": 495.97366627818894, "runs": [ 491.5462200095681, 495.97366627818894, 495.944053630797 ] }, "v8": { "min": 503.0539705616936, "max": 517.1161728455533, "runs": [ 505.02499116243763, 503.0539705616936, 517.1161728455533 ] } }, "median": 499.51381841994123, "notSeparatedFrom": [ "elasticsearch", "mongodb", "sqlitevec", "opensearch", "chroma", "milvus" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "opensearch", "display": "opensearch", "status": "ranked", "perSession": { "v7": { "min": 450.8554892918197, "max": 490.84664260896454, "runs": [ 450.8554892918197, 490.84664260896454, 483.70290668500513 ] }, "v8": { "min": 479.9475378691307, "max": 494.99746066302674, "runs": [ 493.83209611735526, 479.9475378691307, 494.99746066302674 ] } }, "median": 487.27477464698484, "notSeparatedFrom": [ "mongodb", "sqlitevec", "vespa", "chroma", "milvus" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "chroma", "display": "chroma", "status": "ranked", "perSession": { "v7": { "min": 468.8430259599889, "max": 468.93233062978186, "runs": [ 468.9144211182977, 468.93233062978186, 468.8430259599889 ] }, "v8": { "min": 467.6002239561599, "max": 477.8814032451927, "runs": [ 475.3971405919376, 467.6002239561599, 477.8814032451927 ] } }, "median": 468.9233758740398, "notSeparatedFrom": [ "sqlitevec", "vespa", "opensearch", "milvus" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] }, { "target": "milvus", "display": "milvus", "status": "ranked", "perSession": { "v7": { "min": 405.3785108851379, "max": 408.5178047118107, "runs": [ 405.3785108851379, 408.5178047118107, 407.24678682285196 ] }, "v8": { "min": 423.0361730626834, "max": 427.0654717566085, "runs": [ 423.0361730626834, 427.0654717566085, 423.0822115084171 ] } }, "median": 415.77698888724706, "notSeparatedFrom": [ "sqlitevec", "vespa", "opensearch", "chroma" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] }, { "target": "sql-diskann", "display": "sql-diskann", "status": "ranked", "perSession": { "v7": { "min": 263.3209583315056, "max": 271.4620645337917, "runs": [ 271.4620645337917, 263.3209583315056, 267.9145736559255 ] }, "v8": { "min": 275.02814076337216, "max": 277.75152819205755, "runs": [ 275.02814076337216, 276.9902970298951, 277.75152819205755 ] } }, "median": 273.2451026485819, "notSeparatedFrom": [ "duckdb", "sql", "typesense", "clickhouse" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "duckdb", "display": "duckdb", "status": "ranked", "perSession": { "v7": { "min": 269.09912542784235, "max": 270.15952199881895, "runs": [ 269.63042483115726, 269.09912542784235, 270.15952199881895 ] }, "v8": { "min": 271.5965181326375, "max": 273.71913269340615, "runs": [ 271.6474967683173, 273.71913269340615, 271.5965181326375 ] } }, "median": 270.87802006572826, "notSeparatedFrom": [ "sql-diskann", "sql", "typesense", "clickhouse" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "sql", "display": "sql", "status": "ranked", "perSession": { "v7": { "min": 240.98674331925002, "max": 242.88348028008872, "runs": [ 240.98674331925002, 242.88348028008872, 241.0135057349616 ] }, "v8": { "min": 245.3611139438566, "max": 249.65110707893413, "runs": [ 248.42576606652023, 249.65110707893413, 245.3611139438566 ] } }, "median": 244.12229711197267, "notSeparatedFrom": [ "sql-diskann", "duckdb", "typesense", "clickhouse", "weaviate" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "typesense", "display": "typesense", "status": "ranked", "perSession": { "v7": { "min": 234.52840110690008, "max": 235.77537797727783, "runs": [ 235.24331908973784, 234.52840110690008, 235.77537797727783 ] }, "v8": { "min": 235.8028347169999, "max": 237.3686151217086, "runs": [ 235.8028347169999, 237.3686151217086, 237.0923478444733 ] } }, "median": 235.78910634713887, "notSeparatedFrom": [ "sql-diskann", "duckdb", "sql", "clickhouse", "weaviate" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "clickhouse", "display": "clickhouse", "status": "ranked", "perSession": { "v7": { "min": 191.56794302041496, "max": 195.7515348562426, "runs": [ 191.56794302041496, 195.7515348562426, 193.08674169888124 ] }, "v8": { "min": 193.89016782958936, "max": 200.08712439354525, "runs": [ 200.08712439354525, 193.89016782958936, 196.842509158314 ] } }, "median": 194.820851342916, "notSeparatedFrom": [ "sql-diskann", "duckdb", "sql", "typesense", "weaviate" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "weaviate", "display": "weaviate", "status": "ranked", "perSession": { "v7": { "min": 165.4404549129538, "max": 166.74322272171247, "runs": [ 165.64498758267575, 166.74322272171247, 165.4404549129538 ] }, "v8": { "min": 166.6373072363078, "max": 167.3760275684515, "runs": [ 166.6373072363078, 167.3760275684515, 166.87468844725635 ] } }, "median": 166.69026497901012, "notSeparatedFrom": [ "sql", "typesense", "clickhouse" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] } ], "noExactPass": [], "orderedPairs": 127, "comparedPairs": 171 }, { "metric": "qps8", "lowerIsBetter": false, "rows": [ { "target": "mariadb", "display": "mariadb", "status": "ranked", "perSession": { "v7": { "min": 5911.461450602725, "max": 5919.698213785062, "runs": [ 5911.461450602725, 5919.698213785062, 5917.8401180512155 ] }, "v8": { "min": 5926.162255191699, "max": 5955.8473238390025, "runs": [ 5927.248491846133, 5926.162255191699, 5955.8473238390025 ] } }, "median": 5922.93023448838, "notSeparatedFrom": [ "redis", "pgvector" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "redis", "display": "redis", "status": "ranked", "perSession": { "v7": { "min": 5037.759447698817, "max": 5120.058543295747, "runs": [ 5120.058543295747, 5067.34819935126, 5037.759447698817 ] }, "v8": { "min": 5236.9692461509585, "max": 5257.6409555336595, "runs": [ 5236.9692461509585, 5253.391207363199, 5257.6409555336595 ] } }, "median": 5178.513894723353, "notSeparatedFrom": [ "mariadb", "pgvector" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, documented", "note": "redis holds its data in memory: its saved docs page says 'Redis is an in-memory but persistent on disk database', and its compose file sets save \"300 1\" and appendonly no.", "noteSources": [ { "kind": "doc", "ref": "doc:design/engine-docs/redis-faq-2026-10-07.html#Redis is an in-memory but persistent on disk database", "value": "Redis is an in-memory but persistent on disk database" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--save \"300 1\"", "value": "--save \"300 1\"" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--appendonly no", "value": "--appendonly no" } ] }, { "target": "pgvector", "display": "pgvector", "status": "ranked", "perSession": { "v7": { "min": 3909.9337726094227, "max": 4001.304256195787, "runs": [ 3933.8796240634597, 4001.304256195787, 3909.9337726094227 ] }, "v8": { "min": 4006.2675563563803, "max": 4095.1304774444975, "runs": [ 4095.1304774444975, 4006.2675563563803, 4018.5738636969563 ] } }, "median": 4003.7859062760836, "notSeparatedFrom": [ "mariadb", "redis", "qdrant-hnsw", "qdrant", "oracle", "elasticsearch" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "qdrant-hnsw", "display": "qdrant-hnsw", "status": "ranked", "perSession": { "v7": { "min": 3440.089034845817, "max": 3451.9026013313824, "runs": [ 3451.9026013313824, 3440.089034845817, 3448.268759302981 ] }, "v8": { "min": 3467.845761223893, "max": 3477.195519374346, "runs": [ 3477.195519374346, 3472.325466533861, 3467.845761223893 ] } }, "median": 3459.874181277638, "notSeparatedFrom": [ "pgvector", "qdrant", "oracle", "elasticsearch" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "qdrant", "display": "qdrant", "status": "ranked", "perSession": { "v7": { "min": 3273.8173965216006, "max": 3287.037176836421, "runs": [ 3273.8173965216006, 3280.120003527045, 3287.037176836421 ] }, "v8": { "min": 3279.8236730317562, "max": 3295.884651186416, "runs": [ 3279.8236730317562, 3293.3188897188274, 3295.884651186416 ] } }, "median": 3283.578590181733, "notSeparatedFrom": [ "pgvector", "qdrant-hnsw", "oracle", "elasticsearch", "mongodb" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "oracle", "display": "oracle", "status": "ranked", "perSession": { "v7": { "min": 2655.3178479555313, "max": 2803.3946138388146, "runs": [ 2802.2354333862163, 2655.3178479555313, 2803.3946138388146 ] }, "v8": { "min": 2792.378411044751, "max": 2831.8183668715483, "runs": [ 2831.8183668715483, 2820.367533142876, 2792.378411044751 ] } }, "median": 2802.8150236125157, "notSeparatedFrom": [ "pgvector", "qdrant-hnsw", "qdrant", "elasticsearch", "mongodb" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "elasticsearch", "display": "elasticsearch", "status": "ranked", "perSession": { "v7": { "min": 2668.925327549915, "max": 2707.9679162373122, "runs": [ 2671.293756146442, 2668.925327549915, 2707.9679162373122 ] }, "v8": { "min": 2693.80928410206, "max": 2731.8749955193653, "runs": [ 2714.1035322582356, 2731.8749955193653, 2693.80928410206 ] } }, "median": 2700.888600169686, "notSeparatedFrom": [ "pgvector", "qdrant-hnsw", "qdrant", "oracle", "mongodb" ], "flags": [ { "code": "index-not-ready", "text": "Run 20261006-130619-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-130619-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-142724-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-142724-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-154837-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-154837-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-205106-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-205106-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-221429-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-221429-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-233551-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-233551-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "mongodb", "display": "mongodb", "status": "ranked", "perSession": { "v7": { "min": 2066.043956908663, "max": 2351.875618430976, "runs": [ 2066.043956908663, 2115.2776107902896, 2351.875618430976 ] }, "v8": { "min": 2098.320527667094, "max": 2150.0298487495047, "runs": [ 2130.3711985092223, 2150.0298487495047, 2098.320527667094 ] } }, "median": 2122.8244046497557, "notSeparatedFrom": [ "qdrant", "oracle", "elasticsearch", "vespa", "opensearch" ], "flags": [ { "code": "segment-layout-differs", "text": "Run 20261006-130619-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-142724-eshoponweb: mongodb's index state after the searches reads 2 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[mongodb].indexState.afterSearch.detail#2 segment(s)", "value": "2 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-154837-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-205106-eshoponweb: mongodb's index state after the searches reads 3 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].indexState.afterSearch.detail#3 segment(s)", "value": "3 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-221429-eshoponweb: mongodb's index state after the searches reads 2 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[mongodb].indexState.afterSearch.detail#2 segment(s)", "value": "2 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-233551-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": "approximate", "searchConfidence": "recorded (engine's own report)", "note": null, "noteSources": [] }, { "target": "vespa", "display": "vespa", "status": "ranked", "perSession": { "v7": { "min": 1505.711993967193, "max": 1564.7377511477457, "runs": [ 1526.1773279584363, 1505.711993967193, 1564.7377511477457 ] }, "v8": { "min": 1555.6609903569813, "max": 1581.7574849424086, "runs": [ 1570.6761108791757, 1555.6609903569813, 1581.7574849424086 ] } }, "median": 1560.1993707523634, "notSeparatedFrom": [ "mongodb", "opensearch", "milvus", "sqlitevec" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "opensearch", "display": "opensearch", "status": "ranked", "perSession": { "v7": { "min": 1346.3120709689424, "max": 1516.002138980831, "runs": [ 1346.3120709689424, 1516.002138980831, 1476.537174863042 ] }, "v8": { "min": 1462.2057237870592, "max": 1516.024009357268, "runs": [ 1516.024009357268, 1462.2057237870592, 1507.2976452911198 ] } }, "median": 1491.917410077081, "notSeparatedFrom": [ "mongodb", "vespa", "milvus", "sqlitevec" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "milvus", "display": "milvus", "status": "ranked", "perSession": { "v7": { "min": 1254.7267841051075, "max": 1262.7879149446694, "runs": [ 1254.7267841051075, 1258.0478442919007, 1262.7879149446694 ] }, "v8": { "min": 1266.1279678951012, "max": 1273.9740444822246, "runs": [ 1270.1146668039612, 1273.9740444822246, 1266.1279678951012 ] } }, "median": 1264.4579414198852, "notSeparatedFrom": [ "vespa", "opensearch", "sqlitevec" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] }, { "target": "sqlitevec", "display": "sqlitevec", "status": "ranked", "perSession": { "v7": { "min": 1136.4399892495137, "max": 1141.9354931272935, "runs": [ 1141.9354931272935, 1136.4399892495137, 1138.3659888297357 ] }, "v8": { "min": 1130.7841907371057, "max": 1140.0114031136663, "runs": [ 1130.7841907371057, 1136.429319582515, 1140.0114031136663 ] } }, "median": 1137.4029890396246, "notSeparatedFrom": [ "vespa", "opensearch", "milvus", "chroma" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "chroma", "display": "chroma", "status": "ranked", "perSession": { "v7": { "min": 777.0508315446011, "max": 777.2310166184697, "runs": [ 777.0508315446011, 777.2310166184697, 777.109955365809 ] }, "v8": { "min": 778.5321019867957, "max": 791.0136253821347, "runs": [ 791.0136253821347, 778.5321019867957, 790.5422577099188 ] } }, "median": 777.8815593026327, "notSeparatedFrom": [ "sqlitevec", "sql-diskann", "sql", "duckdb", "typesense" ], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] }, { "target": "sql-diskann", "display": "sql-diskann", "status": "ranked", "perSession": { "v7": { "min": 645.3751213907675, "max": 690.6856551446306, "runs": [ 690.6856551446306, 682.3327302695534, 645.3751213907675 ] }, "v8": { "min": 668.3601789755908, "max": 695.2673289383315, "runs": [ 695.2673289383315, 668.3601789755908, 693.8404030823888 ] } }, "median": 686.509192707092, "notSeparatedFrom": [ "chroma", "sql", "duckdb", "typesense" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "sql", "display": "sql", "status": "ranked", "perSession": { "v7": { "min": 602.2388945476348, "max": 613.6218902390702, "runs": [ 611.6123746788679, 602.2388945476348, 613.6218902390702 ] }, "v8": { "min": 580.3678815779276, "max": 606.5416893222192, "runs": [ 580.3678815779276, 606.5416893222192, 589.0811804289342 ] } }, "median": 604.3902919349271, "notSeparatedFrom": [ "chroma", "sql-diskann", "duckdb", "typesense" ], "flags": [], "searchMode": "exact", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "duckdb", "display": "duckdb", "status": "ranked", "perSession": { "v7": { "min": 575.9619635598901, "max": 576.0922659375863, "runs": [ 576.0922659375863, 575.9619635598901, 576.0088999547459 ] }, "v8": { "min": 577.946335436435, "max": 582.7765226868453, "runs": [ 580.9027810920543, 577.946335436435, 582.7765226868453 ] } }, "median": 577.0193006870106, "notSeparatedFrom": [ "chroma", "sql-diskann", "sql", "typesense" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "typesense", "display": "typesense", "status": "ranked", "perSession": { "v7": { "min": 574.0253432239009, "max": 576.1259043070017, "runs": [ 574.9689394061363, 574.0253432239009, 576.1259043070017 ] }, "v8": { "min": 572.0659749640902, "max": 576.3090106241174, "runs": [ 572.7028445385686, 576.3090106241174, 572.0659749640902 ] } }, "median": 574.4971413150186, "notSeparatedFrom": [ "chroma", "sql-diskann", "sql", "duckdb" ], "flags": [], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] }, { "target": "weaviate", "display": "weaviate", "status": "ranked", "perSession": { "v7": { "min": 304.9206911223285, "max": 305.47426215712244, "runs": [ 304.9206911223285, 305.47426215712244, 305.39734408358635 ] }, "v8": { "min": 306.10951285830015, "max": 306.2939485994506, "runs": [ 306.1970954327405, 306.2939485994506, 306.10951285830015 ] } }, "median": 305.7918875077113, "notSeparatedFrom": [], "flags": [], "searchMode": "approximate", "searchConfidence": "recorded, unverified", "note": null, "noteSources": [] }, { "target": "clickhouse", "display": "clickhouse", "status": "not-held", "perSession": { "v7": { "min": 374.7920878371932, "max": 428.4725710440155, "runs": [ 374.7920878371932, 428.4725710440155, 402.40999809529933 ] }, "v8": { "min": 336.45273919969844, "max": 380.2998767710477, "runs": [ 336.45273919969844, 380.2998767710477, 348.66492806816217 ] } }, "median": 377.54598230412046, "notSeparatedFrom": [], "flags": [ { "code": "unsettled", "text": "Run 20261007-205106-eshoponweb: clickhouse was recorded as not settled before its timed eight-searcher pass.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse]#latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up's settled figure, and the timed pass itself within 10% of it). default@8: warm-up 15.0 s and 6,523 searches (0 failed) at 8 searchers; QPS of the last windows 453, 468, 442 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,267 searches in 3.0 s at 8 searchers: 421 QPS against the settled 453 QPS, 8% apart (limit 10%), so the warm-up was EXTENDED once (50.1 s and 20,780 searches (0 failed) at 8 searchers; QPS of its older 10 windows 408 (windows 472, 462, 437, 371, 441, 464, 414, 410, 340, 266) and of its newer 10 418 (windows 295, 431, 418, 408, 444, 472, 407, 408, 432, 470), 2.6% apart (limit 5%) (windows of at least 2 s and 100 searches)); second trial of 1,222 searches in 3.0 s at 8 searchers: 404 QPS against the settled 418 QPS, 3% apart (limit 10%), inside its newer windows' range 295 to 472 QPS; settled after the extension; the timed pass 336 QPS, 24% from the settled 418 QPS (limit 10%), NOT HELD: the engine was still changing when it was timed; default@1: warm-up 15.0 s and 2,629 searches (0 failed) at 1 searcher; p50 of the last windows 5.090, 5.332, 4.942 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 519 searches in 3.0 s at 1 searcher: p50 5.081 ms against the settled p50 5.090 ms, 0% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 5,755 searches (0 failed) at 1 searcher; p50 of its older 6 windows 4.954 ms (windows 5.366, 5.484, 5.357, 4.829, 4.829, 4.833) and of its newer 6 4.847 ms (windows 4.865, 4.832, 4.843, 4.842, 4.856, 4.840), 2.2% apart (limit 5% or 0.1 ms; 0.106 ms) (windows of at least 2 s and 100 searches)); second trial of 559 searches in 3.0 s at 1 searcher: p50 5.137 ms against the settled p50 4.847 ms, 6% apart (limit 10%), outside its newer windows' range 4.832 to 4.865 ms; settled after the extension; the timed pass p50 4.840 ms, 0% from the settled p50 4.847 ms (limit 10% or 0.1 ms; 0.008 ms); exact: warm-up 15.0 s and 2,155 searches (0 failed) at 1 searcher; p50 of the last windows 6.947, 6.647, 6.588 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 423 searches in 3.0 s at 1 searcher: p50 6.698 ms against the settled p50 6.647 ms, 1% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 4,405 searches (0 failed) at 1 searcher; p50 of its older 6 windows 6.629 ms (windows 6.597, 6.601, 6.706, 6.648, 6.614, 6.629) and of its newer 6 6.631 ms (windows 6.650, 6.572, 6.606, 6.609, 6.724, 6.645), 0.0% apart (limit 5% or 0.1 ms; 0.002 ms) (windows of at least 2 s and 100 searches)); second trial of 442 searches in 3.0 s at 1 searcher: p50 6.621 ms against the settled p50 6.631 ms, 0% apart (limit 10%), inside its newer windows' range 6.572 to 6.724 ms; settled after the extension; the timed pass p50 6.630 ms, 0% from the settled p50 6.631 ms (limit 10% or 0.1 ms; 0.001 ms). Its numbers may still include warm-up or a change the engine was still going through; rerun before quoting them.", "value": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up's settled figure, and the timed pass itself within 10% of it). default@8: warm-up 15.0 s and 6,523 searches (0 failed) at 8 searchers; QPS of the last windows 453, 468, 442 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,267 searches in 3.0 s at 8 searchers: 421 QPS against the settled 453 QPS, 8% apart (limit 10%), so the warm-up was EXTENDED once (50.1 s and 20,780 searches (0 failed) at 8 searchers; QPS of its older 10 windows 408 (windows 472, 462, 437, 371, 441, 464, 414, 410, 340, 266) and of its newer 10 418 (windows 295, 431, 418, 408, 444, 472, 407, 408, 432, 470), 2.6% apart (limit 5%) (windows of at least 2 s and 100 searches)); second trial of 1,222 searches in 3.0 s at 8 searchers: 404 QPS against the settled 418 QPS, 3% apart (limit 10%), inside its newer windows' range 295 to 472 QPS; settled after the extension; the timed pass 336 QPS, 24% from the settled 418 QPS (limit 10%), NOT HELD: the engine was still changing when it was timed; default@1: warm-up 15.0 s and 2,629 searches (0 failed) at 1 searcher; p50 of the last windows 5.090, 5.332, 4.942 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 519 searches in 3.0 s at 1 searcher: p50 5.081 ms against the settled p50 5.090 ms, 0% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 5,755 searches (0 failed) at 1 searcher; p50 of its older 6 windows 4.954 ms (windows 5.366, 5.484, 5.357, 4.829, 4.829, 4.833) and of its newer 6 4.847 ms (windows 4.865, 4.832, 4.843, 4.842, 4.856, 4.840), 2.2% apart (limit 5% or 0.1 ms; 0.106 ms) (windows of at least 2 s and 100 searches)); second trial of 559 searches in 3.0 s at 1 searcher: p50 5.137 ms against the settled p50 4.847 ms, 6% apart (limit 10%), outside its newer windows' range 4.832 to 4.865 ms; settled after the extension; the timed pass p50 4.840 ms, 0% from the settled p50 4.847 ms (limit 10% or 0.1 ms; 0.008 ms); exact: warm-up 15.0 s and 2,155 searches (0 failed) at 1 searcher; p50 of the last windows 6.947, 6.647, 6.588 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 423 searches in 3.0 s at 1 searcher: p50 6.698 ms against the settled p50 6.647 ms, 1% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 4,405 searches (0 failed) at 1 searcher; p50 of its older 6 windows 6.629 ms (windows 6.597, 6.601, 6.706, 6.648, 6.614, 6.629) and of its newer 6 6.631 ms (windows 6.650, 6.572, 6.606, 6.609, 6.724, 6.645), 0.0% apart (limit 5% or 0.1 ms; 0.002 ms) (windows of at least 2 s and 100 searches)); second trial of 442 searches in 3.0 s at 1 searcher: p50 6.621 ms against the settled p50 6.631 ms, 0% apart (limit 10%), inside its newer windows' range 6.572 to 6.724 ms; settled after the extension; the timed pass p50 6.630 ms, 0% from the settled p50 6.631 ms (limit 10% or 0.1 ms; 0.001 ms). Its numbers may still include warm-up or a change the engine was still going through; rerun before quoting them." }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "unsettled", "text": "Run 20261007-221429-eshoponweb: clickhouse was recorded as not settled before its timed eight-searcher pass.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse]#latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up's settled figure, and the timed pass itself within 10% of it). default@1: warm-up 15.0 s and 2,959 searches (0 failed) at 1 searcher; p50 of the last windows 4.840, 4.828, 4.892 ms (windows of at least 2 s and 100 searches); trial of 611 searches in 3.0 s at 1 searcher: p50 4.814 ms against the settled p50 4.840 ms, 1% apart (limit 10%); the timed pass p50 4.869 ms, 1% from the settled p50 4.840 ms (limit 10% or 0.1 ms; 0.029 ms); default@8: warm-up 15.0 s and 6,659 searches (0 failed) at 8 searchers; QPS of the last windows 437, 469, 461 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,088 searches in 3.0 s at 8 searchers: 361 QPS against the settled 461 QPS, 28% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 12,488 searches (0 failed) at 8 searchers; QPS of its older 6 windows 434 (windows 459, 453, 386, 410, 446, 452) and of its newer 6 432 (windows 460, 362, 440, 432, 454, 442), 0.6% apart (limit 5%) (windows of at least 2 s and 100 searches)); second trial of 985 searches in 3.0 s at 8 searchers: 327 QPS against the settled 432 QPS, 32% apart (limit 10%), outside its newer windows' range 362 to 460 QPS; still disagreeing after the extension; the timed pass 380 QPS, 13% from the settled 432 QPS (limit 10%), NOT HELD: the engine was still changing when it was timed; exact: warm-up 15.0 s and 1,913 searches (0 failed) at 1 searcher; p50 of the last windows 6.837, 6.787, 6.839 ms (windows of at least 2 s and 100 searches); trial of 400 searches in 3.0 s at 1 searcher: p50 6.790 ms against the settled p50 6.837 ms, 1% apart (limit 10%); the timed pass p50 6.629 ms, 3% from the settled p50 6.837 ms (limit 10% or 0.1 ms; 0.208 ms). Its numbers may still include warm-up or a change the engine was still going through; rerun before quoting them.", "value": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up's settled figure, and the timed pass itself within 10% of it). default@1: warm-up 15.0 s and 2,959 searches (0 failed) at 1 searcher; p50 of the last windows 4.840, 4.828, 4.892 ms (windows of at least 2 s and 100 searches); trial of 611 searches in 3.0 s at 1 searcher: p50 4.814 ms against the settled p50 4.840 ms, 1% apart (limit 10%); the timed pass p50 4.869 ms, 1% from the settled p50 4.840 ms (limit 10% or 0.1 ms; 0.029 ms); default@8: warm-up 15.0 s and 6,659 searches (0 failed) at 8 searchers; QPS of the last windows 437, 469, 461 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,088 searches in 3.0 s at 8 searchers: 361 QPS against the settled 461 QPS, 28% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 12,488 searches (0 failed) at 8 searchers; QPS of its older 6 windows 434 (windows 459, 453, 386, 410, 446, 452) and of its newer 6 432 (windows 460, 362, 440, 432, 454, 442), 0.6% apart (limit 5%) (windows of at least 2 s and 100 searches)); second trial of 985 searches in 3.0 s at 8 searchers: 327 QPS against the settled 432 QPS, 32% apart (limit 10%), outside its newer windows' range 362 to 460 QPS; still disagreeing after the extension; the timed pass 380 QPS, 13% from the settled 432 QPS (limit 10%), NOT HELD: the engine was still changing when it was timed; exact: warm-up 15.0 s and 1,913 searches (0 failed) at 1 searcher; p50 of the last windows 6.837, 6.787, 6.839 ms (windows of at least 2 s and 100 searches); trial of 400 searches in 3.0 s at 1 searcher: p50 6.790 ms against the settled p50 6.837 ms, 1% apart (limit 10%); the timed pass p50 6.629 ms, 3% from the settled p50 6.837 ms (limit 10% or 0.1 ms; 0.208 ms). Its numbers may still include warm-up or a change the engine was still going through; rerun before quoting them." }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "unsettled", "text": "Run 20261007-233551-eshoponweb: clickhouse was recorded as not settled before its timed eight-searcher pass.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse]#latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up's settled figure, and the timed pass itself within 10% of it). exact: warm-up 15.0 s and 2,204 searches (0 failed) at 1 searcher; p50 of the last windows 6.584, 6.565, 6.581 ms (windows of at least 2 s and 100 searches); trial of 443 searches in 3.0 s at 1 searcher: p50 6.622 ms against the settled p50 6.581 ms, 1% apart (limit 10%); the timed pass p50 6.603 ms, 0% from the settled p50 6.581 ms (limit 10% or 0.1 ms; 0.022 ms); default@8: warm-up 15.0 s and 6,591 searches (0 failed) at 8 searchers; QPS of the last windows 472, 465, 405 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,295 searches in 3.0 s at 8 searchers: 431 QPS against the settled 465 QPS, 8% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 12,492 searches (0 failed) at 8 searchers; QPS of its older 6 windows 427 (windows 436, 399, 412, 462, 462, 391) and of its newer 6 416 (windows 400, 458, 446, 411, 409, 370), 2.7% apart (limit 5%) (windows of at least 2 s and 100 searches)); second trial of 1,078 searches in 3.0 s at 8 searchers: 358 QPS against the settled 416 QPS, 16% apart (limit 10%), outside its newer windows' range 370 to 458 QPS; still disagreeing after the extension; the timed pass 349 QPS, 19% from the settled 416 QPS (limit 10%), NOT HELD: the engine was still changing when it was timed; default@1: warm-up 15.0 s and 2,479 searches (0 failed) at 1 searcher; p50 of the last windows 5.497, 5.449, 5.221 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 466 searches in 3.0 s at 1 searcher: p50 6.861 ms against the settled p50 5.449 ms, 21% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 5,806 searches (0 failed) at 1 searcher; p50 of its older 6 windows 4.910 ms (windows 4.850, 4.932, 5.369, 4.842, 4.893, 4.847) and of its newer 6 4.855 ms (windows 4.875, 4.849, 4.863, 4.838, 4.868, 4.838), 1.1% apart (limit 5% or 0.1 ms; 0.056 ms) (windows of at least 2 s and 100 searches)); second trial of 595 searches in 3.0 s at 1 searcher: p50 4.851 ms against the settled p50 4.855 ms, 0% apart (limit 10%), inside its newer windows' range 4.838 to 4.875 ms; settled after the extension; the timed pass p50 4.859 ms, 0% from the settled p50 4.855 ms (limit 10% or 0.1 ms; 0.004 ms). Its numbers may still include warm-up or a change the engine was still going through; rerun before quoting them.", "value": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up's settled figure, and the timed pass itself within 10% of it). exact: warm-up 15.0 s and 2,204 searches (0 failed) at 1 searcher; p50 of the last windows 6.584, 6.565, 6.581 ms (windows of at least 2 s and 100 searches); trial of 443 searches in 3.0 s at 1 searcher: p50 6.622 ms against the settled p50 6.581 ms, 1% apart (limit 10%); the timed pass p50 6.603 ms, 0% from the settled p50 6.581 ms (limit 10% or 0.1 ms; 0.022 ms); default@8: warm-up 15.0 s and 6,591 searches (0 failed) at 8 searchers; QPS of the last windows 472, 465, 405 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,295 searches in 3.0 s at 8 searchers: 431 QPS against the settled 465 QPS, 8% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 12,492 searches (0 failed) at 8 searchers; QPS of its older 6 windows 427 (windows 436, 399, 412, 462, 462, 391) and of its newer 6 416 (windows 400, 458, 446, 411, 409, 370), 2.7% apart (limit 5%) (windows of at least 2 s and 100 searches)); second trial of 1,078 searches in 3.0 s at 8 searchers: 358 QPS against the settled 416 QPS, 16% apart (limit 10%), outside its newer windows' range 370 to 458 QPS; still disagreeing after the extension; the timed pass 349 QPS, 19% from the settled 416 QPS (limit 10%), NOT HELD: the engine was still changing when it was timed; default@1: warm-up 15.0 s and 2,479 searches (0 failed) at 1 searcher; p50 of the last windows 5.497, 5.449, 5.221 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 466 searches in 3.0 s at 1 searcher: p50 6.861 ms against the settled p50 5.449 ms, 21% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 5,806 searches (0 failed) at 1 searcher; p50 of its older 6 windows 4.910 ms (windows 4.850, 4.932, 5.369, 4.842, 4.893, 4.847) and of its newer 6 4.855 ms (windows 4.875, 4.849, 4.863, 4.838, 4.868, 4.838), 1.1% apart (limit 5% or 0.1 ms; 0.056 ms) (windows of at least 2 s and 100 searches)); second trial of 595 searches in 3.0 s at 1 searcher: p50 4.851 ms against the settled p50 4.855 ms, 0% apart (limit 10%), inside its newer windows' range 4.838 to 4.875 ms; settled after the extension; the timed pass p50 4.859 ms, 0% from the settled p50 4.855 ms (limit 10% or 0.1 ms; 0.004 ms). Its numbers may still include warm-up or a change the engine was still going through; rerun before quoting them." }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] }, { "code": "not-held", "text": "Run 20261007-205106-eshoponweb: the timed eight-searcher pass of clickhouse read 336 QPS, 24% from the settled 418 QPS (limit 10%), and the run recorded it as NOT HELD.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].notes#the timed pass 336 QPS, 24% from the settled 418 QPS (limit 10%), NOT HELD", "value": "the timed pass 336 QPS, 24% from the settled 418 QPS (limit 10%), NOT HELD" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:conditions.passes[12].target#clickhouse", "value": "clickhouse" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:conditions.passes[12].pass#default@8", "value": "default@8" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "not-held", "text": "Run 20261007-221429-eshoponweb: the timed eight-searcher pass of clickhouse read 380 QPS, 13% from the settled 432 QPS (limit 10%), and the run recorded it as NOT HELD.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].notes#the timed pass 380 QPS, 13% from the settled 432 QPS (limit 10%), NOT HELD", "value": "the timed pass 380 QPS, 13% from the settled 432 QPS (limit 10%), NOT HELD" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:conditions.passes[22].target#clickhouse", "value": "clickhouse" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:conditions.passes[22].pass#default@8", "value": "default@8" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "not-held", "text": "Run 20261007-233551-eshoponweb: the timed eight-searcher pass of clickhouse read 349 QPS, 19% from the settled 416 QPS (limit 10%), and the run recorded it as NOT HELD.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].notes#the timed pass 349 QPS, 19% from the settled 416 QPS (limit 10%), NOT HELD", "value": "the timed pass 349 QPS, 19% from the settled 416 QPS (limit 10%), NOT HELD" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:conditions.passes[39].target#clickhouse", "value": "clickhouse" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:conditions.passes[39].pass#default@8", "value": "default@8" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": "approximate", "searchConfidence": "measured", "note": null, "noteSources": [] } ], "noExactPass": [], "orderedPairs": 118, "comparedPairs": 153 }, { "metric": "exactP50Ms", "lowerIsBetter": true, "rows": [ { "target": "redis", "display": "redis", "status": "ranked", "perSession": { "v7": { "min": 0.3767, "max": 0.3817, "runs": [ 0.3781, 0.3817, 0.3767 ] }, "v8": { "min": 0.3645, "max": 0.3676, "runs": [ 0.3676, 0.3652, 0.3645 ] } }, "median": 0.37215, "notSeparatedFrom": [], "flags": [], "searchMode": null, "searchConfidence": null, "note": "redis holds its data in memory: its saved docs page says 'Redis is an in-memory but persistent on disk database', and its compose file sets save \"300 1\" and appendonly no.", "noteSources": [ { "kind": "doc", "ref": "doc:design/engine-docs/redis-faq-2026-10-07.html#Redis is an in-memory but persistent on disk database", "value": "Redis is an in-memory but persistent on disk database" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--save \"300 1\"", "value": "--save \"300 1\"" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--appendonly no", "value": "--appendonly no" } ] }, { "target": "qdrant-hnsw", "display": "qdrant-hnsw", "status": "ranked", "perSession": { "v7": { "min": 0.8912, "max": 0.8946, "runs": [ 0.8946, 0.8925, 0.8912 ] }, "v8": { "min": 0.8965, "max": 0.8991, "runs": [ 0.8965, 0.8966, 0.8991 ] } }, "median": 0.89555, "notSeparatedFrom": [ "mongodb", "elasticsearch" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "mongodb", "display": "mongodb", "status": "ranked", "perSession": { "v7": { "min": 1.2423, "max": 1.263, "runs": [ 1.2423, 1.263, 1.2424 ] }, "v8": { "min": 1.2113, "max": 1.2353, "runs": [ 1.2312, 1.2353, 1.2113 ] } }, "median": 1.2388, "notSeparatedFrom": [ "qdrant-hnsw", "elasticsearch", "mariadb" ], "flags": [ { "code": "busy-box", "text": "Run 20261006-130619-eshoponweb: busy box during the exact pass, with 0.307 CPUs of outside load.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:conditions.passes[39].outsideLoadDuring", "value": "0.307" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:conditions.passes[39].target#mongodb", "value": "mongodb" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:conditions.passes[39].pass#exact", "value": "exact" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "busy-box", "text": "Run 20261006-142724-eshoponweb: busy box during the exact pass, with 0.317 CPUs of outside load.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:conditions.passes[47].outsideLoadDuring", "value": "0.317" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:conditions.passes[47].target#mongodb", "value": "mongodb" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:conditions.passes[47].pass#exact", "value": "exact" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "busy-box", "text": "Run 20261006-154837-eshoponweb: busy box during the exact pass, with 0.301 CPUs of outside load.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[49].outsideLoadDuring", "value": "0.301" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[49].target#mongodb", "value": "mongodb" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:conditions.passes[49].pass#exact", "value": "exact" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-130619-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-142724-eshoponweb: mongodb's index state after the searches reads 2 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[mongodb].indexState.afterSearch.detail#2 segment(s)", "value": "2 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261006-154837-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-205106-eshoponweb: mongodb's index state after the searches reads 3 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].indexState.afterSearch.detail#3 segment(s)", "value": "3 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-221429-eshoponweb: mongodb's index state after the searches reads 2 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[mongodb].indexState.afterSearch.detail#2 segment(s)", "value": "2 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "segment-layout-differs", "text": "Run 20261007-233551-eshoponweb: mongodb's index state after the searches reads 4 segment(s).", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[mongodb].indexState.afterSearch.detail#4 segment(s)", "value": "4 segment(s)" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "elasticsearch", "display": "elasticsearch", "status": "ranked", "perSession": { "v7": { "min": 1.2607, "max": 1.2833, "runs": [ 1.2776, 1.2833, 1.2607 ] }, "v8": { "min": 1.2468, "max": 1.2577, "runs": [ 1.2468, 1.2537, 1.2577 ] } }, "median": 1.2591999999999999, "notSeparatedFrom": [ "qdrant-hnsw", "mongodb", "mariadb" ], "flags": [ { "code": "index-not-ready", "text": "Run 20261006-130619-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-130619-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-142724-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-142724-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-154837-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261006-154837-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-205106-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-205106-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-221429-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-221429-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-233551-eshoponweb: elasticsearch's index state after the load read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterLoad.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterLoad.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] }, { "code": "index-not-ready", "text": "Run 20261007-233551-eshoponweb: elasticsearch's index state after the searches read not ready, 0 of 524 vectors indexed.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterSearch.indexedVectors", "value": "0" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[elasticsearch].indexState.afterSearch.totalVectors", "value": "524" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] } ], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "mariadb", "display": "mariadb", "status": "ranked", "perSession": { "v7": { "min": 1.6347, "max": 1.6445, "runs": [ 1.6391, 1.6347, 1.6445 ] }, "v8": { "min": 1.6149, "max": 1.6308, "runs": [ 1.6308, 1.6249, 1.6149 ] } }, "median": 1.6327500000000001, "notSeparatedFrom": [ "mongodb", "elasticsearch", "sqlitevec", "vespa", "pgvector" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "sqlitevec", "display": "sqlitevec", "status": "ranked", "perSession": { "v7": { "min": 1.9056, "max": 1.9363, "runs": [ 1.9056, 1.914, 1.9363 ] }, "v8": { "min": 1.8738, "max": 1.9174, "runs": [ 1.9174, 1.8948, 1.8738 ] } }, "median": 1.9098, "notSeparatedFrom": [ "mariadb", "vespa", "pgvector", "oracle", "opensearch" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "vespa", "display": "vespa", "status": "ranked", "perSession": { "v7": { "min": 1.9348, "max": 1.9631, "runs": [ 1.9631, 1.9413, 1.9348 ] }, "v8": { "min": 1.868, "max": 1.9157, "runs": [ 1.89, 1.9157, 1.868 ] } }, "median": 1.9252500000000001, "notSeparatedFrom": [ "mariadb", "sqlitevec", "pgvector", "oracle", "opensearch" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "pgvector", "display": "pgvector", "status": "ranked", "perSession": { "v7": { "min": 2.2097, "max": 2.2571, "runs": [ 2.2571, 2.2097, 2.2516 ] }, "v8": { "min": 2.1827, "max": 2.1933, "runs": [ 2.1827, 2.1933, 2.1927 ] } }, "median": 2.2015000000000002, "notSeparatedFrom": [ "mariadb", "sqlitevec", "vespa", "oracle", "opensearch" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "oracle", "display": "oracle", "status": "ranked", "perSession": { "v7": { "min": 2.4839, "max": 2.5026, "runs": [ 2.4839, 2.4895, 2.5026 ] }, "v8": { "min": 2.4222, "max": 2.4858, "runs": [ 2.4529, 2.4858, 2.4222 ] } }, "median": 2.48485, "notSeparatedFrom": [ "sqlitevec", "vespa", "pgvector", "opensearch", "sql-diskann" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "opensearch", "display": "opensearch", "status": "ranked", "perSession": { "v7": { "min": 2.6149, "max": 2.7499, "runs": [ 2.7499, 2.6877, 2.6149 ] }, "v8": { "min": 2.5525, "max": 2.6580500000000002, "runs": [ 2.5525, 2.6580500000000002, 2.5624 ] } }, "median": 2.636475, "notSeparatedFrom": [ "sqlitevec", "vespa", "pgvector", "oracle", "sql-diskann" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "sql-diskann", "display": "sql-diskann", "status": "ranked", "perSession": { "v7": { "min": 3.7399, "max": 3.78955, "runs": [ 3.7399, 3.7492, 3.78955 ] }, "v8": { "min": 3.5725, "max": 3.6307, "runs": [ 3.6157, 3.5725, 3.6307 ] } }, "median": 3.6853, "notSeparatedFrom": [ "oracle", "opensearch", "duckdb" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "duckdb", "display": "duckdb", "status": "ranked", "perSession": { "v7": { "min": 4.1946, "max": 4.213850000000001, "runs": [ 4.213850000000001, 4.19955, 4.1946 ] }, "v8": { "min": 4.10105, "max": 4.1627, "runs": [ 4.1627, 4.1409, 4.10105 ] } }, "median": 4.17865, "notSeparatedFrom": [ "sql-diskann", "typesense" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "typesense", "display": "typesense", "status": "ranked", "perSession": { "v7": { "min": 5.6595, "max": 5.6824, "runs": [ 5.6595, 5.6824, 5.66805 ] }, "v8": { "min": 5.6296, "max": 5.6401, "runs": [ 5.6401, 5.62975, 5.6296 ] } }, "median": 5.649800000000001, "notSeparatedFrom": [ "duckdb", "clickhouse" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] }, { "target": "clickhouse", "display": "clickhouse", "status": "ranked", "perSession": { "v7": { "min": 6.6568, "max": 6.6774, "runs": [ 6.67405, 6.6568, 6.6774 ] }, "v8": { "min": 6.6027, "max": 6.63, "runs": [ 6.63, 6.6293, 6.6027 ] } }, "median": 6.6434, "notSeparatedFrom": [ "typesense" ], "flags": [], "searchMode": null, "searchConfidence": null, "note": null, "noteSources": [] } ], "noExactPass": [ { "target": "sql", "mode": "exact", "factText": "no index used, exact scan by design", "factSource": "results:targets[sql].indexState.afterLoad.detail#no index used, exact scan by design" }, { "target": "qdrant", "mode": "exact", "factText": "no index used, exact scan by design", "factSource": "results:targets[qdrant].indexState.afterLoad.detail#no index used, exact scan by design" }, { "target": "milvus", "mode": "approximate", "factText": "approximate, no exact mode", "factSource": "results:targets[milvus].index#approximate only (no exact mode)" }, { "target": "weaviate", "mode": "approximate", "factText": "approximate, no exact mode", "factSource": "results:targets[weaviate].index#approximate only (no exact mode)" }, { "target": "chroma", "mode": "approximate", "factText": "approximate, no exact mode", "factSource": "results:targets[chroma].index#approximate only (no exact mode)" } ], "orderedPairs": 68, "comparedPairs": 91 } ], "guards": { "g2": { "stopped": false, "pairs": [] }, "g3": { "stopped": false, "targets": [], "changedCount": 0, "limit": 3 } }, "drift": { "from": "v7", "to": "v8", "perTarget": [ { "target": "redis", "metric": "p50Ms", "moveBp": -274 }, { "target": "mariadb", "metric": "p50Ms", "moveBp": -93 }, { "target": "pgvector", "metric": "p50Ms", "moveBp": -279 }, { "target": "qdrant", "metric": "p50Ms", "moveBp": 74 }, { "target": "qdrant-hnsw", "metric": "p50Ms", "moveBp": 40 }, { "target": "oracle", "metric": "p50Ms", "moveBp": 11 }, { "target": "elasticsearch", "metric": "p50Ms", "moveBp": -119 }, { "target": "mongodb", "metric": "p50Ms", "moveBp": -182 }, { "target": "sqlitevec", "metric": "p50Ms", "moveBp": -157 }, { "target": "vespa", "metric": "p50Ms", "moveBp": -203 }, { "target": "opensearch", "metric": "p50Ms", "moveBp": -219 }, { "target": "chroma", "metric": "p50Ms", "moveBp": -124 }, { "target": "milvus", "metric": "p50Ms", "moveBp": -344 }, { "target": "sql-diskann", "metric": "p50Ms", "moveBp": -315 }, { "target": "duckdb", "metric": "p50Ms", "moveBp": -85 }, { "target": "sql", "metric": "p50Ms", "moveBp": -289 }, { "target": "typesense", "metric": "p50Ms", "moveBp": -54 }, { "target": "clickhouse", "metric": "p50Ms", "moveBp": -198 }, { "target": "weaviate", "metric": "p50Ms", "moveBp": -67 }, { "target": "redis", "metric": "qps1", "moveBp": 238 }, { "target": "mariadb", "metric": "qps1", "moveBp": 73 }, { "target": "pgvector", "metric": "qps1", "moveBp": 262 }, { "target": "qdrant", "metric": "qps1", "moveBp": -41 }, { "target": "qdrant-hnsw", "metric": "qps1", "moveBp": -23 }, { "target": "oracle", "metric": "qps1", "moveBp": -10 }, { "target": "elasticsearch", "metric": "qps1", "moveBp": 108 }, { "target": "mongodb", "metric": "qps1", "moveBp": 219 }, { "target": "sqlitevec", "metric": "qps1", "moveBp": 210 }, { "target": "vespa", "metric": "qps1", "moveBp": 183 }, { "target": "opensearch", "metric": "qps1", "moveBp": 209 }, { "target": "chroma", "metric": "qps1", "moveBp": 138 }, { "target": "milvus", "metric": "qps1", "moveBp": 389 }, { "target": "sql-diskann", "metric": "qps1", "moveBp": 339 }, { "target": "duckdb", "metric": "qps1", "moveBp": 75 }, { "target": "sql", "metric": "qps1", "moveBp": 308 }, { "target": "typesense", "metric": "qps1", "moveBp": 79 }, { "target": "clickhouse", "metric": "qps1", "moveBp": 195 }, { "target": "weaviate", "metric": "qps1", "moveBp": 74 }, { "target": "mariadb", "metric": "qps8", "moveBp": 16 }, { "target": "redis", "metric": "qps8", "moveBp": 367 }, { "target": "pgvector", "metric": "qps8", "moveBp": 215 }, { "target": "qdrant-hnsw", "metric": "qps8", "moveBp": 70 }, { "target": "qdrant", "metric": "qps8", "moveBp": 40 }, { "target": "oracle", "metric": "qps8", "moveBp": 65 }, { "target": "elasticsearch", "metric": "qps8", "moveBp": 160 }, { "target": "mongodb", "metric": "qps8", "moveBp": 71 }, { "target": "vespa", "metric": "qps8", "moveBp": 292 }, { "target": "opensearch", "metric": "qps8", "moveBp": 208 }, { "target": "milvus", "metric": "qps8", "moveBp": 96 }, { "target": "sqlitevec", "metric": "qps8", "moveBp": -17 }, { "target": "chroma", "metric": "qps8", "moveBp": 173 }, { "target": "sql-diskann", "metric": "qps8", "moveBp": 169 }, { "target": "sql", "metric": "qps8", "moveBp": -368 }, { "target": "duckdb", "metric": "qps8", "moveBp": 85 }, { "target": "typesense", "metric": "qps8", "moveBp": -39 }, { "target": "weaviate", "metric": "qps8", "moveBp": 26 }, { "target": "redis", "metric": "exactP50Ms", "moveBp": -341 }, { "target": "qdrant-hnsw", "metric": "exactP50Ms", "moveBp": 46 }, { "target": "mongodb", "metric": "exactP50Ms", "moveBp": -90 }, { "target": "elasticsearch", "metric": "exactP50Ms", "moveBp": -187 }, { "target": "mariadb", "metric": "exactP50Ms", "moveBp": -87 }, { "target": "sqlitevec", "metric": "exactP50Ms", "moveBp": -100 }, { "target": "vespa", "metric": "exactP50Ms", "moveBp": -264 }, { "target": "pgvector", "metric": "exactP50Ms", "moveBp": -262 }, { "target": "oracle", "metric": "exactP50Ms", "moveBp": -147 }, { "target": "opensearch", "metric": "exactP50Ms", "moveBp": -466 }, { "target": "sql-diskann", "metric": "exactP50Ms", "moveBp": -356 }, { "target": "duckdb", "metric": "exactP50Ms", "moveBp": -140 }, { "target": "typesense", "metric": "exactP50Ms", "moveBp": -68 }, { "target": "clickhouse", "metric": "exactP50Ms", "moveBp": -67 } ], "medianAbsMoveBp": 143, "largest": { "target": "opensearch", "metric": "exactP50Ms", "moveBp": -466 }, "largestAbsMoveBp": 466, "closeFromBp": 3500, "closeCount": 20, "unconfirmedOrders": [ { "metric": "p50Ms", "a": "elasticsearch", "b": "vespa", "session": "v7" }, { "metric": "p50Ms", "a": "mariadb", "b": "oracle", "session": "v7" }, { "metric": "p50Ms", "a": "pgvector", "b": "mongodb", "session": "v8" }, { "metric": "p50Ms", "a": "qdrant", "b": "mongodb", "session": "v8" }, { "metric": "p50Ms", "a": "qdrant-hnsw", "b": "mongodb", "session": "v8" }, { "metric": "qps1", "a": "elasticsearch", "b": "vespa", "session": "v7" }, { "metric": "qps1", "a": "sql", "b": "weaviate", "session": "v8" }, { "metric": "qps1", "a": "pgvector", "b": "mongodb", "session": "v8" }, { "metric": "qps1", "a": "qdrant", "b": "mongodb", "session": "v8" }, { "metric": "qps1", "a": "qdrant-hnsw", "b": "mongodb", "session": "v8" }, { "metric": "qps8", "a": "pgvector", "b": "elasticsearch", "session": "v8" }, { "metric": "qps8", "a": "mariadb", "b": "pgvector", "session": "v7" }, { "metric": "qps8", "a": "sqlitevec", "b": "chroma", "session": "v7" }, { "metric": "qps8", "a": "qdrant", "b": "mongodb", "session": "v8" }, { "metric": "exactP50Ms", "a": "oracle", "b": "sql-diskann", "session": "v7" } ], "closeToLine": [ { "metric": "p50Ms", "a": "oracle", "b": "elasticsearch", "minRatioBp": 13892 }, { "metric": "p50Ms", "a": "oracle", "b": "mongodb", "minRatioBp": 13508 }, { "metric": "p50Ms", "a": "elasticsearch", "b": "sqlitevec", "minRatioBp": 14064 }, { "metric": "p50Ms", "a": "qdrant-hnsw", "b": "elasticsearch", "minRatioBp": 14231 }, { "metric": "p50Ms", "a": "mariadb", "b": "qdrant", "minRatioBp": 13846 }, { "metric": "p50Ms", "a": "mongodb", "b": "opensearch", "minRatioBp": 14111 }, { "metric": "p50Ms", "a": "mariadb", "b": "qdrant-hnsw", "minRatioBp": 14393 }, { "metric": "qps1", "a": "duckdb", "b": "clickhouse", "minRatioBp": 13573 }, { "metric": "qps1", "a": "mongodb", "b": "vespa", "minRatioBp": 13639 }, { "metric": "qps1", "a": "oracle", "b": "elasticsearch", "minRatioBp": 13725 }, { "metric": "qps1", "a": "oracle", "b": "mongodb", "minRatioBp": 13525 }, { "metric": "qps1", "a": "elasticsearch", "b": "sqlitevec", "minRatioBp": 14041 }, { "metric": "qps1", "a": "qdrant-hnsw", "b": "elasticsearch", "minRatioBp": 14306 }, { "metric": "qps1", "a": "mariadb", "b": "qdrant", "minRatioBp": 13826 }, { "metric": "qps1", "a": "mongodb", "b": "opensearch", "minRatioBp": 14078 }, { "metric": "qps1", "a": "mariadb", "b": "qdrant-hnsw", "minRatioBp": 14354 }, { "metric": "qps1", "a": "typesense", "b": "weaviate", "minRatioBp": 14065 }, { "metric": "qps8", "a": "pgvector", "b": "oracle", "minRatioBp": 13947 }, { "metric": "qps8", "a": "mongodb", "b": "opensearch", "minRatioBp": 13628 }, { "metric": "exactP50Ms", "a": "qdrant-hnsw", "b": "elasticsearch", "minRatioBp": 13867 } ], "onLineBp": 25, "onLineCount": 1, "onLine": [ { "metric": "p50Ms", "a": "sql-diskann", "b": "weaviate", "minRatioBp": 14508 } ], "unconfirmedFromFirst": 6, "unconfirmedFromSecond": 9, "sessionDifferences": [ "build", "day", "outside load" ], "outsideLoad": [ { "session": "v7", "runs": [ { "run": "20261006-130619-eshoponweb", "medianCpus": 0.2105, "passes": 52 }, { "run": "20261006-142724-eshoponweb", "medianCpus": 0.2135, "passes": 52 }, { "run": "20261006-154837-eshoponweb", "medianCpus": 0.2095, "passes": 52 } ], "minCpus": 0.2095, "maxCpus": 0.2135 }, { "session": "v8", "runs": [ { "run": "20261007-205106-eshoponweb", "medianCpus": 0.1435, "passes": 52 }, { "run": "20261007-221429-eshoponweb", "medianCpus": 0.14, "passes": 52 }, { "run": "20261007-233551-eshoponweb", "medianCpus": 0.14, "passes": 52 } ], "minCpus": 0.14, "maxCpus": 0.1435 } ], "excluded": [ { "target": "clickhouse", "metric": "qps8", "moveBp": -1336, "absMoveBp": 1336, "fromMin": 374.7920878371932, "fromMax": 428.4725710440155, "toMin": 336.45273919969844, "toMax": 380.2998767710477 } ] }, "recall": [ { "target": "clickhouse", "hits": { "v7": [ 199, 193, 198 ], "v8": [ 199, 199, 199 ] }, "of": 200, "differs": true }, { "target": "vespa", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "oracle", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "elasticsearch", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "sql", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "pgvector", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "sqlitevec", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "qdrant", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "opensearch", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "qdrant-hnsw", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "redis", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "milvus", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "mariadb", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "weaviate", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "mongodb", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "chroma", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "typesense", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false }, { "target": "sql-diskann", "hits": { "v7": [ 193, 193, 193 ], "v8": [ 193, 193, 193 ] }, "of": 200, "differs": false }, { "target": "duckdb", "hits": { "v7": [ 200, 200, 200 ], "v8": [ 200, 200, 200 ] }, "of": 200, "differs": false } ], "recallDerivedSessions": [ "v7" ], "targetCount": 19, "httpClientCount": 8, "clock": { "perSession": { "v7": { "pinned": true, "pinnedMhz": 3500, "noTurbo": 1, "uncore": "0x1e1e", "ceilingBeforeMhz": 3600, "toleranceBp": 100, "pinnedRatio": 35, "kernelMedianMhz": 3492, "kernelMinMhz": 3492, "kernelMaxMhz": 3492, "clockOffPasses": 0, "clockUnreadPasses": 0, "passesEvaluated": 156, "legacyParsed": true }, "v8": { "pinned": true, "pinnedMhz": 3500, "noTurbo": 1, "uncore": "0x1e1e", "ceilingBeforeMhz": 3600, "toleranceBp": 100, "pinnedRatio": 35, "kernelMedianMhz": 3492, "kernelMinMhz": 3492, "kernelMaxMhz": 3500, "clockOffPasses": 0, "clockUnreadPasses": 0, "passesEvaluated": 156, "legacyParsed": false } }, "droppedWarnings": [ { "run": "20261006-130619-eshoponweb", "text": "WARNING: clock off its pinned value during oracle default@8: the engine CPUs averaged 3121 MHz, -10.8% against the pinned 3500 MHz; the client CPUs averaged 3141 MHz, -10.3% against the pinned 3500 MHz (limit 1%); this pass is not comparable with passes at the pinned clock.", "target": "oracle", "pass": "default@8", "engineMedianMhz": 3492, "clientMedianMhz": 3492 }, { "run": "20261006-142724-eshoponweb", "text": "WARNING: clock off its pinned value during oracle default@8: the engine CPUs averaged 3129 MHz, -10.6% against the pinned 3500 MHz; the client CPUs averaged 3230 MHz, -7.7% against the pinned 3500 MHz (limit 1%); this pass is not comparable with passes at the pinned clock.", "target": "oracle", "pass": "default@8", "engineMedianMhz": 3492, "clientMedianMhz": 3492 }, { "run": "20261006-154837-eshoponweb", "text": "WARNING: clock off its pinned value during oracle default@8: the engine CPUs averaged 3168 MHz, -9.5% against the pinned 3500 MHz; the client CPUs averaged 3214 MHz, -8.2% against the pinned 3500 MHz (limit 1%); this pass is not comparable with passes at the pinned clock.", "target": "oracle", "pass": "default@8", "engineMedianMhz": 3492, "clientMedianMhz": 3492 } ] }, "machine": { "cpu": "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz", "logicalCpus": 8, "ramGiB": 62.7, "os": "Ubuntu 24.04.5 LTS, kernel 6.8.0-142-generic", "dotNet": ".NET 10.0.12", "governor": "performance", "partition": "client 0-1,4-5; engines 2-3,6-7", "differences": [] }, "images": [ { "target": "clickhouse", "ref": "clickhouse/clickhouse-server:26.3.39.7", "id": "sha256:3a91276f066905da0edbbd622d3fc2a2df632c87ea7fe32ef4e74fdf8c9567b0", "lastTagTimeUtc": "2026-10-03T14:12:01.743437616Z", "beforeFirstV7Start": true }, { "target": "vespa", "ref": "vespaengine/vespa:8.754.14", "id": "sha256:5c30f5c41e7563498c4f925db6a837a3848f04726a3ed26aed4a7c8ab69f18fd", "lastTagTimeUtc": "2026-10-03T14:13:52.169232039Z", "beforeFirstV7Start": true }, { "target": "oracle", "ref": "gvenzl/oracle-free:23-slim-faststart", "id": "sha256:f5ff19033860d662c821cb04eb10483fa94f14f78eae252d054291ea07028093", "lastTagTimeUtc": "2026-10-03T14:17:05.132133694Z", "beforeFirstV7Start": true }, { "target": "elasticsearch", "ref": "elasticsearch:9.5.3", "id": "sha256:e23d4758358a4e356cc2ef3259a5a6f345cacc1722c10dffe6d7150a1a78e52f", "lastTagTimeUtc": "2026-10-03T14:07:11.337389556Z", "beforeFirstV7Start": true }, { "target": "sql", "ref": "mcr.microsoft.com/mssql/server:2025-CU9-ubuntu-24.04@sha256:2b5b581621126574f3d1f75e78d3eebe8d05aedb59ad0cfdf9aa42cb0634d726", "id": "sha256:2b5b581621126574f3d1f75e78d3eebe8d05aedb59ad0cfdf9aa42cb0634d726", "lastTagTimeUtc": "2026-10-04T22:02:11.293843649Z", "beforeFirstV7Start": true }, { "target": "pgvector", "ref": "pgvector/pgvector:0.8.7-pg17-trixie", "id": "sha256:7a7e9f22015b67edb4bef5c59daeebcd7e74bfa570df6ce60ae01237c8648a84", "lastTagTimeUtc": "2026-10-03T14:04:52.665253039Z", "beforeFirstV7Start": true }, { "target": "qdrant", "ref": "qdrant/qdrant:v1.17.0@sha256:f1c7272cdac52b38c1a0e89313922d940ba50afd90d593a1605dbbc214e66ffb", "id": "sha256:f1c7272cdac52b38c1a0e89313922d940ba50afd90d593a1605dbbc214e66ffb", "lastTagTimeUtc": "2026-10-04T22:03:16.973657523Z", "beforeFirstV7Start": true }, { "target": "opensearch", "ref": "opensearchproject/opensearch:3.9.0", "id": "sha256:adfa61f85025d06b4aeb562e7e74fde7e31c437039c93c3862c17e9acebd6c7c", "lastTagTimeUtc": "2026-10-03T14:09:29.512875356Z", "beforeFirstV7Start": true }, { "target": "qdrant-hnsw", "ref": "qdrant/qdrant:v1.17.0@sha256:f1c7272cdac52b38c1a0e89313922d940ba50afd90d593a1605dbbc214e66ffb", "id": "sha256:f1c7272cdac52b38c1a0e89313922d940ba50afd90d593a1605dbbc214e66ffb", "lastTagTimeUtc": "2026-10-04T22:03:16.973657523Z", "beforeFirstV7Start": true }, { "target": "redis", "ref": "redis:8.10.2", "id": "sha256:6f81e8915c60b065a524e6967e0ad1c639ba6efa84d669f823683ea04d9150ee", "lastTagTimeUtc": "2026-10-03T14:09:34.185138197Z", "beforeFirstV7Start": true }, { "target": "milvus", "ref": "milvusdb/milvus:v2.6.25", "id": "sha256:29f7668e64df1c6d5cdadbfbee99f38c72e4001054f149a298725acae57230ad", "lastTagTimeUtc": "2026-10-03T14:05:28.412151323Z", "beforeFirstV7Start": true }, { "target": "mariadb", "ref": "mariadb:11.8.9", "id": "sha256:6422478cb8e159f080fb1d8ccf65101e26fe51385787fde7d16c3b165a331f15", "lastTagTimeUtc": "2026-10-03T14:10:11.675870329Z", "beforeFirstV7Start": true }, { "target": "weaviate", "ref": "semitechnologies/weaviate:1.39.8", "id": "sha256:f6f4a5961f99e8718a02c822ca6a0d65ed3f9c567734419a7b2144933014f13a", "lastTagTimeUtc": "2026-10-03T14:05:40.948499656Z", "beforeFirstV7Start": true }, { "target": "mongodb", "ref": "mongodb/mongodb-atlas-local:8.0.32", "id": "sha256:1985314b0ded756ba965e0f400d17406b7a89bec659d4763a490f42f006e4fbd", "lastTagTimeUtc": "2026-10-03T14:11:20.531944474Z", "beforeFirstV7Start": true }, { "target": "chroma", "ref": "chromadb/chroma:latest", "id": "sha256:1e0b73a187a28757c572acba508c46f48c9e8b0acaf5c20e6d95cdedce1acdf6", "lastTagTimeUtc": "2026-10-03T14:09:56.392230738Z", "beforeFirstV7Start": true }, { "target": "typesense", "ref": "typesense/typesense:30.2", "id": "sha256:610f2d34b1f93d00762869da2c67736775e5798d19a2c8b91b014b8a0cc1e110", "lastTagTimeUtc": "2026-10-03T14:12:37.929033132Z", "beforeFirstV7Start": true }, { "target": "sql-diskann", "ref": "mcr.microsoft.com/mssql/server:2025-CU9-ubuntu-24.04@sha256:2b5b581621126574f3d1f75e78d3eebe8d05aedb59ad0cfdf9aa42cb0634d726", "id": "sha256:2b5b581621126574f3d1f75e78d3eebe8d05aedb59ad0cfdf9aa42cb0634d726", "lastTagTimeUtc": "2026-10-04T22:02:11.293843649Z", "beforeFirstV7Start": true } ], "engineSettings": [ { "target": "clickhouse", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "max_threads", "value": "\\'auto(4)\\'", "how": "read: ClickHouse system.settings, name max_threads, over HTTP" } ] }, { "target": "vespa", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "6442450944", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" } ] }, { "target": "oracle", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "6442450944", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "cpu_count", "value": "2", "how": "read: docker exec gvb-oracle sqlplus / as sysdba, V$PARAMETER and V$INSTANCE at the instance" }, { "key": "sga_target", "value": "1610612736", "how": "read: docker exec gvb-oracle sqlplus / as sysdba, V$PARAMETER and V$INSTANCE at the instance" }, { "key": "oracle_version", "value": "23.26.3.0.0", "how": "read: docker exec gvb-oracle sqlplus / as sysdba, V$PARAMETER and V$INSTANCE at the instance" } ] }, { "target": "elasticsearch", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "6442450944", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "heap_init_in_bytes", "value": "2147483648", "how": "read: GET /_nodes/jvm (nodes..jvm.mem.heap_init_in_bytes and heap_max_in_bytes)" }, { "key": "heap_max_in_bytes", "value": "2147483648", "how": "read: GET /_nodes/jvm (nodes..jvm.mem.heap_init_in_bytes and heap_max_in_bytes)" } ] }, { "target": "sql", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" } ] }, { "target": "pgvector", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "shared_buffers", "value": "2GB", "how": "read: docker exec gvb-pgvector psql current_setting" }, { "key": "enable_seqscan", "value": "off", "how": "set: src/GenericVectorBuilder.Engines/Sinks/PgVectorSink.cs#SET LOCAL enable_seqscan = off; SET LOCAL jit = off;" }, { "key": "jit", "value": "off", "how": "set: src/GenericVectorBuilder.Engines/Sinks/PgVectorSink.cs#SET LOCAL enable_seqscan = off; SET LOCAL jit = off;" } ] }, { "target": "sqlitevec", "session": "v8", "settings": [ { "key": "sqlite_version", "value": "3.53.3", "how": "read: SELECT sqlite_version() on an in-memory SQLite connection" }, { "key": "synchronous", "value": "NORMAL", "how": "set: src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#PRAGMA synchronous=NORMAL" }, { "key": "journal_mode", "value": "wal", "how": "read: PRAGMA journal_mode on a second, read-only connection to the bench file" } ] }, { "target": "qdrant", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "search_threads", "value": "3", "how": "read: threads named search-* of the qdrant process in container gvb-qdrant (/proc)" } ] }, { "target": "opensearch", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "6442450944", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "heap_init_in_bytes", "value": "2147483648", "how": "read: GET /_nodes/jvm (nodes..jvm.mem.heap_init_in_bytes and heap_max_in_bytes)" }, { "key": "heap_max_in_bytes", "value": "2147483648", "how": "read: GET /_nodes/jvm (nodes..jvm.mem.heap_init_in_bytes and heap_max_in_bytes)" } ] }, { "target": "qdrant-hnsw", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "search_threads", "value": "3", "how": "read: threads named search-* of the qdrant process in container gvb-qdrant (/proc)" } ] }, { "target": "redis", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "12884901888", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "search-workers", "value": "8", "how": "read: docker exec gvb-redis redis-cli CONFIG GET search-workers" }, { "key": "save", "value": "300 1", "how": "read: docker exec gvb-redis redis-cli CONFIG GET save" }, { "key": "appendonly", "value": "no", "how": "read: docker exec gvb-redis redis-cli CONFIG GET appendonly" }, { "key": "maxmemory", "value": "0", "how": "read: docker exec gvb-redis redis-cli CONFIG GET maxmemory" } ] }, { "target": "milvus", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" } ] }, { "target": "mariadb", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "innodb_buffer_pool_size", "value": "2147483648", "how": "read: docker exec gvbbench-mariadb mariadb SHOW GLOBAL VARIABLES" }, { "key": "innodb_flush_log_at_trx_commit", "value": "2", "how": "read: docker exec gvbbench-mariadb mariadb SHOW GLOBAL VARIABLES" }, { "key": "mhnsw_max_cache_size", "value": "4294967296", "how": "read: docker exec gvbbench-mariadb mariadb SHOW GLOBAL VARIABLES" } ] }, { "target": "weaviate", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "GOMEMLIMIT", "value": "6GiB", "how": "read: docker inspect Config.Env, the name GOMEMLIMIT only" } ] }, { "target": "mongodb", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" } ] }, { "target": "chroma", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "6442450944", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" }, { "key": "chroma_version (API version reported)", "value": "1.0.0", "how": "read: GET /api/v2/version" } ] }, { "target": "typesense", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "6442450944", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" } ] }, { "target": "sql-diskann", "session": "v8", "settings": [ { "key": "HostConfig.Memory", "value": "8589934592", "how": "read: docker inspect HostConfig.Memory (0 = no limit)" }, { "key": "HostConfig.CpusetCpus", "value": "2-3,6-7", "how": "read: docker inspect HostConfig.CpusetCpus (empty = every CPU)" }, { "key": "HostConfig.NanoCpus", "value": "0", "how": "read: docker inspect HostConfig.NanoCpus (0 = no limit)" } ] }, { "target": "duckdb", "session": "v8", "settings": [ { "key": "duckdb_version", "value": "v1.5.6", "how": "read: SELECT version() on an in-memory DuckDB connection" }, { "key": "threads", "value": "8", "how": "read: SELECT current_setting('threads') on an in-memory DuckDB connection; DuckDB's own thread count, not a CPU cap (it counts all CPUs and ignores affinity)" }, { "key": "hnsw_ef_search", "value": "100", "how": "set: src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET hnsw_ef_search" }, { "key": "hnsw_enable_experimental_persistence", "value": "true", "how": "set: src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET hnsw_enable_experimental_persistence = true" }, { "key": "memory_limit", "value": "7.4 GiB", "how": "read: SELECT current_setting('memory_limit') on a second connection to the bench file opened with the sink's own connection string" }, { "key": "checkpoint_threshold", "value": "244.1 MiB", "how": "read: SELECT current_setting('checkpoint_threshold') on a second connection to the bench file opened with the sink's own connection string" } ] } ], "why": [ { "target": "redis", "facts": [ { "kind": "index", "text": "HNSW TYPE FLOAT32 M=16 EF_CONSTRUCTION=128, EF_RUNTIME=100", "source": "results:targets[redis].index#HNSW TYPE FLOAT32 M=16 EF_CONSTRUCTION=128, EF_RUNTIME=100 per query", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[redis].index" }, { "kind": "search", "text": "HNSW, EF_RUNTIME=100 per query", "source": "results:targets[redis].index#HNSW TYPE FLOAT32 M=16 EF_CONSTRUCTION=128, EF_RUNTIME=100 per query", "confidence": "recorded, documented", "class": "documented", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[redis].index" }, { "kind": "storage", "text": "Redis is an in-memory but persistent on disk database", "source": "doc:design/engine-docs/redis-faq-2026-10-07.html#Redis is an in-memory but persistent on disk database", "confidence": "documented", "class": "documented", "mode": null, "where": "design/engine-docs/redis-faq-2026-10-07.html, inside one text node" }, { "kind": "protocol", "text": "StackExchange.Redis", "source": "src/GenericVectorBuilder.Engines/Sinks/RedisSink.cs#using StackExchange.Redis;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/RedisSink.cs:12" }, { "kind": "set-by-setup", "text": "save \"300 1\"", "source": "deploy/engines/redis.compose.yaml#--save \"300 1\"", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/redis.compose.yaml:18" }, { "kind": "set-by-setup", "text": "appendonly no", "source": "deploy/engines/redis.compose.yaml#--appendonly no", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/redis.compose.yaml:18" }, { "kind": "set-by-setup", "text": "mem_limit: 12g", "source": "deploy/engines/redis.compose.yaml#mem_limit: 12g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/redis.compose.yaml:23" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 0.2578, "8": 0.2313 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.5545, "8": 0.3582 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 1.2134 } }, { "target": "mariadb", "facts": [ { "kind": "index", "text": "VECTOR INDEX (HNSW variant) DISTANCE=cosine, M=16", "source": "results:targets[mariadb].index#VECTOR INDEX (HNSW variant) DISTANCE=cosine, M=16", "confidence": "recorded, unverified", "class": "unverified", "mode": null, "where": "in all 6 run(s) selected, at targets[mariadb].index" }, { "kind": "search", "text": "EXPLAIN of the search uses key vec_idx", "source": "results:targets[mariadb].load.indexNote#EXPLAIN of the default search uses key vec_idx", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[mariadb].load.indexNote" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[mariadb]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[mariadb]" }, { "kind": "protocol", "text": "MySqlConnector", "source": "src/GenericVectorBuilder.Engines/Sinks/MariaDbSink.cs#using MySqlConnector;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/MariaDbSink.cs:10" }, { "kind": "set-by-setup", "text": "innodb-buffer-pool-size=2G", "source": "deploy/engines/mariadb-bench.compose.yaml#--innodb-buffer-pool-size=2G", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/mariadb-bench.compose.yaml:47" }, { "kind": "set-by-setup", "text": "mhnsw-max-cache-size=4G", "source": "deploy/engines/mariadb-bench.compose.yaml#--mhnsw-max-cache-size=4G", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/mariadb-bench.compose.yaml:48" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/mariadb-bench.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/mariadb-bench.compose.yaml:56" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 0.4536, "8": 0.6206 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.4854, "8": 0.3766 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.6814 } }, { "target": "pgvector", "facts": [ { "kind": "index", "text": "HNSW vector_cosine_ops m=16 ef_construction=128", "source": "results:targets[pgvector].index#HNSW vector_cosine_ops m=16 ef_construction=128", "confidence": "recorded, read-back", "class": "read-back", "mode": null, "where": "in all 6 run(s) selected, at targets[pgvector].index" }, { "kind": "search", "text": "EXPLAIN of the search uses Index Scan", "source": "results:targets[pgvector].load.indexNote#EXPLAIN of the default search uses Index Scan on it = True", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[pgvector].load.indexNote" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[pgvector]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[pgvector]" }, { "kind": "protocol", "text": "Npgsql", "source": "src/GenericVectorBuilder.Engines/Sinks/PgVectorSink.cs#using Npgsql;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/PgVectorSink.cs:9" }, { "kind": "set-by-setup", "text": "shared_buffers=2GB", "source": "deploy/engines/pgvector.compose.yaml#shared_buffers=2GB", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/pgvector.compose.yaml:23" }, { "kind": "set-by-setup", "text": "SET LOCAL enable_seqscan = off; SET LOCAL jit = off", "source": "src/GenericVectorBuilder.Engines/Sinks/PgVectorSink.cs#SET LOCAL enable_seqscan = off; SET LOCAL jit = off", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/PgVectorSink.cs:84" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/pgvector.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/pgvector.compose.yaml:33" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 0.7164, "8": 0.9723 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.6781, "8": 0.5393 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.9073 } }, { "target": "qdrant", "facts": [ { "kind": "index", "text": "exact scan, sink sends exact=true on every search", "source": "results:targets[qdrant].index#exact scan: the builder's sink sends exact=true on every search", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[qdrant].index" }, { "kind": "search", "text": "no index used, exact scan by design", "source": "results:targets[qdrant].indexState.afterLoad.detail#no index used, exact scan by design", "confidence": "measured", "class": "read-back", "mode": "exact", "where": "in all 6 run(s) selected, at targets[qdrant].indexState.afterLoad.detail" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[qdrant]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[qdrant]" }, { "kind": "protocol", "text": "Qdrant.Client.Grpc", "source": "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#using Qdrant.Client.Grpc;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs:3" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/qdrant.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/qdrant.compose.yaml:35" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 0.7933, "8": 1.004 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.7369, "8": 0.5351 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.3018 } }, { "target": "qdrant-hnsw", "facts": [ { "kind": "index", "text": "HNSW m=16 ef_construct=100", "source": "results:targets[qdrant-hnsw].index#HNSW m=16 ef_construct=100", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[qdrant-hnsw].index" }, { "kind": "search", "text": "segment searches walked the graph and none scanned", "source": "results:targets[qdrant-hnsw].indexState.afterSearch.detail#segment searches walked the graph and none scanned", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[qdrant-hnsw].indexState.afterSearch.detail" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[qdrant-hnsw]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[qdrant-hnsw]" }, { "kind": "protocol", "text": "Qdrant.Client.Grpc", "source": "src/GenericVectorBuilder.Bench/Targets/QdrantHnswSink.cs#using Qdrant.Client.Grpc;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Bench/Targets/QdrantHnswSink.cs:5" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/qdrant.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/qdrant.compose.yaml:35" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 0.731, "8": 0.9329 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.7326, "8": 0.5218 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.236 } }, { "target": "oracle", "facts": [ { "kind": "index", "text": "HNSW in-memory neighbor graph NEIGHBORS=16 EFCONSTRUCTION=128", "source": "results:targets[oracle].index#HNSW in-memory neighbor graph NEIGHBORS=16 EFCONSTRUCTION=128", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[oracle].index" }, { "kind": "search", "text": "plan of the search: VECTOR INDEX HNSW SCAN", "source": "results:targets[oracle].load.indexNote#plan of the default search: VECTOR INDEX HNSW SCAN", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[oracle].load.indexNote" }, { "kind": "storage", "text": "HNSW in-memory neighbor graph", "source": "results:targets[oracle].index#HNSW in-memory neighbor graph", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[oracle].index" }, { "kind": "protocol", "text": "Oracle.ManagedDataAccess.Client", "source": "src/GenericVectorBuilder.Engines/Sinks/OracleSink.cs#using Oracle.ManagedDataAccess.Client;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/OracleSink.cs:8" }, { "kind": "cap", "text": "Oracle Free caps itself at 2 CPUs", "source": "results:targets[oracle].index#Oracle Free caps itself at 2 CPUs", "confidence": "recorded, read-back", "class": "read-back", "mode": null, "where": "in all 6 run(s) selected, at targets[oracle].index" }, { "kind": "set-by-setup", "text": "POOL_SIZE=768M", "source": "deploy/engines/oracle-init/01-vector-memory.sh#POOL_SIZE=768M", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/oracle-init/01-vector-memory.sh:13" }, { "kind": "set-by-setup", "text": "mem_limit: 6g", "source": "deploy/engines/oracle.compose.yaml#mem_limit: 6g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/oracle.compose.yaml:45" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 0.5774, "8": 0.7196 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.8853, "8": 0.6429 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 2.0268 } }, { "target": "elasticsearch", "facts": [ { "kind": "index", "text": "total_vex_size_bytes 0, index_options hnsw m=16 ef_construction=128", "source": "results:targets[elasticsearch].load.indexNote#total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128", "confidence": "measured", "class": "read-back", "mode": null, "where": "in all 6 run(s) selected, at targets[elasticsearch].load.indexNote" }, { "kind": "search", "text": "no HNSW graph, searches scan all 524 vectors", "source": "results:targets[elasticsearch].load.indexNote#NO HNSW GRAPH, searches scan all 524 vectors", "confidence": "measured", "class": "read-back", "mode": "exact", "where": "in all 6 run(s) selected, at targets[elasticsearch].load.indexNote" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[elasticsearch]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[elasticsearch]" }, { "kind": "protocol", "text": "HttpClient", "source": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs:70" }, { "kind": "set-by-setup", "text": "-Xms2g -Xmx2g", "source": "deploy/engines/elasticsearch.compose.yaml#-Xms2g -Xmx2g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/elasticsearch.compose.yaml:14" }, { "kind": "set-by-setup", "text": "mem_limit: 6g", "source": "deploy/engines/elasticsearch.compose.yaml#mem_limit: 6g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/elasticsearch.compose.yaml:19" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 1.0453, "8": 1.3789 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.455, "8": 0.493 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.7425 } }, { "target": "mongodb", "facts": [ { "kind": "index", "text": "vectorSearch index, HNSW maxEdges=16 numEdgeCandidates=128", "source": "results:targets[mongodb].index#vectorSearch index, HNSW maxEdges=16 numEdgeCandidates=128", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[mongodb].index" }, { "kind": "search", "text": "searched through the HNSW graph (Approximate)", "source": "results:targets[mongodb].load.indexNote#searched through the HNSW graph (Approximate)", "confidence": "recorded (engine's own report)", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[mongodb].load.indexNote" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[mongodb]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[mongodb]" }, { "kind": "protocol", "text": "MongoDB.Driver", "source": "src/GenericVectorBuilder.Engines/Sinks/MongoDbSink.cs#using MongoDB.Driver;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/MongoDbSink.cs:7" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/mongodb.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/mongodb.compose.yaml:21" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 1.3222, "8": 1.7807 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.3027, "8": 0.2849 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.7991 } }, { "target": "sqlitevec", "facts": [ { "kind": "index", "text": "vec0 brute-force scan, no ANN index (exact)", "source": "results:targets[sqlitevec].index#vec0 brute-force scan, no ANN index (exact)", "confidence": "recorded, read-back", "class": "read-back", "mode": null, "where": "in all 6 run(s) selected, at targets[sqlitevec].index" }, { "kind": "search", "text": "no index, exact scan by design", "source": "results:targets[sqlitevec].indexState.afterLoad.detail#no index, exact scan by design", "confidence": "measured", "class": "read-back", "mode": "exact", "where": "in all 6 run(s) selected, at targets[sqlitevec].indexState.afterLoad.detail" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[sqlitevec].index+engine+durability#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[sqlitevec].index+engine+durability" }, { "kind": "protocol", "text": "Microsoft.Data.Sqlite", "source": "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#using Microsoft.Data.Sqlite;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs:9" }, { "kind": "set-by-setup", "text": "PRAGMA journal_mode=WAL", "source": "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#Run( connection, \"PRAGMA journal_mode=WAL;\" )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs:528" }, { "kind": "set-by-setup", "text": "PRAGMA synchronous=NORMAL", "source": "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#Run( connection, \"PRAGMA synchronous=NORMAL;\" )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs:529" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": null, "8": null }, "engineCpuNullReason": "embedded engine: it runs inside this client process (its CPU is part of clientCpuMsPerSearch), so it has no cgroup of its own", "clientCpuMsPerSearch": { "1": 2.0174, "8": 3.5074 }, "clientIncludesEngine": true, "engineCpusBusyAt8": null } }, { "target": "vespa", "facts": [ { "kind": "index", "text": "HNSW float32 tensor, prenormalized-angular, max-links-per-node=16", "source": "results:targets[vespa].index#HNSW float32 tensor, prenormalized-angular (cosine), max-links-per-node=16", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[vespa].index" }, { "kind": "search", "text": "nearestNeighbor approximate:true", "source": "results:targets[vespa].load.indexNote#nearestNeighbor approximate:true with targetHits above the count", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[vespa].load.indexNote" }, { "kind": "storage", "text": "vectors held in memory", "source": "results:targets[vespa].index#vectors held in memory", "confidence": "recorded, unverified", "class": "unverified", "mode": null, "where": "in all 6 run(s) selected, at targets[vespa].index" }, { "kind": "protocol", "text": "HttpClient", "source": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs:44" }, { "kind": "set-by-setup", "text": "mem_limit: 6g", "source": "deploy/engines/vespa.compose.yaml#mem_limit: 6g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/vespa.compose.yaml:20" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 2.282, "8": 2.4344 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.9156, "8": 0.8725 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.8236 } }, { "target": "opensearch", "facts": [ { "kind": "index", "text": "faiss HNSW float32, no compression, m=16, ef_construction=128, cosinesimil", "source": "results:targets[opensearch].index#faiss HNSW float32, no compression, m=16, ef_construction=128, cosinesimil", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[opensearch].index" }, { "kind": "search", "text": "every segment searched through its HNSW graph", "source": "results:targets[opensearch].load.indexNote#every segment searched through its HNSW graph", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[opensearch].load.indexNote" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[opensearch]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[opensearch]" }, { "kind": "protocol", "text": "HttpClient", "source": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs:70" }, { "kind": "set-by-setup", "text": "-Xms2g -Xmx2g", "source": "deploy/engines/opensearch.compose.yaml#-Xms2g -Xmx2g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/opensearch.compose.yaml:16" }, { "kind": "set-by-setup", "text": "mem_limit: 6g", "source": "deploy/engines/opensearch.compose.yaml#mem_limit: 6g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/opensearch.compose.yaml:21" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 1.7502, "8": 2.554 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.4659, "8": 0.5054 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.8651 } }, { "target": "chroma", "facts": [ { "kind": "index", "text": "HNSW M=16 ef_construction=128, ef_search=100", "source": "results:targets[chroma].index#HNSW M=16 ef_construction=128, ef_search=100", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[chroma].index" }, { "kind": "search", "text": "approximate, no exact mode", "source": "results:targets[chroma].index#approximate only (no exact mode)", "confidence": "recorded, unverified", "class": "unverified", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[chroma].index" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[chroma]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[chroma]" }, { "kind": "protocol", "text": "HttpClient", "source": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs:73" }, { "kind": "set-by-setup", "text": "mem_limit: 6g", "source": "deploy/engines/chroma.compose.yaml#mem_limit: 6g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/chroma.compose.yaml:20" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 2.4635, "8": 2.7195 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.4749, "8": 0.4723 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 2.1455 } }, { "target": "milvus", "facts": [ { "kind": "index", "text": "HNSW M=16 efConstruction=128, ef=100, metric COSINE", "source": "results:targets[milvus].index#HNSW M=16 efConstruction=128, ef=100, metric COSINE", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[milvus].index" }, { "kind": "search", "text": "approximate, no exact mode", "source": "results:targets[milvus].index#approximate only (no exact mode)", "confidence": "recorded, unverified", "class": "unverified", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[milvus].index" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[milvus]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[milvus]" }, { "kind": "protocol", "text": "HttpClient", "source": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs:108" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/milvus.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/milvus.compose.yaml:68" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 2.743, "8": 2.9525 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.5459, "8": 0.63 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.7449 } }, { "target": "sql-diskann", "facts": [ { "kind": "index", "text": "DiskANN (preview) via VECTOR_SEARCH, cosine", "source": "results:targets[sql-diskann].index#DiskANN (preview) via VECTOR_SEARCH, cosine", "confidence": "recorded, unverified", "class": "unverified", "mode": null, "where": "in all 6 run(s) selected, at targets[sql-diskann].index" }, { "kind": "search", "text": "DiskANN index built and used", "source": "results:targets[sql-diskann].indexState.afterLoad.detail#DiskANN index built and used", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[sql-diskann].indexState.afterLoad.detail" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[sql-diskann]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[sql-diskann]" }, { "kind": "protocol", "text": "Microsoft.Data.SqlClient", "source": "src/GenericVectorBuilder.Bench/Targets/SqlDiskAnnSink.cs#using Microsoft.Data.SqlClient;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Bench/Targets/SqlDiskAnnSink.cs:5" }, { "kind": "protocol", "text": "vector as NVarChar value from ToJson", "source": "src/GenericVectorBuilder.Bench/Targets/SqlDiskAnnSink.cs#SqlDbType.NVarChar, -1 ).Value = SqlVectorSink.ToJson( vector )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Bench/Targets/SqlDiskAnnSink.cs:305" }, { "kind": "set-by-setup", "text": "MSSQL_MEMORY_LIMIT_MB: \"6144\"", "source": "deploy/engines/mssql.compose.yaml#MSSQL_MEMORY_LIMIT_MB: \"6144\"", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/mssql.compose.yaml:49" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/mssql.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/mssql.compose.yaml:56" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 3.4805, "8": 5.6305 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 1.0103, "8": 1.0826 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.9066 } }, { "target": "duckdb", "facts": [ { "kind": "index", "text": "HNSW (vss extension) FLOAT[n] metric=cosine m=16 ef_construction=128", "source": "results:targets[duckdb].index#HNSW (vss extension) FLOAT[n] metric=cosine m=16 ef_construction=128", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[duckdb].index" }, { "kind": "search", "text": "EXPLAIN of the search shows HNSW_INDEX_SCAN", "source": "results:targets[duckdb].load.indexNote#EXPLAIN of the default search shows HNSW_INDEX_SCAN", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[duckdb].load.indexNote" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[duckdb].index+engine+durability#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[duckdb].index+engine+durability" }, { "kind": "protocol", "text": "DuckDB.NET.Data", "source": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#using DuckDB.NET.Data;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs:7" }, { "kind": "set-by-setup", "text": "SET hnsw_enable_experimental_persistence = true", "source": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#Run( connection, \"SET hnsw_enable_experimental_persistence = true;\" )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs:750" }, { "kind": "set-by-setup", "text": "CheckpointThreshold 256MB", "source": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs#public string CheckpointThreshold { get; set; } = \"256MB\";", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs:49" }, { "kind": "set-by-setup", "text": "MemoryLimit 8GB", "source": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs#public string MemoryLimit { get; set; } = \"8GB\";", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs:38" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": null, "8": null }, "engineCpuNullReason": "embedded engine: it runs inside this client process (its CPU is part of clientCpuMsPerSearch), so it has no cgroup of its own", "clientCpuMsPerSearch": { "1": 4.9846, "8": 6.8333 }, "clientIncludesEngine": true, "engineCpusBusyAt8": null } }, { "target": "sql", "facts": [ { "kind": "index", "text": "exact VECTOR_DISTANCE cosine, no vector index (full scan)", "source": "results:targets[sql].index#exact VECTOR_DISTANCE cosine, no vector index (full scan)", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[sql].index" }, { "kind": "search", "text": "no index used, exact scan by design", "source": "results:targets[sql].indexState.afterLoad.detail#no index used, exact scan by design", "confidence": "measured", "class": "read-back", "mode": "exact", "where": "in all 6 run(s) selected, at targets[sql].indexState.afterLoad.detail" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[sql]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[sql]" }, { "kind": "protocol", "text": "Microsoft.Data.SqlClient", "source": "src/GenericVectorBuilder.Core/Sinks/SqlVectorSink.cs#using Microsoft.Data.SqlClient;", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Core/Sinks/SqlVectorSink.cs:7" }, { "kind": "protocol", "text": "vector as NVarChar value from ToJson", "source": "src/GenericVectorBuilder.Core/Sinks/SqlVectorSink.cs#SqlDbType.NVarChar, -1 ).Value = ToJson( vector )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Core/Sinks/SqlVectorSink.cs:155" }, { "kind": "protocol", "text": "CAST( @q AS VECTOR(dimension) )", "source": "src/GenericVectorBuilder.Core/Sinks/SqlVectorSink.cs#CAST( @q AS VECTOR({dimension}) )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Core/Sinks/SqlVectorSink.cs:149" }, { "kind": "set-by-setup", "text": "MSSQL_MEMORY_LIMIT_MB: \"6144\"", "source": "deploy/engines/mssql.compose.yaml#MSSQL_MEMORY_LIMIT_MB: \"6144\"", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/mssql.compose.yaml:49" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/mssql.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/mssql.compose.yaml:56" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 3.876, "8": 5.933 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 1.2107, "8": 1.1948 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.495 } }, { "target": "typesense", "facts": [ { "kind": "index", "text": "HNSW float32 (hnswlib), m=16, ef_construction=128", "source": "results:targets[typesense].index#HNSW float32 (hnswlib), m=16, ef_construction=128", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[typesense].index" }, { "kind": "search", "text": "HNSW graph returns 524 of 524 vectors", "source": "results:targets[typesense].load.indexNote#HNSW graph returns 524 of 524 vectors", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[typesense].load.indexNote" }, { "kind": "storage", "text": "index held in memory", "source": "results:targets[typesense].index#index held in memory", "confidence": "recorded, unverified", "class": "unverified", "mode": null, "where": "in all 6 run(s) selected, at targets[typesense].index" }, { "kind": "protocol", "text": "HttpClient", "source": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs:69" }, { "kind": "protocol", "text": "vector as string vectorQuery", "source": "src/GenericVectorBuilder.Engines/Sinks/TypesenseSink.cs#string vectorQuery = $\"vec:([{string.Join( \",\", vector.Select(", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/TypesenseSink.cs:249" }, { "kind": "set-by-setup", "text": "mem_limit: 6g", "source": "deploy/engines/typesense.compose.yaml#mem_limit: 6g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/typesense.compose.yaml:24" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 3.8747, "8": 6.7035 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.523, "8": 0.5817 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.856 } }, { "target": "clickhouse", "facts": [ { "kind": "index", "text": "vector_similarity HNSW cosineDistance, quantization bf16, M=16", "source": "results:targets[clickhouse].index#vector_similarity HNSW cosineDistance, quantization bf16, M=16", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[clickhouse].index" }, { "kind": "search", "text": "plan of the search uses the Skip index vec_idx", "source": "results:targets[clickhouse].load.indexNote#plan of the default search uses the Skip index vec_idx", "confidence": "measured", "class": "read-back", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[clickhouse].load.indexNote" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[clickhouse]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[clickhouse]" }, { "kind": "protocol", "text": "HttpClient over http", "source": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs:117" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/clickhouse.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/clickhouse.compose.yaml:36" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 7.1679, "8": 11.3741 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.8584, "8": 1.0542 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.9658 } }, { "target": "weaviate", "facts": [ { "kind": "index", "text": "HNSW maxConnections(M)=16 efConstruction=128", "source": "results:targets[weaviate].index#HNSW maxConnections(M)=16 efConstruction=128", "confidence": "recorded, documented", "class": "documented", "mode": null, "where": "in all 6 run(s) selected, at targets[weaviate].index" }, { "kind": "search", "text": "approximate, no exact mode", "source": "results:targets[weaviate].index#approximate only (no exact mode)", "confidence": "recorded, unverified", "class": "unverified", "mode": "approximate", "where": "in all 6 run(s) selected, at targets[weaviate].index" }, { "kind": "storage", "text": "not recorded", "source": "absent:results:targets[weaviate]#in memory|in-memory|memory pool|RAM", "confidence": "checked", "class": "measured", "mode": null, "where": "none of 4 words in 6 run(s) at targets[weaviate]" }, { "kind": "protocol", "text": "HttpClient", "source": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs:86" }, { "kind": "protocol", "text": "HttpMethod Post, v1 graphql", "source": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#HttpMethod.Post, \"v1/graphql\"", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs:542" }, { "kind": "set-by-setup", "text": "GOMEMLIMIT: 6GiB", "source": "deploy/engines/weaviate.compose.yaml#GOMEMLIMIT: 6GiB", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/weaviate.compose.yaml:23" }, { "kind": "set-by-setup", "text": "mem_limit: 8g", "source": "deploy/engines/weaviate.compose.yaml#mem_limit: 8g", "confidence": "set-by-code", "class": "documented", "mode": null, "where": "deploy/engines/weaviate.compose.yaml:29" } ], "costs": { "session": "v8", "engineCpuMsPerSearch": { "1": 8.2463, "8": 12.9589 }, "engineCpuNullReason": null, "clientCpuMsPerSearch": { "1": 0.5634, "8": 0.6087 }, "clientIncludesEngine": false, "engineCpusBusyAt8": 3.969 } } ], "searchSettings": [ { "target": "clickhouse", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "M", "value": "16" }, { "key": "ef_construction", "value": "128" }, { "key": "hnsw_candidate_list_size_for_search", "value": "256" } ], "clause": null }, { "target": "vespa", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "ef", "value": "100" }, { "key": "max-links-per-node", "value": "16" }, { "key": "neighbors-to-explore-at-insert", "value": "128" }, { "key": "targetHits", "value": "top" } ], "clause": null }, { "target": "oracle", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "EFCONSTRUCTION", "value": "128" }, { "key": "EFSEARCH", "value": "100" }, { "key": "NEIGHBORS", "value": "16" } ], "clause": null }, { "target": "elasticsearch", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "ef_construction", "value": "128" }, { "key": "k", "value": "top" }, { "key": "m", "value": "16" }, { "key": "num_candidates", "value": "100" } ], "clause": null }, { "target": "sql", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "description", "value": "exact VECTOR_DISTANCE cosine, no vector index (full scan)" } ], "clause": null }, { "target": "pgvector", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "ef_construction", "value": "128" }, { "key": "hnsw.ef_search", "value": "100" }, { "key": "m", "value": "16" } ], "clause": null }, { "target": "sqlitevec", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "chunk_size", "value": "1024" } ], "clause": null }, { "target": "qdrant", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "exact", "value": "true" } ], "clause": null }, { "target": "opensearch", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "approximate_threshold", "value": "0" }, { "key": "ef_construction", "value": "128" }, { "key": "ef_search", "value": "100" }, { "key": "k", "value": "top" }, { "key": "m", "value": "16" } ], "clause": null }, { "target": "qdrant-hnsw", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "ef_construct", "value": "100" }, { "key": "hnsw_ef", "value": "server" }, { "key": "m", "value": "16" } ], "clause": "hnsw_ef=server default" }, { "target": "redis", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "EF_CONSTRUCTION", "value": "128" }, { "key": "EF_RUNTIME", "value": "100" }, { "key": "M", "value": "16" } ], "clause": null }, { "target": "milvus", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "M", "value": "16" }, { "key": "ef", "value": "100" }, { "key": "efConstruction", "value": "128" } ], "clause": null }, { "target": "mariadb", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "DISTANCE", "value": "cosine" }, { "key": "M", "value": "16" }, { "key": "mhnsw_ef_search", "value": "100" } ], "clause": "mhnsw_ef_search=100 per statement (the ef 100 most engines here use, so the search effort matches; MariaDB's own default is 20; recall@10 at ef 100 falls as the set grows (random 1024-dimension vectors, measured 2026-10-04: 0.99 at 524, 0.89 to 0.92 at 2,000; [a statement the run-page notes list marks]))" }, { "target": "weaviate", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "ef", "value": "-1" }, { "key": "efConstruction", "value": "128" } ], "clause": "ef=-1 (dynamic: limit x 8 clamped 100..500)" }, { "target": "mongodb", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "maxEdges", "value": "16" }, { "key": "numCandidates", "value": "20x" }, { "key": "numEdgeCandidates", "value": "128" } ], "clause": "numCandidates=20x hits (min 100)" }, { "target": "chroma", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "M", "value": "16" }, { "key": "ef_construction", "value": "128" }, { "key": "ef_search", "value": "100" } ], "clause": "ef_search=100 (Chroma default)" }, { "target": "typesense", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "ef", "value": "100" }, { "key": "ef_construction", "value": "128" }, { "key": "k", "value": "top" }, { "key": "m", "value": "16" } ], "clause": null }, { "target": "sql-diskann", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "L", "value": "48" }, { "key": "M", "value": "8" }, { "key": "R", "value": "48" }, { "key": "StartId", "value": "306" } ], "clause": null }, { "target": "duckdb", "run": "20261007-205106-eshoponweb", "settings": [ { "key": "checkpoint_threshold", "value": "256MB" }, { "key": "ef_construction", "value": "128" }, { "key": "ef_search", "value": "100" }, { "key": "hnsw_enable_experimental_persistence", "value": "true" }, { "key": "m", "value": "16" }, { "key": "metric", "value": "cosine" } ], "clause": null } ], "observer": { "session": "v8", "copy": "observer/summary.json", "path": "~/gvb-work/v8-run/consolidate-work-report-f5/observer/summary.json", "sha256": "b9e1c73770f297059d2f52de76ebc17f4a8cae34f2b9afe04f2606230ce67f3c", "schema": "v8-observer-summary-1", "runs": [ { "folder": "20261007-205106-eshoponweb", "observerCpuMax": 0.1383, "aperfWorstDeviationBp": 52, "msr620ValuesSeen": [ "0x1e1e" ], "passesWithTimerFired": 6, "timersCovered": true, "cpuWorst": { "run": "20261007-205106-eshoponweb", "target": "oracle", "pass": "default@8", "cpu": 7, "mhz": 3477, "deviationBp": 65.71428571428572, "under": true } }, { "folder": "20261007-221429-eshoponweb", "observerCpuMax": 0.1141, "aperfWorstDeviationBp": 41.4, "msr620ValuesSeen": [ "0x1e1e" ], "passesWithTimerFired": 5, "timersCovered": true, "cpuWorst": { "run": "20261007-221429-eshoponweb", "target": "oracle", "pass": "default@8", "cpu": 7, "mhz": 3483.6, "deviationBp": 46.857142857143124, "under": true } }, { "folder": "20261007-233551-eshoponweb", "observerCpuMax": 0.1147, "aperfWorstDeviationBp": 46, "msr620ValuesSeen": [ "0x1e1e" ], "passesWithTimerFired": 3, "timersCovered": true, "cpuWorst": { "run": "20261007-233551-eshoponweb", "target": "oracle", "pass": "default@8", "cpu": 7, "mhz": 3482.7, "deviationBp": 49.42857142857195, "under": true } } ], "cpuMax": 0.1383, "aperfWorstDeviationBp": 52, "msr620ValuesSeen": [ "0x1e1e" ], "timersCovered": true, "passesWithTimerFired": 14, "pinnedMhz": 3500, "cpuWorst": { "run": "20261007-205106-eshoponweb", "target": "oracle", "pass": "default@8", "cpu": 7, "mhz": 3477, "deviationBp": 65.71428571428572, "under": true }, "dip": { "target": "oracle", "pass": "default@8", "runs": 3, "cpuReadings": 24, "cpusUnder": 24, "everyCpuUnder": true, "minBp": 26.857142857143117, "maxBp": 65.71428571428572, "otherWorstBp": 3.7142857142862344 }, "placement": { "runs": [ { "folder": "20261007-205106-eshoponweb", "file": "design/bench-inputs/observer-placement/analysis-801.txt", "percent": "34.8", "cpus": "2-3,6-7", "cgroupCpu": "0.0755", "placementQuote": "11880 ticks (34.8 percent) on engine CPUs 2-3,6-7", "cgroupQuote": "children included): 0.0755 CPUs" }, { "folder": "20261007-221429-eshoponweb", "file": "design/bench-inputs/observer-placement/analysis-802.txt", "percent": "33.8", "cpus": "2-3,6-7", "cgroupCpu": "0.0753", "placementQuote": "11167 ticks (33.8 percent) on engine CPUs 2-3,6-7", "cgroupQuote": "children included): 0.0753 CPUs" }, { "folder": "20261007-233551-eshoponweb", "file": "design/bench-inputs/observer-placement/analysis-803.txt", "percent": "33.5", "cpus": "2-3,6-7", "cgroupCpu": "0.0755", "placementQuote": "11135 ticks (33.5 percent) on engine CPUs 2-3,6-7", "cgroupQuote": "children included): 0.0755 CPUs" } ], "maxCgroupCpu": "0.0755", "notPinnedQuote": "The observer is NOT pinned" }, "meanDips": [ { "run": "20261007-205106-eshoponweb", "target": "oracle", "pass": "default@8", "engineMeanMhz": 3085.5, "clientMeanMhz": 3189.8 }, { "run": "20261007-221429-eshoponweb", "target": "oracle", "pass": "default@8", "engineMeanMhz": 3206.2, "clientMeanMhz": 3299 }, { "run": "20261007-233551-eshoponweb", "target": "oracle", "pass": "default@8", "engineMeanMhz": 3063, "clientMeanMhz": 3189.2 } ] }, "observerOthers": [ { "session": "v7", "copy": "observer/summary-v7.json", "path": "~/gvb-work/v8-run/consolidate-work-report-f5/observer/summary-v7.json", "sha256": "45b466347e7bc0a2427717e538d2f823855896dd63543732ac65572ee9594538", "schema": "v8-observer-summary-1", "runs": [ { "folder": "20261006-130619-eshoponweb", "observerCpuMax": 0.1234, "aperfWorstDeviationBp": 56.9, "msr620ValuesSeen": [ "0x1e1e" ], "passesWithTimerFired": 0, "timersCovered": false, "cpuWorst": { "run": "20261006-130619-eshoponweb", "target": "oracle", "pass": "default@8", "cpu": 2, "mhz": 3478.1, "deviationBp": 62.57142857142883, "under": true } }, { "folder": "20261006-142724-eshoponweb", "observerCpuMax": 0.1186, "aperfWorstDeviationBp": 62.9, "msr620ValuesSeen": [ "0x1e1e" ], "passesWithTimerFired": 0, "timersCovered": false, "cpuWorst": { "run": "20261006-142724-eshoponweb", "target": "oracle", "pass": "default@8", "cpu": 2, "mhz": 3474.5, "deviationBp": 72.85714285714286, "under": true } }, { "folder": "20261006-154837-eshoponweb", "observerCpuMax": 0.1133, "aperfWorstDeviationBp": 62.3, "msr620ValuesSeen": [ "0x1e1e" ], "passesWithTimerFired": 0, "timersCovered": false, "cpuWorst": { "run": "20261006-154837-eshoponweb", "target": "oracle", "pass": "default@8", "cpu": 2, "mhz": 3474.3, "deviationBp": 73.4285714285709, "under": true } } ], "cpuMax": 0.1234, "aperfWorstDeviationBp": 62.9, "msr620ValuesSeen": [ "0x1e1e" ], "timersCovered": false, "passesWithTimerFired": 0, "pinnedMhz": 3500, "cpuWorst": { "run": "20261006-154837-eshoponweb", "target": "oracle", "pass": "default@8", "cpu": 2, "mhz": 3474.3, "deviationBp": 73.4285714285709, "under": true }, "dip": { "target": "oracle", "pass": "default@8", "runs": 3, "cpuReadings": 24, "cpusUnder": 24, "everyCpuUnder": true, "minBp": 40.85714285714337, "maxBp": 73.4285714285709, "otherWorstBp": 4.571428571428312 }, "placement": null, "meanDips": [ { "run": "20261006-130619-eshoponweb", "target": "oracle", "pass": "default@8", "engineMeanMhz": 3076.6, "clientMeanMhz": 3089.1 }, { "run": "20261006-142724-eshoponweb", "target": "oracle", "pass": "default@8", "engineMeanMhz": 3101.5, "clientMeanMhz": 3165.8 }, { "run": "20261006-154837-eshoponweb", "target": "oracle", "pass": "default@8", "engineMeanMhz": 3120, "clientMeanMhz": 3189.2 } ] } ], "recordedTexts": [ { "target": "clickhouse", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "Acknowledged inserts are not fsynced. The sink creates plain MergeTree tables, so the MergeTree settings are the defaults: fsync_after_insert 0 and fsync_part_directory 0 (system.merge_tree_settings), and async_insert 1 with wait_for_async_insert 1 is the server default for the gvb login (system.settings), so an insert is acknowledged once its part is written, not once it is on disk.", "length": 386, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Measured 2026-10-04 with strace over 30 acknowledged single-row inserts (30 Ok rows in system.asynchronous_insert_log):", "length": 119, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "zero fsync, fdatasync or sync_file_range calls,", "length": 47, "class": "documented", "basis": [ "design/engine-docs/durability-logs-2026-10-04/clickhouse-acknowledged-inserts.strace#SIGUSR1" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "while the control, a table created with fsync_after_insert=1, made 61 fdatasync calls", "length": 85, "class": "documented", "basis": [ "design/engine-docs/durability-logs-2026-10-04/clickhouse-control-fsync-after-insert.strace#fdatasync" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "for 5 inserts.", "length": 14, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a ClickHouse process crash alone should not (inferred, not tested).", "length": 165, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "clickhouse", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "vector_similarity HNSW cosineDistance, quantization bf16, M=16 ef_construction=128,", "length": 83, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#INDEX vec_idx embedding TYPE vector_similarity( 'hnsw', 'cosineDistance', {dimension}, '{_options.Quantization}', {_options.HnswM}, {_options.HnswEfConstruction} )", "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSinkOptions.cs#public string Quantization { get; set; } = \"bf16\";", "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSinkOptions.cs#public int HnswM { get; set; } = 16;", "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSinkOptions.cs#public int HnswEfConstruction { get; set; } = 128;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "hnsw_candidate_list_size_for_search=256, rescoring off;", "length": 55, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#hnsw_candidate_list_size_for_search = {_options.HnswEfSearch}, vector_search_with_rescoring = {_options.Rescoring}", "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSinkOptions.cs#public int HnswEfSearch { get; set; } = 256;", "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSinkOptions.cs#public int Rescoring { get; set; }" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = full scan with skip indexes off", "length": 44, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "vespa", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "Vespa's transaction log server fsyncs after each commit: searchlib.translogserver usefsync=true, and an operation is searchable when it is acknowledged: proton documentdb visibilitydelay=0. Neither is overridden by this sink or by the services.xml it deploys;", "length": 259, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "VespaReadinessTests reads both from the live config server.", "length": 59, "class": "documented", "basis": [ "tests/GenericVectorBuilder.Engines.Tests/VespaReadinessTests.cs#usefsync" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "After a crash the node replays the transaction log over its last flushed data and rebuilds the in-memory HNSW graph. By those settings an acknowledged write survives a crash; this is read from the settings, not shown by pulling power.", "length": 234, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "One node and min-redundancy 1,", "length": 30, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/VespaApplicationPackage.cs#1" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "so a lost disk loses the data.", "length": 30, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "vespa", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW float32 tensor, prenormalized-angular (cosine), max-links-per-node=16, neighbors-to-explore-at-insert=128;", "length": 111, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/VespaApplicationPackage.cs#type tensor(x[{{dimension}}])", "src/GenericVectorBuilder.Engines/Sinks/VespaApplicationPackage.cs#distance-metric: prenormalized-angular", "src/GenericVectorBuilder.Engines/Sinks/VespaApplicationPackage.cs#max-links-per-node: 16", "src/GenericVectorBuilder.Engines/Sinks/VespaApplicationPackage.cs#neighbors-to-explore-at-insert: 128" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "search targetHits=top, ef=100 via exploreAdditionalHits;", "length": 56, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/VespaSink.cs#targetHits:{top},hnsw.exploreAdditionalHits:{extra}", "src/GenericVectorBuilder.Engines/Sinks/VespaSinkOptions.cs#int EfSearch = 100" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = approximate:false;", "length": 31, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/VespaSink.cs#targetHits:{top},approximate:false" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "vectors held in memory", "length": 22, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "oracle", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "By these settings a committed row should survive a crash or power loss. The sink uses a plain COMMIT and commit_logging, commit_wait and commit_write are unset in the database, so every commit waits for its redo to be written.", "length": 226, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Measured 2026-10-04: 50 separate client commits raised V$SYSSTAT 'redo synch writes' by 55 (the 50 commits plus the CREATE and DROP of the probe table), and the log writer and the datafile writer hold their files open with O_DSYNC (open flags 02110002, filesystemio_options none).", "length": 280, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "The database runs NOARCHIVELOG (V$DATABASE.LOG_MODE), so redo serves crash recovery only and there is no point-in-time restore.", "length": 127, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "The HNSW graph lives in the 768 MB vector memory pool (oracle-init/01-vector-memory.sh)", "length": 87, "class": "documented", "basis": [ "deploy/engines/oracle-init/01-vector-memory.sh#POOL_SIZE=768M" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "and is not the durable copy; the table is.", "length": 42, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "length": 98, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "CPU: Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS;", "length": 129, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OracleCpuCap.cs#cpu_count {CpuCount.ToString( CultureInfo.InvariantCulture )} in V$PARAMETER" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the 2 CPU thread limit is Oracle's documented Free edition limit).", "length": 66, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "oracle", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW in-memory neighbor graph NEIGHBORS=16 EFCONSTRUCTION=128,", "length": 62, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OracleSink.cs#ORGANIZATION INMEMORY NEIGHBOR GRAPH DISTANCE COSINE", "src/GenericVectorBuilder.Engines/Sinks/OracleSink.cs#PARAMETERS ( TYPE HNSW, NEIGHBORS {_options.HnswNeighbors}, EFCONSTRUCTION {_options.HnswEfConstruction} )", "src/GenericVectorBuilder.Engines/Sinks/OracleSinkOptions.cs#public int HnswNeighbors { get; set; } = 16;", "src/GenericVectorBuilder.Engines/Sinks/OracleSinkOptions.cs#public int HnswEfConstruction { get; set; } = 128;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "EFSEARCH=100 per query, cosine;", "length": 31, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OracleSink.cs#APPROX FIRST :top ROWS ONLY WITH TARGET ACCURACY PARAMETERS ( EFSEARCH {Math.Max( _options.HnswEfSearch, top )} )", "src/GenericVectorBuilder.Engines/Sinks/OracleSink.cs#VECTOR_DISTANCE( embedding, :query, COSINE )", "src/GenericVectorBuilder.Engines/Sinks/OracleSinkOptions.cs#public int HnswEfSearch { get; set; } = 100;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = FETCH EXACT FIRST (full scan);", "length": 43, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OracleSink.cs#\"EXACT FIRST :top ROWS ONLY\"" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS;", "length": 124, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OracleCpuCap.cs#cpu_count {CpuCount.ToString( CultureInfo.InvariantCulture )} in V$PARAMETER" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the 2 CPU thread limit is Oracle's documented Free edition limit)", "length": 65, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "elasticsearch", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the Elasticsearch default, which neither this sink nor elasticsearch.compose.yaml overrides", "length": 204, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(ElasticsearchReadinessTests reads it back from the live index).", "length": 64, "class": "documented", "basis": [ "tests/GenericVectorBuilder.Engines.Tests/ElasticsearchReadinessTests.cs#index.translog.durability" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power.", "length": 133, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "One node and no replicas,", "length": 25, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchSink.cs#settings = new { number_of_shards = 1, number_of_replicas = 0 }" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "so a lost disk loses the data.", "length": 30, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "elasticsearch", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW float32, no quantization, m=16, ef_construction=128,", "length": 57, "class": "read-back", "basis": [ "results:targets[elasticsearch].load.indexNote#index_options hnsw m=16 ef_construction=128" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "cosine;", "length": 7, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchSink.cs#similarity = \"cosine\"," ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "search k=top, num_candidates=100;", "length": 33, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchSink.cs#knn = new { field = \"vector\", query_vector = vector, k = top, num_candidates = candidates }", "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchSinkOptions.cs#int NumCandidates = 100" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "1 shard, 0 replicas;", "length": 20, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchSink.cs#settings = new { number_of_shards = 1, number_of_replicas = 0 }" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "force-merged to one segment after the load", "length": 42, "class": "read-back", "basis": [ "results:targets[elasticsearch].load.indexNote#1 segment(s), 524 vectors" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(at 1,024 dimensions a segment under 1,043 vectors gets no graph)", "length": 65, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "sql", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging;", "length": 103, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "delayed durability is DISABLED);", "length": 32, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "a database created here copies the model database:", "length": 50, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled;", "length": 77, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases", "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#DBCC TRACESTATUS( -1 ) WITH NO_INFOMSGS;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry,", "length": 200, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#ConfigFiles.ExistsWithSudoAsync( confPath, ct )" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "so SQL Server's own Linux defaults for flushing writes apply (Microsoft's Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here).", "length": 175, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "length": 98, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "length": 143, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/SqlServerContainer.cs#n.StartsWith( \"MSSQL_\"" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "sql", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "exact VECTOR_DISTANCE cosine, no vector index (full scan)", "length": 57, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Targets/TargetFactory.cs#private const string SQL_EXACT_INDEX = \"exact VECTOR_DISTANCE cosine, no vector index (full scan)\"" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "pgvector", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "fsync on, synchronous_commit on, full_page_writes on, wal_sync_method fdatasync (PostgreSQL defaults;", "length": 101, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "pgvector.compose.yaml sets only shared_buffers, maintenance_work_mem and max_wal_size):", "length": 87, "class": "documented", "basis": [ "deploy/engines/pgvector.compose.yaml#shared_buffers=2GB", "deploy/engines/pgvector.compose.yaml#maintenance_work_mem=1GB", "deploy/engines/pgvector.compose.yaml#max_wal_size=4GB" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "every commit is flushed to the write-ahead log before it returns and HNSW index changes are WAL-logged, so a crash loses no committed row (max_wal_size 4GB only spaces out checkpoints; measured with docker kill, which is SIGKILL, right after a 2,000-vector load and a restart: all 2,000 rows were there and the HNSW index was valid and used by the default search)", "length": 363, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "pgvector", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW vector_cosine_ops m=16 ef_construction=128,", "length": 48, "class": "read-back", "basis": [ "results:targets[pgvector].load.indexNote#USING hnsw (embedding vector_cosine_ops) WITH (m='16', ef_construction='128')" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "hnsw.ef_search=100 per query,", "length": 29, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/PgVectorSink.cs#SET LOCAL hnsw.ef_search = ", "src/GenericVectorBuilder.Engines/Sinks/PgVectorSinkOptions.cs#public int HnswEfSearch { get; set; } = 100;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "float32 vector(n), cosine;", "length": 26, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = same query with index scans off (sequential scan)", "length": 62, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/PgVectorSink.cs#EXACT_SETTINGS = \"SET LOCAL enable_indexscan = off; SET LOCAL jit = off\"" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "sqlitevec", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "PRAGMA journal_mode=WAL and PRAGMA synchronous=NORMAL (set by this sink when it opens the file):", "length": 96, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#Run( connection, \"PRAGMA journal_mode=WAL;\" )", "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#Run( connection, \"PRAGMA synchronous=NORMAL;\" )" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "each commit is appended to the -wal file and handed to the operating system, but the WAL is fsynced only when SQLite checkpoints it (measured with strace: 40 single-record commits caused 3 WAL fsyncs), so a crash of the process loses nothing (measured with kill -9 and a reopen: all 50, 300 and 2,000 rows were there), while an operating-system crash or power cut can lose the newest commits since the last checkpoint and leaves the database consistent", "length": 452, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "sqlitevec", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "vec0 brute-force scan, no ANN index (exact),", "length": 44, "class": "read-back", "basis": [ "results:targets[sqlitevec].load.indexNote#EXPLAIN QUERY PLAN: SCAN" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "float32, cosine distance,", "length": 25, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#embedding float[{dimension}] distance_metric=cosine," ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "default chunk_size=1024; score = 1 - cosine distance;", "length": 53, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "searches run concurrently, one WAL reader connection per searcher (opened as searchers arrive, at most 32), writes run one at a time and may overlap searches", "length": 157, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSinkOptions.cs#public int MaxSearchConnections { get; set; } = 32;", "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#return Math.Max( MIN_SEARCH_CONNECTIONS, _options.MaxSearchConnections );" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "qdrant", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30);", "length": 297, "class": "documented", "basis": [ "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-true.strace#MS_SYNC", "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-true.marks.json#\"mode\": \"true\"" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes.", "length": 211, "class": "documented", "basis": [ "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-false.strace#msync", "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-false.marks.json#\"mode\": \"false\"" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel's page cache (inferred, not tested).", "length": 326, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Every upsert here is sent with wait=true in batches of 256 over gRPC;", "length": 69, "class": "documented", "basis": [ "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#_client.UpsertAsync( name, points, wait: true", "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#private const int UPSERT_BATCH = 256;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the strace used one point per request, so one flush per batch is inferred, not measured.", "length": 88, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image's own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process.", "length": 1505, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/QdrantServer.cs#ConfigFiles.ReadContainerFileAsync( container, configPath, ct )" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "length": 98, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "qdrant", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "exact scan: the builder's sink sends exact=true on every search,", "length": 64, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Targets/TargetFactory.cs#private const string QDRANT_EXACT_INDEX = \"exact scan: the builder's sink sends exact=true on every search" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "so no HNSW graph is used whether or not Qdrant has built one (see the index state)", "length": 82, "class": "read-back", "basis": [ "results:targets[qdrant].load.indexNote#0 of 524 vectors in HNSW segments" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "opensearch", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the OpenSearch default, which neither this sink nor opensearch.compose.yaml overrides", "length": 198, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(OpenSearchReadinessTests reads it back from the live index).", "length": 61, "class": "documented", "basis": [ "tests/GenericVectorBuilder.Engines.Tests/OpenSearchReadinessTests.cs#index.translog.durability" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power.", "length": 133, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "One node and no replicas,", "length": 25, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OpenSearchSink.cs#[\"number_of_shards\"] = 1, [\"number_of_replicas\"] = 0" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "so a lost disk loses the data.", "length": 30, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "opensearch", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "faiss HNSW float32, no compression, m=16, ef_construction=128, cosinesimil;", "length": 75, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OpenSearchSink.cs#parameters = new { m = HNSW_M, ef_construction = HNSW_EF_CONSTRUCTION }", "src/GenericVectorBuilder.Engines/Sinks/OpenSearchSink.cs#private const int HNSW_M = 16;", "src/GenericVectorBuilder.Engines/Sinks/OpenSearchSink.cs#private const int HNSW_EF_CONSTRUCTION = 128;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "search k=top, ef_search=100;", "length": 28, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OpenSearchSink.cs#method_parameters = new { ef_search = Math.Max( _options.EfSearch, top ) }", "src/GenericVectorBuilder.Engines/Sinks/OpenSearchSinkOptions.cs#int EfSearch = 100" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "1 shard, 0 replicas;", "length": 20, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/OpenSearchSink.cs#[\"number_of_shards\"] = 1, [\"number_of_replicas\"] = 0" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "graph built at any segment size (approximate_threshold=0);", "length": 58, "class": "read-back", "basis": [ "results:targets[opensearch].load.indexNote#approximate_threshold 0" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "force-merged to one segment after the load", "length": 42, "class": "read-back", "basis": [ "results:targets[opensearch].load.indexNote#force-merged to one segment" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "qdrant-hnsw", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30);", "length": 297, "class": "documented", "basis": [ "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-true.strace#MS_SYNC", "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-true.marks.json#\"mode\": \"true\"" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes.", "length": 211, "class": "documented", "basis": [ "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-false.strace#msync", "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-false.marks.json#\"mode\": \"false\"" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel's page cache (inferred, not tested).", "length": 326, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Every upsert here is sent with wait=true in batches of 256 over gRPC;", "length": 69, "class": "documented", "basis": [ "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#_client.UpsertAsync( name, points, wait: true", "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#private const int UPSERT_BATCH = 256;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the strace used one point per request, so one flush per batch is inferred, not measured.", "length": 88, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image's own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process.", "length": 1505, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/QdrantServer.cs#ConfigFiles.ReadContainerFileAsync( container, configPath, ct )" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "length": 98, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "qdrant-hnsw", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW m=16 ef_construct=100,", "length": 27, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Targets/QdrantHnswSink.cs#var hnsw = new HnswConfigDiff { M = HNSW_M, EfConstruct = HNSW_EF_CONSTRUCT, FullScanThreshold = FULL_SCAN_THRESHOLD_KB };" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "hnsw_ef=server default,", "length": 23, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Targets/QdrantHnswSink.cs#search.HnswEf = _ef.Value;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "cosine;", "length": 7, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Targets/QdrantHnswSink.cs#new VectorParams { Size = (ulong)dimension, Distance = Distance.Cosine }" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "indexing_threshold_kb 1 and full_scan_threshold_kb 10", "length": 53, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Targets/QdrantHnswSink.cs#var optimizers = new OptimizersConfigDiff { IndexingThreshold = INDEXING_THRESHOLD_KB };", "src/GenericVectorBuilder.Bench/Targets/QdrantHnswSink.cs#var hnsw = new HnswConfigDiff { M = HNSW_M, EfConstruct = HNSW_EF_CONSTRUCT, FullScanThreshold = FULL_SCAN_THRESHOLD_KB };" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(server defaults are 10,000 each)", "length": 33, "class": "read-back", "basis": [ "results:targets[qdrant-hnsw].durability#storage.optimizers.indexing_threshold_kb = 10000", "results:targets[qdrant-hnsw].durability#storage.hnsw_index.full_scan_threshold_kb = 10000" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "so a small collection builds and walks its graph", "length": 48, "class": "read-back", "basis": [ "results:targets[qdrant-hnsw].load.indexNote#524 of 524 vectors in HNSW segments" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "redis", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "save \"300 1\" and appendonly no (redis.compose.yaml):", "length": 52, "class": "documented", "basis": [ "deploy/engines/redis.compose.yaml#--save \"300 1\"", "deploy/engines/redis.compose.yaml#--appendonly no" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "an RDB snapshot is written every 5 minutes if at least one key changed, and on a clean stop; there is no append-only log, so a crash loses every write since the last snapshot (up to 5 minutes plus the time a snapshot takes; measured with docker kill, which is SIGKILL, right after a 2,000-vector load: none of the 2,000 hashes were there after the restart, and the restart loaded an older snapshot that still held keys of a collection that had been dropped since)", "length": 463, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "redis", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW TYPE FLOAT32 M=16 EF_CONSTRUCTION=128,", "length": 43, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/RedisSink.cs#[\"M\"] = _options.HnswM,", "src/GenericVectorBuilder.Engines/Sinks/RedisSink.cs#[\"EF_CONSTRUCTION\"] = _options.HnswEfConstruction,", "src/GenericVectorBuilder.Engines/Sinks/RedisSinkOptions.cs#public int HnswM { get; set; } = 16;", "src/GenericVectorBuilder.Engines/Sinks/RedisSinkOptions.cs#public int HnswEfConstruction { get; set; } = 128;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "EF_RUNTIME=100 per query,", "length": 25, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/RedisSink.cs#$\"EF_RUNTIME {Math.Max( _options.HnswEfRuntime, top )}\"", "src/GenericVectorBuilder.Engines/Sinks/RedisSinkOptions.cs#public int HnswEfRuntime { get; set; } = 100;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "cosine;", "length": 7, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/RedisSink.cs#[\"DISTANCE_METRIC\"] = \"COSINE\"," ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = FLAT index built on first exact query", "length": 50, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "milvus", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "Writes are not fsynced. Standalone uses the default message queue rocksmq (RocksDB, /var/lib/milvus/rdb_data, mq.type default) and flushed segments go to local-disk object storage", "length": 179, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(COMMON_STORAGETYPE=local in milvus.compose.yaml).", "length": 50, "class": "documented", "basis": [ "deploy/engines/milvus.compose.yaml#COMMON_STORAGETYPE: local" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Measured 2026-10-04 with strace over 30 acknowledged single-row upserts and one flush: fdatasync ran only on the embedded etcd files (member/wal, member/snap/db), never on rdb_data or the segment files.", "length": 202, "class": "documented", "basis": [ "design/engine-docs/durability-logs-2026-10-04/milvus-acknowledged-upserts.strace#member/wal" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a Milvus process crash alone should not (inferred, not tested). Metadata in embedded etcd is fdatasynced on every commit.", "length": 219, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "milvus", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW M=16 efConstruction=128,", "length": 29, "class": "read-back", "basis": [ "results:targets[milvus].load.indexNote#type HNSW (COSINE, {\"M\":16,\"efConstruction\":128})" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "ef=100, metric COSINE,", "length": 22, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#writer.WriteNumber( \"ef\", _options.Ef.Value );", "src/GenericVectorBuilder.Engines/Sinks/MilvusSinkOptions.cs#int? Ef = 100,", "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#[\"metricType\"] = \"COSINE\"," ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Strong consistency searches;", "length": 28, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "approximate only (no exact mode)", "length": 32, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "mariadb", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "innodb_flush_log_at_trx_commit=2 (set in mariadb-bench.compose.yaml for the benchmark's own container gvbbench-mariadb, and in mariadb.compose.yaml for the daily gvb-mariadb, which has the same server settings):", "length": 211, "class": "documented", "basis": [ "deploy/engines/mariadb-bench.compose.yaml#--innodb-flush-log-at-trx-commit=2", "deploy/engines/mariadb.compose.yaml#--innodb-flush-log-at-trx-commit=2", "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING innodb_flush_log_at_trx_commit = 2" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the InnoDB redo log is written to the operating system at every commit but fsynced about once a second, so a crash of the mariadbd process loses nothing, while an operating-system crash or power cut can lose the last second of commits;", "length": 235, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "innodb_doublewrite is on", "length": 24, "class": "documented", "basis": [ "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING innodb_doublewrite = ON" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(default),", "length": 10, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the binary log is off,", "length": 22, "class": "documented", "basis": [ "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING log_bin = OFF" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "and the vector graph is an InnoDB table under the same log", "length": 58, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(settings read from the running server with SHOW VARIABLES;", "length": 59, "class": "documented", "basis": [ "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING innodb_flush_log_at_trx_commit = 2" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the crash behaviour is InnoDB's documented behaviour for this setting and was not tested on either container)", "length": 109, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "mariadb", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "VECTOR INDEX", "length": 12, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/MariaDbSink.cs#VECTOR INDEX {INDEX_NAME} ( embedding ) M={_options.HnswM} DISTANCE=cosine" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(HNSW variant)", "length": 14, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "DISTANCE=cosine, M=16", "length": 21, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/MariaDbSink.cs#VECTOR INDEX {INDEX_NAME} ( embedding ) M={_options.HnswM} DISTANCE=cosine", "src/GenericVectorBuilder.Engines/Sinks/MariaDbSinkOptions.cs#public int HnswM { get; set; } = 16;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(no ef_construction setting exists),", "length": 36, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "mhnsw_ef_search=100 per statement", "length": 33, "class": "read-back", "basis": [ "results:targets[mariadb].load.indexNote#the server applied mhnsw_ef_search 100" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(the ef 100 most engines here use, so the search effort matches;", "length": 64, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "MariaDB's own default is 20;", "length": 28, "class": "documented", "basis": [ "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING mhnsw_ef_search = 20" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "recall@10 at ef 100 fall...", "length": 176, "class": "unverified", "basis": [], "printed": false, "notPrinted": "mariadb-recall", "unlisted": false }, { "text": "mhnsw_max_cache_size 4G;", "length": 24, "class": "documented", "basis": [ "deploy/engines/mariadb-bench.compose.yaml#--mhnsw-max-cache-size=4G", "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING mhnsw_max_cache_size = 4294967296" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = IGNORE INDEX full scan", "length": 35, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/MariaDbSink.cs#SearchSql( collection, top, $\"IGNORE INDEX ( {INDEX_NAME} )\" )" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "weaviate", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "Weaviate 1.39.8 defaults...", "length": 77, "class": "unverified", "basis": [], "printed": false, "notPrinted": "contradicted", "unlisted": false }, { "text": "every object is appended to the LSM write-ahead log with a plain write and no fsync, and the log is fsynced only when its memtable is flushed, 60 seconds after the last write (PERSISTENCE_MEMTABLES_FLUSH_IDLE_AFTER_SECONDS default; measured with strace: 2,080 writes into the objects log during a 2,000-object load, the first fsync 60 s after the last write), so a power cut can lose the last minute of writes; the HNSW commit log is buffered inside the process (77 writes for 2,000 vectors), so a killed process loses the newest vectors from the vector index while their objects survive (measured with docker kill, which is SIGKILL, one second after the load and a restart, two runs each: 0 to 1 of 50, 264 to 267 of 300 and 1,992 to 1,997 of 2,000 stored objects were still reachable through a vector search)", "length": 810, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "weaviate", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW maxConnections(M)=16 efConstruction=128,", "length": 45, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#[\"maxConnections\"] = _options.MaxConnections,", "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#[\"efConstruction\"] = _options.EfConstruction,", "src/GenericVectorBuilder.Engines/Sinks/WeaviateSinkOptions.cs#int MaxConnections = 16,", "src/GenericVectorBuilder.Engines/Sinks/WeaviateSinkOptions.cs#int EfConstruction = 128," ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "ef=-1", "length": 5, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#[\"ef\"] = _options.Ef,", "src/GenericVectorBuilder.Engines/Sinks/WeaviateSinkOptions.cs#int Ef = -1 );", "design/engine-docs/weaviate-vector-index-reference-2026-10-07.mdx#`ef`" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(dynamic: limit x 8 clamped 100..500),", "length": 38, "class": "documented", "basis": [ "design/engine-docs/weaviate-vector-index-reference-2026-10-07.mdx#`dynamicEfFactor`", "design/engine-docs/weaviate-vector-index-reference-2026-10-07.mdx#`dynamicEfMin`", "design/engine-docs/weaviate-vector-index-reference-2026-10-07.mdx#`dynamicEfMax`", "design/engine-docs/weaviate-vector-index-concepts-2026-10-07.md#The dynamic list size will be set as the query limit multiplied by `dynamicEfFactor`, modified by a minimum of `dynamicEfMin` and a maximum of `dynamicEfMax`." ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "cosine, no quantization;", "length": 24, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "approximate only (no exact mode)", "length": 32, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "mongodb", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "Acknowledged writes are journaled before the acknowledgement. The sink sets no write concern, so the server default applies: getDefaultRWConcern gives w majority and the one-member replica set (--replSet gvbmongo in the image) has writeConcernMajorityJournalDefault true, with WiredTiger journaling on (journalCommitInterval 100 ms is only the interval for unacknowledged work).", "length": 378, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Measured 2026-10-04: 30 acknowledged single-document inserts raised WiredTiger 'log sync operations' by 30 and", "length": 110, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "strace showed fdatasync on /data/db/journal/WiredTigerLog files.", "length": 64, "class": "documented", "basis": [ "design/engine-docs/durability-logs-2026-10-04/mongodb-acknowledged-inserts.strace#WiredTigerLog" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "A crash loses no acknowledged write. mongot's search index is not part of that promise: it follows the collection asynchronously and is rebuilt from it.", "length": 152, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "mongodb", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "vectorSearch index, HNSW maxEdges=16 numEdgeCandidates=128,", "length": 59, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/MongoDbSinkOptions.cs#public int HnswMaxEdges { get; set; } = 16;", "src/GenericVectorBuilder.Engines/Sinks/MongoDbSinkOptions.cs#public int HnswNumEdgeCandidates { get; set; } = 128;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "float32 binData, cosine,", "length": 24, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/MongoDbSink.cs#VectorSimilarity.Cosine" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "numCandidates=20x hits (min 100);", "length": 33, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/MongoDbSinkOptions.cs#public int NumCandidatesFactor { get; set; } = 20;", "src/GenericVectorBuilder.Engines/Sinks/MongoDbSinkOptions.cs#public int MinNumCandidates { get; set; } = 100;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = $vectorSearch exact:true", "length": 37, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "chroma", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "SQLite rollback journal with its default synchronous=FULL under the data directory", "length": 82, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(chroma.compose.yaml sets IS_PERSISTENT=1 and PERSIST_DIRECTORY=/data and no sync setting):", "length": 91, "class": "documented", "basis": [ "deploy/engines/chroma.compose.yaml#IS_PERSISTENT: \"1\"", "deploy/engines/chroma.compose.yaml#PERSIST_DIRECTORY: /data" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "every write is committed to chroma.sqlite3 with fsync of the journal, the directory and the database file before the call returns (strace: 15 database fsyncs and 38 journal fsyncs for one create, four 500-vector upserts and one delete), and the HNSW files are written every sync_threshold=1000 vectors and rebuilt from the SQLite log after a crash; measured with kill -9 and a restart: all 50, 300 and 2,000 vectors were still stored and searchable", "length": 448, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "chroma", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW M=16 ef_construction=128, ef_search=100", "length": 44, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#[\"max_neighbors\"] = _options.MaxNeighbors,", "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#[\"ef_construction\"] = _options.EfConstruction,", "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#[\"ef_search\"] = _options.EfSearch,", "src/GenericVectorBuilder.Engines/Sinks/ChromaSinkOptions.cs#int MaxNeighbors = 16,", "src/GenericVectorBuilder.Engines/Sinks/ChromaSinkOptions.cs#int EfConstruction = 128,", "src/GenericVectorBuilder.Engines/Sinks/ChromaSinkOptions.cs#int EfSearch = 100 );" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(Chroma default),", "length": 17, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "cosine;", "length": 7, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#[\"space\"] = \"cosine\"," ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "approximate only (no exact mode)", "length": 32, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "typesense", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "Every acknowledged write is appended to Typesense's raft log and fsynced before the answer: braft raft_sync=true with raft_sync_policy=0 (sync immediately), read from the running container's brpc /flags page on 2026-10-04. A restart replays the log from the last snapshot", "length": 271, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(typesense.compose.yaml sets TYPESENSE_SNAPSHOT_INTERVAL_SECONDS=300)", "length": 69, "class": "documented", "basis": [ "deploy/engines/typesense.compose.yaml#TYPESENSE_SNAPSHOT_INTERVAL_SECONDS: \"300\"" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "and rebuilds the in-memory HNSW graph, so by those settings a crash loses no acknowledged write, but the restart is slow after a big load. This is read from the settings, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "length": 255, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "typesense", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW float32 (hnswlib), m=16, ef_construction=128, cosine;", "length": 58, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/TypesenseSink.cs#hnsw_params = new { M = HNSW_M, ef_construction = HNSW_EF_CONSTRUCTION }", "src/GenericVectorBuilder.Engines/Sinks/TypesenseSink.cs#private const int HNSW_M = 16;", "src/GenericVectorBuilder.Engines/Sinks/TypesenseSink.cs#private const int HNSW_EF_CONSTRUCTION = 128;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "search k=top, ef=100;", "length": 21, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/TypesenseSink.cs#string options = $\"k:{top}, ef:{Math.Max( _options.Ef, top )}\";", "src/GenericVectorBuilder.Engines/Sinks/TypesenseSinkOptions.cs#int Ef = 100" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = filter ordinal:>=0 with flat_search_cutoff;", "length": 56, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "index held in memory", "length": 20, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "sql-diskann", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging;", "length": 103, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "delayed durability is DISABLED);", "length": 32, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "a database created here copies the model database:", "length": 50, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled;", "length": 77, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases", "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#DBCC TRACESTATUS( -1 ) WITH NO_INFOMSGS;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry,", "length": 200, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#ConfigFiles.ExistsWithSudoAsync( confPath, ct )" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "so SQL Server's own Linux defaults for flushing writes apply (Microsoft's Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here).", "length": 175, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "length": 98, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "length": 143, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Targets/SqlServerContainer.cs#n.StartsWith( \"MSSQL_\"" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "sql-diskann", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "DiskANN (preview) via VECTOR_SEARCH, cosine,", "length": 44, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "build {\"StartId\":\"306\", \"L\":\"48\", \"M\":\"8\", \"R\":\"48\"};", "length": 53, "class": "read-back", "basis": [ "results:targets[sql-diskann].load.indexNote#built DiskANN, parameters {\"StartId\":\"306\", \"L\":\"48\", \"M\":\"8\", \"R\":\"48\"}", "src/GenericVectorBuilder.Bench/Targets/SqlDiskAnnSink.cs#_buildParameters = index?.Build ?? \"unknown\";" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the exact mode scans the same table", "length": 35, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "duckdb", "field": "durability", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "DuckDB's write-ahead log is fsynced at every commit and no DuckDbSink setting changes that", "length": 90, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "(checkpoint_threshold=256MB", "length": 27, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs#CheckpointThreshold { get; set; } = \"256MB\"", "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET checkpoint_threshold" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "only spaces out the checkpoints that write the database file; measured with strace: 42 WAL fsyncs for 40 single-record commits plus 2 setup statements), so an operating-system crash or power cut loses no committed row; measured with kill -9 right after the last commit and a reopen: all 50, 300 and 2,000 rows and an HNSW index that counted the same number were recovered from the log; not tested and documented by DuckDB: the HNSW index is file-backed only through hnsw_enable_experimental_persistence = true,", "length": 510, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "which this sink turns on,", "length": 25, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET hnsw_enable_experimental_persistence = true;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "and WAL recovery for such custom indexes is not complete, so a crash during a checkpoint or a later commit can damage the index while the rows survive", "length": 150, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "duckdb", "field": "index", "run": "20261007-205106-eshoponweb", "note": null, "clauses": [ { "text": "HNSW (vss extension) FLOAT[n] metric=cosine m=16 ef_construction=128,", "length": 69, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#WITH ( metric = 'cosine', m =", "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#Run( connection, \"LOAD vss;\" )", "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#{VECTOR_COLUMN} FLOAT[{dimension.ToString( CultureInfo.InvariantCulture )}] NOT NULL );", "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs#public int HnswM { get; set; } = 16;", "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs#public int HnswEfConstruction { get; set; } = 128;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "ef_search=100 per connection,", "length": 29, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET hnsw_ef_search", "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs#public int HnswEfSearch { get; set; } = 100;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "persistent (hnsw_enable_experimental_persistence=true, checkpoint_threshold=256MB);", "length": 83, "class": "documented", "basis": [ "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET hnsw_enable_experimental_persistence = true;", "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET checkpoint_threshold", "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs#public string CheckpointThreshold { get; set; } = \"256MB\";" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "exact mode = array_cosine_similarity sequential scan;", "length": 53, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "score = 1 - cosine distance;", "length": 28, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "searches run concurrently, one connection per searcher (opened as searchers arrive, at most 32), writes run one at a time and never overlap a search", "length": 148, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 0, "clauses": [ { "text": "Load average at start: 0.38 0.64 1.22 on 8 logical CPUs (1/5/15 min).", "length": 69, "class": "read-back", "basis": [ "src/GenericVectorBuilder.Bench/Report/MachineFacts.cs#File.ReadAllText( \"/proc/loadavg\" )" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "It counts this benchmark's own earlier work and the engines it started, so it is recorded here and never judged; each timed pass is judged by the outside load machine control measures (conditions.passes).", "length": 204, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 1, "clauses": [ { "text": "Order: targets ran one at a time in a random order from seed 801 (runSeed; --seed 801 repeats it, targetOrder lists it).", "length": 120, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Running/RunOrder.cs#return Shuffle( names, Mix( (ulong)(uint)runSeed ^ TARGET_SALT ) );" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Inside each target the timed passes also ran in a random order from the same seed and the target's name (passOrder):", "length": 116, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Running/RunOrder.cs#return Shuffle( passes, Mix( (ulong)(uint)runSeed ^ PASS_SALT ^ StableHash( targetName.ToLowerInvariant() ) ) );" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "default@1 is one searcher for 20 s (every search's latency gives p50/p95/p99, the completed searches give QPS@1, its first answer to each query gives recall and nDCG), default@N is N searchers for 20 s (QPS@N), exact is the engine's exact mode, one searcher for 60 s cycling the queries.", "length": 287, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 2, "clauses": [ { "text": "Preparation, untimed, before any timed pass: a rehearsal of every pass type at its own concurrency for 30 s each, through the same code the passes use.", "length": 151, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 3, "clauses": [ { "text": "Warm-up and settle check, untimed, right before every timed pass: the pass's own search at the pass's own number of searchers for at least 15 s and at least 20 searches (at most 120 s), read in windows of at least 2 s and 100 searches; then a 3 s trial of the same pass. The trial's figure (p50 with one searcher, QPS with several) must lie within 10% of the warm-up's settled figure (the median of its last 3 windows, which must agree within 5%); if not, the warm-up is extended once (at least 30 s, until its windows agree, at most 120 s), where an extension's windows agree when its level test passes: the older and the newer half of its latest windows, 5 to 10 windows a half, each half read as the pass reads it (the p50 of all its searches with one searcher, its searches per second with several), agree within 5%, judged on the windows that stopped it (an extension whose cap runs out with fewer than 10 windows is judged by its last 3 instead); then a second trial is taken, which must lie inside the range of the newer half's windows or within 10% of the newer half's figure, and the warm-up runs again before the pass. After the pass its own figure is held against the settled figure, within 10%. In the level test, the second trial and the hold, two one-searcher p50s no more than 0.1 ms apart agree whatever their percentage (below what this box resolves at one searcher). Each target's notes give every check, and a pass that still disagrees, or whose own figure did not hold, flags its target as unsettled. With machine control on, the check for a quiet box is made when the warm-up is announced, before it starts (a wait between the warm-up and the timed pass let the engine go cold), so by the time the clock opens that check is as old as the warm-up and the trial (about 18 s, more after an extension); each pass's conditions record that lead (quietCheckLeadSeconds). Failed untimed searches are counted per target (warmupErrors) and are not in the timed error counts.", "length": 1985, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 4, "clauses": [ { "text": "Load rows/s counts only time inside each target's upsert calls: one writer, batches of 1000, rows already in memory, the collection dropped and created fresh first. Engines that build or finish their index after the writes do it in a separate timed index step (the load's index seconds), and the run waits for it before searching.", "length": 330, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 5, "clauses": [ { "text": "Index proof: each engine's own report of its index (indexState) is read after the load and again after the last pass. A target whose index was not ready after the load was still measured and carries a WARNING; an engine that reports nothing counts as not ready.", "length": 261, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 6, "clauses": [ { "text": "Search settings: each searched target records the settings its own index description states (searchSettings: the build parameters and the effort per query, such as m, ef_construction and ef_search), read after the load; consolidating never averages runs whose settings differ.", "length": 276, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 7, "clauses": [ { "text": "Durability: each target's crash-safety setting as configured here (durability). Engines that do not force writes to disk on every commit load faster for that reason.", "length": 165, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 8, "clauses": [ { "text": "Engine record: right after each engine was bound, its settings (engineSettings: the container's limits and what the engine itself reports, each entry saying how it was obtained), its image against the id pinned in deploy/bench/image-pins.json (image; another id ends that target with an error) and the size of its data folder (dataFolder: at the start, after any reset, and at the end) were recorded. In run-all, ClickHouse's system log tables are truncated before its load, and the reset is written into its dataFolder.", "length": 520, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 9, "clauses": [ { "text": "Latency is client-side wall time around each search (network and driver included), every search of the default@1 window, one searcher, the queries cycled; a window with fewer than 200 searches, a p50 above 1.25 x its mean, or a mean above its p99 is flagged.", "length": 258, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 10, "clauses": [ { "text": "QPS: N searchers back to back for 20 s per level; completed searches divided by the window's elapsed time.", "length": 106, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 11, "clauses": [ { "text": "Recall@10: share of the exact top 10 (brute force in memory) that the engine returned.", "length": 86, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "A hit whose exact similarity ties the 10th best (within 1e-5) also counts", "length": 73, "class": "documented", "basis": [ "src/GenericVectorBuilder.Bench/Stats/BenchMath.cs#public const double TIE_TOLERANCE = 1e-5;" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": ", because duplicate rows embed to identical vectors.", "length": 52, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 12, "clauses": [ { "text": "Routes: a container engine (the benchmark's own SQL Server and Qdrant containers included) is reached at its container's own address on its Docker network, never through the published localhost port (docker-proxy);", "length": 214, "class": "read-back", "basis": [ "results:conditions.connections#this target" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "the native comparison ta...", "length": 109, "class": "unverified", "basis": [], "printed": false, "notPrinted": "absent-targets", "unlisted": false }, { "text": "Each target's addresses and the connections the client held open after its passes are in its notes and in conditions.connections.", "length": 129, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 13, "clauses": [ { "text": "RAM of compose engines, the benchmark's own SQL Server and Qdrant containers (sql, sql-diskann, qdrant, qdrant-hnsw) included, is docker stats of the engine's containers;", "length": 170, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "for the native compariso...", "length": 149, "class": "unverified", "basis": [], "printed": false, "notPrinted": "absent-targets", "unlisted": false }, { "text": "Disk is the table's reserved pages (SQL), the collection folder (Qdrant), or for other engines what the load added to the engine's data folder (engines that keep data in memory until a snapshot show almost nothing); when the folder was smaller after the load than before it, no figure is given.", "length": 294, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 14, "clauses": [ { "text": "Engines: run-all starts an engine that is down and stops it afterwards only if it was not running when the run began; an engine that was already running is left running.", "length": 169, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 16, "clauses": [ { "text": "Machine control on: governor performance on every CPU during the run (before: schedutil; at the end: performance; after putting it back: schedutil). CPU clock pinned for the run: turbo off (intel_pstate/no_turbo 0 -> 1)", "length": 219, "class": "read-back", "basis": [ "results:conditions.governor#performance", "results:conditions.clientCpus#0-1,4-5", "results:conditions.engineCpus#2-3,6-7" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": ", so every CPU's clock i...", "length": 83, "class": "unverified", "basis": [], "printed": false, "notPrinted": "clock-held", "unlisted": false }, { "text": "the uncore (L3 and memory) clock is held at 3000 MHz (min 3000 MHz, max 3000 MHz (MSR 0x620 = 0x1e1e); was min 1200 MHz, max 3000 MHz (MSR 0x620 = 0xc1e)) (before: turbo on (intel_pstate/no_turbo 0), ceiling 3600 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; while pinned: turbo off (intel_pstate/no_turbo 1), ceiling 3500 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; at the end: turbo off (intel_pstate/no_turbo 1), ceiling 3500 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; after putting it back: turbo on (intel_pstate/no_turbo 0), ceiling 3600 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7). Uncore limit at the end: min 3000 MHz, max 3000 MHz (MSR 0x620 = 0x1e1e); after putting it back: min 1200 MHz, max 3000 MHz (MSR 0x620 = 0xc1e). Each target's clock note gives every pass's median MHz on the engine CPUs and on the client CPUs (the median of each CPU's median); a pass whose median on either is more than 100 bp (1%) off the pinned 3500 MHz, or that has no reading on one, is flagged (conditions.clock.rule). Figures from runs made with turbo on are not comparable with these in absolute terms. engine CPUs 2-3,6-7 (cores 2,6 and 3,7), client CPUs 0-1,4-5 (cores 0,4 and 1,5); the client process was pinned; each engine was pinned to the engine CPUs for its turn and put back after (conditions.engines); an engine run-all started (and stopped) was asked to be created on them, and its notes say whether the host did so or it was moved there after the start.", "length": 1464, "class": "read-back", "basis": [ "results:conditions.governor#performance", "results:conditions.clientCpus#0-1,4-5", "results:conditions.engineCpus#2-3,6-7" ], "printed": true, "notPrinted": null, "unlisted": false }, { "text": "Busy box: right before each pass's warm-up the run waits, up to 10 min, while processes outside the benchmark (everything but this client and the engine under test's cgroups) use more than 0.3 CPUs on average over the last 60 s or the last 5 s, neither window reaching back past the start of the target or of its engine (so the engine's own start-up is not outside work); if the box does not clear the pass runs anyway, flagged 'busy box', as is a pass whose own outside load is above the limit. Outside load counts busy = user + nice + system + irq + softirq + steal (guest time is already in user); CONFIG_IRQ_TIME_ACCOUNTING is not set, so task and cgroup run time include the interrupt and softirq time that hit them and it is added; CONFIG_PARAVIRT_TIME_ACCOUNTING is not set, so steal is added (/boot/config-6.8.0-142-generic) (conditions.cpuAccounting); each pass also records this client's own CPU time per search and the engine's (conditions.passes[].clientCpuMsPerSearch and engineCpuMsPerSearch; how in conditions.engineCpu.rule). CPU clocks were sampled every 250 ms; each pass's min/median/max per CPU is in conditions.passes. CPU idle states, recorded and left as found: driver intel_idle, governor menu, intel_idle max_cstate 9; POLL on, C1 on, C1E on, C3 OFF on CPUs 0-7 (default disabled), C6 on (conditions.cpuIdle). How each target was reached: conditions.connections. Client build Release, .NET 10.0.12.", "length": 1423, "class": "unverified", "basis": [], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 17, "clauses": [ { "text": "duckdb is embedded: it ran inside the client process on the client CPUs 0-1,4-5, sharing them with the client.", "length": 110, "class": "read-back", "basis": [ "results:conditions.engines#embedded" ], "printed": true, "notPrinted": null, "unlisted": false } ] }, { "target": "", "field": "notes", "run": "20261007-205106-eshoponweb", "note": 18, "clauses": [ { "text": "sqlitevec is embedded: it ran inside the client process on the client CPUs 0-1,4-5, sharing them with the client.", "length": 113, "class": "read-back", "basis": [ "results:conditions.engines#embedded" ], "printed": true, "notPrinted": null, "unlisted": false } ] } ], "runPageNotes": [ { "runs": [ "20261005-023459-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261005-023459-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261005-023459-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "1.27 GiB added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (2.63 GiB at the end, 1.36 GiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 1,362,545,922 bytes.", "sources": [ { "kind": "results", "ref": "results@20261005-023459-eshoponweb:targets[clickhouse].disk.text", "value": "1.27 GiB added by this load (2.63 GiB (whole engine data folder), 1.36 GiB before)" }, { "kind": "results", "ref": "results@20261005-023459-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261005-023459-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261005-023459-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261005-023459-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261005-031556-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261005-031556-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261005-031556-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "906.61 MiB added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (3.52 GiB at the end, 2.64 GiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 950,649,750 bytes.", "sources": [ { "kind": "results", "ref": "results@20261005-031556-eshoponweb:targets[clickhouse].disk.text", "value": "906.61 MiB added by this load (3.52 GiB (whole engine data folder), 2.64 GiB before)" }, { "kind": "results", "ref": "results@20261005-031556-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261005-031556-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261005-031556-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261005-031556-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261005-035711-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261005-035711-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261005-035711-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "2.42 GiB added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (5.95 GiB at the end, 3.53 GiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 2,594,340,474 bytes.", "sources": [ { "kind": "results", "ref": "results@20261005-035711-eshoponweb:targets[clickhouse].disk.text", "value": "2.42 GiB added by this load (5.95 GiB (whole engine data folder), 3.53 GiB before)" }, { "kind": "results", "ref": "results@20261005-035711-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261005-035711-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261005-035711-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261005-035711-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261005-073329-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261005-073329-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261005-073329-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261005-073329-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261005-085448-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261005-085448-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261005-085448-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261005-085448-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261005-101815-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261005-101815-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261005-101815-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261005-101815-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261006-130619-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261006-130619-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "4.21 GiB added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (4.21 GiB at the end, 99.17 KiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 4,515,349,223 bytes.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[clickhouse].disk.text", "value": "4.21 GiB added by this load (4.21 GiB (whole engine data folder), 99.17 KiB before)" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261006-130619-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261006-130619-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261006-130619-eshoponweb" ], "target": "", "field": "notes", "statement": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "kind": "unbacked", "note": "No saved reading backs the clause for this run. The APERF and MPERF analyses kept in design/bench-inputs/observer-v8 are of the three v8 runs, not of this one, and the summary does not print the clause. In the v8 runs, made under the same pin, the observer read a pass under it (see the correction to the clause on those pages).", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:notes[14]#so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "value": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs" }, { "kind": "file", "ref": "design/bench-inputs/observer-v8/SOURCES.txt#Byte copies of the run operator's analysis of the three v8 runs (seeds 801, 802, 803)", "value": "Byte copies of the run operator's analysis of the three v8 runs (seeds 801, 802, 803)" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"clock-held\"", "value": "\"drop\": \"clock-held\"" } ] }, { "runs": [ "20261006-142724-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261006-142724-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "2.57 GiB added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (6.79 GiB at the end, 4.22 GiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 2,758,116,325 bytes.", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[clickhouse].disk.text", "value": "2.57 GiB added by this load (6.79 GiB (whole engine data folder), 4.22 GiB before)" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261006-142724-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261006-142724-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261006-142724-eshoponweb" ], "target": "", "field": "notes", "statement": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "kind": "unbacked", "note": "No saved reading backs the clause for this run. The APERF and MPERF analyses kept in design/bench-inputs/observer-v8 are of the three v8 runs, not of this one, and the summary does not print the clause. In the v8 runs, made under the same pin, the observer read a pass under it (see the correction to the clause on those pages).", "sources": [ { "kind": "results", "ref": "results@20261006-142724-eshoponweb:notes[14]#so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "value": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs" }, { "kind": "file", "ref": "design/bench-inputs/observer-v8/SOURCES.txt#Byte copies of the run operator's analysis of the three v8 runs (seeds 801, 802, 803)", "value": "Byte copies of the run operator's analysis of the three v8 runs (seeds 801, 802, 803)" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"clock-held\"", "value": "\"drop\": \"clock-held\"" } ] }, { "runs": [ "20261006-154837-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261006-154837-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "0 B added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (6.79 GiB at the end, 6.79 GiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 0 bytes.", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[clickhouse].disk.text", "value": "0 B added by this load (6.79 GiB (whole engine data folder), 6.79 GiB before)" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261006-154837-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261006-154837-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261006-154837-eshoponweb" ], "target": "", "field": "notes", "statement": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "kind": "unbacked", "note": "No saved reading backs the clause for this run. The APERF and MPERF analyses kept in design/bench-inputs/observer-v8 are of the three v8 runs, not of this one, and the summary does not print the clause. In the v8 runs, made under the same pin, the observer read a pass under it (see the correction to the clause on those pages).", "sources": [ { "kind": "results", "ref": "results@20261006-154837-eshoponweb:notes[14]#so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "value": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs" }, { "kind": "file", "ref": "design/bench-inputs/observer-v8/SOURCES.txt#Byte copies of the run operator's analysis of the three v8 runs (seeds 801, 802, 803)", "value": "Byte copies of the run operator's analysis of the three v8 runs (seeds 801, 802, 803)" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"clock-held\"", "value": "\"drop\": \"clock-held\"" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "5.29 GiB added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (5.31 GiB at the end, 14.01 MiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 5,685,109,857 bytes.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].disk.text", "value": "5.29 GiB added by this load (5.31 GiB (whole engine data folder), 14.01 MiB before)" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "clickhouse", "field": "dataFolder.reset", "statement": "(the folder was steady)", "kind": "misleading", "note": "The words describe the folder after the reset: it stayed within the tolerance over a stability window before the after reading was taken. They do not say the reset left the folder unchanged; the same sentence records 4.25 GiB before and 14.01 MiB after.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/ClickHouseStartState.cs#the folder was steady", "value": "the folder was steady" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/ClickHouseStartState.cs#True when the folder stayed within the tolerance over a full stability window before the deadline.", "value": "True when the folder stayed within the tolerance over a full stability window before the deadline." }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].dataFolder.reset", "value": "truncated 9 MergeTree log tables in database system (asynchronous_insert_log, asynchronous_metric_log, background_schedule_pool_log, metric_log, part_log, processors_profile_log, query_log, text_log, trace_log); ClickHouse's own active bytes of those tables 2.09 GiB before and 0 B after; data folder 4.25 GiB before and 14.01 MiB after (the folder was steady); not truncated, name does not end in _log: none" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "mariadb", "field": "dataFolder.bytesAtStart", "statement": "166.78 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 162,269,780 bytes (154.75 MiB) when the container was created, before the engine ran. The tool recorded 174,877,277 bytes (166.78 MiB) as the start, 8 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-801.txt#X12", "value": "('mariadb', 174877277, 162269780)" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].dataFolder.bytesAtStart", "value": "174877277" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "milvus", "field": "dataFolder.bytesAtStart", "statement": "218 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 164,473,352 bytes (156.85 MiB) when the container was created, before the engine ran. The tool recorded 228,593,016 bytes (218 MiB) as the start, 39 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-801.txt#X12", "value": "('milvus', 228593016, 164473352)" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[milvus].dataFolder.bytesAtStart", "value": "228593016" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "sql", "field": "dataFolder.bytesAtStart", "statement": "105.06 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 380,452,110 bytes (362.83 MiB) when the container was created, before the engine ran. The tool recorded 110,158,310 bytes (105.06 MiB) as the start, 71 percent below the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-801.txt#X12", "value": "('sql', 110158310, 380452110)" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sql].dataFolder.bytesAtStart", "value": "110158310" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "sql-diskann", "field": "dataFolder.bytesAtStart", "statement": "105.05 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 380,452,110 bytes (362.83 MiB) when the container was created, before the engine ran. The tool recorded 110,154,214 bytes (105.05 MiB) as the start, 71 percent below the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-801.txt#X12", "value": "('sql-diskann', 110154214, 380452110)" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].dataFolder.bytesAtStart", "value": "110154214" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "typesense", "field": "dataFolder.bytesAtStart", "statement": "707.22 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 293,394,812 bytes (279.8 MiB) when the container was created, before the engine ran. The tool recorded 741,572,783 bytes (707.22 MiB) as the start, 153 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-801.txt#X12", "value": "('typesense', 741572783, 293394812)" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[typesense].dataFolder.bytesAtStart", "value": "741572783" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261007-205106-eshoponweb" ], "target": "", "field": "notes", "statement": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "kind": "false", "note": "The observer's APERF and MPERF readings of this run put oracle's default@8 pass (eight searchers at once) under the pin: CPU 7 read 3476.97 MHz against 3500 MHz, 65.8 basis points under it (0.658 percent). That is inside the observer's own limit of 100 basis points: the pin held to within that, and it did not hold at the ceiling. The clause says every CPU's clock is held at its ceiling whatever the engine runs; for that pass it was not. The summary does not print the clause.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:notes[16]#so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "value": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs" }, { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-801.txt#worst deviation 65.8 bp (oracle/default@8 cpu7 3476.97 MHz)", "value": "worst deviation 65.8 bp (oracle/default@8 cpu7 3476.97 MHz)" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"clock-held\"", "value": "\"drop\": \"clock-held\"" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "4.18 GiB added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (6.9 GiB at the end, 2.72 GiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 4,491,193,144 bytes.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].disk.text", "value": "4.18 GiB added by this load (6.9 GiB (whole engine data folder), 2.72 GiB before)" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "clickhouse", "field": "dataFolder.reset", "statement": "(the folder was steady)", "kind": "misleading", "note": "The words describe the folder after the reset: it stayed within the tolerance over a stability window before the after reading was taken. They do not say the reset left the folder unchanged; the same sentence records 5.32 GiB before and 2.72 GiB after.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/ClickHouseStartState.cs#the folder was steady", "value": "the folder was steady" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/ClickHouseStartState.cs#True when the folder stayed within the tolerance over a full stability window before the deadline.", "value": "True when the folder stayed within the tolerance over a full stability window before the deadline." }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].dataFolder.reset", "value": "truncated 9 MergeTree log tables in database system (asynchronous_insert_log, asynchronous_metric_log, background_schedule_pool_log, metric_log, part_log, processors_profile_log, query_log, text_log, trace_log); ClickHouse's own active bytes of those tables 183.97 KiB before and 0 B after; data folder 5.32 GiB before and 2.72 GiB after (the folder was steady); not truncated, name does not end in _log: none" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "mariadb", "field": "dataFolder.bytesAtStart", "statement": "166.77 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 162,258,887 bytes (154.74 MiB) when the container was created, before the engine ran. The tool recorded 174,866,384 bytes (166.77 MiB) as the start, 8 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-802.txt#X12", "value": "('mariadb', 174866384, 162258887)" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[mariadb].dataFolder.bytesAtStart", "value": "174866384" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "milvus", "field": "dataFolder.bytesAtStart", "statement": "226.87 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 173,771,992 bytes (165.72 MiB) when the container was created, before the engine ran. The tool recorded 237,892,829 bytes (226.87 MiB) as the start, 37 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-802.txt#X12", "value": "('milvus', 237892829, 173771992)" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[milvus].dataFolder.bytesAtStart", "value": "237892829" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "sql", "field": "dataFolder.bytesAtStart", "statement": "105.07 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 380,458,862 bytes (362.83 MiB) when the container was created, before the engine ran. The tool recorded 110,169,158 bytes (105.07 MiB) as the start, 71 percent below the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-802.txt#X12", "value": "('sql', 110169158, 380458862)" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[sql].dataFolder.bytesAtStart", "value": "110169158" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "sql-diskann", "field": "dataFolder.bytesAtStart", "statement": "105.06 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 380,458,862 bytes (362.83 MiB) when the container was created, before the engine ran. The tool recorded 110,162,406 bytes (105.06 MiB) as the start, 71 percent below the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-802.txt#X12", "value": "('sql-diskann', 110162406, 380458862)" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[sql-diskann].dataFolder.bytesAtStart", "value": "110162406" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "typesense", "field": "dataFolder.bytesAtStart", "statement": "856.64 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 301,621,308 bytes (287.65 MiB) when the container was created, before the engine ran. The tool recorded 898,253,504 bytes (856.64 MiB) as the start, 198 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-802.txt#X12", "value": "('typesense', 898253504, 301621308)" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[typesense].dataFolder.bytesAtStart", "value": "898253504" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261007-221429-eshoponweb" ], "target": "", "field": "notes", "statement": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "kind": "false", "note": "The observer's APERF and MPERF readings of this run put oracle's default@8 pass (eight searchers at once) under the pin: CPU 7 read 3483.59 MHz against 3500 MHz, 46.9 basis points under it (0.469 percent). That is inside the observer's own limit of 100 basis points: the pin held to within that, and it did not hold at the ceiling. The clause says every CPU's clock is held at its ceiling whatever the engine runs; for that pass it was not. The summary does not print the clause.", "sources": [ { "kind": "results", "ref": "results@20261007-221429-eshoponweb:notes[16]#so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "value": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs" }, { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-802.txt#worst deviation 46.9 bp (oracle/default@8 cpu7 3483.59 MHz)", "value": "worst deviation 46.9 bp (oracle/default@8 cpu7 3483.59 MHz)" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"clock-held\"", "value": "\"drop\": \"clock-held\"" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "weaviate", "field": "durability", "statement": "weaviate.compose.yaml sets no persistence variable", "kind": "false", "note": "deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH to /var/lib/weaviate. The compose file does set a persistence variable.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH: /var/lib/weaviate", "value": "PERSISTENCE_DATA_PATH: /var/lib/weaviate" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "clickhouse", "field": "disk.text", "statement": "4.67 GiB added by this load", "kind": "misleading", "note": "The figure is the growth of the whole data folder over the engine's turn, as the same line says (6.76 GiB at the end, 2.09 GiB before). The load itself is 524 vectors of 1024 dimensions, 2,146,304 bytes as 4-byte floats (2.05 MiB); the figure is 5,014,957,314 bytes.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].disk.text", "value": "4.67 GiB added by this load (6.76 GiB (whole engine data folder), 2.09 GiB before)" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:rows", "value": "524" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:dimension", "value": "1024" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "clickhouse", "field": "dataFolder.reset", "statement": "(the folder was steady)", "kind": "misleading", "note": "The words describe the folder after the reset: it stayed within the tolerance over a stability window before the after reading was taken. They do not say the reset left the folder unchanged; the same sentence records 6.92 GiB before and 2.09 GiB after.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/ClickHouseStartState.cs#the folder was steady", "value": "the folder was steady" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/ClickHouseStartState.cs#True when the folder stayed within the tolerance over a full stability window before the deadline.", "value": "True when the folder stayed within the tolerance over a full stability window before the deadline." }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].dataFolder.reset", "value": "truncated 9 MergeTree log tables in database system (asynchronous_insert_log, asynchronous_metric_log, background_schedule_pool_log, metric_log, part_log, processors_profile_log, query_log, text_log, trace_log); ClickHouse's own active bytes of those tables 185.06 KiB before and 0 B after; data folder 6.92 GiB before and 2.09 GiB after (the folder was steady); not truncated, name does not end in _log: none" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "mariadb", "field": "dataFolder.bytesAtStart", "statement": "166.77 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 162,258,911 bytes (154.74 MiB) when the container was created, before the engine ran. The tool recorded 174,866,408 bytes (166.77 MiB) as the start, 8 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-803.txt#X12", "value": "('mariadb', 174866408, 162258911)" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[mariadb].dataFolder.bytesAtStart", "value": "174866408" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "milvus", "field": "dataFolder.bytesAtStart", "statement": "235.67 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 183,071,822 bytes (174.59 MiB) when the container was created, before the engine ran. The tool recorded 247,123,030 bytes (235.67 MiB) as the start, 35 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-803.txt#X12", "value": "('milvus', 247123030, 183071822)" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[milvus].dataFolder.bytesAtStart", "value": "247123030" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "sql", "field": "dataFolder.bytesAtStart", "statement": "105.07 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 380,464,398 bytes (362.84 MiB) when the container was created, before the engine ran. The tool recorded 110,170,598 bytes (105.07 MiB) as the start, 71 percent below the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-803.txt#X12", "value": "('sql', 110170598, 380464398)" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[sql].dataFolder.bytesAtStart", "value": "110170598" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "sql-diskann", "field": "dataFolder.bytesAtStart", "statement": "105.06 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 380,464,398 bytes (362.84 MiB) when the container was created, before the engine ran. The tool recorded 110,167,942 bytes (105.06 MiB) as the start, 71 percent below the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-803.txt#X12", "value": "('sql-diskann', 110167942, 380464398)" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[sql-diskann].dataFolder.bytesAtStart", "value": "110167942" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "typesense", "field": "dataFolder.bytesAtStart", "statement": "688.52 MiB at the start", "kind": "misleading", "note": "A witness read this data folder at 308,306,092 bytes (294.02 MiB) when the container was created, before the engine ran. The tool recorded 721,960,355 bytes (688.52 MiB) as the start, 134 percent above the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-803.txt#X12", "value": "('typesense', 721960355, 308306092)" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[typesense].dataFolder.bytesAtStart", "value": "721960355" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "vespa", "field": "dataFolder.bytesAtStart", "statement": "1.9 GiB at the start", "kind": "misleading", "note": "A witness read this data folder at 2,280,491,799 bytes (2.12 GiB) when the container was created, before the engine ran. The tool recorded 2,042,392,842 bytes (1.9 GiB) as the start, 10 percent below the witness. The two readings disagree; this line prints the tool's.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-803.txt#X12", "value": "('vespa', 2042392842, 2280491799)" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[vespa].dataFolder.bytesAtStart", "value": "2042392842" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "vespa", "field": "disk.text", "statement": "235.33 MiB added by this load", "kind": "misleading", "note": "The figure rests on the tool's start reading (2,042,392,842 bytes), which the witness disputes (above). From the witness's reading and the recorded end (2,291,891,892 bytes) the folder grew by 11,400,093 bytes (10.87 MiB) over the turn.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-803.txt#X12", "value": "('vespa', 2042392842, 2280491799)" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[vespa].dataFolder.bytesAtStart", "value": "2042392842" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[vespa].dataFolder.bytesAtEnd", "value": "2291891892" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "", "field": "framing line", "statement": "Measured end to end through each engine's .NET client", "kind": "false", "note": "The statement is false for 8 of the 19 engines of this run: chroma, clickhouse, elasticsearch, milvus, opensearch, typesense, vespa and weaviate are reached through the benchmark's own HttpClient REST code, not through a .NET client of the engine. For those engines the figure is the cost of the benchmark's own request code. A code line of each is cited.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "", "field": "index", "statement": "an earlier run gave about 0.09 at 100,000", "kind": "unbacked", "note": "No log was saved for this figure, and the summary does not print the clause. A saved re-run of the MariaDB effort test (10 repeats, 524 and 2,000 random 1024-dimension vectors) read recall@10 at ef 100 of 0.988 to 0.996 at 524 vectors and 0.894 to 0.932 at 2,000. The 100,000-row figure was not re-measured the same way and is dropped.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#for which no log was saved", "value": "for which no log was saved" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/SOURCES.txt#The 100,000-row figure was not re-measured the same way, so the report drops it.", "value": "The 100,000-row figure was not re-measured the same way, so the report drops it." }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.872, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.886, ef 100 0.996, ef 3200 0.996" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"mariadb-recall\"", "value": "\"drop\": \"mariadb-recall\"" } ] }, { "runs": [ "20261007-233551-eshoponweb" ], "target": "", "field": "notes", "statement": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "kind": "false", "note": "The observer's APERF and MPERF readings of this run put oracle's default@8 pass (eight searchers at once) under the pin: CPU 7 read 3482.66 MHz against 3500 MHz, 49.5 basis points under it (0.495 percent). That is inside the observer's own limit of 100 basis points: the pin held to within that, and it did not hold at the ceiling. The clause says every CPU's clock is held at its ceiling whatever the engine runs; for that pass it was not. The summary does not print the clause.", "sources": [ { "kind": "results", "ref": "results@20261007-233551-eshoponweb:notes[16]#so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs", "value": "so every CPU's clock is held at its ceiling of 3500 MHz whatever the engine runs" }, { "kind": "file", "ref": "design/bench-inputs/observer-v8/analysis-803.txt#worst deviation 49.5 bp (oracle/default@8 cpu7 3482.66 MHz)", "value": "worst deviation 49.5 bp (oracle/default@8 cpu7 3482.66 MHz)" }, { "kind": "file", "ref": "deploy/bench/recorded-text-classes.json#\"drop\": \"clock-held\"", "value": "\"drop\": \"clock-held\"" } ] } ], "sentences": [ { "slot": "headline", "text": "redis had the lowest p50 latency: every other engine took at least 1.45 times as long, in each session; redis holds its data in memory.", "sources": [ { "kind": "consolidated", "ref": "consolidated:threshold.tBp|bp-ratio", "value": "1.45" }, { "kind": "consolidated", "ref": "consolidated:metrics[metric=p50Ms].rows[0].target", "value": "redis" }, { "kind": "doc", "ref": "doc:design/engine-docs/redis-faq-2026-10-07.html#Redis is an in-memory but persistent on disk database", "value": "Redis is an in-memory but persistent on disk database" } ] }, { "slot": "subtitle.data", "text": "Data: eshoponweb, 524 vectors of 1024 dimensions; 20 queries, top 10 hits each.", "sources": [ { "kind": "results", "ref": "results:pipeline", "value": "eshoponweb" }, { "kind": "results", "ref": "results:rows", "value": "524" }, { "kind": "results", "ref": "results:dimension", "value": "1024" }, { "kind": "results", "ref": "results:queryCount", "value": "20" }, { "kind": "results", "ref": "results:top", "value": "10" } ] }, { "slot": "subtitle.machine", "text": "Machine: Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz, 8 logical CPUs, 62.7 GiB of RAM.", "sources": [ { "kind": "results", "ref": "results:machine.cpu", "value": "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz" }, { "kind": "results", "ref": "results:machine.logicalCpus", "value": "8" }, { "kind": "results", "ref": "results:machine.ramGiB", "value": "62.7" } ] }, { "slot": "subtitle.scope", "text": "At 524 vectors each figure is the cost of one request through the benchmark's client for that engine, and does not show how an index scales.", "sources": [ { "kind": "results", "ref": "results:rows", "value": "524" } ] }, { "slot": "subtitle.clients", "text": "Of the 19 engines, 8 are reached through the benchmark's own HttpClient REST code: elasticsearch, vespa, opensearch, chroma, milvus, typesense, clickhouse and weaviate.", "sources": [ { "kind": "consolidated", "ref": "consolidated:targetCount", "value": "19" }, { "kind": "consolidated", "ref": "consolidated:httpClientCount", "value": "8" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaRest.cs#_data = new HttpClient( handler )", "value": "_data = new HttpClient( handler )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MilvusSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/TypesenseRest.cs#new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )", "value": "new HttpClient { BaseAddress = new Uri( _baseUrl + \"/\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ClickHouseSink.cs#new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"", "value": "new HttpClient { BaseAddress = new Uri( $\"http://{options.Host}:{options.Port}/\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#new HttpClient { BaseAddress = new Uri( options.BaseUrl", "value": "new HttpClient { BaseAddress = new Uri( options.BaseUrl" } ] }, { "slot": "subtitle.session.v7", "text": "Session v7: runs 20261006-130619-eshoponweb, 20261006-142724-eshoponweb and 20261006-154837-eshoponweb, started from 2026-10-06T13:06:19Z to 2026-10-06T15:48:37Z.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:startedUtc", "value": "2026-10-06T13:06:19Z" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:startedUtc", "value": "2026-10-06T14:27:24Z" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:startedUtc", "value": "2026-10-06T15:48:37Z" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] }, { "slot": "subtitle.session.v8", "text": "Session v8: runs 20261007-205106-eshoponweb, 20261007-221429-eshoponweb and 20261007-233551-eshoponweb, started from 2026-10-07T20:51:06Z to 2026-10-07T23:35:51Z.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:startedUtc", "value": "2026-10-07T20:51:06Z" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:startedUtc", "value": "2026-10-07T22:14:29Z" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:startedUtc", "value": "2026-10-07T23:35:51Z" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" } ] }, { "slot": "subtitle.build.v8", "text": "The runs of v8 were measured by build 9b924200abc3.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:conditions.build.commit#9b924200abc3", "value": "9b924200abc3" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:conditions.build.commit#9b924200abc3", "value": "9b924200abc3" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:conditions.build.commit#9b924200abc3", "value": "9b924200abc3" }, { "kind": "consolidated", "ref": "consolidated:build.measured[1].session", "value": "v8" } ] }, { "slot": "subtitle.build.consolidated", "text": "This report was consolidated by build 9e5878ce5dc0, not by the build that measured the runs of v8.", "sources": [ { "kind": "consolidated", "ref": "consolidated:build.consolidatedBy.commitShort", "value": "9e5878ce5dc0" }, { "kind": "consolidated", "ref": "consolidated:build.measured[session=v8].session", "value": "v8" } ] }, { "slot": "subtitle.questions", "text": "The runs of v8 read a question file with the same SHA256 hash as design/bench-inputs/questions_golden.json.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:queriesFileSha256", "value": "e3a917ec8c5e687d2a2d6b83bc8ffd7eb901f0ffac8d8b277995b86898e9a712" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:queriesFileSha256", "value": "e3a917ec8c5e687d2a2d6b83bc8ffd7eb901f0ffac8d8b277995b86898e9a712" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:queriesFileSha256", "value": "e3a917ec8c5e687d2a2d6b83bc8ffd7eb901f0ffac8d8b277995b86898e9a712" }, { "kind": "consolidated", "ref": "consolidated:queries.copySha256", "value": "e3a917ec8c5e687d2a2d6b83bc8ffd7eb901f0ffac8d8b277995b86898e9a712" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Data/GoldenQueries.cs#Convert.ToHexString( SHA256.HashData( bytes ) ).ToLowerInvariant()", "value": "Convert.ToHexString( SHA256.HashData( bytes ) ).ToLowerInvariant()" } ] }, { "slot": "subtitle.truth", "text": "The runs of v7 record no question-file hash, and truthNdcg reads 0.5181699774768911 in all 6 claim runs.", "sources": [ { "kind": "results", "ref": "results:truthNdcg", "value": "0.5181699774768911" }, { "kind": "consolidated", "ref": "consolidated:claimRunCount", "value": "6" } ] }, { "slot": "subtitle.queries.v7", "text": "golden: 20 labelled questions from ~/ForClaude/evalkit/questions_golden.json, embedded with qwen3-emb-0.6b (cached in ~/gvb-data/bench-cache/golden-eshoponweb-68df42ca69efa079.json, no embedding calls)", "sources": [ { "kind": "quote", "ref": "results@20261006-130619-eshoponweb:queries", "value": "golden: 20 labelled questions from ~/ForClaude/evalkit/questions_golden.json, embedded with qwen3-emb-0.6b (cached in ~/gvb-data/bench-cache/golden-eshoponweb-68df42ca69efa079.json, no embedding calls)" } ] }, { "slot": "rule.claim", "text": "An engine is shown ahead of another when, in each session separately, its slowest run beat the other's fastest run by at least 1.45 times; every other pair is not separated by this test.", "sources": [ { "kind": "consolidated", "ref": "consolidated:threshold.tBp|bp-ratio", "value": "1.45" } ] }, { "slot": "rule.threshold", "text": "The threshold is 45%: the smallest multiple of 5% at least 5% above the largest move in the basis, 35.82%, and never below 35%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:threshold.tBp|bp-pct", "value": "45" }, { "kind": "consolidated", "ref": "consolidated:basis.stepBp|bp-pct", "value": "5" }, { "kind": "consolidated", "ref": "consolidated:basis.marginBp|bp-pct", "value": "5" }, { "kind": "consolidated", "ref": "consolidated:basis.maxBp|bp-pct", "value": "35.82" }, { "kind": "consolidated", "ref": "consolidated:basis.floorBp|bp-pct", "value": "35" } ] }, { "slot": "basis.runs", "text": "The basis holds the 12 runs of sessions v5, v6, v7 and v8, all with machine control on.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.runCount", "value": "12" } ] }, { "slot": "basis.clock", "text": "The clock was held in the runs of v7 and v8 and not in those of v5 and v6.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v7].session", "value": "v7" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v7].held", "value": "3" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v7].runs", "value": "3" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v8].session", "value": "v8" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v8].held", "value": "3" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v8].runs", "value": "3" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v5].session", "value": "v5" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v5].held", "value": "0" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v5].runs", "value": "3" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v6].session", "value": "v6" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v6].held", "value": "0" }, { "kind": "consolidated", "ref": "consolidated:basis.clockHeld[session=v6].runs", "value": "3" } ] }, { "slot": "basis.identity", "text": "Basis moves are taken between runs in which the engine recorded the same setup: its engine text, index text, search settings, durability text, engine files, engine settings, image id and hosting.", "sources": [] }, { "slot": "basis.identity.missing", "text": "A field other than the engine text and the index text that one of two runs did not record is not compared.", "sources": [] }, { "slot": "basis.max", "text": "The largest move was 35.82%, the QPS@8 ratio of clickhouse/oracle between runs 20261006-142724-eshoponweb (seed 702) and 20261007-205106-eshoponweb (seed 801).", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.maxMove.moveBp|bp-pct", "value": "35.82" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMove.pair", "value": "clickhouse/oracle" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMove.runs[0]", "value": "20261006-142724-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMove.runs[1]", "value": "20261007-205106-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMove.seeds[0]", "value": "702" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMove.seeds[1]", "value": "801" } ] }, { "slot": "basis.max.rounding", "text": "Each move is rounded up to the next basis point before it is printed: the largest, 35.815% unrounded, prints as 35.82%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.maxMove.moveBpExact|bp-pct", "value": "35.815" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMove.moveBp|bp-pct", "value": "35.82" }, { "kind": "consolidated", "ref": "consolidated:basis.moveRule", "value": "moveBp = ceiling(10000 x hi / lo) - 10000, with hi and lo the two values (one engine) or the two same-run ratios (pair)" } ] }, { "slot": "basis.max.differences", "text": "Those two runs differ in build.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.maxMove.differenceNames[0]", "value": "build" } ] }, { "slot": "basis.max.notHeld.0", "text": "Run 20261007-205106-eshoponweb: the timed eight-searcher pass of clickhouse read 336 QPS, 24% from the settled 418 QPS (limit 10%), and the run recorded it as NOT HELD.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.maxMoveNotHeld[0].run", "value": "20261007-205106-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMoveNotHeld[0].target", "value": "clickhouse" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMoveNotHeld[0].timed", "value": "336 QPS" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMoveNotHeld[0].offPercent", "value": "24" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMoveNotHeld[0].settled", "value": "418 QPS" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMoveNotHeld[0].limitPercent", "value": "10" } ] }, { "slot": "basis.max.without", "text": "The largest move includes a NOT HELD pass; with clickhouse left out of the basis, the largest move is 29.08%, the QPS@8 ratio of mongodb/oracle, and the threshold would be 35%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.maxMoveWithout.maxBp|bp-pct", "value": "29.08" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMoveWithout.tBp|bp-pct", "value": "35" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMoveWithout.maxMove.pair", "value": "mongodb/oracle" }, { "kind": "consolidated", "ref": "consolidated:basis.maxMoveWithout.targets[0]", "value": "clickhouse" } ] }, { "slot": "basis.all.p50Ms", "text": "p50, between basis runs of one recorded setup: largest moves 15.28% (redis) for one engine and 26.34% (mongodb/redis) for a pair.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.perMetric.p50Ms.oneEngine.moveBp|bp-pct", "value": "15.28" }, { "kind": "consolidated", "ref": "consolidated:basis.perMetric.p50Ms.pair.moveBp|bp-pct", "value": "26.34" } ] }, { "slot": "basis.same.p50Ms", "text": "p50, between those runs that also share turbo, uncore, warm-up and rehearsal: largest moves 12.02% (mongodb) for one engine and 18.88% (mongodb/redis) for a pair.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.perMetric.p50Ms.oneEngineSameSettings.moveBp|bp-pct", "value": "12.02" }, { "kind": "consolidated", "ref": "consolidated:basis.perMetric.p50Ms.pairSameSettings.moveBp|bp-pct", "value": "18.88" } ] }, { "slot": "basis.all.qps1", "text": "QPS@1, between basis runs of one recorded setup: largest moves 11.49% (mongodb) for one engine and 19.33% (mongodb/pgvector) for a pair.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.perMetric.qps1.oneEngine.moveBp|bp-pct", "value": "11.49" }, { "kind": "consolidated", "ref": "consolidated:basis.perMetric.qps1.pair.moveBp|bp-pct", "value": "19.33" } ] }, { "slot": "basis.same.qps1", "text": "QPS@1, between those runs that also share turbo, uncore, warm-up and rehearsal: largest moves 11.49% (mongodb) for one engine and 14.01% (mongodb/sqlitevec) for a pair.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.perMetric.qps1.oneEngineSameSettings.moveBp|bp-pct", "value": "11.49" }, { "kind": "consolidated", "ref": "consolidated:basis.perMetric.qps1.pairSameSettings.moveBp|bp-pct", "value": "14.01" } ] }, { "slot": "basis.all.qps8", "text": "QPS@8, between basis runs of one recorded setup: largest moves 27.36% (clickhouse) for one engine and 35.82% (clickhouse/oracle) for a pair.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.perMetric.qps8.oneEngine.moveBp|bp-pct", "value": "27.36" }, { "kind": "consolidated", "ref": "consolidated:basis.perMetric.qps8.pair.moveBp|bp-pct", "value": "35.82" } ] }, { "slot": "basis.same.qps8", "text": "QPS@8, between those runs that also share turbo, uncore, warm-up and rehearsal: largest moves 27.36% (clickhouse) for one engine and 35.82% (clickhouse/oracle) for a pair.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.perMetric.qps8.oneEngineSameSettings.moveBp|bp-pct", "value": "27.36" }, { "kind": "consolidated", "ref": "consolidated:basis.perMetric.qps8.pairSameSettings.moveBp|bp-pct", "value": "35.82" } ] }, { "slot": "basis.all.exactP50Ms", "text": "exact p50, between basis runs of one recorded setup: largest moves 20.23% (redis) for one engine and 19.66% (qdrant-hnsw/redis) for a pair.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.perMetric.exactP50Ms.oneEngine.moveBp|bp-pct", "value": "20.23" }, { "kind": "consolidated", "ref": "consolidated:basis.perMetric.exactP50Ms.pair.moveBp|bp-pct", "value": "19.66" } ] }, { "slot": "basis.same.exactP50Ms", "text": "exact p50, between those runs that also share turbo, uncore, warm-up and rehearsal: largest moves 7.74% (opensearch) for one engine and 8.41% (opensearch/sqlitevec) for a pair.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.perMetric.exactP50Ms.oneEngineSameSettings.moveBp|bp-pct", "value": "7.74" }, { "kind": "consolidated", "ref": "consolidated:basis.perMetric.exactP50Ms.pairSameSettings.moveBp|bp-pct", "value": "8.41" } ] }, { "slot": "basis.splits", "text": "Changes of recorded setup left out of the basis moves: 4, in clickhouse, duckdb, mariadb and sqlitevec.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplitCount", "value": "4" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].target", "value": "clickhouse" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].target", "value": "duckdb" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].target", "value": "mariadb" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].target", "value": "sqlitevec" } ] }, { "slot": "basis.splits.each", "text": "Each is listed with the field that changed, the move it hides and the threshold it would give.", "sources": [] }, { "slot": "basis.split.0.fields", "text": "Between its v6 runs and its v7 and v8 runs, clickhouse recorded a different engine.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].target", "value": "clickhouse" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].before.sessions[0]", "value": "v6" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].after.sessions[0]", "value": "v7" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].after.sessions[1]", "value": "v8" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].fields[0]", "value": "engine" } ] }, { "slot": "basis.split.0.moves", "text": "Counted across that change, the largest one-engine move is 47.36% (QPS@8) and the largest pair move 52.08% (clickhouse/redis QPS@8), and the threshold would be 60%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].oneEngine.moveBp|bp-pct", "value": "47.36" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].pair.moveBp|bp-pct", "value": "52.08" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].pair.pair", "value": "clickhouse/redis" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].tBpIfCounted|bp-pct", "value": "60" } ] }, { "slot": "basis.split.0.conditions", "text": "The two runs of that one-engine move also differ in turbo, uncore, warmup, build and median MHz.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].oneEngine.differenceNames[0]", "value": "turbo" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].oneEngine.differenceNames[1]", "value": "uncore" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].oneEngine.differenceNames[2]", "value": "warmup" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].oneEngine.differenceNames[3]", "value": "build" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[0].oneEngine.differenceNames[4]", "value": "median MHz" } ] }, { "slot": "basis.split.1.fields", "text": "Between its v5 runs and its v6, v7 and v8 runs, duckdb recorded a different index.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].target", "value": "duckdb" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].before.sessions[0]", "value": "v5" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].after.sessions[0]", "value": "v6" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].after.sessions[1]", "value": "v7" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].after.sessions[2]", "value": "v8" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].fields[0]", "value": "index" } ] }, { "slot": "basis.split.1.moves", "text": "Counted across that change, the largest one-engine move is 115.56% (QPS@8) and the largest pair move 144.95% (duckdb/oracle QPS@8), and the threshold would be 150%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].oneEngine.moveBp|bp-pct", "value": "115.56" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].pair.moveBp|bp-pct", "value": "144.95" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].pair.pair", "value": "duckdb/oracle" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].tBpIfCounted|bp-pct", "value": "150" } ] }, { "slot": "basis.split.1.conditions", "text": "The two runs of that one-engine move also differ in warmup, rehearsal, build, median MHz and whether searchSettings was recorded.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].oneEngine.differenceNames[0]", "value": "warmup" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].oneEngine.differenceNames[1]", "value": "rehearsal" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].oneEngine.differenceNames[2]", "value": "build" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].oneEngine.differenceNames[3]", "value": "median MHz" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[1].oneEngine.differenceNames[4]", "value": "whether searchSettings was recorded" } ] }, { "slot": "basis.split.2.fields", "text": "Between its v5 runs and its v6, v7 and v8 runs, mariadb recorded a different durability.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].target", "value": "mariadb" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].before.sessions[0]", "value": "v5" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].after.sessions[0]", "value": "v6" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].after.sessions[1]", "value": "v7" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].after.sessions[2]", "value": "v8" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].fields[0]", "value": "durability" } ] }, { "slot": "basis.split.2.moves", "text": "Counted across that change, the largest one-engine move is 4.09% (exact p50) and the largest pair move 17.85% (mariadb/oracle QPS@8), and the threshold would be 45%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].oneEngine.moveBp|bp-pct", "value": "4.09" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].pair.moveBp|bp-pct", "value": "17.85" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].pair.pair", "value": "mariadb/oracle" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].tBpIfCounted|bp-pct", "value": "45" } ] }, { "slot": "basis.split.2.conditions", "text": "The two runs of that one-engine move also differ in turbo, uncore, warmup, rehearsal, build, median MHz and whether searchSettings was recorded.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].oneEngine.differenceNames[0]", "value": "turbo" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].oneEngine.differenceNames[1]", "value": "uncore" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].oneEngine.differenceNames[2]", "value": "warmup" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].oneEngine.differenceNames[3]", "value": "rehearsal" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].oneEngine.differenceNames[4]", "value": "build" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].oneEngine.differenceNames[5]", "value": "median MHz" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[2].oneEngine.differenceNames[6]", "value": "whether searchSettings was recorded" } ] }, { "slot": "basis.split.3.fields", "text": "Between its v5 runs and its v6, v7 and v8 runs, sqlitevec recorded a different index.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].target", "value": "sqlitevec" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].before.sessions[0]", "value": "v5" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].after.sessions[0]", "value": "v6" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].after.sessions[1]", "value": "v7" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].after.sessions[2]", "value": "v8" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].fields[0]", "value": "index" } ] }, { "slot": "basis.split.3.moves", "text": "Counted across that change, the largest one-engine move is 119.36% (QPS@8) and the largest pair move 155.08% (oracle/sqlitevec QPS@8), and the threshold would be 165%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].oneEngine.moveBp|bp-pct", "value": "119.36" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].pair.moveBp|bp-pct", "value": "155.08" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].pair.pair", "value": "oracle/sqlitevec" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].tBpIfCounted|bp-pct", "value": "165" } ] }, { "slot": "basis.split.3.conditions", "text": "The two runs of that one-engine move also differ in warmup, rehearsal, build, median MHz and whether searchSettings was recorded.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].oneEngine.differenceNames[0]", "value": "warmup" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].oneEngine.differenceNames[1]", "value": "rehearsal" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].oneEngine.differenceNames[2]", "value": "build" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].oneEngine.differenceNames[3]", "value": "median MHz" }, { "kind": "consolidated", "ref": "consolidated:basis.setupSplits[3].oneEngine.differenceNames[4]", "value": "whether searchSettings was recorded" } ] }, { "slot": "basis.exclusion.0", "text": "vespa QPS@8, seed list 502 and 503, is left out of the basis (kind: method defect); design/verdicts/v5-verdict.txt, item 2, holds the words: The @8 pass ran first in runs 502 and 503.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.exclusions[0].seeds[0]", "value": "502" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[0].seeds[1]", "value": "503" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[0].item", "value": "2" }, { "kind": "file", "ref": "design/verdicts/v5-verdict.txt#The @8 pass ran first in runs 502 and 503", "value": "The @8 pass ran first in runs 502 and 503" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[0].source", "value": "design/verdicts/v5-verdict.txt" } ] }, { "slot": "basis.exclusion.1", "text": "elasticsearch QPS@8, seed list 503, is left out of the basis (kind: method defect); design/verdicts/v5-verdict.txt, item 2, holds the words: run 503, the one run where @8 was its first pass.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.exclusions[1].seeds[0]", "value": "503" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[1].item", "value": "2" }, { "kind": "file", "ref": "design/verdicts/v5-verdict.txt#run 503, the one run where @8 was its first pass", "value": "run 503, the one run where @8 was its first pass" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[1].source", "value": "design/verdicts/v5-verdict.txt" } ] }, { "slot": "basis.exclusion.2", "text": "clickhouse every metric, seed list 501, 502 and 503, is left out of the basis (kind: unrecorded config change); design/verdicts/v6-verdict.md, item 1, holds the words: after the v5 runs ended at 04:38.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.exclusions[2].seeds[0]", "value": "501" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[2].seeds[1]", "value": "502" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[2].seeds[2]", "value": "503" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[2].item", "value": "1" }, { "kind": "file", "ref": "design/verdicts/v6-verdict.md#after the v5 runs ended at 04:38", "value": "after the v5 runs ended at 04:38" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusions[2].source", "value": "design/verdicts/v6-verdict.md" } ] }, { "slot": "basis.kept.method-defect", "text": "With the method defect rows kept in, the largest moves would be 74.42% for one engine and 95.64% for a pair, and the threshold 105%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.exclusionsKept[kind=method defect].oneEngine.moveBp|bp-pct", "value": "74.42" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusionsKept[kind=method defect].pair.moveBp|bp-pct", "value": "95.64" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusionsKept[kind=method defect].tBpIfKept|bp-pct", "value": "105" } ] }, { "slot": "basis.kept.unrecorded-config-change", "text": "With the unrecorded config change rows kept in, the largest moves would be 27.36% for one engine and 35.82% for a pair, and the threshold 45%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.exclusionsKept[kind=unrecorded config change].oneEngine.moveBp|bp-pct", "value": "27.36" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusionsKept[kind=unrecorded config change].pair.moveBp|bp-pct", "value": "35.82" }, { "kind": "consolidated", "ref": "consolidated:basis.exclusionsKept[kind=unrecorded config change].tBpIfKept|bp-pct", "value": "45" } ] }, { "slot": "basis.noMachineControl", "text": "In the 6 runs without machine control, one engine moved up to 112.82% and a pair up to 148.82%; the threshold rests on runs with machine control on.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.noMachineControlCount", "value": "6" }, { "kind": "consolidated", "ref": "consolidated:basis.noMachineControl.oneEngine.moveBp|bp-pct", "value": "112.82" }, { "kind": "consolidated", "ref": "consolidated:basis.noMachineControl.pair.moveBp|bp-pct", "value": "148.82" } ] }, { "slot": "basis.leftOut", "text": "6 other runs of this pipeline are not in the basis; each is listed with its reason.", "sources": [ { "kind": "consolidated", "ref": "consolidated:basis.leftOutCount", "value": "6" } ] }, { "slot": "table.p50Ms.caption", "text": "p50 is the median latency of one searcher's searches, in ms; rows are in order of the median of 6 runs.", "sources": [ { "kind": "consolidated", "ref": "consolidated:runsShown", "value": "6" } ] }, { "slot": "table.qps1.caption", "text": "QPS@1 is searches per second of the same one-searcher pass as p50, so it is no second confirmation of the p50 order.", "sources": [] }, { "slot": "table.qps8.caption", "text": "QPS@8 is searches per second with 8 searchers at once.", "sources": [ { "kind": "results", "ref": "results:concurrency[1]", "value": "8" } ] }, { "slot": "table.qps8.notHeld.clickhouse", "text": "clickhouse is shown and not ranked in this table: its timed eight-searcher pass was recorded NOT HELD in 3 of the 6 runs.", "sources": [ { "kind": "consolidated", "ref": "consolidated:notHeld.unranked[0].target", "value": "clickhouse" }, { "kind": "consolidated", "ref": "consolidated:notHeld.unranked[0].runs", "value": "3" }, { "kind": "consolidated", "ref": "consolidated:runsShown", "value": "6" } ] }, { "slot": "table.exactP50Ms.caption", "text": "Exact p50 is the median latency of each engine's own exact mode, which is a different operation per engine; the why table's CPU figures come from the other passes.", "sources": [] }, { "slot": "table.exactP50Ms.absent.sql", "text": "sql has no exact pass; its search fact says: no index used, exact scan by design.", "sources": [ { "kind": "results", "ref": "results:targets[sql].indexState.afterLoad.detail#no index used, exact scan by design", "value": "no index used, exact scan by design" } ] }, { "slot": "table.exactP50Ms.absent.qdrant", "text": "qdrant has no exact pass; its search fact says: no index used, exact scan by design.", "sources": [ { "kind": "results", "ref": "results:targets[qdrant].indexState.afterLoad.detail#no index used, exact scan by design", "value": "no index used, exact scan by design" } ] }, { "slot": "table.exactP50Ms.absent.milvus", "text": "milvus has no exact pass in these runs.", "sources": [] }, { "slot": "table.exactP50Ms.absent.weaviate", "text": "weaviate has no exact pass in these runs.", "sources": [] }, { "slot": "table.exactP50Ms.absent.chroma", "text": "chroma has no exact pass in these runs.", "sources": [] }, { "slot": "recall.caption", "text": "Recall counts hits of the exact top 10 over 20 queries, 200 per run; it is printed, never ranked.", "sources": [ { "kind": "results", "ref": "results:top", "value": "10" }, { "kind": "results", "ref": "results:queryCount", "value": "20" }, { "kind": "consolidated", "ref": "consolidated:recall[0].of", "value": "200" } ] }, { "slot": "recall.derived.v7", "text": "The recall hits of v7 are the recall of each run times 20 queries times 10 hits, rounded; those runs record no hit count.", "sources": [ { "kind": "consolidated", "ref": "consolidated:recallDerivedSessions[0]", "value": "v7" }, { "kind": "results", "ref": "results:queryCount", "value": "20" }, { "kind": "results", "ref": "results:top", "value": "10" } ] }, { "slot": "recall.differs.clickhouse", "text": "clickhouse's recall hits differ between runs: v7 199, 193, 198; v8 199, 199, 199.", "sources": [ { "kind": "consolidated", "ref": "consolidated:recall[target=clickhouse].hits.v7[0]", "value": "199" }, { "kind": "consolidated", "ref": "consolidated:recall[target=clickhouse].hits.v7[1]", "value": "193" }, { "kind": "consolidated", "ref": "consolidated:recall[target=clickhouse].hits.v7[2]", "value": "198" }, { "kind": "consolidated", "ref": "consolidated:recall[target=clickhouse].hits.v8[0]", "value": "199" }, { "kind": "consolidated", "ref": "consolidated:recall[target=clickhouse].hits.v8[1]", "value": "199" }, { "kind": "consolidated", "ref": "consolidated:recall[target=clickhouse].hits.v8[2]", "value": "199" } ] }, { "slot": "why.framing.1", "text": "CPU per search is cost summed over every thread.", "sources": [] }, { "slot": "why.framing.2", "text": "It can exceed the time per search, so it is not a split of the latency.", "sources": [] }, { "slot": "why.framing.3", "text": "This test did not isolate causes.", "sources": [] }, { "slot": "why.costs", "text": "Costs are medians over the 3 runs of v8, and a cost is left blank unless every one of them recorded it.", "sources": [ { "kind": "consolidated", "ref": "consolidated:runsPerSession", "value": "3" } ] }, { "slot": "why.range.client", "text": "At one searcher, client CPU per search ran from 0.303 ms (mongodb) to 4.985 ms (duckdb).", "sources": [ { "kind": "consolidated", "ref": "consolidated:why[target=mongodb].costs.clientCpuMsPerSearch.1", "value": "0.303" }, { "kind": "consolidated", "ref": "consolidated:why[target=duckdb].costs.clientCpuMsPerSearch.1", "value": "4.985" } ] }, { "slot": "why.range.engine", "text": "At one searcher, engine CPU per search ran from 0.258 ms (redis) to 8.246 ms (weaviate) where measured.", "sources": [ { "kind": "consolidated", "ref": "consolidated:why[target=redis].costs.engineCpuMsPerSearch.1", "value": "0.258" }, { "kind": "consolidated", "ref": "consolidated:why[target=weaviate].costs.engineCpuMsPerSearch.1", "value": "8.246" } ] }, { "slot": "why.embedded", "text": "Hosting embedded is recorded for sqlitevec and duckdb: each runs inside the test's own process, so its client CPU per search includes the engine's own work.", "sources": [ { "kind": "results", "ref": "results:targets[sqlitevec].hosting#embedded", "value": "embedded" }, { "kind": "results", "ref": "results:targets[duckdb].hosting#embedded", "value": "embedded" } ] }, { "slot": "why.settings.redis", "text": "redis recorded these search and index settings: EF_CONSTRUCTION=128, EF_RUNTIME=100, M=16.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[redis].searchSettings#EF_CONSTRUCTION=128", "value": "EF_CONSTRUCTION=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[redis].searchSettings#EF_RUNTIME=100", "value": "EF_RUNTIME=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[redis].searchSettings#M=16", "value": "M=16" } ] }, { "slot": "why.settings.mariadb", "text": "mariadb recorded these search and index settings: DISTANCE=cosine, M=16, mhnsw_ef_search=100.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].searchSettings#DISTANCE=cosine", "value": "DISTANCE=cosine" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].searchSettings#M=16", "value": "M=16" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].searchSettings#mhnsw_ef_search=100", "value": "mhnsw_ef_search=100" } ] }, { "slot": "why.effort.mariadb", "text": "mariadb: the clause of its recorded index text that states how hard a query searches, with how each part is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "why.effort.mariadb.g0", "text": "Read back in these runs from the engine or the machine:", "sources": [ { "kind": "results", "ref": "results:targets[mariadb].load.indexNote#the server applied mhnsw_ef_search 100", "value": "the server applied mhnsw_ef_search 100" } ] }, { "slot": "why.effort.mariadb.g0.q0", "text": "mhnsw_ef_search=100 per statement", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].index", "value": "mhnsw_ef_search=100 per statement" } ] }, { "slot": "why.effort.mariadb.g1", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "why.effort.mariadb.g1.q1", "text": "(the ef 100 most engines here use, so the search effort matches;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].index", "value": "(the ef 100 most engines here use, so the search effort matches;" } ] }, { "slot": "why.effort.mariadb.g2", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING mhnsw_ef_search = 20", "value": "SETTING mhnsw_ef_search = 20" } ] }, { "slot": "why.effort.mariadb.g2.q2", "text": "MariaDB's own default is 20;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].index", "value": "MariaDB's own default is 20;" } ] }, { "slot": "why.effort.mariadb.dropped.0", "text": "A clause of the mariadb text is not printed: it cites figures that no saved source backs.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].index#recall@10 at ef 100 falls as the set grows (rand", "value": "recall@10 at ef 100 falls as the set grows (rand" } ] }, { "slot": "why.effort.mariadb.recall.524", "text": "In the saved re-run of the mariadb effort test, recall@10 at ef 100 on 524 random 1024-dim vectors read 0.988 to 0.996.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.898, ef 100 0.988, ef 3200 0.988", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.898, ef 100 0.988, ef 3200 0.988" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.882, ef 100 0.996, ef 3200 0.996", "value": "RECALL@10 on 524 random 1024-dim vectors, 50 queries: ef 20 0.882, ef 100 0.996, ef 3200 0.996" } ] }, { "slot": "why.effort.mariadb.recall.2000", "text": "In the saved re-run of the mariadb effort test, recall@10 at ef 100 on 2000 random 1024-dim vectors read 0.894 to 0.932.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.468, ef 100 0.894, ef 3200 0.994" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992", "value": "RECALL@10 on 2000 random 1024-dim vectors, 50 queries: ef 20 0.482, ef 100 0.932, ef 3200 0.992" } ] }, { "slot": "why.settings.pgvector", "text": "pgvector recorded these search and index settings: ef_construction=128, hnsw.ef_search=100, m=16.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[pgvector].searchSettings#ef_construction=128", "value": "ef_construction=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[pgvector].searchSettings#hnsw.ef_search=100", "value": "hnsw.ef_search=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[pgvector].searchSettings#m=16", "value": "m=16" } ] }, { "slot": "why.settings.qdrant", "text": "qdrant recorded these search and index settings: exact=true.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[qdrant].searchSettings#exact=true", "value": "exact=true" } ] }, { "slot": "why.settings.qdrant-hnsw", "text": "qdrant-hnsw recorded these search and index settings: ef_construct=100, hnsw_ef=server, m=16.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].searchSettings#ef_construct=100", "value": "ef_construct=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].searchSettings#hnsw_ef=server", "value": "hnsw_ef=server" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].searchSettings#m=16", "value": "m=16" } ] }, { "slot": "why.effort.qdrant-hnsw", "text": "qdrant-hnsw: the clause of its recorded index text that states how hard a query searches, with how each part is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "why.effort.qdrant-hnsw.g0", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/QdrantHnswSink.cs#search.HnswEf = _ef.Value;", "value": "search.HnswEf = _ef.Value;" } ] }, { "slot": "why.effort.qdrant-hnsw.g0.q0", "text": "hnsw_ef=server default", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].index", "value": "hnsw_ef=server default" } ] }, { "slot": "why.settings.oracle", "text": "oracle recorded these search and index settings: EFCONSTRUCTION=128, EFSEARCH=100, NEIGHBORS=16.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[oracle].searchSettings#EFCONSTRUCTION=128", "value": "EFCONSTRUCTION=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[oracle].searchSettings#EFSEARCH=100", "value": "EFSEARCH=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[oracle].searchSettings#NEIGHBORS=16", "value": "NEIGHBORS=16" } ] }, { "slot": "why.settings.elasticsearch", "text": "elasticsearch recorded these search and index settings: ef_construction=128, k=top, m=16, num_candidates=100.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].searchSettings#ef_construction=128", "value": "ef_construction=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].searchSettings#k=top", "value": "k=top" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].searchSettings#m=16", "value": "m=16" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].searchSettings#num_candidates=100", "value": "num_candidates=100" } ] }, { "slot": "why.settings.mongodb", "text": "mongodb recorded these search and index settings: maxEdges=16, numCandidates=20x, numEdgeCandidates=128.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].searchSettings#maxEdges=16", "value": "maxEdges=16" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].searchSettings#numCandidates=20x", "value": "numCandidates=20x" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].searchSettings#numEdgeCandidates=128", "value": "numEdgeCandidates=128" } ] }, { "slot": "why.effort.mongodb", "text": "mongodb: the clause of its recorded index text that states how hard a query searches, with how each part is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "why.effort.mongodb.g0", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MongoDbSinkOptions.cs#public int NumCandidatesFactor { get; set; } = 20;", "value": "public int NumCandidatesFactor { get; set; } = 20;" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/MongoDbSinkOptions.cs#public int MinNumCandidates { get; set; } = 100;", "value": "public int MinNumCandidates { get; set; } = 100;" } ] }, { "slot": "why.effort.mongodb.g0.q0", "text": "numCandidates=20x hits (min 100)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].index", "value": "numCandidates=20x hits (min 100)" } ] }, { "slot": "why.settings.sqlitevec", "text": "sqlitevec recorded these search and index settings: chunk_size=1024.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sqlitevec].searchSettings#chunk_size=1024", "value": "chunk_size=1024" } ] }, { "slot": "why.settings.vespa", "text": "vespa recorded these search and index settings: ef=100, max-links-per-node=16, neighbors-to-explore-at-insert=128, targetHits=top.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[vespa].searchSettings#ef=100", "value": "ef=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[vespa].searchSettings#max-links-per-node=16", "value": "max-links-per-node=16" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[vespa].searchSettings#neighbors-to-explore-at-insert=128", "value": "neighbors-to-explore-at-insert=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[vespa].searchSettings#targetHits=top", "value": "targetHits=top" } ] }, { "slot": "why.settings.opensearch", "text": "opensearch recorded these search and index settings: approximate_threshold=0, ef_construction=128, ef_search=100, k=top, m=16.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].searchSettings#approximate_threshold=0", "value": "approximate_threshold=0" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].searchSettings#ef_construction=128", "value": "ef_construction=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].searchSettings#ef_search=100", "value": "ef_search=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].searchSettings#k=top", "value": "k=top" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].searchSettings#m=16", "value": "m=16" } ] }, { "slot": "why.settings.chroma", "text": "chroma recorded these search and index settings: M=16, ef_construction=128, ef_search=100.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[chroma].searchSettings#M=16", "value": "M=16" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[chroma].searchSettings#ef_construction=128", "value": "ef_construction=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[chroma].searchSettings#ef_search=100", "value": "ef_search=100" } ] }, { "slot": "why.effort.chroma", "text": "chroma: the clause of its recorded index text that states how hard a query searches, with how each part is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "why.effort.chroma.g0", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#[\"max_neighbors\"] = _options.MaxNeighbors,", "value": "[\"max_neighbors\"] = _options.MaxNeighbors," }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#[\"ef_construction\"] = _options.EfConstruction,", "value": "[\"ef_construction\"] = _options.EfConstruction," }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSink.cs#[\"ef_search\"] = _options.EfSearch,", "value": "[\"ef_search\"] = _options.EfSearch," }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSinkOptions.cs#int MaxNeighbors = 16,", "value": "int MaxNeighbors = 16," }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSinkOptions.cs#int EfConstruction = 128,", "value": "int EfConstruction = 128," }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ChromaSinkOptions.cs#int EfSearch = 100 );", "value": "int EfSearch = 100 );" } ] }, { "slot": "why.effort.chroma.g0.q0", "text": "ef_search=100", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[chroma].index", "value": "ef_search=100" } ] }, { "slot": "why.effort.chroma.g1", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "why.effort.chroma.g1.q1", "text": "(Chroma default)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[chroma].index", "value": "(Chroma default)" } ] }, { "slot": "why.settings.milvus", "text": "milvus recorded these search and index settings: M=16, ef=100, efConstruction=128.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[milvus].searchSettings#M=16", "value": "M=16" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[milvus].searchSettings#ef=100", "value": "ef=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[milvus].searchSettings#efConstruction=128", "value": "efConstruction=128" } ] }, { "slot": "why.settings.sql-diskann", "text": "sql-diskann recorded these search and index settings: L=48, M=8, R=48, StartId=306.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].searchSettings#L=48", "value": "L=48" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].searchSettings#M=8", "value": "M=8" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].searchSettings#R=48", "value": "R=48" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].searchSettings#StartId=306", "value": "StartId=306" } ] }, { "slot": "why.noeffort.sql-diskann", "text": "sql-diskann recorded no setting for how hard a query searches.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].searchSettings#L=48", "value": "L=48" } ] }, { "slot": "why.settings.duckdb", "text": "duckdb recorded these search and index settings: checkpoint_threshold=256MB, ef_construction=128, ef_search=100, hnsw_enable_experimental_persistence=true, m=16, metric=cosine.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].searchSettings#checkpoint_threshold=256MB", "value": "checkpoint_threshold=256MB" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].searchSettings#ef_construction=128", "value": "ef_construction=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].searchSettings#ef_search=100", "value": "ef_search=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].searchSettings#hnsw_enable_experimental_persistence=true", "value": "hnsw_enable_experimental_persistence=true" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].searchSettings#m=16", "value": "m=16" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].searchSettings#metric=cosine", "value": "metric=cosine" } ] }, { "slot": "why.settings.sql", "text": "sql recorded these search and index settings: description=exact VECTOR_DISTANCE cosine, no vector index (full scan).", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sql].searchSettings#description=exact VECTOR_DISTANCE cosine, no vector index (full scan)", "value": "description=exact VECTOR_DISTANCE cosine, no vector index (full scan)" } ] }, { "slot": "why.settings.typesense", "text": "typesense recorded these search and index settings: ef=100, ef_construction=128, k=top, m=16.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[typesense].searchSettings#ef=100", "value": "ef=100" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[typesense].searchSettings#ef_construction=128", "value": "ef_construction=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[typesense].searchSettings#k=top", "value": "k=top" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[typesense].searchSettings#m=16", "value": "m=16" } ] }, { "slot": "why.settings.clickhouse", "text": "clickhouse recorded these search and index settings: M=16, ef_construction=128, hnsw_candidate_list_size_for_search=256.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].searchSettings#M=16", "value": "M=16" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].searchSettings#ef_construction=128", "value": "ef_construction=128" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].searchSettings#hnsw_candidate_list_size_for_search=256", "value": "hnsw_candidate_list_size_for_search=256" } ] }, { "slot": "why.settings.weaviate", "text": "weaviate recorded these search and index settings: ef=-1, efConstruction=128.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[weaviate].searchSettings#ef=-1", "value": "ef=-1" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[weaviate].searchSettings#efConstruction=128", "value": "efConstruction=128" } ] }, { "slot": "why.effort.weaviate", "text": "weaviate: the clause of its recorded index text that states how hard a query searches, with how each part is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "why.effort.weaviate.g0", "text": "Documented on a saved page, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSink.cs#[\"ef\"] = _options.Ef,", "value": "[\"ef\"] = _options.Ef," }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/WeaviateSinkOptions.cs#int Ef = -1 );", "value": "int Ef = -1 );" }, { "kind": "file", "ref": "design/engine-docs/weaviate-vector-index-reference-2026-10-07.mdx#`ef`", "value": "`ef`" }, { "kind": "file", "ref": "design/engine-docs/weaviate-vector-index-reference-2026-10-07.mdx#`dynamicEfFactor`", "value": "`dynamicEfFactor`" }, { "kind": "file", "ref": "design/engine-docs/weaviate-vector-index-reference-2026-10-07.mdx#`dynamicEfMin`", "value": "`dynamicEfMin`" }, { "kind": "file", "ref": "design/engine-docs/weaviate-vector-index-reference-2026-10-07.mdx#`dynamicEfMax`", "value": "`dynamicEfMax`" }, { "kind": "file", "ref": "design/engine-docs/weaviate-vector-index-concepts-2026-10-07.md#The dynamic list size will be set as the query limit multiplied by `dynamicEfFactor`, modified by a minimum of `dynamicEfMin` and a maximum of `dynamicEfMax`.", "value": "The dynamic list size will be set as the query limit multiplied by `dynamicEfFactor`, modified by a minimum of `dynamicEfMin` and a maximum of `dynamicEfMax`." } ] }, { "slot": "why.effort.weaviate.g0.q0", "text": "ef=-1", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[weaviate].index", "value": "ef=-1" } ] }, { "slot": "why.effort.weaviate.g0.q1", "text": "(dynamic: limit x 8 clamped 100..500)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[weaviate].index", "value": "(dynamic: limit x 8 clamped 100..500)" } ] }, { "slot": "drift.median", "text": "From v7 to v8, a ranked figure's session median moved 1.43% in the middle case and 4.66% at most (opensearch, exact p50).", "sources": [ { "kind": "consolidated", "ref": "consolidated:drift.medianAbsMoveBp|bp-pct", "value": "1.43" }, { "kind": "consolidated", "ref": "consolidated:drift.largestAbsMoveBp|bp-pct", "value": "4.66" }, { "kind": "consolidated", "ref": "consolidated:drift.largest.target", "value": "opensearch" } ] }, { "slot": "drift.excluded.clickhouse.qps8", "text": "The NOT HELD clickhouse QPS@8 cell is left out of these figures; its session median moved 13.36%, with v7 runs at 375 to 428 and v8 runs at 336 to 380.", "sources": [ { "kind": "consolidated", "ref": "consolidated:drift.excluded[0].target", "value": "clickhouse" }, { "kind": "consolidated", "ref": "consolidated:drift.excluded[0].absMoveBp|bp-pct", "value": "13.36" }, { "kind": "consolidated", "ref": "consolidated:drift.excluded[0].fromMin", "value": "375" }, { "kind": "consolidated", "ref": "consolidated:drift.excluded[0].fromMax", "value": "428" }, { "kind": "consolidated", "ref": "consolidated:drift.excluded[0].toMin", "value": "336" }, { "kind": "consolidated", "ref": "consolidated:drift.excluded[0].toMax", "value": "380" } ] }, { "slot": "drift.unconfirmed", "text": "6 orders held in v7 and not in v8, and 9 the other way; they are listed and not published.", "sources": [ { "kind": "consolidated", "ref": "consolidated:drift.unconfirmedFromFirst", "value": "6" }, { "kind": "consolidated", "ref": "consolidated:drift.unconfirmedFromSecond", "value": "9" } ] }, { "slot": "drift.close", "text": "20 unordered pairs sat between 1.35 and 1.45 times in each session, always in one direction; they are listed as close to the line.", "sources": [ { "kind": "consolidated", "ref": "consolidated:drift.closeCount", "value": "20" }, { "kind": "consolidated", "ref": "consolidated:drift.closeFromBp|bp-ratio", "value": "1.35" }, { "kind": "consolidated", "ref": "consolidated:threshold.tBp|bp-ratio", "value": "1.45" } ] }, { "slot": "drift.onLine", "text": "1 ordered pair cleared 1.45 times by 25 bp or less in its lowest session; it is listed as on the line.", "sources": [ { "kind": "consolidated", "ref": "consolidated:drift.onLineCount", "value": "1" }, { "kind": "consolidated", "ref": "consolidated:threshold.tBp|bp-ratio", "value": "1.45" }, { "kind": "consolidated", "ref": "consolidated:drift.onLineBp", "value": "25" } ] }, { "slot": "drift.sessions", "text": "The sessions differ in build, day and outside load, and this test does not separate those from the engines' own drift.", "sources": [ { "kind": "consolidated", "ref": "consolidated:drift.sessionDifferences[0]", "value": "build" }, { "kind": "consolidated", "ref": "consolidated:drift.sessionDifferences[1]", "value": "day" }, { "kind": "consolidated", "ref": "consolidated:drift.sessionDifferences[2]", "value": "outside load" } ] }, { "slot": "drift.outsideLoad", "text": "The median outside load of a pass, per run, was 0.2095 to 0.2135 CPUs in v7 and 0.14 to 0.1435 in v8.", "sources": [ { "kind": "consolidated", "ref": "consolidated:drift.outsideLoad[0].minCpus", "value": "0.2095" }, { "kind": "consolidated", "ref": "consolidated:drift.outsideLoad[0].maxCpus", "value": "0.2135" }, { "kind": "consolidated", "ref": "consolidated:drift.outsideLoad[0].session", "value": "v7" }, { "kind": "consolidated", "ref": "consolidated:drift.outsideLoad[1].minCpus", "value": "0.14" }, { "kind": "consolidated", "ref": "consolidated:drift.outsideLoad[1].maxCpus", "value": "0.1435" }, { "kind": "consolidated", "ref": "consolidated:drift.outsideLoad[1].session", "value": "v8" } ] }, { "slot": "drift.scope", "text": "Both sessions ran with the clock held; drift under other conditions is not measured here.", "sources": [] }, { "slot": "disclosure.clock.held.v7", "text": "In v7, every run held the clock: turbo off, MSR 0x620 at 0x1e1e, top ratio 35 (nominally 3500 MHz); the median of the kernel's pass medians is 3492 MHz; the ceiling before was 3600 MHz.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Running/MachineControlSampler.cs#LIMIT_MSR = \"0x620\"", "value": "LIMIT_MSR = \"0x620\"" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.uncore", "value": "0x1e1e" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.pinnedMhz", "value": "3500" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.kernelMedianMhz", "value": "3492" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.ceilingBeforeMhz", "value": "3600" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.noTurbo", "value": "1" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.pinnedRatio", "value": "35" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Stats/ConsolidateClock.cs#public const int RATIO_STEP_MHZ = 100;", "value": "public const int RATIO_STEP_MHZ = 100;" } ] }, { "slot": "disclosure.clock.rule.v7", "text": "The median rule: a pass is off when the median, over its engine CPUs or over its client CPUs, of each CPU's median MHz is more than 1% from the pinned 3500 MHz.", "sources": [ { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.toleranceBp|bp-pct", "value": "1" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.pinnedMhz", "value": "3500" } ] }, { "slot": "disclosure.clock.checked.v7", "text": "In v7, by the median rule, 0 of 156 passes were off the pinned clock and 0 had a CPU group not read.", "sources": [ { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.clockOffPasses", "value": "0" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.passesEvaluated", "value": "156" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.clockUnreadPasses", "value": "0" } ] }, { "slot": "disclosure.clock.held.v8", "text": "In v8, every run held the clock: turbo off, MSR 0x620 at 0x1e1e, top ratio 35 (nominally 3500 MHz); the median of the kernel's pass medians is 3492 MHz; the ceiling before was 3600 MHz.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Running/MachineControlSampler.cs#LIMIT_MSR = \"0x620\"", "value": "LIMIT_MSR = \"0x620\"" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.uncore", "value": "0x1e1e" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.pinnedMhz", "value": "3500" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.kernelMedianMhz", "value": "3492" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.ceilingBeforeMhz", "value": "3600" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.noTurbo", "value": "1" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.pinnedRatio", "value": "35" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Stats/ConsolidateClock.cs#public const int RATIO_STEP_MHZ = 100;", "value": "public const int RATIO_STEP_MHZ = 100;" } ] }, { "slot": "disclosure.clock.checked.v8", "text": "In v8, by the median rule, 0 of 156 passes were off the pinned clock and 0 had a CPU group not read.", "sources": [ { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.clockOffPasses", "value": "0" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.passesEvaluated", "value": "156" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v8.clockUnreadPasses", "value": "0" } ] }, { "slot": "disclosure.clock.dropped.0", "text": "Run 20261006-130619-eshoponweb flagged oracle's eight-searcher pass as off its clock by a mean; the medians read 3492 MHz on engine CPUs and 3492 MHz on client CPUs, within 1%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[0].run", "value": "20261006-130619-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[0].engineMedianMhz", "value": "3492" }, { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[0].clientMedianMhz", "value": "3492" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.toleranceBp|bp-pct", "value": "1" } ] }, { "slot": "disclosure.clock.dropped.1", "text": "Run 20261006-142724-eshoponweb flagged oracle's eight-searcher pass as off its clock by a mean; the medians read 3492 MHz on engine CPUs and 3492 MHz on client CPUs, within 1%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[1].run", "value": "20261006-142724-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[1].engineMedianMhz", "value": "3492" }, { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[1].clientMedianMhz", "value": "3492" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.toleranceBp|bp-pct", "value": "1" } ] }, { "slot": "disclosure.clock.dropped.2", "text": "Run 20261006-154837-eshoponweb flagged oracle's eight-searcher pass as off its clock by a mean; the medians read 3492 MHz on engine CPUs and 3492 MHz on client CPUs, within 1%.", "sources": [ { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[2].run", "value": "20261006-154837-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[2].engineMedianMhz", "value": "3492" }, { "kind": "consolidated", "ref": "consolidated:clock.droppedWarnings[2].clientMedianMhz", "value": "3492" }, { "kind": "consolidated", "ref": "consolidated:clock.perSession.v7.toleranceBp|bp-pct", "value": "1" } ] }, { "slot": "disclosure.governor", "text": "Every claim run used the performance governor, with the client on CPUs 0-1,4-5.", "sources": [ { "kind": "results", "ref": "results:conditions.governor", "value": "performance" }, { "kind": "results", "ref": "results:conditions.clientCpus", "value": "0-1,4-5" } ] }, { "slot": "disclosure.engines.0", "text": "The container engines clickhouse, vespa, oracle, elasticsearch, sql, pgvector, qdrant, opensearch, qdrant-hnsw, redis, milvus, mariadb, weaviate, mongodb, chroma, typesense and sql-diskann ran on CPUs 2-3,6-7.", "sources": [ { "kind": "results", "ref": "results:conditions.engines[target=clickhouse].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=clickhouse].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=vespa].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=vespa].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=oracle].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=oracle].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=elasticsearch].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=elasticsearch].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=sql].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=sql].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=pgvector].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=pgvector].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=qdrant].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=qdrant].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=opensearch].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=opensearch].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=qdrant-hnsw].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=qdrant-hnsw].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=redis].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=redis].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=milvus].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=milvus].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=mariadb].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=mariadb].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=weaviate].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=weaviate].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=mongodb].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=mongodb].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=chroma].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=chroma].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=typesense].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=typesense].hosting", "value": "compose" }, { "kind": "results", "ref": "results:conditions.engines[target=sql-diskann].cpus", "value": "2-3,6-7" }, { "kind": "results", "ref": "results:conditions.engines[target=sql-diskann].hosting", "value": "compose" } ] }, { "slot": "disclosure.engines.1", "text": "The embedded engines sqlitevec and duckdb ran inside the client process on CPUs 0-1,4-5.", "sources": [ { "kind": "results", "ref": "results:conditions.engines[target=sqlitevec].cpus", "value": "0-1,4-5" }, { "kind": "results", "ref": "results:conditions.engines[target=sqlitevec].hosting", "value": "embedded" }, { "kind": "results", "ref": "results:conditions.engines[target=duckdb].cpus", "value": "0-1,4-5" }, { "kind": "results", "ref": "results:conditions.engines[target=duckdb].hosting", "value": "embedded" } ] }, { "slot": "disclosure.boot", "text": "Every v8 run recorded boot a2f94313-eeda-460c-8474-8a9ee898f123 at 2026-10-03T15:53:39Z, before the first v7 run started at 2026-10-06T13:06:19Z, so v7 ran on that boot too.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:conditions.boot.bootId", "value": "a2f94313-eeda-460c-8474-8a9ee898f123" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:conditions.boot.bootId", "value": "a2f94313-eeda-460c-8474-8a9ee898f123" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:conditions.boot.bootId", "value": "a2f94313-eeda-460c-8474-8a9ee898f123" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:conditions.boot.bootTimeUtc", "value": "2026-10-03T15:53:39Z" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:conditions.boot.bootTimeUtc", "value": "2026-10-03T15:53:39Z" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:conditions.boot.bootTimeUtc", "value": "2026-10-03T15:53:39Z" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:startedUtc", "value": "2026-10-06T13:06:19Z" } ] }, { "slot": "disclosure.images", "text": "Each of the 17 container targets ran one image id, equal to its pin, in every v8 run; the ids are in the images table.", "sources": [ { "kind": "consolidated", "ref": "consolidated:imageTargets", "value": "17" }, { "kind": "consolidated", "ref": "consolidated:images[target=clickhouse].id", "value": "sha256:3a91276f066905da0edbbd622d3fc2a2df632c87ea7fe32ef4e74fdf8c9567b0" }, { "kind": "consolidated", "ref": "consolidated:images[target=vespa].id", "value": "sha256:5c30f5c41e7563498c4f925db6a837a3848f04726a3ed26aed4a7c8ab69f18fd" }, { "kind": "consolidated", "ref": "consolidated:images[target=oracle].id", "value": "sha256:f5ff19033860d662c821cb04eb10483fa94f14f78eae252d054291ea07028093" }, { "kind": "consolidated", "ref": "consolidated:images[target=elasticsearch].id", "value": "sha256:e23d4758358a4e356cc2ef3259a5a6f345cacc1722c10dffe6d7150a1a78e52f" }, { "kind": "consolidated", "ref": "consolidated:images[target=sql].id", "value": "sha256:2b5b581621126574f3d1f75e78d3eebe8d05aedb59ad0cfdf9aa42cb0634d726" }, { "kind": "consolidated", "ref": "consolidated:images[target=pgvector].id", "value": "sha256:7a7e9f22015b67edb4bef5c59daeebcd7e74bfa570df6ce60ae01237c8648a84" }, { "kind": "consolidated", "ref": "consolidated:images[target=qdrant].id", "value": "sha256:f1c7272cdac52b38c1a0e89313922d940ba50afd90d593a1605dbbc214e66ffb" }, { "kind": "consolidated", "ref": "consolidated:images[target=opensearch].id", "value": "sha256:adfa61f85025d06b4aeb562e7e74fde7e31c437039c93c3862c17e9acebd6c7c" }, { "kind": "consolidated", "ref": "consolidated:images[target=qdrant-hnsw].id", "value": "sha256:f1c7272cdac52b38c1a0e89313922d940ba50afd90d593a1605dbbc214e66ffb" }, { "kind": "consolidated", "ref": "consolidated:images[target=redis].id", "value": "sha256:6f81e8915c60b065a524e6967e0ad1c639ba6efa84d669f823683ea04d9150ee" }, { "kind": "consolidated", "ref": "consolidated:images[target=milvus].id", "value": "sha256:29f7668e64df1c6d5cdadbfbee99f38c72e4001054f149a298725acae57230ad" }, { "kind": "consolidated", "ref": "consolidated:images[target=mariadb].id", "value": "sha256:6422478cb8e159f080fb1d8ccf65101e26fe51385787fde7d16c3b165a331f15" }, { "kind": "consolidated", "ref": "consolidated:images[target=weaviate].id", "value": "sha256:f6f4a5961f99e8718a02c822ca6a0d65ed3f9c567734419a7b2144933014f13a" }, { "kind": "consolidated", "ref": "consolidated:images[target=mongodb].id", "value": "sha256:1985314b0ded756ba965e0f400d17406b7a89bec659d4763a490f42f006e4fbd" }, { "kind": "consolidated", "ref": "consolidated:images[target=chroma].id", "value": "sha256:1e0b73a187a28757c572acba508c46f48c9e8b0acaf5c20e6d95cdedce1acdf6" }, { "kind": "consolidated", "ref": "consolidated:images[target=typesense].id", "value": "sha256:610f2d34b1f93d00762869da2c67736775e5798d19a2c8b91b014b8a0cc1e110" }, { "kind": "consolidated", "ref": "consolidated:images[target=sql-diskann].id", "value": "sha256:2b5b581621126574f3d1f75e78d3eebe8d05aedb59ad0cfdf9aa42cb0634d726" } ] }, { "slot": "disclosure.images.tagged", "text": "Each of those images was last tagged on this box before the first v7 run started at 2026-10-06T13:06:19Z.", "sources": [ { "kind": "results", "ref": "results@20261006-130619-eshoponweb:startedUtc", "value": "2026-10-06T13:06:19Z" }, { "kind": "consolidated", "ref": "consolidated:images[target=clickhouse].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=vespa].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=oracle].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=elasticsearch].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=sql].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=pgvector].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=qdrant].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=opensearch].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=qdrant-hnsw].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=redis].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=milvus].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=mariadb].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=weaviate].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=mongodb].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=chroma].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=typesense].beforeFirstV7Start", "value": "true" }, { "kind": "consolidated", "ref": "consolidated:images[target=sql-diskann].beforeFirstV7Start", "value": "true" } ] }, { "slot": "disclosure.memory.redis", "text": "redis holds its data in memory: its saved docs page says 'Redis is an in-memory but persistent on disk database'; its compose file sets save \"300 1\" and appendonly no.", "sources": [ { "kind": "doc", "ref": "doc:design/engine-docs/redis-faq-2026-10-07.html#Redis is an in-memory but persistent on disk database", "value": "Redis is an in-memory but persistent on disk database" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--save \"300 1\"", "value": "--save \"300 1\"" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--appendonly no", "value": "--appendonly no" } ] }, { "slot": "disclosure.durability", "text": "Each engine's durability text, as run 20261007-205106-eshoponweb recorded it, follows clause by clause with how each clause is backed; the program wrote the text, and its figures were not re-derived for this report.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.logs", "text": "Strace logs of measurements cited in those texts are saved for clickhouse, milvus, mongodb, qdrant and qdrant-hnsw, in design/engine-docs/durability-logs-2026-10-04, each listed with its hash in SHA256SUMS.", "sources": [ { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/index.json#\"folder\": \"design/engine-docs/durability-logs-2026-10-04\"", "value": "\"folder\": \"design/engine-docs/durability-logs-2026-10-04\"" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/index.json#\"hashes\": \"SHA256SUMS\"", "value": "\"hashes\": \"SHA256SUMS\"" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/index.json#\"target\": \"clickhouse\"", "value": "\"target\": \"clickhouse\"" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/index.json#\"target\": \"milvus\"", "value": "\"target\": \"milvus\"" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/index.json#\"target\": \"mongodb\"", "value": "\"target\": \"mongodb\"" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/index.json#\"target\": \"qdrant\"", "value": "\"target\": \"qdrant\"" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/index.json#\"target\": \"qdrant-hnsw\"", "value": "\"target\": \"qdrant-hnsw\"" } ] }, { "slot": "disclosure.durability.nologs", "text": "No log is saved for the measurements cited in the durability texts of oracle, pgvector, sqlitevec, redis, weaviate, chroma, typesense and duckdb.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability#Measured 2026-10-04: 50 separate client commits ", "value": "Measured 2026-10-04: 50 separate client commits " }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[pgvector].durability#every commit is flushed to the write-ahead log b", "value": "every commit is flushed to the write-ahead log b" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[sqlitevec].durability#each commit is appended to the -wal file and han", "value": "each commit is appended to the -wal file and han" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[redis].durability#an RDB snapshot is written every 5 minutes if at", "value": "an RDB snapshot is written every 5 minutes if at" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[weaviate].durability#every object is appended to the LSM write-ahead ", "value": "every object is appended to the LSM write-ahead " }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[chroma].durability#every write is committed to chroma.sqlite3 with ", "value": "every write is committed to chroma.sqlite3 with " }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[typesense].durability#Every acknowledged write is appended to Typesens", "value": "Every acknowledged write is appended to Typesens" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].durability#only spaces out the checkpoints that write the d", "value": "only spaces out the checkpoints that write the d" } ] }, { "slot": "disclosure.durability.clickhouse", "text": "clickhouse: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.clickhouse.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.clickhouse.g0.q0", "text": "Acknowledged inserts are not fsynced. The sink creates plain MergeTree tables, so the MergeTree settings are the defaults: fsync_after_insert 0 and fsync_part_directory 0 (system.merge_tree_settings), and async_insert 1 with wait_for_async_insert 1 is the server default for the gvb login (system.settings), so an insert is acknowledged once its part is written, not once it is on disk.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].durability", "value": "Acknowledged inserts are not fsynced. The sink creates plain MergeTree tables, so the MergeTree settings are the defaults: fsync_after_insert 0 and fsync_part_directory 0 (system.merge_tree_settings), and async_insert 1 with wait_for_async_insert 1 is the server default for the gvb login (system.settings), so an insert is acknowledged once its part is written, not once it is on disk." } ] }, { "slot": "disclosure.durability.clickhouse.g1", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.clickhouse.g1.q1", "text": "Measured 2026-10-04 with strace over 30 acknowledged single-row inserts (30 Ok rows in system.asynchronous_insert_log):", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].durability", "value": "Measured 2026-10-04 with strace over 30 acknowledged single-row inserts (30 Ok rows in system.asynchronous_insert_log):" } ] }, { "slot": "disclosure.durability.clickhouse.g2", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/clickhouse-acknowledged-inserts.strace#SIGUSR1", "value": "SIGUSR1" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/clickhouse-control-fsync-after-insert.strace#fdatasync", "value": "fdatasync" } ] }, { "slot": "disclosure.durability.clickhouse.g2.q2", "text": "zero fsync, fdatasync or sync_file_range calls,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].durability", "value": "zero fsync, fdatasync or sync_file_range calls," } ] }, { "slot": "disclosure.durability.clickhouse.g2.q3", "text": "while the control, a table created with fsync_after_insert=1, made 61 fdatasync calls", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].durability", "value": "while the control, a table created with fsync_after_insert=1, made 61 fdatasync calls" } ] }, { "slot": "disclosure.durability.clickhouse.g3", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.clickhouse.g3.q4", "text": "for 5 inserts.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].durability", "value": "for 5 inserts." } ] }, { "slot": "disclosure.durability.clickhouse.g4", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.clickhouse.g4.q5", "text": "A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a ClickHouse process crash alone should not (inferred, not tested).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].durability", "value": "A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a ClickHouse process crash alone should not (inferred, not tested)." } ] }, { "slot": "disclosure.durability.vespa", "text": "vespa: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.vespa.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.vespa.g0.q0", "text": "Vespa's transaction log server fsyncs after each commit: searchlib.translogserver usefsync=true, and an operation is searchable when it is acknowledged: proton documentdb visibilitydelay=0. Neither is overridden by this sink or by the services.xml it deploys;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[vespa].durability", "value": "Vespa's transaction log server fsyncs after each commit: searchlib.translogserver usefsync=true, and an operation is searchable when it is acknowledged: proton documentdb visibilitydelay=0. Neither is overridden by this sink or by the services.xml it deploys;" } ] }, { "slot": "disclosure.durability.vespa.g1", "text": "Described by a test saved in this repository, which these runs did not run:", "sources": [ { "kind": "file", "ref": "tests/GenericVectorBuilder.Engines.Tests/VespaReadinessTests.cs#usefsync", "value": "usefsync" } ] }, { "slot": "disclosure.durability.vespa.g1.q1", "text": "VespaReadinessTests reads both from the live config server.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[vespa].durability", "value": "VespaReadinessTests reads both from the live config server." } ] }, { "slot": "disclosure.durability.vespa.g2", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.vespa.g2.q2", "text": "After a crash the node replays the transaction log over its last flushed data and rebuilds the in-memory HNSW graph. By those settings an acknowledged write survives a crash; this is read from the settings, not shown by pulling power.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[vespa].durability", "value": "After a crash the node replays the transaction log over its last flushed data and rebuilds the in-memory HNSW graph. By those settings an acknowledged write survives a crash; this is read from the settings, not shown by pulling power." } ] }, { "slot": "disclosure.durability.vespa.g3", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/VespaApplicationPackage.cs#1", "value": "1" } ] }, { "slot": "disclosure.durability.vespa.g3.q3", "text": "One node and min-redundancy 1,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[vespa].durability", "value": "One node and min-redundancy 1," } ] }, { "slot": "disclosure.durability.vespa.g4", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.vespa.g4.q4", "text": "so a lost disk loses the data.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[vespa].durability", "value": "so a lost disk loses the data." } ] }, { "slot": "disclosure.durability.oracle", "text": "oracle: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.oracle.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.oracle.g0.q0", "text": "By these settings a committed row should survive a crash or power loss. The sink uses a plain COMMIT and commit_logging, commit_wait and commit_write are unset in the database, so every commit waits for its redo to be written.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability", "value": "By these settings a committed row should survive a crash or power loss. The sink uses a plain COMMIT and commit_logging, commit_wait and commit_write are unset in the database, so every commit waits for its redo to be written." } ] }, { "slot": "disclosure.durability.oracle.g1", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.oracle.g1.q1", "text": "Measured 2026-10-04: 50 separate client commits raised V$SYSSTAT 'redo synch writes' by 55 (the 50 commits plus the CREATE and DROP of the probe table), and the log writer and the datafile writer hold their files open with O_DSYNC (open flags 02110002, filesystemio_options none).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability", "value": "Measured 2026-10-04: 50 separate client commits raised V$SYSSTAT 'redo synch writes' by 55 (the 50 commits plus the CREATE and DROP of the probe table), and the log writer and the datafile writer hold their files open with O_DSYNC (open flags 02110002, filesystemio_options none)." } ] }, { "slot": "disclosure.durability.oracle.g2", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.oracle.g2.q2", "text": "The database runs NOARCHIVELOG (V$DATABASE.LOG_MODE), so redo serves crash recovery only and there is no point-in-time restore.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability", "value": "The database runs NOARCHIVELOG (V$DATABASE.LOG_MODE), so redo serves crash recovery only and there is no point-in-time restore." } ] }, { "slot": "disclosure.durability.oracle.g3", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "deploy/engines/oracle-init/01-vector-memory.sh#POOL_SIZE=768M", "value": "POOL_SIZE=768M" } ] }, { "slot": "disclosure.durability.oracle.g3.q3", "text": "The HNSW graph lives in the 768 MB vector memory pool (oracle-init/01-vector-memory.sh)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability", "value": "The HNSW graph lives in the 768 MB vector memory pool (oracle-init/01-vector-memory.sh)" } ] }, { "slot": "disclosure.durability.oracle.g4", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.oracle.g4.q4", "text": "and is not the durable copy; the table is.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability", "value": "and is not the durable copy; the table is." } ] }, { "slot": "disclosure.durability.oracle.g4.q5", "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability", "value": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked." } ] }, { "slot": "disclosure.durability.oracle.g5", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OracleCpuCap.cs#cpu_count {CpuCount.ToString( CultureInfo.InvariantCulture )} in V$PARAMETER", "value": "cpu_count {CpuCount.ToString( CultureInfo.InvariantCulture )} in V$PARAMETER" } ] }, { "slot": "disclosure.durability.oracle.g5.q6", "text": "CPU: Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability", "value": "CPU: Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS;" } ] }, { "slot": "disclosure.durability.oracle.g6", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.oracle.g6.q7", "text": "the 2 CPU thread limit is Oracle's documented Free edition limit).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[oracle].durability", "value": "the 2 CPU thread limit is Oracle's documented Free edition limit)." } ] }, { "slot": "disclosure.durability.elasticsearch", "text": "elasticsearch: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.elasticsearch.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.elasticsearch.g0.q0", "text": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the Elasticsearch default, which neither this sink nor elasticsearch.compose.yaml overrides", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].durability", "value": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the Elasticsearch default, which neither this sink nor elasticsearch.compose.yaml overrides" } ] }, { "slot": "disclosure.durability.elasticsearch.g1", "text": "Described by a test saved in this repository, which these runs did not run:", "sources": [ { "kind": "file", "ref": "tests/GenericVectorBuilder.Engines.Tests/ElasticsearchReadinessTests.cs#index.translog.durability", "value": "index.translog.durability" } ] }, { "slot": "disclosure.durability.elasticsearch.g1.q1", "text": "(ElasticsearchReadinessTests reads it back from the live index).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].durability", "value": "(ElasticsearchReadinessTests reads it back from the live index)." } ] }, { "slot": "disclosure.durability.elasticsearch.g2", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.elasticsearch.g2.q2", "text": "By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].durability", "value": "By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power." } ] }, { "slot": "disclosure.durability.elasticsearch.g3", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/ElasticsearchSink.cs#settings = new { number_of_shards = 1, number_of_replicas = 0 }", "value": "settings = new { number_of_shards = 1, number_of_replicas = 0 }" } ] }, { "slot": "disclosure.durability.elasticsearch.g3.q3", "text": "One node and no replicas,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].durability", "value": "One node and no replicas," } ] }, { "slot": "disclosure.durability.elasticsearch.g4", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.elasticsearch.g4.q4", "text": "so a lost disk loses the data.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[elasticsearch].durability", "value": "so a lost disk loses the data." } ] }, { "slot": "disclosure.durability.sql", "text": "sql: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.sql.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.sql.g0.q0", "text": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql].durability", "value": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging;" } ] }, { "slot": "disclosure.durability.sql.g1", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases", "value": "SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases" } ] }, { "slot": "disclosure.durability.sql.g1.q1", "text": "delayed durability is DISABLED);", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql].durability", "value": "delayed durability is DISABLED);" } ] }, { "slot": "disclosure.durability.sql.g2", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.sql.g2.q2", "text": "a database created here copies the model database:", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql].durability", "value": "a database created here copies the model database:" } ] }, { "slot": "disclosure.durability.sql.g3", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases", "value": "SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#DBCC TRACESTATUS( -1 ) WITH NO_INFOMSGS;", "value": "DBCC TRACESTATUS( -1 ) WITH NO_INFOMSGS;" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#ConfigFiles.ExistsWithSudoAsync( confPath, ct )", "value": "ConfigFiles.ExistsWithSudoAsync( confPath, ct )" } ] }, { "slot": "disclosure.durability.sql.g3.q3", "text": "recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql].durability", "value": "recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled;" } ] }, { "slot": "disclosure.durability.sql.g3.q4", "text": "mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql].durability", "value": "mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry," } ] }, { "slot": "disclosure.durability.sql.g4", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.sql.g4.q5", "text": "so SQL Server's own Linux defaults for flushing writes apply (Microsoft's Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql].durability", "value": "so SQL Server's own Linux defaults for flushing writes apply (Microsoft's Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here)." } ] }, { "slot": "disclosure.durability.sql.g4.q6", "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql].durability", "value": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked." } ] }, { "slot": "disclosure.durability.sql.g5", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlServerContainer.cs#n.StartsWith( \"MSSQL_\"", "value": "n.StartsWith( \"MSSQL_\"" } ] }, { "slot": "disclosure.durability.sql.g5.q7", "text": "Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql].durability", "value": "Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD." } ] }, { "slot": "disclosure.durability.pgvector", "text": "pgvector: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.pgvector.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.pgvector.g0.q0", "text": "fsync on, synchronous_commit on, full_page_writes on, wal_sync_method fdatasync (PostgreSQL defaults;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[pgvector].durability", "value": "fsync on, synchronous_commit on, full_page_writes on, wal_sync_method fdatasync (PostgreSQL defaults;" } ] }, { "slot": "disclosure.durability.pgvector.g1", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "deploy/engines/pgvector.compose.yaml#shared_buffers=2GB", "value": "shared_buffers=2GB" }, { "kind": "file", "ref": "deploy/engines/pgvector.compose.yaml#maintenance_work_mem=1GB", "value": "maintenance_work_mem=1GB" }, { "kind": "file", "ref": "deploy/engines/pgvector.compose.yaml#max_wal_size=4GB", "value": "max_wal_size=4GB" } ] }, { "slot": "disclosure.durability.pgvector.g1.q1", "text": "pgvector.compose.yaml sets only shared_buffers, maintenance_work_mem and max_wal_size):", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[pgvector].durability", "value": "pgvector.compose.yaml sets only shared_buffers, maintenance_work_mem and max_wal_size):" } ] }, { "slot": "disclosure.durability.pgvector.g2", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.pgvector.g2.q2", "text": "every commit is flushed to the write-ahead log before it returns and HNSW index changes are WAL-logged, so a crash loses no committed row (max_wal_size 4GB only spaces out checkpoints; measured with docker kill, which is SIGKILL, right after a 2,000-vector load and a restart: all 2,000 rows were there and the HNSW index was valid and used by the default search)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[pgvector].durability", "value": "every commit is flushed to the write-ahead log before it returns and HNSW index changes are WAL-logged, so a crash loses no committed row (max_wal_size 4GB only spaces out checkpoints; measured with docker kill, which is SIGKILL, right after a 2,000-vector load and a restart: all 2,000 rows were there and the HNSW index was valid and used by the default search)" } ] }, { "slot": "disclosure.durability.sqlitevec", "text": "sqlitevec: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.sqlitevec.g0", "text": "Set by a line of code or a compose file saved in this repository; the v8 engine settings also list journal_mode, read from the running engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#Run( connection, \"PRAGMA journal_mode=WAL;\" )", "value": "Run( connection, \"PRAGMA journal_mode=WAL;\" )" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/SqliteVecSink.cs#Run( connection, \"PRAGMA synchronous=NORMAL;\" )", "value": "Run( connection, \"PRAGMA synchronous=NORMAL;\" )" }, { "kind": "consolidated", "ref": "consolidated:engineSettings[6].settings[2].key", "value": "journal_mode" }, { "kind": "consolidated", "ref": "consolidated:engineSettings[6].settings[2].how", "value": "read: PRAGMA journal_mode on a second, read-only connection to the bench file" } ] }, { "slot": "disclosure.durability.sqlitevec.g0.q0", "text": "PRAGMA journal_mode=WAL and PRAGMA synchronous=NORMAL (set by this sink when it opens the file):", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sqlitevec].durability", "value": "PRAGMA journal_mode=WAL and PRAGMA synchronous=NORMAL (set by this sink when it opens the file):" } ] }, { "slot": "disclosure.durability.sqlitevec.g1", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.sqlitevec.g1.q1", "text": "each commit is appended to the -wal file and handed to the operating system, but the WAL is fsynced only when SQLite checkpoints it (measured with strace: 40 single-record commits caused 3 WAL fsyncs), so a crash of the process loses nothing (measured with kill -9 and a reopen: all 50, 300 and 2,000 rows were there), while an operating-system crash or power cut can lose the newest commits since the last checkpoint and leaves the database consistent", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sqlitevec].durability", "value": "each commit is appended to the -wal file and handed to the operating system, but the WAL is fsynced only when SQLite checkpoints it (measured with strace: 40 single-record commits caused 3 WAL fsyncs), so a crash of the process loses nothing (measured with kill -9 and a reopen: all 50, 300 and 2,000 rows were there), while an operating-system crash or power cut can lose the newest commits since the last checkpoint and leaves the database consistent" } ] }, { "slot": "disclosure.durability.qdrant", "text": "qdrant: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.qdrant.g0", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-true.strace#MS_SYNC", "value": "MS_SYNC" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-true.marks.json#\"mode\": \"true\"", "value": "\"mode\": \"true\"" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-false.strace#msync", "value": "msync" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-false.marks.json#\"mode\": \"false\"", "value": "\"mode\": \"false\"" } ] }, { "slot": "disclosure.durability.qdrant.g0.q0", "text": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30);", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant].durability", "value": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30);" } ] }, { "slot": "disclosure.durability.qdrant.g0.q1", "text": "with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant].durability", "value": "with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes." } ] }, { "slot": "disclosure.durability.qdrant.g1", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.qdrant.g1.q2", "text": "So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel's page cache (inferred, not tested).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant].durability", "value": "So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel's page cache (inferred, not tested)." } ] }, { "slot": "disclosure.durability.qdrant.g2", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#_client.UpsertAsync( name, points, wait: true", "value": "_client.UpsertAsync( name, points, wait: true" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#private const int UPSERT_BATCH = 256;", "value": "private const int UPSERT_BATCH = 256;" } ] }, { "slot": "disclosure.durability.qdrant.g2.q3", "text": "Every upsert here is sent with wait=true in batches of 256 over gRPC;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant].durability", "value": "Every upsert here is sent with wait=true in batches of 256 over gRPC;" } ] }, { "slot": "disclosure.durability.qdrant.g3", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.qdrant.g3.q4", "text": "the strace used one point per request, so one flush per batch is inferred, not measured.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant].durability", "value": "the strace used one point per request, so one flush per batch is inferred, not measured." } ] }, { "slot": "disclosure.durability.qdrant.g4", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/QdrantServer.cs#ConfigFiles.ReadContainerFileAsync( container, configPath, ct )", "value": "ConfigFiles.ReadContainerFileAsync( container, configPath, ct )" } ] }, { "slot": "disclosure.durability.qdrant.g4.q5", "text": "Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image's own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant].durability", "value": "Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image's own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process." } ] }, { "slot": "disclosure.durability.qdrant.g5", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.qdrant.g5.q6", "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant].durability", "value": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked." } ] }, { "slot": "disclosure.durability.opensearch", "text": "opensearch: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.opensearch.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.opensearch.g0.q0", "text": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the OpenSearch default, which neither this sink nor opensearch.compose.yaml overrides", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].durability", "value": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the OpenSearch default, which neither this sink nor opensearch.compose.yaml overrides" } ] }, { "slot": "disclosure.durability.opensearch.g1", "text": "Described by a test saved in this repository, which these runs did not run:", "sources": [ { "kind": "file", "ref": "tests/GenericVectorBuilder.Engines.Tests/OpenSearchReadinessTests.cs#index.translog.durability", "value": "index.translog.durability" } ] }, { "slot": "disclosure.durability.opensearch.g1.q1", "text": "(OpenSearchReadinessTests reads it back from the live index).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].durability", "value": "(OpenSearchReadinessTests reads it back from the live index)." } ] }, { "slot": "disclosure.durability.opensearch.g2", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.opensearch.g2.q2", "text": "By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].durability", "value": "By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power." } ] }, { "slot": "disclosure.durability.opensearch.g3", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/OpenSearchSink.cs#[\"number_of_shards\"] = 1, [\"number_of_replicas\"] = 0", "value": "[\"number_of_shards\"] = 1, [\"number_of_replicas\"] = 0" } ] }, { "slot": "disclosure.durability.opensearch.g3.q3", "text": "One node and no replicas,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].durability", "value": "One node and no replicas," } ] }, { "slot": "disclosure.durability.opensearch.g4", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.opensearch.g4.q4", "text": "so a lost disk loses the data.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[opensearch].durability", "value": "so a lost disk loses the data." } ] }, { "slot": "disclosure.durability.qdrant-hnsw", "text": "qdrant-hnsw: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g0", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-true.strace#MS_SYNC", "value": "MS_SYNC" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-true.marks.json#\"mode\": \"true\"", "value": "\"mode\": \"true\"" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-false.strace#msync", "value": "msync" }, { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/qdrant-wait-false.marks.json#\"mode\": \"false\"", "value": "\"mode\": \"false\"" } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g0.q0", "text": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30);", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].durability", "value": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30);" } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g0.q1", "text": "with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].durability", "value": "with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes." } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g1", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.qdrant-hnsw.g1.q2", "text": "So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel's page cache (inferred, not tested).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].durability", "value": "So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel's page cache (inferred, not tested)." } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g2", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#_client.UpsertAsync( name, points, wait: true", "value": "_client.UpsertAsync( name, points, wait: true" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Core/Sinks/QdrantSink.cs#private const int UPSERT_BATCH = 256;", "value": "private const int UPSERT_BATCH = 256;" } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g2.q3", "text": "Every upsert here is sent with wait=true in batches of 256 over gRPC;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].durability", "value": "Every upsert here is sent with wait=true in batches of 256 over gRPC;" } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g3", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.qdrant-hnsw.g3.q4", "text": "the strace used one point per request, so one flush per batch is inferred, not measured.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].durability", "value": "the strace used one point per request, so one flush per batch is inferred, not measured." } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g4", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/QdrantServer.cs#ConfigFiles.ReadContainerFileAsync( container, configPath, ct )", "value": "ConfigFiles.ReadContainerFileAsync( container, configPath, ct )" } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g4.q5", "text": "Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image's own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].durability", "value": "Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image's own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process." } ] }, { "slot": "disclosure.durability.qdrant-hnsw.g5", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.qdrant-hnsw.g5.q6", "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[qdrant-hnsw].durability", "value": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked." } ] }, { "slot": "disclosure.durability.redis", "text": "redis: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.redis.g0", "text": "Set by a line of code or a compose file saved in this repository; the v8 engine settings also list appendonly, read from the running engine:", "sources": [ { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--save \"300 1\"", "value": "--save \"300 1\"" }, { "kind": "file", "ref": "deploy/engines/redis.compose.yaml#--appendonly no", "value": "--appendonly no" }, { "kind": "consolidated", "ref": "consolidated:engineSettings[10].settings[5].key", "value": "appendonly" }, { "kind": "consolidated", "ref": "consolidated:engineSettings[10].settings[5].how", "value": "read: docker exec gvb-redis redis-cli CONFIG GET appendonly" } ] }, { "slot": "disclosure.durability.redis.g0.q0", "text": "save \"300 1\" and appendonly no (redis.compose.yaml):", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[redis].durability", "value": "save \"300 1\" and appendonly no (redis.compose.yaml):" } ] }, { "slot": "disclosure.durability.redis.g1", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.redis.g1.q1", "text": "an RDB snapshot is written every 5 minutes if at least one key changed, and on a clean stop; there is no append-only log, so a crash loses every write since the last snapshot (up to 5 minutes plus the time a snapshot takes; measured with docker kill, which is SIGKILL, right after a 2,000-vector load: none of the 2,000 hashes were there after the restart, and the restart loaded an older snapshot that still held keys of a collection that had been dropped since)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[redis].durability", "value": "an RDB snapshot is written every 5 minutes if at least one key changed, and on a clean stop; there is no append-only log, so a crash loses every write since the last snapshot (up to 5 minutes plus the time a snapshot takes; measured with docker kill, which is SIGKILL, right after a 2,000-vector load: none of the 2,000 hashes were there after the restart, and the restart loaded an older snapshot that still held keys of a collection that had been dropped since)" } ] }, { "slot": "disclosure.durability.milvus", "text": "milvus: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.milvus.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.milvus.g0.q0", "text": "Writes are not fsynced. Standalone uses the default message queue rocksmq (RocksDB, /var/lib/milvus/rdb_data, mq.type default) and flushed segments go to local-disk object storage", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[milvus].durability", "value": "Writes are not fsynced. Standalone uses the default message queue rocksmq (RocksDB, /var/lib/milvus/rdb_data, mq.type default) and flushed segments go to local-disk object storage" } ] }, { "slot": "disclosure.durability.milvus.g1", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "deploy/engines/milvus.compose.yaml#COMMON_STORAGETYPE: local", "value": "COMMON_STORAGETYPE: local" } ] }, { "slot": "disclosure.durability.milvus.g1.q1", "text": "(COMMON_STORAGETYPE=local in milvus.compose.yaml).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[milvus].durability", "value": "(COMMON_STORAGETYPE=local in milvus.compose.yaml)." } ] }, { "slot": "disclosure.durability.milvus.g2", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/milvus-acknowledged-upserts.strace#member/wal", "value": "member/wal" } ] }, { "slot": "disclosure.durability.milvus.g2.q2", "text": "Measured 2026-10-04 with strace over 30 acknowledged single-row upserts and one flush: fdatasync ran only on the embedded etcd files (member/wal, member/snap/db), never on rdb_data or the segment files.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[milvus].durability", "value": "Measured 2026-10-04 with strace over 30 acknowledged single-row upserts and one flush: fdatasync ran only on the embedded etcd files (member/wal, member/snap/db), never on rdb_data or the segment files." } ] }, { "slot": "disclosure.durability.milvus.g3", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.milvus.g3.q3", "text": "A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a Milvus process crash alone should not (inferred, not tested). Metadata in embedded etcd is fdatasynced on every commit.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[milvus].durability", "value": "A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a Milvus process crash alone should not (inferred, not tested). Metadata in embedded etcd is fdatasynced on every commit." } ] }, { "slot": "disclosure.durability.mariadb", "text": "mariadb: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.mariadb.g0", "text": "Backed by a saved log or measurement; the v8 engine settings also list innodb_flush_log_at_trx_commit, read from the running engine:", "sources": [ { "kind": "file", "ref": "deploy/engines/mariadb-bench.compose.yaml#--innodb-flush-log-at-trx-commit=2", "value": "--innodb-flush-log-at-trx-commit=2" }, { "kind": "file", "ref": "deploy/engines/mariadb.compose.yaml#--innodb-flush-log-at-trx-commit=2", "value": "--innodb-flush-log-at-trx-commit=2" }, { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING innodb_flush_log_at_trx_commit = 2", "value": "SETTING innodb_flush_log_at_trx_commit = 2" }, { "kind": "consolidated", "ref": "consolidated:engineSettings[12].settings[4].key", "value": "innodb_flush_log_at_trx_commit" }, { "kind": "consolidated", "ref": "consolidated:engineSettings[12].settings[4].how", "value": "read: docker exec gvbbench-mariadb mariadb SHOW GLOBAL VARIABLES" } ] }, { "slot": "disclosure.durability.mariadb.g0.q0", "text": "innodb_flush_log_at_trx_commit=2 (set in mariadb-bench.compose.yaml for the benchmark's own container gvbbench-mariadb, and in mariadb.compose.yaml for the daily gvb-mariadb, which has the same server settings):", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].durability", "value": "innodb_flush_log_at_trx_commit=2 (set in mariadb-bench.compose.yaml for the benchmark's own container gvbbench-mariadb, and in mariadb.compose.yaml for the daily gvb-mariadb, which has the same server settings):" } ] }, { "slot": "disclosure.durability.mariadb.g1", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.mariadb.g1.q1", "text": "the InnoDB redo log is written to the operating system at every commit but fsynced about once a second, so a crash of the mariadbd process loses nothing, while an operating-system crash or power cut can lose the last second of commits;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].durability", "value": "the InnoDB redo log is written to the operating system at every commit but fsynced about once a second, so a crash of the mariadbd process loses nothing, while an operating-system crash or power cut can lose the last second of commits;" } ] }, { "slot": "disclosure.durability.mariadb.g2", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING innodb_doublewrite = ON", "value": "SETTING innodb_doublewrite = ON" } ] }, { "slot": "disclosure.durability.mariadb.g2.q2", "text": "innodb_doublewrite is on", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].durability", "value": "innodb_doublewrite is on" } ] }, { "slot": "disclosure.durability.mariadb.g3", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.mariadb.g3.q3", "text": "(default),", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].durability", "value": "(default)," } ] }, { "slot": "disclosure.durability.mariadb.g4", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING log_bin = OFF", "value": "SETTING log_bin = OFF" } ] }, { "slot": "disclosure.durability.mariadb.g4.q4", "text": "the binary log is off,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].durability", "value": "the binary log is off," } ] }, { "slot": "disclosure.durability.mariadb.g5", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.mariadb.g5.q5", "text": "and the vector graph is an InnoDB table under the same log", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].durability", "value": "and the vector graph is an InnoDB table under the same log" } ] }, { "slot": "disclosure.durability.mariadb.g6", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/bench-inputs/mariadb-effort-2026-10-07/run.log#SETTING innodb_flush_log_at_trx_commit = 2", "value": "SETTING innodb_flush_log_at_trx_commit = 2" } ] }, { "slot": "disclosure.durability.mariadb.g6.q6", "text": "(settings read from the running server with SHOW VARIABLES;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].durability", "value": "(settings read from the running server with SHOW VARIABLES;" } ] }, { "slot": "disclosure.durability.mariadb.g7", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.mariadb.g7.q7", "text": "the crash behaviour is InnoDB's documented behaviour for this setting and was not tested on either container)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mariadb].durability", "value": "the crash behaviour is InnoDB's documented behaviour for this setting and was not tested on either container)" } ] }, { "slot": "disclosure.durability.weaviate", "text": "weaviate: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.weaviate.correction", "text": "The recorded weaviate text says 'sets no persistence variable'; deploy/engines/weaviate.compose.yaml sets PERSISTENCE_DATA_PATH. That clause is not printed.", "sources": [ { "kind": "file", "ref": "deploy/engines/weaviate.compose.yaml#PERSISTENCE_DATA_PATH:", "value": "PERSISTENCE_DATA_PATH:" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[weaviate].durability#sets no persistence variable", "value": "sets no persistence variable" } ] }, { "slot": "disclosure.durability.weaviate.g0", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.weaviate.g0.q0", "text": "every object is appended to the LSM write-ahead log with a plain write and no fsync, and the log is fsynced only when its memtable is flushed, 60 seconds after the last write (PERSISTENCE_MEMTABLES_FLUSH_IDLE_AFTER_SECONDS default; measured with strace: 2,080 writes into the objects log during a 2,000-object load, the first fsync 60 s after the last write), so a power cut can lose the last minute of writes; the HNSW commit log is buffered inside the process (77 writes for 2,000 vectors), so a killed process loses the newest vectors from the vector index while their objects survive (measured with docker kill, which is SIGKILL, one second after the load and a restart, two runs each: 0 to 1 of 50, 264 to 267 of 300 and 1,992 to 1,997 of 2,000 stored objects were still reachable through a vector search)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[weaviate].durability", "value": "every object is appended to the LSM write-ahead log with a plain write and no fsync, and the log is fsynced only when its memtable is flushed, 60 seconds after the last write (PERSISTENCE_MEMTABLES_FLUSH_IDLE_AFTER_SECONDS default; measured with strace: 2,080 writes into the objects log during a 2,000-object load, the first fsync 60 s after the last write), so a power cut can lose the last minute of writes; the HNSW commit log is buffered inside the process (77 writes for 2,000 vectors), so a killed process loses the newest vectors from the vector index while their objects survive (measured with docker kill, which is SIGKILL, one second after the load and a restart, two runs each: 0 to 1 of 50, 264 to 267 of 300 and 1,992 to 1,997 of 2,000 stored objects were still reachable through a vector search)" } ] }, { "slot": "disclosure.durability.mongodb", "text": "mongodb: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.mongodb.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.mongodb.g0.q0", "text": "Acknowledged writes are journaled before the acknowledgement. The sink sets no write concern, so the server default applies: getDefaultRWConcern gives w majority and the one-member replica set (--replSet gvbmongo in the image) has writeConcernMajorityJournalDefault true, with WiredTiger journaling on (journalCommitInterval 100 ms is only the interval for unacknowledged work).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].durability", "value": "Acknowledged writes are journaled before the acknowledgement. The sink sets no write concern, so the server default applies: getDefaultRWConcern gives w majority and the one-member replica set (--replSet gvbmongo in the image) has writeConcernMajorityJournalDefault true, with WiredTiger journaling on (journalCommitInterval 100 ms is only the interval for unacknowledged work)." } ] }, { "slot": "disclosure.durability.mongodb.g1", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.mongodb.g1.q1", "text": "Measured 2026-10-04: 30 acknowledged single-document inserts raised WiredTiger 'log sync operations' by 30 and", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].durability", "value": "Measured 2026-10-04: 30 acknowledged single-document inserts raised WiredTiger 'log sync operations' by 30 and" } ] }, { "slot": "disclosure.durability.mongodb.g2", "text": "Backed by a saved log or measurement, not read back from the engine in these runs:", "sources": [ { "kind": "file", "ref": "design/engine-docs/durability-logs-2026-10-04/mongodb-acknowledged-inserts.strace#WiredTigerLog", "value": "WiredTigerLog" } ] }, { "slot": "disclosure.durability.mongodb.g2.q2", "text": "strace showed fdatasync on /data/db/journal/WiredTigerLog files.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].durability", "value": "strace showed fdatasync on /data/db/journal/WiredTigerLog files." } ] }, { "slot": "disclosure.durability.mongodb.g3", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.mongodb.g3.q3", "text": "A crash loses no acknowledged write. mongot's search index is not part of that promise: it follows the collection asynchronously and is rebuilt from it.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[mongodb].durability", "value": "A crash loses no acknowledged write. mongot's search index is not part of that promise: it follows the collection asynchronously and is rebuilt from it." } ] }, { "slot": "disclosure.durability.chroma", "text": "chroma: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.chroma.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.chroma.g0.q0", "text": "SQLite rollback journal with its default synchronous=FULL under the data directory", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[chroma].durability", "value": "SQLite rollback journal with its default synchronous=FULL under the data directory" } ] }, { "slot": "disclosure.durability.chroma.g1", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "deploy/engines/chroma.compose.yaml#IS_PERSISTENT: \"1\"", "value": "IS_PERSISTENT: \"1\"" }, { "kind": "file", "ref": "deploy/engines/chroma.compose.yaml#PERSIST_DIRECTORY: /data", "value": "PERSIST_DIRECTORY: /data" } ] }, { "slot": "disclosure.durability.chroma.g1.q1", "text": "(chroma.compose.yaml sets IS_PERSISTENT=1 and PERSIST_DIRECTORY=/data and no sync setting):", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[chroma].durability", "value": "(chroma.compose.yaml sets IS_PERSISTENT=1 and PERSIST_DIRECTORY=/data and no sync setting):" } ] }, { "slot": "disclosure.durability.chroma.g2", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.chroma.g2.q2", "text": "every write is committed to chroma.sqlite3 with fsync of the journal, the directory and the database file before the call returns (strace: 15 database fsyncs and 38 journal fsyncs for one create, four 500-vector upserts and one delete), and the HNSW files are written every sync_threshold=1000 vectors and rebuilt from the SQLite log after a crash; measured with kill -9 and a restart: all 50, 300 and 2,000 vectors were still stored and searchable", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[chroma].durability", "value": "every write is committed to chroma.sqlite3 with fsync of the journal, the directory and the database file before the call returns (strace: 15 database fsyncs and 38 journal fsyncs for one create, four 500-vector upserts and one delete), and the HNSW files are written every sync_threshold=1000 vectors and rebuilt from the SQLite log after a crash; measured with kill -9 and a restart: all 50, 300 and 2,000 vectors were still stored and searchable" } ] }, { "slot": "disclosure.durability.typesense", "text": "typesense: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.typesense.g0", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.typesense.g0.q0", "text": "Every acknowledged write is appended to Typesense's raft log and fsynced before the answer: braft raft_sync=true with raft_sync_policy=0 (sync immediately), read from the running container's brpc /flags page on 2026-10-04. A restart replays the log from the last snapshot", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[typesense].durability", "value": "Every acknowledged write is appended to Typesense's raft log and fsynced before the answer: braft raft_sync=true with raft_sync_policy=0 (sync immediately), read from the running container's brpc /flags page on 2026-10-04. A restart replays the log from the last snapshot" } ] }, { "slot": "disclosure.durability.typesense.g1", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "deploy/engines/typesense.compose.yaml#TYPESENSE_SNAPSHOT_INTERVAL_SECONDS: \"300\"", "value": "TYPESENSE_SNAPSHOT_INTERVAL_SECONDS: \"300\"" } ] }, { "slot": "disclosure.durability.typesense.g1.q1", "text": "(typesense.compose.yaml sets TYPESENSE_SNAPSHOT_INTERVAL_SECONDS=300)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[typesense].durability", "value": "(typesense.compose.yaml sets TYPESENSE_SNAPSHOT_INTERVAL_SECONDS=300)" } ] }, { "slot": "disclosure.durability.typesense.g2", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.typesense.g2.q2", "text": "and rebuilds the in-memory HNSW graph, so by those settings a crash loses no acknowledged write, but the restart is slow after a big load. This is read from the settings, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[typesense].durability", "value": "and rebuilds the in-memory HNSW graph, so by those settings a crash loses no acknowledged write, but the restart is slow after a big load. This is read from the settings, not shown by pulling power. One node and no replicas, so a lost disk loses the data." } ] }, { "slot": "disclosure.durability.sql-diskann", "text": "sql-diskann: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.sql-diskann.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.sql-diskann.g0.q0", "text": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].durability", "value": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging;" } ] }, { "slot": "disclosure.durability.sql-diskann.g1", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases", "value": "SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases" } ] }, { "slot": "disclosure.durability.sql-diskann.g1.q1", "text": "delayed durability is DISABLED);", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].durability", "value": "delayed durability is DISABLED);" } ] }, { "slot": "disclosure.durability.sql-diskann.g2", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.sql-diskann.g2.q2", "text": "a database created here copies the model database:", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].durability", "value": "a database created here copies the model database:" } ] }, { "slot": "disclosure.durability.sql-diskann.g3", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases", "value": "SELECT name, recovery_model_desc, delayed_durability_desc, page_verify_option_desc FROM sys.databases" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#DBCC TRACESTATUS( -1 ) WITH NO_INFOMSGS;", "value": "DBCC TRACESTATUS( -1 ) WITH NO_INFOMSGS;" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlDurability.cs#ConfigFiles.ExistsWithSudoAsync( confPath, ct )", "value": "ConfigFiles.ExistsWithSudoAsync( confPath, ct )" } ] }, { "slot": "disclosure.durability.sql-diskann.g3.q3", "text": "recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].durability", "value": "recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled;" } ] }, { "slot": "disclosure.durability.sql-diskann.g3.q4", "text": "mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].durability", "value": "mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry," } ] }, { "slot": "disclosure.durability.sql-diskann.g4", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.sql-diskann.g4.q5", "text": "so SQL Server's own Linux defaults for flushing writes apply (Microsoft's Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].durability", "value": "so SQL Server's own Linux defaults for flushing writes apply (Microsoft's Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here)." } ] }, { "slot": "disclosure.durability.sql-diskann.g4.q6", "text": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].durability", "value": "Not tested by cutting power; whether the disk's own write cache reaches the media was not checked." } ] }, { "slot": "disclosure.durability.sql-diskann.g5", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Targets/SqlServerContainer.cs#n.StartsWith( \"MSSQL_\"", "value": "n.StartsWith( \"MSSQL_\"" } ] }, { "slot": "disclosure.durability.sql-diskann.g5.q7", "text": "Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[sql-diskann].durability", "value": "Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD." } ] }, { "slot": "disclosure.durability.duckdb", "text": "duckdb: its recorded durability text, clause by clause, with how each clause is backed.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.durability.duckdb.g0", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.duckdb.g0.q0", "text": "DuckDB's write-ahead log is fsynced at every commit and no DuckDbSink setting changes that", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].durability", "value": "DuckDB's write-ahead log is fsynced at every commit and no DuckDbSink setting changes that" } ] }, { "slot": "disclosure.durability.duckdb.g1", "text": "Set by a line of code or a compose file saved in this repository; the v8 engine settings also list checkpoint_threshold, read from the running engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSinkOptions.cs#CheckpointThreshold { get; set; } = \"256MB\"", "value": "CheckpointThreshold { get; set; } = \"256MB\"" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET checkpoint_threshold", "value": "SET checkpoint_threshold" }, { "kind": "consolidated", "ref": "consolidated:engineSettings[18].settings[5].key", "value": "checkpoint_threshold" }, { "kind": "consolidated", "ref": "consolidated:engineSettings[18].settings[5].how", "value": "read: SELECT current_setting('checkpoint_threshold') on a second connection to the bench file opened with the sink's own connection string" } ] }, { "slot": "disclosure.durability.duckdb.g1.q1", "text": "(checkpoint_threshold=256MB", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].durability", "value": "(checkpoint_threshold=256MB" } ] }, { "slot": "disclosure.durability.duckdb.g2", "text": "Typed in the program's own text; it cites a measurement that these runs did not repeat, and no saved source backs this clause:", "sources": [] }, { "slot": "disclosure.durability.duckdb.g2.q2", "text": "only spaces out the checkpoints that write the database file; measured with strace: 42 WAL fsyncs for 40 single-record commits plus 2 setup statements), so an operating-system crash or power cut loses no committed row; measured with kill -9 right after the last commit and a reopen: all 50, 300 and 2,000 rows and an HNSW index that counted the same number were recovered from the log; not tested and documented by DuckDB: the HNSW index is file-backed only through hnsw_enable_experimental_persistence = true,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].durability", "value": "only spaces out the checkpoints that write the database file; measured with strace: 42 WAL fsyncs for 40 single-record commits plus 2 setup statements), so an operating-system crash or power cut loses no committed row; measured with kill -9 right after the last commit and a reopen: all 50, 300 and 2,000 rows and an HNSW index that counted the same number were recovered from the log; not tested and documented by DuckDB: the HNSW index is file-backed only through hnsw_enable_experimental_persistence = true," } ] }, { "slot": "disclosure.durability.duckdb.g3", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Engines/Sinks/DuckDbSink.cs#SET hnsw_enable_experimental_persistence = true;", "value": "SET hnsw_enable_experimental_persistence = true;" } ] }, { "slot": "disclosure.durability.duckdb.g3.q3", "text": "which this sink turns on,", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].durability", "value": "which this sink turns on," } ] }, { "slot": "disclosure.durability.duckdb.g4", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "disclosure.durability.duckdb.g4.q4", "text": "and WAL recovery for such custom indexes is not complete, so a crash during a checkpoint or a later commit can damage the index while the rows survive", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:targets[duckdb].durability", "value": "and WAL recovery for such custom indexes is not complete, so a crash during a checkpoint or a later commit can damage the index while the rows survive" } ] }, { "slot": "disclosure.settings", "text": "The engine settings table lists what run 20261007-205106-eshoponweb read from each running engine or set from a repository file, as each row's how column says.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "disclosure.scan.qdrant", "text": "qdrant's measured search fact says: no index used, exact scan by design.", "sources": [ { "kind": "results", "ref": "results:targets[qdrant].indexState.afterLoad.detail#no index used, exact scan by design", "value": "no index used, exact scan by design" } ] }, { "slot": "disclosure.cap.oracle", "text": "oracle's recorded index text says: Oracle Free caps itself at 2 CPUs.", "sources": [ { "kind": "results", "ref": "results:targets[oracle].index#Oracle Free caps itself at 2 CPUs", "value": "Oracle Free caps itself at 2 CPUs" } ] }, { "slot": "disclosure.scan.elasticsearch", "text": "elasticsearch's measured search fact says: no HNSW graph, searches scan all 524 vectors.", "sources": [ { "kind": "results", "ref": "results:targets[elasticsearch].load.indexNote#NO HNSW GRAPH, searches scan all 524 vectors", "value": "NO HNSW GRAPH, searches scan all 524 vectors" } ] }, { "slot": "disclosure.scan.sqlitevec", "text": "sqlitevec's measured search fact says: no index, exact scan by design.", "sources": [ { "kind": "results", "ref": "results:targets[sqlitevec].indexState.afterLoad.detail#no index, exact scan by design", "value": "no index, exact scan by design" } ] }, { "slot": "disclosure.scan.sql", "text": "sql's measured search fact says: no index used, exact scan by design.", "sources": [ { "kind": "results", "ref": "results:targets[sql].indexState.afterLoad.detail#no index used, exact scan by design", "value": "no index used, exact scan by design" } ] }, { "slot": "disclosure.graph.mongodb", "text": "mongodb's search fact is the engine's own report; its segments held at most 278 vectors, below the 1,043 at which elasticsearch's recorded text says a segment gets a graph, and no graph was checked.", "sources": [ { "kind": "results", "ref": "results:targets[mongodb].load.indexNote#searched through the HNSW graph (Approximate)", "value": "searched through the HNSW graph (Approximate)" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[mongodb].load.indexNote#278", "value": "278" }, { "kind": "results", "ref": "results:targets[elasticsearch].index#under 1,043 vectors gets no graph", "value": "under 1,043 vectors gets no graph" } ] }, { "slot": "disclosure.disk.clickhouse.20261006-130619-eshoponweb", "text": "Run 20261006-130619-eshoponweb: clickhouse's whole engine data folder was recorded as 4.21 GiB, against 99.17 KiB before; the run's page prints a correction to its recorded \"added by this load\" figure.", "sources": [ { "kind": "file", "ref": "deploy/bench/run-page-notes.json#results@20261006-130619-eshoponweb:targets[clickhouse].disk.text", "value": "results@20261006-130619-eshoponweb:targets[clickhouse].disk.text" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" }, { "kind": "results", "ref": "results@20261006-130619-eshoponweb:targets[clickhouse].disk.text#4.21 GiB (whole engine data folder), 99.17 KiB before", "value": "4.21 GiB (whole engine data folder), 99.17 KiB before" } ] }, { "slot": "disclosure.disk.clickhouse.20261006-142724-eshoponweb", "text": "Run 20261006-142724-eshoponweb: clickhouse's whole engine data folder was recorded as 6.79 GiB, against 4.22 GiB before; the run's page prints a correction to its recorded \"added by this load\" figure.", "sources": [ { "kind": "file", "ref": "deploy/bench/run-page-notes.json#results@20261006-142724-eshoponweb:targets[clickhouse].disk.text", "value": "results@20261006-142724-eshoponweb:targets[clickhouse].disk.text" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" }, { "kind": "results", "ref": "results@20261006-142724-eshoponweb:targets[clickhouse].disk.text#6.79 GiB (whole engine data folder), 4.22 GiB before", "value": "6.79 GiB (whole engine data folder), 4.22 GiB before" } ] }, { "slot": "disclosure.disk.clickhouse.20261006-154837-eshoponweb", "text": "Run 20261006-154837-eshoponweb: clickhouse's whole engine data folder was recorded as 6.79 GiB, against 6.79 GiB before; the run's page prints a correction to its recorded \"added by this load\" figure.", "sources": [ { "kind": "file", "ref": "deploy/bench/run-page-notes.json#results@20261006-154837-eshoponweb:targets[clickhouse].disk.text", "value": "results@20261006-154837-eshoponweb:targets[clickhouse].disk.text" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" }, { "kind": "results", "ref": "results@20261006-154837-eshoponweb:targets[clickhouse].disk.text#6.79 GiB (whole engine data folder), 6.79 GiB before", "value": "6.79 GiB (whole engine data folder), 6.79 GiB before" } ] }, { "slot": "disclosure.dataFolder.clickhouse.20261007-205106-eshoponweb", "text": "Run 20261007-205106-eshoponweb: clickhouse's data folder held 4559215474 bytes at its start, 14691380 bytes after its system log tables were truncated and it stopped changing, and 5699801237 bytes at its end.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].dataFolder.bytesAtStart", "value": "4559215474" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].dataFolder.bytesAfterReset", "value": "14691380" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].dataFolder.bytesAtEnd", "value": "5699801237" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].dataFolder.reset#the folder was steady", "value": "the folder was steady" } ] }, { "slot": "disclosure.dataFolder.clickhouse.20261007-205106-eshoponweb.tables", "text": "Run 20261007-205106-eshoponweb: the reset truncated 9 MergeTree log tables, whose active bytes by clickhouse's own count were 2.09 GiB before and 0 B after.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].dataFolder.reset#truncated 9 MergeTree log tables in database system", "value": "truncated 9 MergeTree log tables in database system" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].dataFolder.reset#2.09 GiB", "value": "2.09 GiB" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:targets[clickhouse].dataFolder.reset#0 B", "value": "0 B" } ] }, { "slot": "disclosure.dataFolder.clickhouse.20261007-221429-eshoponweb", "text": "Run 20261007-221429-eshoponweb: clickhouse's data folder held 5707585756 bytes at its start, 2917655627 bytes after its system log tables were truncated and it stopped changing, and 7408848771 bytes at its end.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].dataFolder.bytesAtStart", "value": "5707585756" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].dataFolder.bytesAfterReset", "value": "2917655627" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].dataFolder.bytesAtEnd", "value": "7408848771" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].dataFolder.reset#the folder was steady", "value": "the folder was steady" } ] }, { "slot": "disclosure.dataFolder.clickhouse.20261007-221429-eshoponweb.tables", "text": "Run 20261007-221429-eshoponweb: the reset truncated 9 MergeTree log tables, whose active bytes by clickhouse's own count were 183.97 KiB before and 0 B after.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-221429-eshoponweb].folder", "value": "20261007-221429-eshoponweb" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].dataFolder.reset#truncated 9 MergeTree log tables in database system", "value": "truncated 9 MergeTree log tables in database system" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].dataFolder.reset#183.97 KiB", "value": "183.97 KiB" }, { "kind": "results", "ref": "results@20261007-221429-eshoponweb:targets[clickhouse].dataFolder.reset#0 B", "value": "0 B" } ] }, { "slot": "disclosure.dataFolder.clickhouse.20261007-233551-eshoponweb", "text": "Run 20261007-233551-eshoponweb: clickhouse's data folder held 7426117183 bytes at its start, 2246745378 bytes after its system log tables were truncated and it stopped changing, and 7261702692 bytes at its end.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].dataFolder.bytesAtStart", "value": "7426117183" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].dataFolder.bytesAfterReset", "value": "2246745378" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].dataFolder.bytesAtEnd", "value": "7261702692" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].dataFolder.reset#the folder was steady", "value": "the folder was steady" } ] }, { "slot": "disclosure.dataFolder.clickhouse.20261007-233551-eshoponweb.tables", "text": "Run 20261007-233551-eshoponweb: the reset truncated 9 MergeTree log tables, whose active bytes by clickhouse's own count were 185.06 KiB before and 0 B after.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-233551-eshoponweb].folder", "value": "20261007-233551-eshoponweb" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].dataFolder.reset#truncated 9 MergeTree log tables in database system", "value": "truncated 9 MergeTree log tables in database system" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].dataFolder.reset#185.06 KiB", "value": "185.06 KiB" }, { "kind": "results", "ref": "results@20261007-233551-eshoponweb:targets[clickhouse].dataFolder.reset#0 B", "value": "0 B" } ] }, { "slot": "disclosure.layouts", "text": "Segment layouts differed between runs for mongodb; each run's layout is in its row flags.", "sources": [] }, { "slot": "disclosure.observer", "text": "An observer process ran beside the v8 runs; its own CPU, at most 0.138 CPUs in any one pass, counts as outside load, and its summary is observer/summary.json beside this report.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.cpuMax", "value": "0.138" }, { "kind": "consolidated", "ref": "consolidated:observer.copy", "value": "observer/summary.json" } ] }, { "slot": "disclosure.observer.clock", "text": "By APERF and MPERF, the largest deviation of one CPU in a pass was 0.66%: oracle's eight-searcher pass on CPU 7, at 3477.0 MHz against the pin of 3500 MHz.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.cpuWorst.deviationBp|bp-pct", "value": "0.66" }, { "kind": "consolidated", "ref": "consolidated:observer.cpuWorst.target", "value": "oracle" }, { "kind": "consolidated", "ref": "consolidated:observer.cpuWorst.cpu", "value": "7" }, { "kind": "consolidated", "ref": "consolidated:observer.cpuWorst.mhz", "value": "3477.0" }, { "kind": "consolidated", "ref": "consolidated:observer.pinnedMhz", "value": "3500" } ] }, { "slot": "disclosure.observer.msr", "text": "The observer read MSR 0x620 as 0x1e1e.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Running/MachineControlSampler.cs#LIMIT_MSR = \"0x620\"", "value": "LIMIT_MSR = \"0x620\"" }, { "kind": "consolidated", "ref": "consolidated:observer.msr620ValuesSeen[0]", "value": "0x1e1e" } ] }, { "slot": "disclosure.observer.dip", "text": "oracle's eight-searcher pass ran 0.27% to 0.66% under the pin on every CPU in all 3 runs of v8; no other pass was more than 0.04% from it, and why is not known.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.dip.target", "value": "oracle" }, { "kind": "consolidated", "ref": "consolidated:observer.dip.runs", "value": "3" }, { "kind": "consolidated", "ref": "consolidated:observer.dip.maxBp|bp-pct", "value": "0.66" }, { "kind": "consolidated", "ref": "consolidated:observer.dip.otherWorstBp|bp-pct", "value": "0.04" }, { "kind": "consolidated", "ref": "consolidated:observer.session", "value": "v8" }, { "kind": "consolidated", "ref": "consolidated:observer.dip.minBp|bp-pct", "value": "0.27" } ] }, { "slot": "disclosure.observer.meandip.0", "text": "Run 20261007-205106-eshoponweb: oracle's eight-searcher pass averaged 3085.5 MHz on the engine CPUs and 3189.8 MHz on the client CPUs, as in v7; a mean rule flags it and the median rule does not.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.meanDips[0].run", "value": "20261007-205106-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:observer.meanDips[0].engineMeanMhz", "value": "3085.5" }, { "kind": "consolidated", "ref": "consolidated:observer.meanDips[0].clientMeanMhz", "value": "3189.8" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].name", "value": "v7" } ] }, { "slot": "disclosure.observer.meandip.1", "text": "Run 20261007-221429-eshoponweb: oracle's eight-searcher pass averaged 3206.2 MHz on the engine CPUs and 3299.0 MHz on the client CPUs, as in v7; a mean rule flags it and the median rule does not.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.meanDips[1].run", "value": "20261007-221429-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:observer.meanDips[1].engineMeanMhz", "value": "3206.2" }, { "kind": "consolidated", "ref": "consolidated:observer.meanDips[1].clientMeanMhz", "value": "3299.0" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].name", "value": "v7" } ] }, { "slot": "disclosure.observer.meandip.2", "text": "Run 20261007-233551-eshoponweb: oracle's eight-searcher pass averaged 3063.0 MHz on the engine CPUs and 3189.2 MHz on the client CPUs, as in v7; a mean rule flags it and the median rule does not.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.meanDips[2].run", "value": "20261007-233551-eshoponweb" }, { "kind": "consolidated", "ref": "consolidated:observer.meanDips[2].engineMeanMhz", "value": "3063.0" }, { "kind": "consolidated", "ref": "consolidated:observer.meanDips[2].clientMeanMhz", "value": "3189.2" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].name", "value": "v7" } ] }, { "slot": "disclosure.observer.placement", "text": "The observer was not pinned: in the v8 runs 34.8, 33.8 and 33.5 percent of its resident-thread ticks were seen on engine CPUs 2-3,6-7.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.session", "value": "v8" }, { "kind": "doc", "ref": "doc:design/bench-inputs/observer-placement/analysis-801.txt#The observer is NOT pinned", "value": "The observer is NOT pinned" }, { "kind": "doc", "ref": "doc:design/bench-inputs/observer-placement/analysis-801.txt#11880 ticks (34.8 percent) on engine CPUs 2-3,6-7", "value": "11880 ticks (34.8 percent) on engine CPUs 2-3,6-7" }, { "kind": "doc", "ref": "doc:design/bench-inputs/observer-placement/analysis-802.txt#11167 ticks (33.8 percent) on engine CPUs 2-3,6-7", "value": "11167 ticks (33.8 percent) on engine CPUs 2-3,6-7" }, { "kind": "doc", "ref": "doc:design/bench-inputs/observer-placement/analysis-803.txt#11135 ticks (33.5 percent) on engine CPUs 2-3,6-7", "value": "11135 ticks (33.5 percent) on engine CPUs 2-3,6-7" } ] }, { "slot": "disclosure.observer.cgroup", "text": "Averaged over a whole run, the observer's CPU by cgroup was at most 0.0755 CPUs in any of the v8 runs.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.session", "value": "v8" }, { "kind": "doc", "ref": "doc:design/bench-inputs/observer-placement/analysis-801.txt#children included): 0.0755 CPUs", "value": "children included): 0.0755 CPUs" }, { "kind": "doc", "ref": "doc:design/bench-inputs/observer-placement/analysis-802.txt#children included): 0.0753 CPUs", "value": "children included): 0.0753 CPUs" }, { "kind": "doc", "ref": "doc:design/bench-inputs/observer-placement/analysis-803.txt#children included): 0.0755 CPUs", "value": "children included): 0.0755 CPUs" } ] }, { "slot": "disclosure.timers", "text": "The observer saw a systemd timer fire during 14 timed passes of the v8 runs.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observer.passesWithTimerFired", "value": "14" } ] }, { "slot": "disclosure.observer.v7", "text": "An observer process ran beside the v7 runs; its own CPU, at most 0.123 CPUs in any one pass, counts as outside load, and its summary is observer/summary-v7.json beside this report.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observerOthers[0].cpuMax", "value": "0.123" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].copy", "value": "observer/summary-v7.json" } ] }, { "slot": "disclosure.observer.clock.v7", "text": "By APERF and MPERF, the largest deviation of one CPU in a pass was 0.73%: oracle's eight-searcher pass on CPU 2, at 3474.3 MHz against the pin of 3500 MHz.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observerOthers[0].cpuWorst.deviationBp|bp-pct", "value": "0.73" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].cpuWorst.target", "value": "oracle" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].cpuWorst.cpu", "value": "2" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].cpuWorst.mhz", "value": "3474.3" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].pinnedMhz", "value": "3500" } ] }, { "slot": "disclosure.observer.msr.v7", "text": "The observer read MSR 0x620 as 0x1e1e.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Running/MachineControlSampler.cs#LIMIT_MSR = \"0x620\"", "value": "LIMIT_MSR = \"0x620\"" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].msr620ValuesSeen[0]", "value": "0x1e1e" } ] }, { "slot": "disclosure.observer.dip.v7", "text": "oracle's eight-searcher pass ran 0.41% to 0.73% under the pin on every CPU in all 3 runs of v7; no other pass was more than 0.05% from it, and why is not known.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observerOthers[0].dip.target", "value": "oracle" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].dip.runs", "value": "3" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].dip.maxBp|bp-pct", "value": "0.73" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].dip.otherWorstBp|bp-pct", "value": "0.05" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].session", "value": "v7" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].dip.minBp|bp-pct", "value": "0.41" } ] }, { "slot": "disclosure.timers.v7", "text": "The observer summary covers systemd timers for no v7 run.", "sources": [ { "kind": "consolidated", "ref": "consolidated:observerOthers[0].runs[0].timersCovered", "value": "false" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].runs[1].timersCovered", "value": "false" }, { "kind": "consolidated", "ref": "consolidated:observerOthers[0].runs[2].timersCovered", "value": "false" } ] }, { "slot": "disclosure.cstates", "text": "CPU idle states were recorded and left as found: driver intel_idle, governor menu.", "sources": [ { "kind": "results", "ref": "results:conditions.cpuIdle.driver", "value": "intel_idle" }, { "kind": "results", "ref": "results:conditions.cpuIdle.governor", "value": "menu" } ] }, { "slot": "disclosure.busy", "text": "Outside load is busy CPU less this client's and the followed engine cgroups'; dockerd's cgroup is one of them for a compose engine, so its CPU counts as benchmark work, not outside load.", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Running/MachineControlPinning.cs#AddGroup( dockerd );", "value": "AddGroup( dockerd );" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Running/MachineControlSampler.cs#double outside = ( reading.Busy - _last.Busy ) - ( reading.Self - _last.Self ) - engine;", "value": "double outside = ( reading.Busy - _last.Busy ) - ( reading.Self - _last.Self ) - engine;" } ] }, { "slot": "disclosure.dockerd.v7", "text": "The runs of v7 did not record how dockerd's CPU was counted.", "sources": [] }, { "slot": "disclosure.dockerd.v8", "text": "engine CPU per search is the CPU time (cpu.stat usage_usec) of the cgroups each target's turn followed (conditions.engines[].cgroups, dockerd's left out), read with every 250 ms sample, interpolated to the pass's timed window and divided by its completed searches; engineCpusBusy is that CPU time over the window's length. A followed cgroup counts every process in it. dockerd's cgroup (/sys/fs/cgroup/system.slice/docker.service, holding docker-proxy, dockerd, for every container on the box) is left out of engine CPU per search and given as dockerdCpuMsPerSearch; it is subtracted from outside load as benchmark work during the turns of oracle, mongodb, redis, typesense, clickhouse, sql, opensearch, milvus, sql-diskann, qdrant-hnsw, weaviate, pgvector, vespa, elasticsearch, mariadb, chroma, qdrant, and counts as outside load during the turns of duckdb, sqlitevec. containerd-shim runs in /sys/fs/cgroup/system.slice/containerd.service, which is in neither engine figure and counts as outside load. An embedded engine runs in this client process, so its CPU is in clientCpuMsPerSearch and its engine figure is null with the reason; a target whose engine has no cgroup followed, or whose containers could not all be pinned, also gets null with the reason, never 0.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:conditions.engineCpu.rule", "value": "engine CPU per search is the CPU time (cpu.stat usage_usec) of the cgroups each target's turn followed (conditions.engines[].cgroups, dockerd's left out), read with every 250 ms sample, interpolated to the pass's timed window and divided by its completed searches; engineCpusBusy is that CPU time over the window's length. A followed cgroup counts every process in it. dockerd's cgroup (/sys/fs/cgroup/system.slice/docker.service, holding docker-proxy, dockerd, for every container on the box) is left out of engine CPU per search and given as dockerdCpuMsPerSearch; it is subtracted from outside load as benchmark work during the turns of oracle, mongodb, redis, typesense, clickhouse, sql, opensearch, milvus, sql-diskann, qdrant-hnsw, weaviate, pgvector, vespa, elasticsearch, mariadb, chroma, qdrant, and counts as outside load during the turns of duckdb, sqlitevec. containerd-shim runs in /sys/fs/cgroup/system.slice/containerd.service, which is in neither engine figure and counts as outside load. An embedded engine runs in this client process, so its CPU is in clientCpuMsPerSearch and its engine figure is null with the reason; a target whose engine has no cgroup followed, or whose containers could not all be pinned, also gets null with the reason, never 0." } ] }, { "slot": "disclosure.dockerd.test", "text": "The test OutsideLoad_IsTheV7AccountingBitForBit holds the formula that turns CPU counters into outside load equal to the formula of v7, on fixed counters; it does not make the measured load of two sessions equal.", "sources": [ { "kind": "file", "ref": "tests/GenericVectorBuilder.Engines.Tests/Bench/MachineControlV8Tests.cs#OutsideLoad_IsTheV7AccountingBitForBit", "value": "OutsideLoad_IsTheV7AccountingBitForBit" } ] }, { "slot": "disclosure.searchSettings", "text": "Every order here was measured at the search settings each run recorded for the engine, listed in the why section; this report makes no claim at other settings.", "sources": [] }, { "slot": "disclosure.route.0", "text": "A route each run recorded in conditions.connections reads: container address on its Docker network, not the published port (no docker-proxy).", "sources": [ { "kind": "results", "ref": "results:conditions.connections#container address on its Docker network, not the published port (no docker-proxy)", "value": "container address on its Docker network, not the published port (no docker-proxy)" } ] }, { "slot": "disclosure.route.1", "text": "A route each run recorded in conditions.connections reads: in this process (embedded), no network.", "sources": [ { "kind": "results", "ref": "results:conditions.connections#in this process (embedded), no network", "value": "in this process (embedded), no network" } ] }, { "slot": "method.intro", "text": "The method, quoted from the notes of run 20261007-205106-eshoponweb:", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v8].runs[folder=20261007-205106-eshoponweb].folder", "value": "20261007-205106-eshoponweb" } ] }, { "slot": "method.g0", "text": "Read from the running engine or the machine by the benchmark's own code, as the cited line shows:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Report/MachineFacts.cs#File.ReadAllText( \"/proc/loadavg\" )", "value": "File.ReadAllText( \"/proc/loadavg\" )" } ] }, { "slot": "method.g0.q0", "text": "Load average at start: 0.38 0.64 1.22 on 8 logical CPUs (1/5/15 min).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[0]", "value": "Load average at start: 0.38 0.64 1.22 on 8 logical CPUs (1/5/15 min)." } ] }, { "slot": "method.g1", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "method.g1.q1", "text": "It counts this benchmark's own earlier work and the engines it started, so it is recorded here and never judged; each timed pass is judged by the outside load machine control measures (conditions.passes).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[0]", "value": "It counts this benchmark's own earlier work and the engines it started, so it is recorded here and never judged; each timed pass is judged by the outside load machine control measures (conditions.passes)." } ] }, { "slot": "method.g2", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Running/RunOrder.cs#return Shuffle( names, Mix( (ulong)(uint)runSeed ^ TARGET_SALT ) );", "value": "return Shuffle( names, Mix( (ulong)(uint)runSeed ^ TARGET_SALT ) );" }, { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Running/RunOrder.cs#return Shuffle( passes, Mix( (ulong)(uint)runSeed ^ PASS_SALT ^ StableHash( targetName.ToLowerInvariant() ) ) );", "value": "return Shuffle( passes, Mix( (ulong)(uint)runSeed ^ PASS_SALT ^ StableHash( targetName.ToLowerInvariant() ) ) );" } ] }, { "slot": "method.g2.q2", "text": "Order: targets ran one at a time in a random order from seed 801 (runSeed; --seed 801 repeats it, targetOrder lists it).", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[1]", "value": "Order: targets ran one at a time in a random order from seed 801 (runSeed; --seed 801 repeats it, targetOrder lists it)." } ] }, { "slot": "method.g2.q3", "text": "Inside each target the timed passes also ran in a random order from the same seed and the target's name (passOrder):", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[1]", "value": "Inside each target the timed passes also ran in a random order from the same seed and the target's name (passOrder):" } ] }, { "slot": "method.g3", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "method.g3.q4", "text": "default@1 is one searcher for 20 s (every search's latency gives p50/p95/p99, the completed searches give QPS@1, its first answer to each query gives recall and nDCG), default@N is N searchers for 20 s (QPS@N), exact is the engine's exact mode, one searcher for 60 s cycling the queries.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[1]", "value": "default@1 is one searcher for 20 s (every search's latency gives p50/p95/p99, the completed searches give QPS@1, its first answer to each query gives recall and nDCG), default@N is N searchers for 20 s (QPS@N), exact is the engine's exact mode, one searcher for 60 s cycling the queries." } ] }, { "slot": "method.g3.q5", "text": "Preparation, untimed, before any timed pass: a rehearsal of every pass type at its own concurrency for 30 s each, through the same code the passes use.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[2]", "value": "Preparation, untimed, before any timed pass: a rehearsal of every pass type at its own concurrency for 30 s each, through the same code the passes use." } ] }, { "slot": "method.g3.q6", "text": "Warm-up and settle check, untimed, right before every timed pass: the pass's own search at the pass's own number of searchers for at least 15 s and at least 20 searches (at most 120 s), read in windows of at least 2 s and 100 searches; then a 3 s trial of the same pass. The trial's figure (p50 with one searcher, QPS with several) must lie within 10% of the warm-up's settled figure (the median of its last 3 windows, which must agree within 5%); if not, the warm-up is extended once (at least 30 s, until its windows agree, at most 120 s), where an extension's windows agree when its level test passes: the older and the newer half of its latest windows, 5 to 10 windows a half, each half read as the pass reads it (the p50 of all its searches with one searcher, its searches per second with several), agree within 5%, judged on the windows that stopped it (an extension whose cap runs out with fewer than 10 windows is judged by its last 3 instead); then a second trial is taken, which must lie inside the range of the newer half's windows or within 10% of the newer half's figure, and the warm-up runs again before the pass. After the pass its own figure is held against the settled figure, within 10%. In the level test, the second trial and the hold, two one-searcher p50s no more than 0.1 ms apart agree whatever their percentage (below what this box resolves at one searcher). Each target's notes give every check, and a pass that still disagrees, or whose own figure did not hold, flags its target as unsettled. With machine control on, the check for a quiet box is made when the warm-up is announced, before it starts (a wait between the warm-up and the timed pass let the engine go cold), so by the time the clock opens that check is as old as the warm-up and the trial (about 18 s, more after an extension); each pass's conditions record that lead (quietCheckLeadSeconds). Failed untimed searches are counted per target (warmupErrors) and are not in the timed error counts.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[3]", "value": "Warm-up and settle check, untimed, right before every timed pass: the pass's own search at the pass's own number of searchers for at least 15 s and at least 20 searches (at most 120 s), read in windows of at least 2 s and 100 searches; then a 3 s trial of the same pass. The trial's figure (p50 with one searcher, QPS with several) must lie within 10% of the warm-up's settled figure (the median of its last 3 windows, which must agree within 5%); if not, the warm-up is extended once (at least 30 s, until its windows agree, at most 120 s), where an extension's windows agree when its level test passes: the older and the newer half of its latest windows, 5 to 10 windows a half, each half read as the pass reads it (the p50 of all its searches with one searcher, its searches per second with several), agree within 5%, judged on the windows that stopped it (an extension whose cap runs out with fewer than 10 windows is judged by its last 3 instead); then a second trial is taken, which must lie inside the range of the newer half's windows or within 10% of the newer half's figure, and the warm-up runs again before the pass. After the pass its own figure is held against the settled figure, within 10%. In the level test, the second trial and the hold, two one-searcher p50s no more than 0.1 ms apart agree whatever their percentage (below what this box resolves at one searcher). Each target's notes give every check, and a pass that still disagrees, or whose own figure did not hold, flags its target as unsettled. With machine control on, the check for a quiet box is made when the warm-up is announced, before it starts (a wait between the warm-up and the timed pass let the engine go cold), so by the time the clock opens that check is as old as the warm-up and the trial (about 18 s, more after an extension); each pass's conditions record that lead (quietCheckLeadSeconds). Failed untimed searches are counted per target (warmupErrors) and are not in the timed error counts." } ] }, { "slot": "method.g3.q7", "text": "Load rows/s counts only time inside each target's upsert calls: one writer, batches of 1000, rows already in memory, the collection dropped and created fresh first. Engines that build or finish their index after the writes do it in a separate timed index step (the load's index seconds), and the run waits for it before searching.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[4]", "value": "Load rows/s counts only time inside each target's upsert calls: one writer, batches of 1000, rows already in memory, the collection dropped and created fresh first. Engines that build or finish their index after the writes do it in a separate timed index step (the load's index seconds), and the run waits for it before searching." } ] }, { "slot": "method.g3.q8", "text": "Index proof: each engine's own report of its index (indexState) is read after the load and again after the last pass. A target whose index was not ready after the load was still measured and carries a WARNING; an engine that reports nothing counts as not ready.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[5]", "value": "Index proof: each engine's own report of its index (indexState) is read after the load and again after the last pass. A target whose index was not ready after the load was still measured and carries a WARNING; an engine that reports nothing counts as not ready." } ] }, { "slot": "method.g3.q9", "text": "Search settings: each searched target records the settings its own index description states (searchSettings: the build parameters and the effort per query, such as m, ef_construction and ef_search), read after the load; consolidating never averages runs whose settings differ.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[6]", "value": "Search settings: each searched target records the settings its own index description states (searchSettings: the build parameters and the effort per query, such as m, ef_construction and ef_search), read after the load; consolidating never averages runs whose settings differ." } ] }, { "slot": "method.g3.q10", "text": "Durability: each target's crash-safety setting as configured here (durability). Engines that do not force writes to disk on every commit load faster for that reason.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[7]", "value": "Durability: each target's crash-safety setting as configured here (durability). Engines that do not force writes to disk on every commit load faster for that reason." } ] }, { "slot": "method.g3.q11", "text": "Engine record: right after each engine was bound, its settings (engineSettings: the container's limits and what the engine itself reports, each entry saying how it was obtained), its image against the id pinned in deploy/bench/image-pins.json (image; another id ends that target with an error) and the size of its data folder (dataFolder: at the start, after any reset, and at the end) were recorded. In run-all, ClickHouse's system log tables are truncated before its load, and the reset is written into its dataFolder.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[8]", "value": "Engine record: right after each engine was bound, its settings (engineSettings: the container's limits and what the engine itself reports, each entry saying how it was obtained), its image against the id pinned in deploy/bench/image-pins.json (image; another id ends that target with an error) and the size of its data folder (dataFolder: at the start, after any reset, and at the end) were recorded. In run-all, ClickHouse's system log tables are truncated before its load, and the reset is written into its dataFolder." } ] }, { "slot": "method.g3.q12", "text": "Latency is client-side wall time around each search (network and driver included), every search of the default@1 window, one searcher, the queries cycled; a window with fewer than 200 searches, a p50 above 1.25 x its mean, or a mean above its p99 is flagged.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[9]", "value": "Latency is client-side wall time around each search (network and driver included), every search of the default@1 window, one searcher, the queries cycled; a window with fewer than 200 searches, a p50 above 1.25 x its mean, or a mean above its p99 is flagged." } ] }, { "slot": "method.g3.q13", "text": "QPS: N searchers back to back for 20 s per level; completed searches divided by the window's elapsed time.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[10]", "value": "QPS: N searchers back to back for 20 s per level; completed searches divided by the window's elapsed time." } ] }, { "slot": "method.g3.q14", "text": "Recall@10: share of the exact top 10 (brute force in memory) that the engine returned.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[11]", "value": "Recall@10: share of the exact top 10 (brute force in memory) that the engine returned." } ] }, { "slot": "method.g4", "text": "Set by a line of code or a compose file saved in this repository, not read back from the engine:", "sources": [ { "kind": "file", "ref": "src/GenericVectorBuilder.Bench/Stats/BenchMath.cs#public const double TIE_TOLERANCE = 1e-5;", "value": "public const double TIE_TOLERANCE = 1e-5;" } ] }, { "slot": "method.g4.q15", "text": "A hit whose exact similarity ties the 10th best (within 1e-5) also counts", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[11]", "value": "A hit whose exact similarity ties the 10th best (within 1e-5) also counts" } ] }, { "slot": "method.tie", "text": "A saved check of this data found 0 pairs of rows within 1e-5 of identical, and 0 of 20 queries with a row outside the exact top 10 that the tie rule would count.", "sources": [ { "kind": "file", "ref": "design/bench-inputs/tie-check-eshoponweb-2026-10-07.json#\"duplicatePairs\": 0", "value": "\"duplicatePairs\": 0" }, { "kind": "file", "ref": "design/bench-inputs/tie-check-eshoponweb-2026-10-07.json#\"queriesWithRowsOutsideTopKWithinEpsilon\": 0", "value": "\"queriesWithRowsOutsideTopKWithinEpsilon\": 0" }, { "kind": "file", "ref": "design/bench-inputs/tie-check-eshoponweb-2026-10-07.json#\"queries\": 20", "value": "\"queries\": 20" }, { "kind": "file", "ref": "design/bench-inputs/tie-check-eshoponweb-2026-10-07.json#\"top\": 10", "value": "\"top\": 10" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:notes[11]#within 1e-5", "value": "within 1e-5" } ] }, { "slot": "method.g5", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "method.g5.q16", "text": ", because duplicate rows embed to identical vectors.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[11]", "value": ", because duplicate rows embed to identical vectors." } ] }, { "slot": "method.g6", "text": "Read back in these runs from the engine or the machine:", "sources": [ { "kind": "results", "ref": "results:conditions.connections#this target", "value": "this target" } ] }, { "slot": "method.g6.q17", "text": "Routes: a container engine (the benchmark's own SQL Server and Qdrant containers included) is reached at its container's own address on its Docker network, never through the published localhost port (docker-proxy);", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[12]", "value": "Routes: a container engine (the benchmark's own SQL Server and Qdrant containers included) is reached at its container's own address on its Docker network, never through the published localhost port (docker-proxy);" } ] }, { "slot": "method.dropped.0", "text": "A clause of this note names sql-native and qdrant-native, which are not targets of this report, so it is not printed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:notes[12]#sql-native", "value": "sql-native" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:notes[12]#qdrant-native", "value": "qdrant-native" } ] }, { "slot": "method.g7", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "method.g7.q18", "text": "Each target's addresses and the connections the client held open after its passes are in its notes and in conditions.connections.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[12]", "value": "Each target's addresses and the connections the client held open after its passes are in its notes and in conditions.connections." } ] }, { "slot": "method.g7.q19", "text": "RAM of compose engines, the benchmark's own SQL Server and Qdrant containers (sql, sql-diskann, qdrant, qdrant-hnsw) included, is docker stats of the engine's containers;", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[13]", "value": "RAM of compose engines, the benchmark's own SQL Server and Qdrant containers (sql, sql-diskann, qdrant, qdrant-hnsw) included, is docker stats of the engine's containers;" } ] }, { "slot": "method.dropped.1", "text": "A clause of this note names sql-native and qdrant-native, which are not targets of this report, so it is not printed.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:notes[13]#sql-native", "value": "sql-native" }, { "kind": "results", "ref": "results@20261007-205106-eshoponweb:notes[13]#qdrant-native", "value": "qdrant-native" } ] }, { "slot": "method.g8", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "method.g8.q20", "text": "Disk is the table's reserved pages (SQL), the collection folder (Qdrant), or for other engines what the load added to the engine's data folder (engines that keep data in memory until a snapshot show almost nothing); when the folder was smaller after the load than before it, no figure is given.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[13]", "value": "Disk is the table's reserved pages (SQL), the collection folder (Qdrant), or for other engines what the load added to the engine's data folder (engines that keep data in memory until a snapshot show almost nothing); when the folder was smaller after the load than before it, no figure is given." } ] }, { "slot": "method.g8.q21", "text": "Engines: run-all starts an engine that is down and stops it afterwards only if it was not running when the run began; an engine that was already running is left running.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[14]", "value": "Engines: run-all starts an engine that is down and stops it afterwards only if it was not running when the run began; an engine that was already running is left running." } ] }, { "slot": "method.g9", "text": "Read back in these runs from the engine or the machine:", "sources": [ { "kind": "results", "ref": "results:conditions.governor#performance", "value": "performance" }, { "kind": "results", "ref": "results:conditions.clientCpus#0-1,4-5", "value": "0-1,4-5" }, { "kind": "results", "ref": "results:conditions.engineCpus#2-3,6-7", "value": "2-3,6-7" } ] }, { "slot": "method.g9.q22", "text": "Machine control on: governor performance on every CPU during the run (before: schedutil; at the end: performance; after putting it back: schedutil). CPU clock pinned for the run: turbo off (intel_pstate/no_turbo 0 -> 1)", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[16]", "value": "Machine control on: governor performance on every CPU during the run (before: schedutil; at the end: performance; after putting it back: schedutil). CPU clock pinned for the run: turbo off (intel_pstate/no_turbo 0 -> 1)" } ] }, { "slot": "method.dropped.2", "text": "A clause of the machine control note, that every CPU's clock is held at its ceiling for any engine, is not printed: the observer found oracle's eight-searcher pass 0.27% to 0.66% under the pin.", "sources": [ { "kind": "results", "ref": "results@20261007-205106-eshoponweb:notes[16]#so every CPU's clock is held at its ceiling of", "value": "so every CPU's clock is held at its ceiling of" }, { "kind": "consolidated", "ref": "consolidated:observer.dip.target", "value": "oracle" }, { "kind": "consolidated", "ref": "consolidated:observer.dip.minBp|bp-pct", "value": "0.27" }, { "kind": "consolidated", "ref": "consolidated:observer.dip.maxBp|bp-pct", "value": "0.66" } ] }, { "slot": "method.g10", "text": "Read back in these runs from the engine or the machine:", "sources": [ { "kind": "results", "ref": "results:conditions.governor#performance", "value": "performance" }, { "kind": "results", "ref": "results:conditions.clientCpus#0-1,4-5", "value": "0-1,4-5" }, { "kind": "results", "ref": "results:conditions.engineCpus#2-3,6-7", "value": "2-3,6-7" } ] }, { "slot": "method.g10.q23", "text": "the uncore (L3 and memory) clock is held at 3000 MHz (min 3000 MHz, max 3000 MHz (MSR 0x620 = 0x1e1e); was min 1200 MHz, max 3000 MHz (MSR 0x620 = 0xc1e)) (before: turbo on (intel_pstate/no_turbo 0), ceiling 3600 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; while pinned: turbo off (intel_pstate/no_turbo 1), ceiling 3500 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; at the end: turbo off (intel_pstate/no_turbo 1), ceiling 3500 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; after putting it back: turbo on (intel_pstate/no_turbo 0), ceiling 3600 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7). Uncore limit at the end: min 3000 MHz, max 3000 MHz (MSR 0x620 = 0x1e1e); after putting it back: min 1200 MHz, max 3000 MHz (MSR 0x620 = 0xc1e). Each target's clock note gives every pass's median MHz on the engine CPUs and on the client CPUs (the median of each CPU's median); a pass whose median on either is more than 100 bp (1%) off the pinned 3500 MHz, or that has no reading on one, is flagged (conditions.clock.rule). Figures from runs made with turbo on are not comparable with these in absolute terms. engine CPUs 2-3,6-7 (cores 2,6 and 3,7), client CPUs 0-1,4-5 (cores 0,4 and 1,5); the client process was pinned; each engine was pinned to the engine CPUs for its turn and put back after (conditions.engines); an engine run-all started (and stopped) was asked to be created on them, and its notes say whether the host did so or it was moved there after the start.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[16]", "value": "the uncore (L3 and memory) clock is held at 3000 MHz (min 3000 MHz, max 3000 MHz (MSR 0x620 = 0x1e1e); was min 1200 MHz, max 3000 MHz (MSR 0x620 = 0xc1e)) (before: turbo on (intel_pstate/no_turbo 0), ceiling 3600 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; while pinned: turbo off (intel_pstate/no_turbo 1), ceiling 3500 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; at the end: turbo off (intel_pstate/no_turbo 1), ceiling 3500 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7; after putting it back: turbo on (intel_pstate/no_turbo 0), ceiling 3600 MHz on CPUs 0-7, floor 1200 MHz on CPUs 0-7). Uncore limit at the end: min 3000 MHz, max 3000 MHz (MSR 0x620 = 0x1e1e); after putting it back: min 1200 MHz, max 3000 MHz (MSR 0x620 = 0xc1e). Each target's clock note gives every pass's median MHz on the engine CPUs and on the client CPUs (the median of each CPU's median); a pass whose median on either is more than 100 bp (1%) off the pinned 3500 MHz, or that has no reading on one, is flagged (conditions.clock.rule). Figures from runs made with turbo on are not comparable with these in absolute terms. engine CPUs 2-3,6-7 (cores 2,6 and 3,7), client CPUs 0-1,4-5 (cores 0,4 and 1,5); the client process was pinned; each engine was pinned to the engine CPUs for its turn and put back after (conditions.engines); an engine run-all started (and stopped) was asked to be created on them, and its notes say whether the host did so or it was moved there after the start." } ] }, { "slot": "method.g11", "text": "Typed in the program's own text; not read back, measured or backed by a saved source:", "sources": [] }, { "slot": "method.g11.q24", "text": "Busy box: right before each pass's warm-up the run waits, up to 10 min, while processes outside the benchmark (everything but this client and the engine under test's cgroups) use more than 0.3 CPUs on average over the last 60 s or the last 5 s, neither window reaching back past the start of the target or of its engine (so the engine's own start-up is not outside work); if the box does not clear the pass runs anyway, flagged 'busy box', as is a pass whose own outside load is above the limit. Outside load counts busy = user + nice + system + irq + softirq + steal (guest time is already in user); CONFIG_IRQ_TIME_ACCOUNTING is not set, so task and cgroup run time include the interrupt and softirq time that hit them and it is added; CONFIG_PARAVIRT_TIME_ACCOUNTING is not set, so steal is added (/boot/config-6.8.0-142-generic) (conditions.cpuAccounting); each pass also records this client's own CPU time per search and the engine's (conditions.passes[].clientCpuMsPerSearch and engineCpuMsPerSearch; how in conditions.engineCpu.rule). CPU clocks were sampled every 250 ms; each pass's min/median/max per CPU is in conditions.passes. CPU idle states, recorded and left as found: driver intel_idle, governor menu, intel_idle max_cstate 9; POLL on, C1 on, C1E on, C3 OFF on CPUs 0-7 (default disabled), C6 on (conditions.cpuIdle). How each target was reached: conditions.connections. Client build Release, .NET 10.0.12.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[16]", "value": "Busy box: right before each pass's warm-up the run waits, up to 10 min, while processes outside the benchmark (everything but this client and the engine under test's cgroups) use more than 0.3 CPUs on average over the last 60 s or the last 5 s, neither window reaching back past the start of the target or of its engine (so the engine's own start-up is not outside work); if the box does not clear the pass runs anyway, flagged 'busy box', as is a pass whose own outside load is above the limit. Outside load counts busy = user + nice + system + irq + softirq + steal (guest time is already in user); CONFIG_IRQ_TIME_ACCOUNTING is not set, so task and cgroup run time include the interrupt and softirq time that hit them and it is added; CONFIG_PARAVIRT_TIME_ACCOUNTING is not set, so steal is added (/boot/config-6.8.0-142-generic) (conditions.cpuAccounting); each pass also records this client's own CPU time per search and the engine's (conditions.passes[].clientCpuMsPerSearch and engineCpuMsPerSearch; how in conditions.engineCpu.rule). CPU clocks were sampled every 250 ms; each pass's min/median/max per CPU is in conditions.passes. CPU idle states, recorded and left as found: driver intel_idle, governor menu, intel_idle max_cstate 9; POLL on, C1 on, C1E on, C3 OFF on CPUs 0-7 (default disabled), C6 on (conditions.cpuIdle). How each target was reached: conditions.connections. Client build Release, .NET 10.0.12." } ] }, { "slot": "method.g12", "text": "Read back in these runs from the engine or the machine:", "sources": [ { "kind": "results", "ref": "results:conditions.engines#embedded", "value": "embedded" } ] }, { "slot": "method.g12.q25", "text": "duckdb is embedded: it ran inside the client process on the client CPUs 0-1,4-5, sharing them with the client.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[17]", "value": "duckdb is embedded: it ran inside the client process on the client CPUs 0-1,4-5, sharing them with the client." } ] }, { "slot": "method.g13", "text": "Read back in these runs from the engine or the machine:", "sources": [ { "kind": "results", "ref": "results:conditions.engines#embedded", "value": "embedded" } ] }, { "slot": "method.g13.q26", "text": "sqlitevec is embedded: it ran inside the client process on the client CPUs 0-1,4-5, sharing them with the client.", "sources": [ { "kind": "quote", "ref": "results@20261007-205106-eshoponweb:notes[18]", "value": "sqlitevec is embedded: it ran inside the client process on the client CPUs 0-1,4-5, sharing them with the client." } ] }, { "slot": "runs.caption", "text": "Runs used: 6 claim runs, which are among the 12 runs of the basis; each is listed below with its seed and start time.", "sources": [ { "kind": "consolidated", "ref": "consolidated:claimRunCount", "value": "6" }, { "kind": "consolidated", "ref": "consolidated:basis.runCount", "value": "12" } ] }, { "slot": "runs.reuse.v7", "text": "The runs of v7 were also used by the set blocked-2026-10-06-v7, which was not published; its verdict, design/verdicts/v7-verdict.md, holds the word BLOCK.", "sources": [ { "kind": "file", "ref": "design/verdicts/v7-verdict.md#BLOCK", "value": "BLOCK" }, { "kind": "consolidated", "ref": "consolidated:reuse[session=v7].verdict", "value": "design/verdicts/v7-verdict.md" }, { "kind": "consolidated", "ref": "consolidated:reuse[session=v7].blockedFolders[0]", "value": "blocked-2026-10-06-v7" } ] }, { "slot": "runs.reuse.v5", "text": "The runs of v5 were also used by the set blocked-2026-10-05-v5, which was not published; its verdict, design/verdicts/v5-verdict.txt, holds the word BLOCK.", "sources": [ { "kind": "file", "ref": "design/verdicts/v5-verdict.txt#BLOCK", "value": "BLOCK" }, { "kind": "consolidated", "ref": "consolidated:reuse[session=v5].verdict", "value": "design/verdicts/v5-verdict.txt" }, { "kind": "consolidated", "ref": "consolidated:reuse[session=v5].blockedFolders[0]", "value": "blocked-2026-10-05-v5" } ] }, { "slot": "runs.reuse.v6", "text": "The runs of v6 were also used by the set blocked-2026-10-05-v6, which was not published; its verdict, design/verdicts/v6-verdict.md, holds the word BLOCK.", "sources": [ { "kind": "file", "ref": "design/verdicts/v6-verdict.md#BLOCK", "value": "BLOCK" }, { "kind": "consolidated", "ref": "consolidated:reuse[session=v6].verdict", "value": "design/verdicts/v6-verdict.md" }, { "kind": "consolidated", "ref": "consolidated:reuse[session=v6].blockedFolders[0]", "value": "blocked-2026-10-05-v6" } ] }, { "slot": "runs.stale.20261006-130619-eshoponweb", "text": "The results.md of run 20261006-130619-eshoponweb holds a framing sentence this report retired, on line 13.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].staleLines[0].line", "value": "13" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-130619-eshoponweb].folder", "value": "20261006-130619-eshoponweb" } ] }, { "slot": "runs.stale.20261006-142724-eshoponweb", "text": "The results.md of run 20261006-142724-eshoponweb holds a framing sentence this report retired, on line 13.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].staleLines[0].line", "value": "13" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-142724-eshoponweb].folder", "value": "20261006-142724-eshoponweb" } ] }, { "slot": "runs.stale.20261006-154837-eshoponweb", "text": "The results.md of run 20261006-154837-eshoponweb holds a framing sentence this report retired, on line 13.", "sources": [ { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].staleLines[0].line", "value": "13" }, { "kind": "consolidated", "ref": "consolidated:sessions[name=v7].runs[folder=20261006-154837-eshoponweb].folder", "value": "20261006-154837-eshoponweb" } ] } ], "notHeld": { "cells": [ { "run": "20261007-205106-eshoponweb", "seed": 801, "target": "clickhouse", "pass": "default@8", "metric": "qps8", "token": "the timed pass 336 QPS, 24% from the settled 418 QPS (limit 10%), NOT HELD", "timed": "336 QPS", "settled": "418 QPS", "offPercent": "24", "limitPercent": "10" }, { "run": "20261007-221429-eshoponweb", "seed": 802, "target": "clickhouse", "pass": "default@8", "metric": "qps8", "token": "the timed pass 380 QPS, 13% from the settled 432 QPS (limit 10%), NOT HELD", "timed": "380 QPS", "settled": "432 QPS", "offPercent": "13", "limitPercent": "10" }, { "run": "20261007-233551-eshoponweb", "seed": 803, "target": "clickhouse", "pass": "default@8", "metric": "qps8", "token": "the timed pass 349 QPS, 19% from the settled 416 QPS (limit 10%), NOT HELD", "timed": "349 QPS", "settled": "416 QPS", "offPercent": "19", "limitPercent": "10" } ], "unranked": [ { "metric": "qps8", "target": "clickhouse", "pass": "default@8", "runs": 3 } ] }, "build": { "measured": [ { "session": "v7", "commit": null, "commitShort": null }, { "session": "v8", "commit": "9b924200abc3f2c3e41c2a9520e5c7bb16748e6c", "commitShort": "9b924200abc3" } ], "consolidatedBy": { "informationalVersion": "1.0.0+9e5878ce5dc045797e3df7467d807e90d5a70651", "commit": "9e5878ce5dc045797e3df7467d807e90d5a70651", "commitShort": "9e5878ce5dc0", "assemblySha256": "cab9532222b37951bbf27d42bb1a98b03c05e30f98955beb07d127b66b65c1cc" } }, "audit": { "sentencesChecked": 523, "rowSentencesChecked": 87, "factsChecked": 116, "failures": [], "factsSha256": "74e8debc9d9f7617433290b3aa4c2dc65f2acafe19dad752d77418d084e20a9e", "exclusionsSha256": "d676e39775d262b8fafd7353386aae87a726499bf9eb6763f91493890ca98075", "observerSha256": "b9e1c73770f297059d2f52de76ebc17f4a8cae34f2b9afe04f2606230ce67f3c", "observerSha256Others": { "v7": "45b466347e7bc0a2427717e538d2f823855896dd63543732ac65572ee9594538" }, "classesSha256": "dcd7598b3b81bf41c3fd790c968ee312b828374b671fe9f6820e063a9252b310", "runPageNotesSha256": "c344f4030ace2fc06ca98a8924416383c68340f938bec9243426e2a23b2fa142", "unlistedClauses": [] } }