{ "createdUtc": "2026-10-05T11:41:53Z", "commandLine": "consolidate --targets sql,sql-diskann,qdrant,qdrant-hnsw,pgvector,mariadb,oracle,redis,mongodb,clickhouse,milvus,weaviate,chroma,elasticsearch,opensearch,typesense,vespa,duckdb,sqlitevec --runs ~/ForClaude/GenericVectorBuilder/bench-results/20261005-073329-eshoponweb,~/ForClaude/GenericVectorBuilder/bench-results/20261005-085448-eshoponweb,~/ForClaude/GenericVectorBuilder/bench-results/20261005-101815-eshoponweb --out ~/ForClaude/GenericVectorBuilder/bench-results/published-2026-10-05b", "targets": [ "sql", "sql-diskann", "qdrant", "qdrant-hnsw", "pgvector", "mariadb", "oracle", "redis", "mongodb", "clickhouse", "milvus", "weaviate", "chroma", "elasticsearch", "opensearch", "typesense", "vespa", "duckdb", "sqlitevec" ], "settings": { "pipeline": "eshoponweb", "command": "run-all", "host": "bench-host", "rows": 524, "dimension": 1024, "queryKind": "golden", "queryCount": 20, "top": 10, "concurrency": [ 1, 8 ], "secondsPerLevel": 20, "buildConfiguration": "Release", "machineControl": "on", "governor": "performance", "cpuPartition": "client 0-1,4-5; engines 2-3,6-7", "warmupSearches": 20, "exactSeconds": 60, "searchSettings": { "sql": "description=exact VECTOR_DISTANCE cosine, no vector index (full scan)", "sql-diskann": "L=48, M=8, R=48, StartId=306", "qdrant": "exact=true", "qdrant-hnsw": "ef_construct=100, hnsw_ef=server, m=16", "pgvector": "ef_construction=128, hnsw.ef_search=100, m=16", "mariadb": "DISTANCE=cosine, M=16, mhnsw_ef_search=100", "oracle": "EFCONSTRUCTION=128, EFSEARCH=100, NEIGHBORS=16", "redis": "EF_CONSTRUCTION=128, EF_RUNTIME=100, M=16", "mongodb": "maxEdges=16, numCandidates=20x, numEdgeCandidates=128", "clickhouse": "M=16, ef_construction=128, hnsw_candidate_list_size_for_search=256", "milvus": "M=16, ef=100, efConstruction=128", "weaviate": "ef=-1, efConstruction=128", "chroma": "M=16, ef_construction=128, ef_search=100", "elasticsearch": "ef_construction=128, k=top, m=16, num_candidates=100", "opensearch": "approximate_threshold=0, ef_construction=128, ef_search=100, k=top, m=16", "typesense": "ef=100, ef_construction=128, k=top, m=16", "vespa": "ef=100, max-links-per-node=16, neighbors-to-explore-at-insert=128, targetHits=top", "duckdb": "checkpoint_threshold=256MB, ef_construction=128, ef_search=100, hnsw_enable_experimental_persistence=true, m=16, metric=cosine", "sqlitevec": "chunk_size=1024" }, "derived": {} }, "runs": [ { "name": "20261005-073329-eshoponweb", "folder": "~/ForClaude/GenericVectorBuilder/bench-results/20261005-073329-eshoponweb", "startedUtc": "2026-10-05T07:33:29Z", "runSeed": 601, "targetOrder": [ "qdrant-hnsw", "milvus", "oracle", "weaviate", "clickhouse", "sqlitevec", "mongodb", "sql", "elasticsearch", "duckdb", "chroma", "opensearch", "vespa", "typesense", "qdrant", "mariadb", "redis", "pgvector", "sql-diskann" ], "loadAverage": "1.98 5.18 4.13" }, { "name": "20261005-085448-eshoponweb", "folder": "~/ForClaude/GenericVectorBuilder/bench-results/20261005-085448-eshoponweb", "startedUtc": "2026-10-05T08:54:48Z", "runSeed": 602, "targetOrder": [ "sql", "opensearch", "pgvector", "duckdb", "milvus", "sqlitevec", "qdrant-hnsw", "typesense", "qdrant", "oracle", "weaviate", "sql-diskann", "chroma", "redis", "vespa", "mariadb", "elasticsearch", "clickhouse", "mongodb" ], "loadAverage": "3.32 2.91 3.31" }, { "name": "20261005-101815-eshoponweb", "folder": "~/ForClaude/GenericVectorBuilder/bench-results/20261005-101815-eshoponweb", "startedUtc": "2026-10-05T10:18:15Z", "runSeed": 603, "targetOrder": [ "sql", "opensearch", "duckdb", "mongodb", "chroma", "mariadb", "qdrant-hnsw", "typesense", "clickhouse", "qdrant", "pgvector", "vespa", "sqlitevec", "sql-diskann", "milvus", "elasticsearch", "redis", "weaviate", "oracle" ], "loadAverage": "1.45 3.00 3.58" } ], "dropped": [], "withheld": [], "targetSummaries": [ { "name": "sql", "engines": [ "Microsoft SQL Server 2025 (RTM-CU9) (KB5122048) - 17.0.5005.3 (X64), Enterprise Developer Edition (64-bit), in container gvb-mssql (image mcr.microsoft.com/mssql/server:2025-CU9-ubuntu-24.04@sha256:2b5b581621126574f3d1f75e78d3eebe8d05aedb59ad0cfdf9aa42cb0634d726; cpuset 2-3,6-7 from its creation; SQL Server counts 4 CPU(s) and runs 4 visible scheduler(s), affinity AUTO)" ], "indexes": [ "exact VECTOR_DISTANCE cosine, no vector index (full scan)" ], "hosting": [ "compose" ], "p50Ms": { "median": 3.89, "min": 3.8233, "max": 3.8929, "n": 3, "perRun": [ 3.89, 3.8233, 3.8929 ] }, "p95Ms": { "median": 4.48134, "min": 4.35044, "max": 4.5092, "n": 3, "perRun": [ 4.48134, 4.35044, 4.5092 ] }, "qps": { "1": { "median": 250.7237818141357, "min": 249.9475055238949, "max": 255.63820101883195, "n": 3, "perRun": [ 250.7237818141357, 255.63820101883195, 249.9475055238949 ] }, "8": { "median": 613.8945213079639, "min": 612.512633572833, "max": 621.4294884199052, "n": 3, "perRun": [ 621.4294884199052, 613.8945213079639, 612.512633572833 ] } }, "qpsRatio": { "8/1": { "median": 2.450565098815427, "min": 2.4014193452360453, "max": 2.478542258430745, "n": 3, "perRun": [ 2.478542258430745, 2.4014193452360453, 2.450565098815427 ] } }, "clientCpuMsPerSearch": { "1": { "median": 1.1818, "min": 1.1806, "max": 1.1886, "n": 3, "perRun": [ 1.1886, 1.1818, 1.1806 ] }, "8": { "median": 1.1806, "min": 1.1771, "max": 1.1856, "n": 3, "perRun": [ 1.1856, 1.1806, 1.1771 ] } }, "loadRowsPerSecond": { "median": 574.9604083798944, "min": 562.8099683956425, "max": 601.1596185481604, "n": 3, "perRun": [ 601.1596185481604, 574.9604083798944, 562.8099683956425 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": null, "min": null, "max": null, "n": 0, "perRun": [ null, null, null ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 16, 16, 16 ], "median": 16 }, "p95Ms": { "perRun": [ 17, 16, 17 ], "median": 17 }, "qps@1": { "perRun": [ 16, 16, 16 ], "median": 16 }, "qps@8": { "perRun": [ 15, 15, 15 ], "median": 15 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 8, "passOrder": [ "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "no index used, exact scan by design: dbo.gvb_gvbbench_eshoponweb in GvbBench holds 524 rows; sys.vector_indexes lists no index on the table; indexes: PK_gvb_gvbbench_eshoponweb CLUSTERED, IX_gvb_gvbbench_eshoponweb_DocKey NONCLUSTERED; a real search under SET STATISTICS XML ON ran: Clustered Index Scan of GvbBench.gvb_gvbbench_eshoponweb through PK_gvb_gvbbench_eshoponweb (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "afterSearch": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "no index used, exact scan by design: dbo.gvb_gvbbench_eshoponweb in GvbBench holds 524 rows; sys.vector_indexes lists no index on the table; indexes: PK_gvb_gvbbench_eshoponweb CLUSTERED, IX_gvb_gvbbench_eshoponweb_DocKey NONCLUSTERED; a real search under SET STATISTICS XML ON ran: Clustered Index Scan of GvbBench.gvb_gvbbench_eshoponweb through PK_gvb_gvbbench_eshoponweb (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging; delayed durability is DISABLED); a database created here copies the model database: recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled; mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry, so SQL Server\u0027s own Linux defaults for flushing writes apply (Microsoft\u0027s Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here). Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "loadIndexNote": null, "searchSettings": "description=exact VECTOR_DISTANCE cosine, no vector index (full scan)", "settled": true, "settleDetail": null, "meanMs": 3.9884529212362914, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 1, "passOrder": [ "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "no index used, exact scan by design: dbo.gvb_gvbbench_eshoponweb in GvbBench holds 524 rows; sys.vector_indexes lists no index on the table; indexes: PK_gvb_gvbbench_eshoponweb CLUSTERED, IX_gvb_gvbbench_eshoponweb_DocKey NONCLUSTERED; a real search under SET STATISTICS XML ON ran: Clustered Index Scan of GvbBench.gvb_gvbbench_eshoponweb through PK_gvb_gvbbench_eshoponweb (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "afterSearch": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "no index used, exact scan by design: dbo.gvb_gvbbench_eshoponweb in GvbBench holds 524 rows; sys.vector_indexes lists no index on the table; indexes: PK_gvb_gvbbench_eshoponweb CLUSTERED, IX_gvb_gvbbench_eshoponweb_DocKey NONCLUSTERED; a real search under SET STATISTICS XML ON ran: Clustered Index Scan of GvbBench.gvb_gvbbench_eshoponweb through PK_gvb_gvbbench_eshoponweb (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging; delayed durability is DISABLED); a database created here copies the model database: recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled; mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry, so SQL Server\u0027s own Linux defaults for flushing writes apply (Microsoft\u0027s Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here). Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "loadIndexNote": null, "searchSettings": "description=exact VECTOR_DISTANCE cosine, no vector index (full scan)", "settled": true, "settleDetail": null, "meanMs": 3.9117784275376497, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 1, "passOrder": [ "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "no index used, exact scan by design: dbo.gvb_gvbbench_eshoponweb in GvbBench holds 524 rows; sys.vector_indexes lists no index on the table; indexes: PK_gvb_gvbbench_eshoponweb CLUSTERED, IX_gvb_gvbbench_eshoponweb_DocKey NONCLUSTERED; a real search under SET STATISTICS XML ON ran: Clustered Index Scan of GvbBench.gvb_gvbbench_eshoponweb through PK_gvb_gvbbench_eshoponweb (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "afterSearch": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "no index used, exact scan by design: dbo.gvb_gvbbench_eshoponweb in GvbBench holds 524 rows; sys.vector_indexes lists no index on the table; indexes: PK_gvb_gvbbench_eshoponweb CLUSTERED, IX_gvb_gvbbench_eshoponweb_DocKey NONCLUSTERED; a real search under SET STATISTICS XML ON ran: Clustered Index Scan of GvbBench.gvb_gvbbench_eshoponweb through PK_gvb_gvbbench_eshoponweb (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging; delayed durability is DISABLED); a database created here copies the model database: recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled; mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry, so SQL Server\u0027s own Linux defaults for flushing writes apply (Microsoft\u0027s Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here). Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "loadIndexNote": null, "searchSettings": "description=exact VECTOR_DISTANCE cosine, no vector index (full scan)", "settled": true, "settleDetail": null, "meanMs": 4.000840088017603, "meanSource": "1000 / QPS@1" } ], "engineNotes": [ { "kind": "exact-by-design", "text": "Exact by design: every search compares the query with every stored vector, so its time grows with the collection and index quality plays no part in its speed. The results say: no index used, exact scan by design.", "source": "results" } ] }, { "name": "sql-diskann", "engines": [ "Microsoft SQL Server 2025 (RTM-CU9) (KB5122048) - 17.0.5005.3 (X64), Enterprise Developer Edition (64-bit), in container gvb-mssql (image mcr.microsoft.com/mssql/server:2025-CU9-ubuntu-24.04@sha256:2b5b581621126574f3d1f75e78d3eebe8d05aedb59ad0cfdf9aa42cb0634d726; cpuset 2-3,6-7 from its creation; SQL Server counts 4 CPU(s) and runs 4 visible scheduler(s), affinity AUTO)" ], "indexes": [ "DiskANN (preview) via VECTOR_SEARCH, cosine, build {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}; the exact mode scans the same table" ], "hosting": [ "compose" ], "p50Ms": { "median": 3.6036, "min": 3.56355, "max": 3.6159, "n": 3, "perRun": [ 3.6036, 3.56355, 3.6159 ] }, "p95Ms": { "median": 4.06184, "min": 4.05883, "max": 4.0776650000000005, "n": 3, "perRun": [ 4.06184, 4.05883, 4.0776650000000005 ] }, "qps": { "1": { "median": 273.3524626399846, "min": 272.4901658299152, "max": 276.38487760142203, "n": 3, "perRun": [ 272.4901658299152, 276.38487760142203, 273.3524626399846 ] }, "8": { "median": 690.4279825257992, "min": 690.3636272262703, "max": 690.7239886382225, "n": 3, "perRun": [ 690.4279825257992, 690.7239886382225, 690.3636272262703 ] } }, "qpsRatio": { "8/1": { "median": 2.5255438365503404, "min": 2.49913813893365, "max": 2.5337721103548203, "n": 3, "perRun": [ 2.5337721103548203, 2.49913813893365, 2.5255438365503404 ] } }, "clientCpuMsPerSearch": { "1": { "median": 1.0073, "min": 0.998, "max": 1.0258, "n": 3, "perRun": [ 1.0258, 0.998, 1.0073 ] }, "8": { "median": 1.0876, "min": 1.0682, "max": 1.0954, "n": 3, "perRun": [ 1.0954, 1.0876, 1.0682 ] } }, "loadRowsPerSecond": { "median": 637.2814255450368, "min": 630.2190768998369, "max": 637.3201030220649, "n": 3, "perRun": [ 637.2814255450368, 637.3201030220649, 630.2190768998369 ] }, "recall": { "median": 0.9650000000000002, "min": 0.9650000000000002, "max": 0.9650000000000002, "n": 3, "perRun": [ 0.9650000000000002, 0.9650000000000002, 0.9650000000000002 ] }, "ndcg": { "median": 0.5256995680516536, "min": 0.5256995680516536, "max": 0.5256995680516536, "n": 3, "perRun": [ 0.5256995680516536, 0.5256995680516536, 0.5256995680516536 ] }, "exactP50Ms": { "median": 3.5627, "min": 3.5483, "max": 3.577, "n": 3, "perRun": [ 3.5483, 3.5627, 3.577 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 15, 15, 15 ], "median": 15 }, "p95Ms": { "perRun": [ 15, 15, 15 ], "median": 15 }, "qps@1": { "perRun": [ 15, 15, 15 ], "median": 15 }, "qps@8": { "perRun": [ 14, 14, 14 ], "median": 14 }, "recall": { "perRun": [ 19, 19, 19 ], "median": 19 }, "ndcg": { "perRun": [ 2, 2, 1 ], "median": 2 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 19, "passOrder": [ "default@1", "exact", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "DiskANN index built and used. sys.vector_indexes: vix_gvb_gvbbench_eshoponweb on dbo.gvb_gvbbench_eshoponweb_ann in GvbBenchDiskAnn, DiskANN, COSINE, enabled, build {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}; graph table rows 524 of 524 table rows. Real VECTOR_SEARCH plan: Vector Index Seek on index vix_gvb_gvbbench_eshoponweb of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann (IndexKind DiskANN), 10 rows returned, 10 hits. The exact mode searches the SAME table (dbo.gvb_gvbbench_eshoponweb_ann, database GvbBenchDiskAnn) and its real plan is: Clustered Index Scan of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann through PK_gvb_gvbbench_eshoponweb_ann (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "DiskANN index built and used. sys.vector_indexes: vix_gvb_gvbbench_eshoponweb on dbo.gvb_gvbbench_eshoponweb_ann in GvbBenchDiskAnn, DiskANN, COSINE, enabled, build {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}; graph table rows 524 of 524 table rows. Real VECTOR_SEARCH plan: Vector Index Seek on index vix_gvb_gvbbench_eshoponweb of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann (IndexKind DiskANN), 10 rows returned, 10 hits. The exact mode searches the SAME table (dbo.gvb_gvbbench_eshoponweb_ann, database GvbBenchDiskAnn) and its real plan is: Clustered Index Scan of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann through PK_gvb_gvbbench_eshoponweb_ann (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging; delayed durability is DISABLED); a database created here copies the model database: recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled; mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry, so SQL Server\u0027s own Linux defaults for flushing writes apply (Microsoft\u0027s Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here). Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "loadIndexNote": "copied 524 rows to dbo.gvb_gvbbench_eshoponweb_ann (INT key) and built DiskANN, parameters {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}, graph covers 524 rows", "searchSettings": "L=48, M=8, R=48, StartId=306", "settled": true, "settleDetail": null, "meanMs": 3.669857211009174, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 12, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "DiskANN index built and used. sys.vector_indexes: vix_gvb_gvbbench_eshoponweb on dbo.gvb_gvbbench_eshoponweb_ann in GvbBenchDiskAnn, DiskANN, COSINE, enabled, build {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}; graph table rows 524 of 524 table rows. Real VECTOR_SEARCH plan: Vector Index Seek on index vix_gvb_gvbbench_eshoponweb of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann (IndexKind DiskANN), 10 rows returned, 10 hits. The exact mode searches the SAME table (dbo.gvb_gvbbench_eshoponweb_ann, database GvbBenchDiskAnn) and its real plan is: Clustered Index Scan of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann through PK_gvb_gvbbench_eshoponweb_ann (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "DiskANN index built and used. sys.vector_indexes: vix_gvb_gvbbench_eshoponweb on dbo.gvb_gvbbench_eshoponweb_ann in GvbBenchDiskAnn, DiskANN, COSINE, enabled, build {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}; graph table rows 524 of 524 table rows. Real VECTOR_SEARCH plan: Vector Index Seek on index vix_gvb_gvbbench_eshoponweb of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann (IndexKind DiskANN), 10 rows returned, 10 hits. The exact mode searches the SAME table (dbo.gvb_gvbbench_eshoponweb_ann, database GvbBenchDiskAnn) and its real plan is: Clustered Index Scan of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann through PK_gvb_gvbbench_eshoponweb_ann (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging; delayed durability is DISABLED); a database created here copies the model database: recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled; mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry, so SQL Server\u0027s own Linux defaults for flushing writes apply (Microsoft\u0027s Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here). Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "loadIndexNote": "copied 524 rows to dbo.gvb_gvbbench_eshoponweb_ann (INT key) and built DiskANN, parameters {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}, graph covers 524 rows", "searchSettings": "L=48, M=8, R=48, StartId=306", "settled": true, "settleDetail": null, "meanMs": 3.618142963096961, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 14, "passOrder": [ "default@1", "default@8", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "DiskANN index built and used. sys.vector_indexes: vix_gvb_gvbbench_eshoponweb on dbo.gvb_gvbbench_eshoponweb_ann in GvbBenchDiskAnn, DiskANN, COSINE, enabled, build {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}; graph table rows 524 of 524 table rows. Real VECTOR_SEARCH plan: Vector Index Seek on index vix_gvb_gvbbench_eshoponweb of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann (IndexKind DiskANN), 10 rows returned, 10 hits. The exact mode searches the SAME table (dbo.gvb_gvbbench_eshoponweb_ann, database GvbBenchDiskAnn) and its real plan is: Clustered Index Scan of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann through PK_gvb_gvbbench_eshoponweb_ann (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "DiskANN index built and used. sys.vector_indexes: vix_gvb_gvbbench_eshoponweb on dbo.gvb_gvbbench_eshoponweb_ann in GvbBenchDiskAnn, DiskANN, COSINE, enabled, build {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}; graph table rows 524 of 524 table rows. Real VECTOR_SEARCH plan: Vector Index Seek on index vix_gvb_gvbbench_eshoponweb of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann (IndexKind DiskANN), 10 rows returned, 10 hits. The exact mode searches the SAME table (dbo.gvb_gvbbench_eshoponweb_ann, database GvbBenchDiskAnn) and its real plan is: Clustered Index Scan of GvbBenchDiskAnn.gvb_gvbbench_eshoponweb_ann through PK_gvb_gvbbench_eshoponweb_ann (IndexKind Clustered), 524 rows read, no vector index operator, 10 hits" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "A commit returns after its transaction-log records are written to disk (SQL Server write-ahead logging; delayed durability is DISABLED); a database created here copies the model database: recovery model FULL, page_verify CHECKSUM; no global trace flags are enabled; mssql.conf of container gvb-mssql (/var/opt/mssql/mssql.conf, on the host at ~/gvb-data/engines/mssql/mssql.conf) does not exist, so it sets: nothing; it has no [control] or [traceflag] entry, so SQL Server\u0027s own Linux defaults for flushing writes apply (Microsoft\u0027s Linux performance guide names trace flag 3982 as that default; read from the guide, not tested here). Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. Container settings from its environment (names only): MSSQL_AGENT_ENABLED, MSSQL_MEMORY_LIMIT_MB, MSSQL_PID, MSSQL_RPC_PORT, MSSQL_SA_PASSWORD.", "loadIndexNote": "copied 524 rows to dbo.gvb_gvbbench_eshoponweb_ann (INT key) and built DiskANN, parameters {\u0022StartId\u0022:\u0022306\u0022, \u0022L\u0022:\u002248\u0022, \u0022M\u0022:\u00228\u0022, \u0022R\u0022:\u002248\u0022}, graph covers 524 rows", "searchSettings": "L=48, M=8, R=48, StartId=306", "settled": true, "settleDetail": null, "meanMs": 3.658280559619605, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "qdrant", "engines": [ "Qdrant 1.17.0 in container gvb-qdrant (image qdrant/qdrant:v1.17.0@sha256:f1c7272cdac52b38c1a0e89313922d940ba50afd90d593a1605dbbc214e66ffb; cpuset 2-3,6-7 from its creation; 3 search thread(s))" ], "indexes": [ "exact scan: the builder\u0027s sink sends exact=true on every search, so no HNSW graph is used whether or not Qdrant has built one (see the index state)" ], "hosting": [ "compose" ], "p50Ms": { "median": 0.8786, "min": 0.8765, "max": 0.8794, "n": 3, "perRun": [ 0.8765, 0.8794, 0.8786 ] }, "p95Ms": { "median": 0.9915, "min": 0.9914, "max": 0.9947, "n": 3, "perRun": [ 0.9915, 0.9947, 0.9914 ] }, "qps": { "1": { "median": 1122.7265630829957, "min": 1122.0466731316142, "max": 1125.4144425306883, "n": 3, "perRun": [ 1125.4144425306883, 1122.0466731316142, 1122.7265630829957 ] }, "8": { "median": 3295.513777572424, "min": 3291.3846853471437, "max": 3303.0483656232514, "n": 3, "perRun": [ 3303.0483656232514, 3291.3846853471437, 3295.513777572424 ] } }, "qpsRatio": { "8/1": { "median": 2.93496177123494, "min": 2.9333759139991407, "max": 2.9352772847228064, "n": 3, "perRun": [ 2.93496177123494, 2.9333759139991407, 2.9352772847228064 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.7359, "min": 0.7239, "max": 0.7361, "n": 3, "perRun": [ 0.7239, 0.7361, 0.7359 ] }, "8": { "median": 0.5339, "min": 0.5251, "max": 0.5346, "n": 3, "perRun": [ 0.5251, 0.5346, 0.5339 ] } }, "loadRowsPerSecond": { "median": 9284.590414900706, "min": 8596.942835252059, "max": 9364.95255865159, "n": 3, "perRun": [ 9284.590414900706, 8596.942835252059, 9364.95255865159 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": null, "min": null, "max": null, "n": 0, "perRun": [ null, null, null ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 4, 4, 4 ], "median": 4 }, "p95Ms": { "perRun": [ 3, 4, 3 ], "median": 3 }, "qps@1": { "perRun": [ 4, 4, 4 ], "median": 4 }, "qps@8": { "perRun": [ 5, 5, 5 ], "median": 5 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 15, "passOrder": [ "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 0 of 524 points, 2 segments, indexing_threshold_kb 10,000, full_scan_threshold_kb 10,000; no index used, exact scan by design (the builder\u0027s sink sends exact=true on every search); no graph was built; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 0, unfiltered_exact 0" }, "afterSearch": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 0 of 524 points, 2 segments, indexing_threshold_kb 10,000, full_scan_threshold_kb 10,000; no index used, exact scan by design (the builder\u0027s sink sends exact=true on every search); no graph was built; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 603,262, unfiltered_exact 0" }, "segmentLayoutAfterLoad": "2 segments", "segmentLayoutAfterSearch": "2 segments", "durability": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30); with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes. So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel\u0027s page cache (inferred, not tested). Every upsert here is sent with wait=true in batches of 256 over gRPC; the strace used one point per request, so one flush per batch is inferred, not measured. Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image\u0027s own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked.", "loadIndexNote": "status Green, 0 of 524 vectors in HNSW segments (2 segments), waited 1.0 s", "searchSettings": "exact=true", "settled": true, "settleDetail": null, "meanMs": 0.8885615487138477, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 9, "passOrder": [ "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 0 of 524 points, 2 segments, indexing_threshold_kb 10,000, full_scan_threshold_kb 10,000; no index used, exact scan by design (the builder\u0027s sink sends exact=true on every search); no graph was built; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 0, unfiltered_exact 0" }, "afterSearch": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 0 of 524 points, 2 segments, indexing_threshold_kb 10,000, full_scan_threshold_kb 10,000; no index used, exact scan by design (the builder\u0027s sink sends exact=true on every search); no graph was built; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 601,820, unfiltered_exact 0" }, "segmentLayoutAfterLoad": "2 segments", "segmentLayoutAfterSearch": "2 segments", "durability": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30); with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes. So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel\u0027s page cache (inferred, not tested). Every upsert here is sent with wait=true in batches of 256 over gRPC; the strace used one point per request, so one flush per batch is inferred, not measured. Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image\u0027s own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked.", "loadIndexNote": "status Green, 0 of 524 vectors in HNSW segments (2 segments), waited 1.0 s", "searchSettings": "exact=true", "settled": true, "settleDetail": null, "meanMs": 0.8912285236843278, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 10, "passOrder": [ "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 0 of 524 points, 2 segments, indexing_threshold_kb 10,000, full_scan_threshold_kb 10,000; no index used, exact scan by design (the builder\u0027s sink sends exact=true on every search); no graph was built; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 0, unfiltered_exact 0" }, "afterSearch": { "ready": true, "indexedVectors": 0, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 0 of 524 points, 2 segments, indexing_threshold_kb 10,000, full_scan_threshold_kb 10,000; no index used, exact scan by design (the builder\u0027s sink sends exact=true on every search); no graph was built; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 599,122, unfiltered_exact 0" }, "segmentLayoutAfterLoad": "2 segments", "segmentLayoutAfterSearch": "2 segments", "durability": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30); with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes. So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel\u0027s page cache (inferred, not tested). Every upsert here is sent with wait=true in batches of 256 over gRPC; the strace used one point per request, so one flush per batch is inferred, not measured. Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image\u0027s own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked.", "loadIndexNote": "status Green, 0 of 524 vectors in HNSW segments (2 segments), waited 1.0 s", "searchSettings": "exact=true", "settled": true, "settleDetail": null, "meanMs": 0.8906888220886217, "meanSource": "1000 / QPS@1" } ], "engineNotes": [ { "kind": "exact-by-design", "text": "Exact by design: every search compares the query with every stored vector, so its time grows with the collection and index quality plays no part in its speed. The results say: no index used, exact scan by design (the builder\u0027s sink sends exact=true on every search).", "source": "results" } ] }, { "name": "qdrant-hnsw", "engines": [ "Qdrant 1.17.0 in container gvb-qdrant (image qdrant/qdrant:v1.17.0@sha256:f1c7272cdac52b38c1a0e89313922d940ba50afd90d593a1605dbbc214e66ffb; cpuset 2-3,6-7 from its creation; 3 search thread(s))" ], "indexes": [ "HNSW m=16 ef_construct=100, hnsw_ef=server default, cosine; indexing_threshold_kb 1 and full_scan_threshold_kb 10 (server defaults are 10,000 each) so a small collection builds and walks its graph" ], "hosting": [ "compose" ], "p50Ms": { "median": 0.9199, "min": 0.9042, "max": 0.9202, "n": 3, "perRun": [ 0.9042, 0.9202, 0.9199 ] }, "p95Ms": { "median": 1.0291, "min": 1.012805, "max": 1.0293, "n": 3, "perRun": [ 1.012805, 1.0293, 1.0291 ] }, "qps": { "1": { "median": 1074.8272888993856, "min": 1073.0969363082468, "max": 1092.976987369531, "n": 3, "perRun": [ 1092.976987369531, 1074.8272888993856, 1073.0969363082468 ] }, "8": { "median": 3465.8944897862284, "min": 3453.7536264850546, "max": 3473.1995748210193, "n": 3, "perRun": [ 3473.1995748210193, 3453.7536264850546, 3465.8944897862284 ] } }, "qpsRatio": { "8/1": { "median": 3.2133103263703604, "min": 3.1777426377292475, "max": 3.2298055958577923, "n": 3, "perRun": [ 3.1777426377292475, 3.2133103263703604, 3.2298055958577923 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.7286, "min": 0.7158, "max": 0.7334, "n": 3, "perRun": [ 0.7158, 0.7286, 0.7334 ] }, "8": { "median": 0.5214, "min": 0.5134, "max": 0.5244, "n": 3, "perRun": [ 0.5134, 0.5244, 0.5214 ] } }, "loadRowsPerSecond": { "median": 5186.455038185771, "min": 4928.309091507602, "max": 5445.677805017714, "n": 3, "perRun": [ 4928.309091507602, 5445.677805017714, 5186.455038185771 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 0.8911, "min": 0.8842, "max": 0.8979, "n": 3, "perRun": [ 0.8842, 0.8979, 0.8911 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 6, 6, 6 ], "median": 6 }, "p95Ms": { "perRun": [ 5, 5, 5 ], "median": 5 }, "qps@1": { "perRun": [ 6, 6, 6 ], "median": 6 }, "qps@8": { "perRun": [ 4, 4, 4 ], "median": 4 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 1, "passOrder": [ "default@8", "default@1", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 524 of 524 points, 2 segments, indexing_threshold_kb 1, full_scan_threshold_kb 10; every one of 1 non-empty segments has a complete HNSW graph above its full-scan threshold; no search has run yet, so the walk is proven by the segment settings only; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 0, unfiltered_exact 0" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 524 of 524 points, 2 segments, indexing_threshold_kb 1, full_scan_threshold_kb 10; every one of 1 non-empty segments has a complete HNSW graph above its full-scan threshold; 309,093 segment searches walked the graph and none scanned; searches counted since the collection was created: unfiltered_hnsw 309,093, unfiltered_plain 0, unfiltered_exact 120,597" }, "segmentLayoutAfterLoad": "2 segments", "segmentLayoutAfterSearch": "2 segments", "durability": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30); with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes. So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel\u0027s page cache (inferred, not tested). Every upsert here is sent with wait=true in batches of 256 over gRPC; the strace used one point per request, so one flush per batch is inferred, not measured. Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image\u0027s own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked.", "loadIndexNote": "indexing_threshold_kb 1, full_scan_threshold_kb 10; status Green, 524 of 524 vectors in HNSW segments (2 segments), waited 2.0 s", "searchSettings": "ef_construct=100, hnsw_ef=server, m=16", "settled": true, "settleDetail": null, "meanMs": 0.9149323467520586, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 7, "passOrder": [ "exact", "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 524 of 524 points, 2 segments, indexing_threshold_kb 1, full_scan_threshold_kb 10; every one of 1 non-empty segments has a complete HNSW graph above its full-scan threshold; no search has run yet, so the walk is proven by the segment settings only; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 0, unfiltered_exact 0" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 524 of 524 points, 2 segments, indexing_threshold_kb 1, full_scan_threshold_kb 10; every one of 1 non-empty segments has a complete HNSW graph above its full-scan threshold; 308,968 segment searches walked the graph and none scanned; searches counted since the collection was created: unfiltered_hnsw 308,968, unfiltered_plain 0, unfiltered_exact 119,134" }, "segmentLayoutAfterLoad": "2 segments", "segmentLayoutAfterSearch": "2 segments", "durability": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30); with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes. So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel\u0027s page cache (inferred, not tested). Every upsert here is sent with wait=true in batches of 256 over gRPC; the strace used one point per request, so one flush per batch is inferred, not measured. Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image\u0027s own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked.", "loadIndexNote": "indexing_threshold_kb 1, full_scan_threshold_kb 10; status Green, 524 of 524 vectors in HNSW segments (2 segments), waited 2.0 s", "searchSettings": "ef_construct=100, hnsw_ef=server, m=16", "settled": true, "settleDetail": null, "meanMs": 0.9303820347025166, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 7, "passOrder": [ "exact", "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 524 of 524 points, 2 segments, indexing_threshold_kb 1, full_scan_threshold_kb 10; every one of 1 non-empty segments has a complete HNSW graph above its full-scan threshold; no search has run yet, so the walk is proven by the segment settings only; searches counted since the collection was created: unfiltered_hnsw 0, unfiltered_plain 0, unfiltered_exact 0" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "status Green, optimizer ok, indexed_vectors_count 524 of 524 points, 2 segments, indexing_threshold_kb 1, full_scan_threshold_kb 10; every one of 1 non-empty segments has a complete HNSW graph above its full-scan threshold; 308,694 segment searches walked the graph and none scanned; searches counted since the collection was created: unfiltered_hnsw 308,694, unfiltered_plain 0, unfiltered_exact 119,553" }, "segmentLayoutAfterLoad": "2 segments", "segmentLayoutAfterSearch": "2 segments", "durability": "What was measured (strace -f on the Qdrant server process, 30 single-point REST upserts per setting, every sync call timed against the request that caused it): with wait=true each upsert had exactly one msync(MS_SYNC) of the write-ahead-log segment inside the request, before the reply (30 of 30); with wait=false the 30 upserts were acknowledged within 0.26 s with no sync call, and the first WAL msync (one call covering all 30 records) came 2.7 s after the last reply, followed by the segment-file flushes. So the log is flushed to disk under both settings, but with wait=false the flush comes after the acknowledgement: an operating-system crash or power cut in that gap loses acknowledged writes, while a crash of the Qdrant process alone should not, because the bytes are already in the kernel\u0027s page cache (inferred, not tested). Every upsert here is sent with wait=true in batches of 256 over gRPC; the strace used one point per request, so one flush per batch is inferred, not measured. Segment files are flushed every 5 s; log segments 32 MB, 0 created ahead. /qdrant/config/config.yaml in container gvb-qdrant (the image\u0027s own file; its storage keys are listed) sets: storage.collection.quantization = null, storage.collection.replication_factor = 1, storage.collection.vectors.on_disk = null, storage.collection.write_consistency_factor = 1, storage.hnsw_index.ef_construct = 100, storage.hnsw_index.full_scan_threshold_kb = 10000, storage.hnsw_index.m = 16, storage.hnsw_index.max_indexing_threads = 0, storage.hnsw_index.on_disk = false, storage.hnsw_index.payload_m = null, storage.max_collections = null, storage.node_type = Normal, storage.on_disk_payload = true, storage.optimizers.default_segment_number = 0, storage.optimizers.deleted_threshold = 0.2, storage.optimizers.flush_interval_sec = 5, storage.optimizers.indexing_threshold_kb = 10000, storage.optimizers.max_optimization_threads = null, storage.optimizers.max_segment_size_kb = null, storage.optimizers.vacuum_min_vector_number = 1000, storage.performance.max_search_threads = 0, storage.performance.optimizer_cpu_budget = 0, storage.performance.update_rate_limit = null, storage.shard_transfer_method = null, storage.snapshots_config.snapshots_storage = local, storage.snapshots_path = ./snapshots, storage.storage_path = ./storage, storage.temp_path = null, storage.update_concurrency = null, storage.wal.wal_capacity_mb = 32, storage.wal.wal_segments_ahead = 0; no QDRANT__ environment overrides in the server process. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked.", "loadIndexNote": "indexing_threshold_kb 1, full_scan_threshold_kb 10; status Green, 524 of 524 vectors in HNSW segments (2 segments), waited 2.0 s", "searchSettings": "ef_construct=100, hnsw_ef=server, m=16", "settled": true, "settleDetail": null, "meanMs": 0.9318822616717921, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "pgvector", "engines": [ "PostgreSQL 17 \u002B pgvector 0.8.7" ], "indexes": [ "HNSW vector_cosine_ops m=16 ef_construction=128, hnsw.ef_search=100 per query, float32 vector(n), cosine; exact mode = same query with index scans off (sequential scan)" ], "hosting": [ "compose" ], "p50Ms": { "median": 0.8493, "min": 0.8165, "max": 0.8508, "n": 3, "perRun": [ 0.8493, 0.8165, 0.8508 ] }, "p95Ms": { "median": 0.9953, "min": 0.9537, "max": 1.001215, "n": 3, "perRun": [ 0.9953, 0.9537, 1.001215 ] }, "qps": { "1": { "median": 1153.3591710853436, "min": 1150.8681669865011, "max": 1200.4241008500242, "n": 3, "perRun": [ 1153.3591710853436, 1200.4241008500242, 1150.8681669865011 ] }, "8": { "median": 4049.9862214772998, "min": 4007.5671515102313, "max": 4142.489789207402, "n": 3, "perRun": [ 4007.5671515102313, 4142.489789207402, 4049.9862214772998 ] } }, "qpsRatio": { "8/1": { "median": 3.474691364129872, "min": 3.450855232141784, "max": 3.5190705049058875, "n": 3, "perRun": [ 3.474691364129872, 3.450855232141784, 3.5190705049058875 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.6812, "min": 0.6645, "max": 0.6832, "n": 3, "perRun": [ 0.6812, 0.6645, 0.6832 ] }, "8": { "median": 0.5386, "min": 0.531, "max": 0.5397, "n": 3, "perRun": [ 0.5397, 0.531, 0.5386 ] } }, "loadRowsPerSecond": { "median": 612.3678676088717, "min": 611.1128540008241, "max": 615.8665318498785, "n": 3, "perRun": [ 615.8665318498785, 611.1128540008241, 612.3678676088717 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 2.1735, "min": 2.1681, "max": 2.1829, "n": 3, "perRun": [ 2.1829, 2.1681, 2.1735 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 3, 3, 3 ], "median": 3 }, "p95Ms": { "perRun": [ 4, 3, 4 ], "median": 4 }, "qps@1": { "perRun": [ 3, 3, 3 ], "median": 3 }, "qps@8": { "perRun": [ 3, 3, 3 ], "median": 3 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 18, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "fsync on, synchronous_commit on, full_page_writes on, wal_sync_method fdatasync (PostgreSQL defaults; pgvector.compose.yaml sets only shared_buffers, maintenance_work_mem and max_wal_size): every commit is flushed to the write-ahead log before it returns and HNSW index changes are WAL-logged, so a crash loses no committed row (max_wal_size 4GB only spaces out checkpoints; measured with docker kill, which is SIGKILL, right after a 2,000-vector load and a restart: all 2,000 rows were there and the HNSW index was valid and used by the default search)", "loadIndexNote": "HNSW is maintained inside every insert, nothing to build; confirmed in 0.0 s: pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported", "searchSettings": "ef_construction=128, hnsw.ef_search=100, m=16", "settled": true, "settleDetail": null, "meanMs": 0.8670325992717184, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 3, "passOrder": [ "default@1", "exact", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "fsync on, synchronous_commit on, full_page_writes on, wal_sync_method fdatasync (PostgreSQL defaults; pgvector.compose.yaml sets only shared_buffers, maintenance_work_mem and max_wal_size): every commit is flushed to the write-ahead log before it returns and HNSW index changes are WAL-logged, so a crash loses no committed row (max_wal_size 4GB only spaces out checkpoints; measured with docker kill, which is SIGKILL, right after a 2,000-vector load and a restart: all 2,000 rows were there and the HNSW index was valid and used by the default search)", "loadIndexNote": "HNSW is maintained inside every insert, nothing to build; confirmed in 0.0 s: pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported", "searchSettings": "ef_construction=128, hnsw.ef_search=100, m=16", "settled": true, "settleDetail": null, "meanMs": 0.833038922903911, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 11, "passOrder": [ "default@8", "exact", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "fsync on, synchronous_commit on, full_page_writes on, wal_sync_method fdatasync (PostgreSQL defaults; pgvector.compose.yaml sets only shared_buffers, maintenance_work_mem and max_wal_size): every commit is flushed to the write-ahead log before it returns and HNSW index changes are WAL-logged, so a crash loses no committed row (max_wal_size 4GB only spaces out checkpoints; measured with docker kill, which is SIGKILL, right after a 2,000-vector load and a restart: all 2,000 rows were there and the HNSW index was valid and used by the default search)", "loadIndexNote": "HNSW is maintained inside every insert, nothing to build; confirmed in 0.0 s: pg_indexes lists gvb_gvbbench_eshoponweb_hnsw (USING hnsw (embedding vector_cosine_ops) WITH (m=\u002716\u0027, ef_construction=\u0027128\u0027)), pg_index valid and ready = True, EXPLAIN of the default search uses Index Scan on it = True, that search returned 10 of 10 rows; PostgreSQL keeps no entry count for an HNSW index, so indexed vectors are not reported", "searchSettings": "ef_construction=128, hnsw.ef_search=100, m=16", "settled": true, "settleDetail": null, "meanMs": 0.8689092536275959, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "mariadb", "engines": [ "MariaDB 11.8.9 (InnoDB \u002B VECTOR INDEX)" ], "indexes": [ "VECTOR INDEX (HNSW variant) DISTANCE=cosine, M=16 (no ef_construction setting exists), mhnsw_ef_search=100 per statement (the ef 100 most engines here use, so the search effort matches; MariaDB\u0027s own default is 20; recall@10 at ef 100 falls as the set grows (random 1024-dimension vectors, measured 2026-10-04: 0.99 at 524, 0.89 to 0.92 at 2,000; an earlier run gave about 0.09 at 100,000)), mhnsw_max_cache_size 4G; exact mode = IGNORE INDEX full scan" ], "hosting": [ "compose" ], "p50Ms": { "median": 0.6236, "min": 0.6219, "max": 0.625, "n": 3, "perRun": [ 0.625, 0.6219, 0.6236 ] }, "p95Ms": { "median": 0.7083, "min": 0.6988, "max": 0.7094599999999999, "n": 3, "perRun": [ 0.7094599999999999, 0.6988, 0.7083 ] }, "qps": { "1": { "median": 1573.403230588971, "min": 1569.4076887687108, "max": 1581.5353075369928, "n": 3, "perRun": [ 1569.4076887687108, 1581.5353075369928, 1573.403230588971 ] }, "8": { "median": 5940.1061883415, "min": 5930.118349029013, "max": 5944.1738027383435, "n": 3, "perRun": [ 5930.118349029013, 5940.1061883415, 5944.1738027383435 ] } }, "qpsRatio": { "8/1": { "median": 3.777908731325831, "min": 3.7559112085789192, "max": 3.77857098029227, "n": 3, "perRun": [ 3.77857098029227, 3.7559112085789192, 3.777908731325831 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.4862, "min": 0.4703, "max": 0.4865, "n": 3, "perRun": [ 0.4703, 0.4862, 0.4865 ] }, "8": { "median": 0.3784, "min": 0.3776, "max": 0.3807, "n": 3, "perRun": [ 0.3807, 0.3784, 0.3776 ] } }, "loadRowsPerSecond": { "median": 1039.2742533687162, "min": 1020.0613360545351, "max": 1052.365422244555, "n": 3, "perRun": [ 1052.365422244555, 1020.0613360545351, 1039.2742533687162 ] }, "recall": { "median": 1, "min": 0.9949999999999999, "max": 1, "n": 3, "perRun": [ 1, 0.9949999999999999, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 1.5793, "min": 1.5776, "max": 1.5836, "n": 3, "perRun": [ 1.5776, 1.5836, 1.5793 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 2, 2, 2 ], "median": 2 }, "p95Ms": { "perRun": [ 2, 2, 2 ], "median": 2 }, "qps@1": { "perRun": [ 2, 2, 2 ], "median": 2 }, "qps@8": { "perRun": [ 1, 1, 1 ], "median": 1 }, "recall": { "perRun": [ 1, 17, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 16, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 518 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 518 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "innodb_flush_log_at_trx_commit=2 (set in mariadb-bench.compose.yaml for the benchmark\u0027s own container gvbbench-mariadb, and in mariadb.compose.yaml for the daily gvb-mariadb, which has the same server settings): the InnoDB redo log is written to the operating system at every commit but fsynced about once a second, so a crash of the mariadbd process loses nothing, while an operating-system crash or power cut can lose the last second of commits; innodb_doublewrite is on (default), the binary log is off, and the vector graph is an InnoDB table under the same log (settings read from the running server with SHOW VARIABLES; the crash behaviour is InnoDB\u0027s documented behaviour for this setting and was not tested on either container)", "loadIndexNote": "VECTOR INDEX is maintained inside every INSERT, nothing to build; confirmed after 2 poll(s) in 0.5 s (poll 1 of 2 said: not ready: a search at mhnsw_ef_search 100 returned only 405 of 524 requested rows through the index (EXPLAIN key \u0027vec_idx\u0027) (information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 405 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported)): information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 518 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported", "searchSettings": "DISTANCE=cosine, M=16, mhnsw_ef_search=100", "settled": true, "settleDetail": null, "meanMs": 0.6371830641307464, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 16, "passOrder": [ "default@1", "exact", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 515 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 515 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "innodb_flush_log_at_trx_commit=2 (set in mariadb-bench.compose.yaml for the benchmark\u0027s own container gvbbench-mariadb, and in mariadb.compose.yaml for the daily gvb-mariadb, which has the same server settings): the InnoDB redo log is written to the operating system at every commit but fsynced about once a second, so a crash of the mariadbd process loses nothing, while an operating-system crash or power cut can lose the last second of commits; innodb_doublewrite is on (default), the binary log is off, and the vector graph is an InnoDB table under the same log (settings read from the running server with SHOW VARIABLES; the crash behaviour is InnoDB\u0027s documented behaviour for this setting and was not tested on either container)", "loadIndexNote": "VECTOR INDEX is maintained inside every INSERT, nothing to build; confirmed after 2 poll(s) in 0.5 s (poll 1 of 2 said: not ready: a search at mhnsw_ef_search 100 returned only 397 of 524 requested rows through the index (EXPLAIN key \u0027vec_idx\u0027) (information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 397 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported)): information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 515 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported", "searchSettings": "DISTANCE=cosine, M=16, mhnsw_ef_search=100", "settled": true, "settleDetail": null, "meanMs": 0.632296980809965, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 6, "passOrder": [ "default@1", "default@8", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 518 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 518 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "innodb_flush_log_at_trx_commit=2 (set in mariadb-bench.compose.yaml for the benchmark\u0027s own container gvbbench-mariadb, and in mariadb.compose.yaml for the daily gvb-mariadb, which has the same server settings): the InnoDB redo log is written to the operating system at every commit but fsynced about once a second, so a crash of the mariadbd process loses nothing, while an operating-system crash or power cut can lose the last second of commits; innodb_doublewrite is on (default), the binary log is off, and the vector graph is an InnoDB table under the same log (settings read from the running server with SHOW VARIABLES; the crash behaviour is InnoDB\u0027s documented behaviour for this setting and was not tested on either container)", "loadIndexNote": "VECTOR INDEX is maintained inside every INSERT, nothing to build; confirmed after 2 poll(s) in 0.5 s (poll 1 of 2 said: not ready: a search at mhnsw_ef_search 100 returned only 389 of 524 requested rows through the index (EXPLAIN key \u0027vec_idx\u0027) (information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 389 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported)): information_schema lists vec_idx as INDEX_TYPE VECTOR, EXPLAIN of the default search uses key vec_idx, the server applied mhnsw_ef_search 100 (asked for 100, read back from the server), a search at that effort returned 518 of 524 requested rows through the index; MariaDB has no per-index row count (the graph is a hidden InnoDB table), so indexed vectors are not reported", "searchSettings": "DISTANCE=cosine, M=16, mhnsw_ef_search=100", "settled": true, "settleDetail": null, "meanMs": 0.6355649845880071, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "oracle", "engines": [ "Oracle AI Database Free 23.26 (23ai line, VECTOR FLOAT32)" ], "indexes": [ "HNSW in-memory neighbor graph NEIGHBORS=16 EFCONSTRUCTION=128, EFSEARCH=100 per query, cosine; exact mode = FETCH EXACT FIRST (full scan); Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit)" ], "hosting": [ "compose" ], "p50Ms": { "median": 0.8857, "min": 0.8841, "max": 0.8963, "n": 3, "perRun": [ 0.8841, 0.8963, 0.8857 ] }, "p95Ms": { "median": 1.0625899999999997, "min": 1.0505, "max": 1.06575, "n": 3, "perRun": [ 1.0505, 1.0625899999999997, 1.06575 ] }, "qps": { "1": { "median": 1096.7649802941794, "min": 1089.134321911436, "max": 1103.8255668210784, "n": 3, "perRun": [ 1103.8255668210784, 1089.134321911436, 1096.7649802941794 ] }, "8": { "median": 2879.2116939366606, "min": 2826.271911110787, "max": 2896.5483183004194, "n": 3, "perRun": [ 2879.2116939366606, 2896.5483183004194, 2826.271911110787 ] } }, "qpsRatio": { "8/1": { "median": 2.6083937358223546, "min": 2.5769166247017763, "max": 2.659495950156968, "n": 3, "perRun": [ 2.6083937358223546, 2.659495950156968, 2.5769166247017763 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.8676, "min": 0.8532, "max": 0.8704, "n": 3, "perRun": [ 0.8532, 0.8676, 0.8704 ] }, "8": { "median": 0.6435, "min": 0.6372, "max": 0.6488, "n": 3, "perRun": [ 0.6372, 0.6435, 0.6488 ] } }, "loadRowsPerSecond": { "median": 2007.1967964526245, "min": 1937.9235740636298, "max": 2019.6733139871242, "n": 3, "perRun": [ 1937.9235740636298, 2007.1967964526245, 2019.6733139871242 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 2.3729, "min": 2.3597, "max": 2.3801, "n": 3, "perRun": [ 2.3729, 2.3801, 2.3597 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 5, 5, 5 ], "median": 5 }, "p95Ms": { "perRun": [ 6, 6, 6 ], "median": 6 }, "qps@1": { "perRun": [ 5, 5, 5 ], "median": 5 }, "qps@8": { "perRun": [ 6, 6, 6 ], "median": 6 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 3, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 0 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit)" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 671982 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit)" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "By these settings a committed row should survive a crash or power loss. The sink uses a plain COMMIT and commit_logging, commit_wait and commit_write are unset in the database, so every commit waits for its redo to be written. Measured 2026-10-04: 50 separate client commits raised V$SYSSTAT \u0027redo synch writes\u0027 by 55 (the 50 commits plus the CREATE and DROP of the probe table), and the log writer and the datafile writer hold their files open with O_DSYNC (open flags 02110002, filesystemio_options none). The database runs NOARCHIVELOG (V$DATABASE.LOG_MODE), so redo serves crash recovery only and there is no point-in-time restore. The HNSW graph lives in the 768 MB vector memory pool (oracle-init/01-vector-memory.sh) and is not the durable copy; the table is. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. CPU: Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit).", "loadIndexNote": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 0 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit); graph rebuilt in 2.2 s", "searchSettings": "EFCONSTRUCTION=128, EFSEARCH=100, NEIGHBORS=16", "settled": false, "settleDetail": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 6,134 searches (0 failed) at 1 searcher; p50 of the last windows 2.383, 2.353, 2.360 ms (windows of at least 2 s and 100 searches); trial of 1,220 searches in 3.0 s at 1 searcher: p50 2.382 ms against the settled p50 2.360 ms, 1% apart (limit 10%); default@8: warm-up 15.0 s and 43,925 searches (0 failed) at 8 searchers; QPS of the last windows 2,416, 3,480, 2,581 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 9,374 searches in 3.0 s at 8 searchers: 3,124 QPS against the settled 2,581 QPS, 17% apart (limit 10%), so the warm-up was EXTENDED once (120.3 s and 343,792 searches (0 failed) at 8 searchers; QPS of the last windows 3,449, 2,488, 3,050 (windows of at least 2 s and 100 searches), the 120 s cap ran out); second trial of 9,654 searches in 3.0 s at 8 searchers: 3,217 QPS against the settled 3,050 QPS, 5% apart (limit 10%); still disagreeing after the extension; default@1: warm-up 15.0 s and 16,210 searches (0 failed) at 1 searcher; p50 of the last windows 0.894, 0.885, 0.895 ms (windows of at least 2 s and 100 searches); trial of 3,375 searches in 3.0 s at 1 searcher: p50 0.867 ms against the settled p50 0.894 ms, 3% apart (limit 10%). Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 0.9059402409747701, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 10, "passOrder": [ "exact", "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 0 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit)" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 707301 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit)" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "By these settings a committed row should survive a crash or power loss. The sink uses a plain COMMIT and commit_logging, commit_wait and commit_write are unset in the database, so every commit waits for its redo to be written. Measured 2026-10-04: 50 separate client commits raised V$SYSSTAT \u0027redo synch writes\u0027 by 55 (the 50 commits plus the CREATE and DROP of the probe table), and the log writer and the datafile writer hold their files open with O_DSYNC (open flags 02110002, filesystemio_options none). The database runs NOARCHIVELOG (V$DATABASE.LOG_MODE), so redo serves crash recovery only and there is no point-in-time restore. The HNSW graph lives in the 768 MB vector memory pool (oracle-init/01-vector-memory.sh) and is not the durable copy; the table is. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. CPU: Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit).", "loadIndexNote": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 0 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit); graph rebuilt in 2.3 s", "searchSettings": "EFCONSTRUCTION=128, EFSEARCH=100, NEIGHBORS=16", "settled": false, "settleDetail": "latency had NOT settled when timing began for default@1, default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 5,949 searches (0 failed) at 1 searcher; p50 of the last windows 2.363, 2.376, 2.356 ms (windows of at least 2 s and 100 searches); trial of 1,229 searches in 3.0 s at 1 searcher: p50 2.387 ms against the settled p50 2.363 ms, 1% apart (limit 10%); default@1: warm-up 15.0 s and 16,397 searches (0 failed) at 1 searcher; p50 of the last windows 0.860, 0.892, 0.913 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 3,246 searches in 3.0 s at 1 searcher: p50 0.900 ms against the settled p50 0.892 ms, 1% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 32,617 searches (0 failed) at 1 searcher; p50 of the last windows 0.883, 0.857, 0.926 ms (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 3,341 searches in 3.0 s at 1 searcher: p50 0.874 ms against the settled p50 0.883 ms, 1% apart (limit 10%); still disagreeing after the extension; default@8: warm-up 15.0 s and 48,930 searches (0 failed) at 8 searchers; QPS of the last windows 3,514, 2,364, 3,100 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 7,829 searches in 3.1 s at 8 searchers: 2,519 QPS against the settled 3,100 QPS, 23% apart (limit 10%), so the warm-up was EXTENDED once (120.0 s and 315,148 searches (0 failed) at 8 searchers; QPS of the last windows 2,207, 2,452, 2,930 (windows of at least 2 s and 100 searches), the 120 s cap ran out); second trial of 7,269 searches in 3.3 s at 8 searchers: 2,230 QPS against the settled 2,452 QPS, 10% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 0.9181603957214342, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 19, "passOrder": [ "exact", "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 0 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit)" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 678204 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit)" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "By these settings a committed row should survive a crash or power loss. The sink uses a plain COMMIT and commit_logging, commit_wait and commit_write are unset in the database, so every commit waits for its redo to be written. Measured 2026-10-04: 50 separate client commits raised V$SYSSTAT \u0027redo synch writes\u0027 by 55 (the 50 commits plus the CREATE and DROP of the probe table), and the log writer and the datafile writer hold their files open with O_DSYNC (open flags 02110002, filesystemio_options none). The database runs NOARCHIVELOG (V$DATABASE.LOG_MODE), so redo serves crash recovery only and there is no point-in-time restore. The HNSW graph lives in the 768 MB vector memory pool (oracle-init/01-vector-memory.sh) and is not the durable copy; the table is. Not tested by cutting power; whether the disk\u0027s own write cache reaches the media was not checked. CPU: Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit).", "loadIndexNote": "HNSW graph holds 524 of 524 rows, change log waiting: 0 inserts and 0 deletes, USER_INDEXES status VALID, plan of the default search: VECTOR INDEX HNSW SCAN \u003E TABLE ACCESS BY INDEX ROWID, index used by 0 queries so far; Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit); graph rebuilt in 2.2 s", "searchSettings": "EFCONSTRUCTION=128, EFSEARCH=100, NEIGHBORS=16", "settled": false, "settleDetail": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 6,064 searches (0 failed) at 1 searcher; p50 of the last windows 2.371, 2.350, 2.331 ms (windows of at least 2 s and 100 searches); trial of 1,244 searches in 3.0 s at 1 searcher: p50 2.356 ms against the settled p50 2.350 ms, 0% apart (limit 10%); default@1: warm-up 15.0 s and 16,495 searches (0 failed) at 1 searcher; p50 of the last windows 0.888, 0.870, 0.904 ms (windows of at least 2 s and 100 searches); trial of 3,251 searches in 3.0 s at 1 searcher: p50 0.896 ms against the settled p50 0.888 ms, 1% apart (limit 10%); default@8: warm-up 15.3 s and 44,258 searches (0 failed) at 8 searchers; QPS of the last windows 3,062, 2,834, 2,622 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 9,357 searches in 3.0 s at 8 searchers: 3,118 QPS against the settled 2,834 QPS, 9% apart (limit 10%), so the warm-up was EXTENDED once (120.2 s and 344,269 searches (0 failed) at 8 searchers; QPS of the last windows 3,394, 2,474, 2,998 (windows of at least 2 s and 100 searches), the 120 s cap ran out); second trial of 9,022 searches in 3.0 s at 8 searchers: 3,006 QPS against the settled 2,998 QPS, 0% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 0.9117723650619984, "meanSource": "1000 / QPS@1" } ], "engineNotes": [ { "kind": "cpu-cap", "text": "The engine cannot use more CPUs than its own cap, whatever it is pinned to. The results say: Oracle Free caps itself at 2 CPUs (cpu_count 2 in V$PARAMETER, edition FREE in V$INSTANCE, 8 host CPUs in V$OSSTAT NUM_CPUS; the 2 CPU thread limit is Oracle\u0027s documented Free edition limit).", "source": "results" } ] }, { "name": "redis", "engines": [ "Redis 8.10.2 (query engine, HASH \u002B vector index)" ], "indexes": [ "HNSW TYPE FLOAT32 M=16 EF_CONSTRUCTION=128, EF_RUNTIME=100 per query, cosine; exact mode = FLAT index built on first exact query" ], "hosting": [ "compose" ], "p50Ms": { "median": 0.3725, "min": 0.3582, "max": 0.3744, "n": 3, "perRun": [ 0.3582, 0.3725, 0.3744 ] }, "p95Ms": { "median": 0.4719, "min": 0.4696, "max": 0.4758, "n": 3, "perRun": [ 0.4696, 0.4719, 0.4758 ] }, "qps": { "1": { "median": 2620.4265995904652, "min": 2594.784937273439, "max": 2653.8854965157616, "n": 3, "perRun": [ 2653.8854965157616, 2620.4265995904652, 2594.784937273439 ] }, "8": { "median": 5074.549465810664, "min": 5039.114772889733, "max": 5132.6717990784855, "n": 3, "perRun": [ 5074.549465810664, 5132.6717990784855, 5039.114772889733 ] } }, "qpsRatio": { "8/1": { "median": 1.94201634998882, "min": 1.912120727315835, "max": 1.9587161112929659, "n": 3, "perRun": [ 1.912120727315835, 1.9587161112929659, 1.94201634998882 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.5472, "min": 0.5375, "max": 0.554, "n": 3, "perRun": [ 0.5375, 0.5472, 0.554 ] }, "8": { "median": 0.3635, "min": 0.3605, "max": 0.368, "n": 3, "perRun": [ 0.3635, 0.3605, 0.368 ] } }, "loadRowsPerSecond": { "median": 11833.624657299133, "min": 11413.98343665457, "max": 11939.155874338105, "n": 3, "perRun": [ 11833.624657299133, 11413.98343665457, 11939.155874338105 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 0.3503, "min": 0.3495, "max": 0.3582, "n": 3, "perRun": [ 0.3503, 0.3495, 0.3582 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 1, 1, 1 ], "median": 1 }, "p95Ms": { "perRun": [ 1, 1, 1 ], "median": 1 }, "qps@1": { "perRun": [ 1, 1, 1 ], "median": 1 }, "qps@8": { "perRun": [ 2, 2, 2 ], "median": 2 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 17, "passOrder": [ "default@1", "default@8", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "save \u0022300 1\u0022 and appendonly no (redis.compose.yaml): an RDB snapshot is written every 5 minutes if at least one key changed, and on a clean stop; there is no append-only log, so a crash loses every write since the last snapshot (up to 5 minutes plus the time a snapshot takes; measured with docker kill, which is SIGKILL, right after a 2,000-vector load: none of the 2,000 hashes were there after the restart, and the restart loaded an older snapshot that still held keys of a collection that had been dropped since)", "loadIndexNote": "HNSW index finished after 0.0 s of waiting: FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524", "searchSettings": "EF_CONSTRUCTION=128, EF_RUNTIME=100, M=16", "settled": false, "settleDetail": "latency had NOT settled when timing began for exact (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). default@1: warm-up 15.0 s and 39,275 searches (0 failed) at 1 searcher; p50 of the last windows 0.383, 0.380, 0.382 ms (windows of at least 2 s and 100 searches); trial of 8,168 searches in 3.0 s at 1 searcher: p50 0.349 ms against the settled p50 0.382 ms, 10% apart (limit 10%); default@8: warm-up 15.0 s and 76,531 searches (0 failed) at 8 searchers; QPS of the last windows 5,217, 5,302, 5,056 (windows of at least 2 s and 100 searches); trial of 15,028 searches in 3.0 s at 8 searchers: 5,007 QPS against the settled 5,217 QPS, 4% apart (limit 10%); exact: warm-up 15.0 s and 41,741 searches (0 failed) at 1 searcher; p50 of the last windows 0.364, 0.362, 0.324 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 7,959 searches in 3.0 s at 1 searcher: p50 0.380 ms against the settled p50 0.362 ms, 5% apart (limit 10%), so the warm-up was EXTENDED once (36.0 s and 99,849 searches (0 failed) at 1 searcher; p50 of the last windows 0.362, 0.353, 0.342 ms (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 8,351 searches in 3.0 s at 1 searcher: p50 0.359 ms against the settled p50 0.353 ms, 2% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 0.3768060081389653, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 14, "passOrder": [ "default@1", "exact", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "save \u0022300 1\u0022 and appendonly no (redis.compose.yaml): an RDB snapshot is written every 5 minutes if at least one key changed, and on a clean stop; there is no append-only log, so a crash loses every write since the last snapshot (up to 5 minutes plus the time a snapshot takes; measured with docker kill, which is SIGKILL, right after a 2,000-vector load: none of the 2,000 hashes were there after the restart, and the restart loaded an older snapshot that still held keys of a collection that had been dropped since)", "loadIndexNote": "HNSW index finished after 0.0 s of waiting: FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524", "searchSettings": "EF_CONSTRUCTION=128, EF_RUNTIME=100, M=16", "settled": false, "settleDetail": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). default@1: warm-up 15.0 s and 39,692 searches (0 failed) at 1 searcher; p50 of the last windows 0.352, 0.350, 0.369 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 8,035 searches in 3.0 s at 1 searcher: p50 0.355 ms against the settled p50 0.352 ms, 1% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 77,908 searches (0 failed) at 1 searcher; p50 of the last windows 0.379, 0.384, 0.376 ms (windows of at least 2 s and 100 searches)); second trial of 7,691 searches in 3.0 s at 1 searcher: p50 0.388 ms against the settled p50 0.379 ms, 3% apart (limit 10%); settled after the extension; exact: warm-up 15.0 s and 41,325 searches (0 failed) at 1 searcher; p50 of the last windows 0.367, 0.376, 0.359 ms (windows of at least 2 s and 100 searches); trial of 8,543 searches in 3.0 s at 1 searcher: p50 0.342 ms against the settled p50 0.367 ms, 7% apart (limit 10%); default@8: warm-up 15.0 s and 75,695 searches (0 failed) at 8 searchers; QPS of the last windows 4,921, 5,016, 5,277 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 15,098 searches in 3.0 s at 8 searchers: 5,030 QPS against the settled 5,016 QPS, 0% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 153,806 searches (0 failed) at 8 searchers; QPS of the last windows 5,068, 5,484, 4,940 (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 15,130 searches in 3.0 s at 8 searchers: 5,041 QPS against the settled 5,068 QPS, 1% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 0.38161725276193026, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 17, "passOrder": [ "exact", "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "save \u0022300 1\u0022 and appendonly no (redis.compose.yaml): an RDB snapshot is written every 5 minutes if at least one key changed, and on a clean stop; there is no append-only log, so a crash loses every write since the last snapshot (up to 5 minutes plus the time a snapshot takes; measured with docker kill, which is SIGKILL, right after a 2,000-vector load: none of the 2,000 hashes were there after the restart, and the restart loaded an older snapshot that still held keys of a collection that had been dropped since)", "loadIndexNote": "HNSW index finished after 0.0 s of waiting: FT.INFO indexing 0, percent_indexed 1, hash_indexing_failures 0, flat_buffer_size 0, num_docs 524, hashes counted with SCAN under gvb_gvbbench_eshoponweb: 524", "searchSettings": "EF_CONSTRUCTION=128, EF_RUNTIME=100, M=16", "settled": false, "settleDetail": "latency had NOT settled when timing began for exact (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 41,314 searches (0 failed) at 1 searcher; p50 of the last windows 0.336, 0.374, 0.372 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 8,416 searches in 3.0 s at 1 searcher: p50 0.335 ms against the settled p50 0.372 ms, 11% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 81,883 searches (0 failed) at 1 searcher; p50 of the last windows 0.363, 0.369, 0.338 ms (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 8,081 searches in 3.0 s at 1 searcher: p50 0.369 ms against the settled p50 0.363 ms, 2% apart (limit 10%); still disagreeing after the extension; default@1: warm-up 15.0 s and 38,516 searches (0 failed) at 1 searcher; p50 of the last windows 0.372, 0.407, 0.378 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 7,842 searches in 3.0 s at 1 searcher: p50 0.377 ms against the settled p50 0.378 ms, 0% apart (limit 10%), so the warm-up was EXTENDED once (32.0 s and 82,511 searches (0 failed) at 1 searcher; p50 of the last windows 0.385, 0.380, 0.391 ms (windows of at least 2 s and 100 searches)); second trial of 7,681 searches in 3.0 s at 1 searcher: p50 0.390 ms against the settled p50 0.385 ms, 1% apart (limit 10%); settled after the extension; default@8: warm-up 15.0 s and 75,402 searches (0 failed) at 8 searchers; QPS of the last windows 4,844, 5,008, 5,010 (windows of at least 2 s and 100 searches); trial of 14,794 searches in 3.0 s at 8 searchers: 4,929 QPS against the settled 5,008 QPS, 2% apart (limit 10%). Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 0.3853883940958841, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "mongodb", "engines": [ "MongoDB 8.0.32 Atlas Local (mongod \u002B mongot Vector Search)" ], "indexes": [ "vectorSearch index, HNSW maxEdges=16 numEdgeCandidates=128, float32 binData, cosine, numCandidates=20x hits (min 100); exact mode = $vectorSearch exact:true" ], "hosting": [ "compose" ], "p50Ms": { "median": 1.394, "min": 1.389, "max": 1.4123, "n": 3, "perRun": [ 1.389, 1.4123, 1.394 ] }, "p95Ms": { "median": 1.6168099999999999, "min": 1.613405, "max": 1.6446349999999998, "n": 3, "perRun": [ 1.613405, 1.6446349999999998, 1.6168099999999999 ] }, "qps": { "1": { "median": 703.9109540593784, "min": 693.6669606426645, "max": 704.9536880674624, "n": 3, "perRun": [ 704.9536880674624, 693.6669606426645, 703.9109540593784 ] }, "8": { "median": 2088.8625589929193, "min": 2088.0560852254434, "max": 2102.412694793193, "n": 3, "perRun": [ 2102.412694793193, 2088.0560852254434, 2088.8625589929193 ] } }, "qpsRatio": { "8/1": { "median": 2.982341578432876, "min": 2.9675096643214243, "max": 3.0101708798281432, "n": 3, "perRun": [ 2.982341578432876, 3.0101708798281432, 2.9675096643214243 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.3048, "min": 0.3032, "max": 0.3081, "n": 3, "perRun": [ 0.3048, 0.3081, 0.3032 ] }, "8": { "median": 0.2917, "min": 0.286, "max": 0.2928, "n": 3, "perRun": [ 0.2917, 0.2928, 0.286 ] } }, "loadRowsPerSecond": { "median": 3084.2724472053783, "min": 2969.4155861450695, "max": 3172.542732455355, "n": 3, "perRun": [ 3084.2724472053783, 2969.4155861450695, 3172.542732455355 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 1.2278, "min": 1.2168, "max": 1.2476, "n": 3, "perRun": [ 1.2168, 1.2476, 1.2278 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 8, 8, 8 ], "median": 8 }, "p95Ms": { "perRun": [ 8, 8, 8 ], "median": 8 }, "qps@1": { "perRun": [ 8, 8, 8 ], "median": 8 }, "qps@8": { "perRun": [ 8, 8, 8 ], "median": 8 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 7, "passOrder": [ "default@1", "exact", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index status READY, queryable True; mongot holds 524 of 524 documents in 2 segment(s) (269 \u002B 255 documents), 2 searched through the HNSW graph (Approximate)" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index status READY, queryable True; mongot holds 524 of 524 documents in 2 segment(s) (269 \u002B 255 documents), 2 searched through the HNSW graph (Approximate)" }, "segmentLayoutAfterLoad": "2 segments", "segmentLayoutAfterSearch": "2 segments", "durability": "Acknowledged writes are journaled before the acknowledgement. The sink sets no write concern, so the server default applies: getDefaultRWConcern gives w majority and the one-member replica set (--replSet gvbmongo in the image) has writeConcernMajorityJournalDefault true, with WiredTiger journaling on (journalCommitInterval 100 ms is only the interval for unacknowledged work). Measured 2026-10-04: 30 acknowledged single-document inserts raised WiredTiger \u0027log sync operations\u0027 by 30 and strace showed fdatasync on /data/db/journal/WiredTigerLog files. A crash loses no acknowledged write. mongot\u0027s search index is not part of that promise: it follows the collection asynchronously and is rebuilt from it.", "loadIndexNote": "index status READY, queryable True; mongot holds 524 of 524 documents in 2 segment(s) (269 \u002B 255 documents), 2 searched through the HNSW graph (Approximate); segment layout unchanged for 10.2 s over 11 readings; wait took 10.2 s", "searchSettings": "maxEdges=16, numCandidates=20x, numEdgeCandidates=128", "settled": true, "settleDetail": null, "meanMs": 1.4185329007092198, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 19, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index status READY, queryable True; mongot holds 524 of 524 documents in 4 segment(s) (268 \u002B 254 \u002B 1 \u002B 1 documents), 4 searched through the HNSW graph (Approximate)" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index status READY, queryable True; mongot holds 524 of 524 documents in 4 segment(s) (268 \u002B 254 \u002B 1 \u002B 1 documents), 4 searched through the HNSW graph (Approximate)" }, "segmentLayoutAfterLoad": "4 segments", "segmentLayoutAfterSearch": "4 segments", "durability": "Acknowledged writes are journaled before the acknowledgement. The sink sets no write concern, so the server default applies: getDefaultRWConcern gives w majority and the one-member replica set (--replSet gvbmongo in the image) has writeConcernMajorityJournalDefault true, with WiredTiger journaling on (journalCommitInterval 100 ms is only the interval for unacknowledged work). Measured 2026-10-04: 30 acknowledged single-document inserts raised WiredTiger \u0027log sync operations\u0027 by 30 and strace showed fdatasync on /data/db/journal/WiredTigerLog files. A crash loses no acknowledged write. mongot\u0027s search index is not part of that promise: it follows the collection asynchronously and is rebuilt from it.", "loadIndexNote": "index status READY, queryable True; mongot holds 524 of 524 documents in 4 segment(s) (268 \u002B 254 \u002B 1 \u002B 1 documents), 4 searched through the HNSW graph (Approximate); segment layout unchanged for 10.2 s over 11 readings; wait took 10.2 s", "searchSettings": "maxEdges=16, numCandidates=20x, numEdgeCandidates=128", "settled": true, "settleDetail": null, "meanMs": 1.4416139974052185, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 4, "passOrder": [ "default@1", "exact", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index status READY, queryable True; mongot holds 524 of 524 documents in 2 segment(s) (264 \u002B 260 documents), 2 searched through the HNSW graph (Approximate)" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index status READY, queryable True; mongot holds 524 of 524 documents in 2 segment(s) (264 \u002B 260 documents), 2 searched through the HNSW graph (Approximate)" }, "segmentLayoutAfterLoad": "2 segments", "segmentLayoutAfterSearch": "2 segments", "durability": "Acknowledged writes are journaled before the acknowledgement. The sink sets no write concern, so the server default applies: getDefaultRWConcern gives w majority and the one-member replica set (--replSet gvbmongo in the image) has writeConcernMajorityJournalDefault true, with WiredTiger journaling on (journalCommitInterval 100 ms is only the interval for unacknowledged work). Measured 2026-10-04: 30 acknowledged single-document inserts raised WiredTiger \u0027log sync operations\u0027 by 30 and strace showed fdatasync on /data/db/journal/WiredTigerLog files. A crash loses no acknowledged write. mongot\u0027s search index is not part of that promise: it follows the collection asynchronously and is rebuilt from it.", "loadIndexNote": "index status READY, queryable True; mongot holds 524 of 524 documents in 2 segment(s) (264 \u002B 260 documents), 2 searched through the HNSW graph (Approximate); segment layout unchanged for 10.2 s over 11 readings; wait took 10.3 s", "searchSettings": "maxEdges=16, numCandidates=20x, numEdgeCandidates=128", "settled": true, "settleDetail": null, "meanMs": 1.420634235386036, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "clickhouse", "engines": [ "ClickHouse 26.3.39.7 (MergeTree \u002B vector_similarity index)" ], "indexes": [ "vector_similarity HNSW cosineDistance, quantization bf16, M=16 ef_construction=128, hnsw_candidate_list_size_for_search=256, rescoring off; exact mode = full scan with skip indexes off" ], "hosting": [ "compose" ], "p50Ms": { "median": 4.5007, "min": 4.4976, "max": 4.50825, "n": 3, "perRun": [ 4.5007, 4.50825, 4.4976 ] }, "p95Ms": { "median": 5.7463999999999995, "min": 5.554369999999999, "max": 6.25531, "n": 3, "perRun": [ 5.554369999999999, 5.7463999999999995, 6.25531 ] }, "qps": { "1": { "median": 214.68461033370824, "min": 214.6841724093891, "max": 215.15501646493257, "n": 3, "perRun": [ 215.15501646493257, 214.68461033370824, 214.6841724093891 ] }, "8": { "median": 492.78898362481993, "min": 479.84989575485946, "max": 495.78019393955714, "n": 3, "perRun": [ 495.78019393955714, 492.78898362481993, 479.84989575485946 ] } }, "qpsRatio": { "8/1": { "median": 2.2954089855757385, "min": 2.23514332877701, "max": 2.3042929794777187, "n": 3, "perRun": [ 2.3042929794777187, 2.2954089855757385, 2.23514332877701 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.8191, "min": 0.8151, "max": 0.8256, "n": 3, "perRun": [ 0.8256, 0.8151, 0.8191 ] }, "8": { "median": 1.0216, "min": 1.0193, "max": 1.0232, "n": 3, "perRun": [ 1.0216, 1.0193, 1.0232 ] } }, "loadRowsPerSecond": { "median": 2976.7247393093544, "min": 2519.7431977753167, "max": 3361.1310847537425, "n": 3, "perRun": [ 2519.7431977753167, 2976.7247393093544, 3361.1310847537425 ] }, "recall": { "median": 0.9949999999999999, "min": 0.99, "max": 0.9949999999999999, "n": 3, "perRun": [ 0.9949999999999999, 0.9949999999999999, 0.99 ] }, "ndcg": { "median": 0.5276466892383727, "min": 0.514829573372199, "max": 0.5276466892383727, "n": 3, "perRun": [ 0.5276466892383727, 0.5276466892383727, 0.514829573372199 ] }, "exactP50Ms": { "median": 6.0876, "min": 6.0564, "max": 6.1026, "n": 3, "perRun": [ 6.0876, 6.1026, 6.0564 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 18, 18, 18 ], "median": 18 }, "p95Ms": { "perRun": [ 18, 18, 18 ], "median": 18 }, "qps@1": { "perRun": [ 18, 18, 18 ], "median": 18 }, "qps@8": { "perRun": [ 18, 18, 18 ], "median": 18 }, "recall": { "perRun": [ 18, 17, 18 ], "median": 18 }, "ndcg": { "perRun": [ 1, 1, 19 ], "median": 1 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 5, "passOrder": [ "exact", "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Acknowledged inserts are not fsynced. The sink creates plain MergeTree tables, so the MergeTree settings are the defaults: fsync_after_insert 0 and fsync_part_directory 0 (system.merge_tree_settings), and async_insert 1 with wait_for_async_insert 1 is the server default for the gvb login (system.settings), so an insert is acknowledged once its part is written, not once it is on disk. Measured 2026-10-04 with strace over 30 acknowledged single-row inserts (30 Ok rows in system.asynchronous_insert_log): zero fsync, fdatasync or sync_file_range calls, while the control, a table created with fsync_after_insert=1, made 61 fdatasync calls for 5 inserts. A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a ClickHouse process crash alone should not (inferred, not tested).", "loadIndexNote": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts; wait took 6.2 s", "searchSettings": "M=16, ef_construction=128, hnsw_candidate_list_size_for_search=256", "settled": false, "settleDetail": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 2,395 searches (0 failed) at 1 searcher; p50 of the last windows 6.058, 6.063, 6.057 ms (windows of at least 2 s and 100 searches); trial of 488 searches in 3.0 s at 1 searcher: p50 6.032 ms against the settled p50 6.058 ms, 0% apart (limit 10%); default@1: warm-up 15.0 s and 3,279 searches (0 failed) at 1 searcher; p50 of the last windows 4.472, 4.478, 4.471 ms (windows of at least 2 s and 100 searches); trial of 654 searches in 3.0 s at 1 searcher: p50 4.486 ms against the settled p50 4.472 ms, 0% apart (limit 10%); default@8: warm-up 15.0 s and 7,362 searches (0 failed) at 8 searchers; QPS of the last windows 434, 504, 522 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,512 searches in 3.0 s at 8 searchers: 502 QPS against the settled 504 QPS, 0% apart (limit 10%), so the warm-up was EXTENDED once (38.1 s and 18,460 searches (0 failed) at 8 searchers; QPS of the last windows 525, 524, 488 (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 1,302 searches in 3.0 s at 8 searchers: 432 QPS against the settled 524 QPS, 21% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 4.647811686802974, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 18, "passOrder": [ "default@8", "exact", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Acknowledged inserts are not fsynced. The sink creates plain MergeTree tables, so the MergeTree settings are the defaults: fsync_after_insert 0 and fsync_part_directory 0 (system.merge_tree_settings), and async_insert 1 with wait_for_async_insert 1 is the server default for the gvb login (system.settings), so an insert is acknowledged once its part is written, not once it is on disk. Measured 2026-10-04 with strace over 30 acknowledged single-row inserts (30 Ok rows in system.asynchronous_insert_log): zero fsync, fdatasync or sync_file_range calls, while the control, a table created with fsync_after_insert=1, made 61 fdatasync calls for 5 inserts. A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a ClickHouse process crash alone should not (inferred, not tested).", "loadIndexNote": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts; wait took 6.2 s", "searchSettings": "M=16, ef_construction=128, hnsw_candidate_list_size_for_search=256", "settled": false, "settleDetail": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). default@8: warm-up 15.0 s and 7,401 searches (0 failed) at 8 searchers; QPS of the last windows 458, 442, 444 (windows of at least 2 s and 100 searches); trial of 1,581 searches in 3.0 s at 8 searchers: 525 QPS against the settled 444 QPS, 16% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 14,706 searches (0 failed) at 8 searchers; QPS of the last windows 530, 525, 452 (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 1,311 searches in 3.0 s at 8 searchers: 435 QPS against the settled 525 QPS, 21% apart (limit 10%); still disagreeing after the extension; exact: warm-up 15.0 s and 2,370 searches (0 failed) at 1 searcher; p50 of the last windows 6.089, 6.058, 6.067 ms (windows of at least 2 s and 100 searches); trial of 487 searches in 3.0 s at 1 searcher: p50 6.077 ms against the settled p50 6.067 ms, 0% apart (limit 10%); default@1: warm-up 15.0 s and 3,267 searches (0 failed) at 1 searcher; p50 of the last windows 4.489, 4.504, 4.493 ms (windows of at least 2 s and 100 searches); trial of 657 searches in 3.0 s at 1 searcher: p50 4.468 ms against the settled p50 4.493 ms, 1% apart (limit 10%). Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 4.6579957382394035, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 9, "passOrder": [ "default@1", "exact", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Acknowledged inserts are not fsynced. The sink creates plain MergeTree tables, so the MergeTree settings are the defaults: fsync_after_insert 0 and fsync_part_directory 0 (system.merge_tree_settings), and async_insert 1 with wait_for_async_insert 1 is the server default for the gvb login (system.settings), so an insert is acknowledged once its part is written, not once it is on disk. Measured 2026-10-04 with strace over 30 acknowledged single-row inserts (30 Ok rows in system.asynchronous_insert_log): zero fsync, fdatasync or sync_file_range calls, while the control, a table created with fsync_after_insert=1, made 61 fdatasync calls for 5 inserts. A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a ClickHouse process crash alone should not (inferred, not tested).", "loadIndexNote": "1 data part(s), index files on 1 of them, 524 of 524 live rows in indexed parts; 0 merge(s) running, 0 unfinished mutation(s); plan of the default search uses the Skip index vec_idx on 1 of 1 parts; wait took 6.2 s", "searchSettings": "M=16, ef_construction=128, hnsw_candidate_list_size_for_search=256", "settled": false, "settleDetail": "latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). default@1: warm-up 15.0 s and 3,232 searches (0 failed) at 1 searcher; p50 of the last windows 4.436, 4.452, 4.476 ms (windows of at least 2 s and 100 searches); trial of 655 searches in 3.0 s at 1 searcher: p50 4.479 ms against the settled p50 4.452 ms, 1% apart (limit 10%); exact: warm-up 15.0 s and 2,442 searches (0 failed) at 1 searcher; p50 of the last windows 6.029, 6.056, 6.035 ms (windows of at least 2 s and 100 searches); trial of 457 searches in 3.0 s at 1 searcher: p50 6.120 ms against the settled p50 6.035 ms, 1% apart (limit 10%); default@8: warm-up 15.0 s and 7,381 searches (0 failed) at 8 searchers; QPS of the last windows 435, 510, 524 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,509 searches in 3.0 s at 8 searchers: 501 QPS against the settled 510 QPS, 2% apart (limit 10%), so the warm-up was EXTENDED once (32.0 s and 15,463 searches (0 failed) at 8 searchers; QPS of the last windows 440, 445, 524 (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 1,583 searches in 3.0 s at 8 searchers: 526 QPS against the settled 445 QPS, 15% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them.", "meanMs": 4.658005239869586, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "milvus", "engines": [ "Milvus 2.6.25 (standalone, embedded etcd, local storage, REST API v2)" ], "indexes": [ "HNSW M=16 efConstruction=128, ef=100, metric COSINE, Strong consistency searches; approximate only (no exact mode)" ], "hosting": [ "compose" ], "p50Ms": { "median": 2.2078, "min": 2.1592, "max": 2.2176, "n": 3, "perRun": [ 2.1592, 2.2176, 2.2078 ] }, "p95Ms": { "median": 2.73669, "min": 2.6742999999999992, "max": 2.789575, "n": 3, "perRun": [ 2.6742999999999992, 2.73669, 2.789575 ] }, "qps": { "1": { "median": 430.0783928615426, "min": 428.75784238514746, "max": 440.8275266126933, "n": 3, "perRun": [ 440.8275266126933, 430.0783928615426, 428.75784238514746 ] }, "8": { "median": 1285.0924096519475, "min": 1281.516014398511, "max": 1290.5351276847628, "n": 3, "perRun": [ 1290.5351276847628, 1281.516014398511, 1285.0924096519475 ] } }, "qpsRatio": { "8/1": { "median": 2.9797265700141233, "min": 2.927528454498292, "max": 2.9972452573766946, "n": 3, "perRun": [ 2.927528454498292, 2.9797265700141233, 2.9972452573766946 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.5463, "min": 0.5437, "max": 0.5468, "n": 3, "perRun": [ 0.5463, 0.5468, 0.5437 ] }, "8": { "median": 0.6304, "min": 0.6246, "max": 0.6307, "n": 3, "perRun": [ 0.6246, 0.6304, 0.6307 ] } }, "loadRowsPerSecond": { "median": 1175.4555899867335, "min": 1073.4343836601981, "max": 1197.959353970698, "n": 3, "perRun": [ 1073.4343836601981, 1175.4555899867335, 1197.959353970698 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": null, "min": null, "max": null, "n": 0, "perRun": [ null, null, null ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 13, 13, 13 ], "median": 13 }, "p95Ms": { "perRun": [ 13, 13, 13 ], "median": 13 }, "qps@1": { "perRun": [ 13, 13, 13 ], "median": 13 }, "qps@8": { "perRun": [ 11, 11, 11 ], "median": 11 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 2, "passOrder": [ "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; search ledger since the index finished: the sink sent 0 searches (searchParams.params.ef=100, Strong consistency), the query node counted 0 search requests for this collection, segment searches Sealed \u002B0 and Growing \u002B0, the same sealed segments with the same loaded index builds at both readings, segments compacted away \u002B0" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; search ledger since the index finished: the sink sent 117453 searches (searchParams.params.ef=100, Strong consistency), the query node counted 117453 search requests for this collection, segment searches Sealed \u002B115475 and Growing \u002B0, the same sealed segments with the same loaded index builds at both readings, segments compacted away \u002B0" }, "segmentLayoutAfterLoad": "1 sealed segment covering 524 rows", "segmentLayoutAfterSearch": "1 sealed segment covering 524 rows", "durability": "Writes are not fsynced. Standalone uses the default message queue rocksmq (RocksDB, /var/lib/milvus/rdb_data, mq.type default) and flushed segments go to local-disk object storage (COMMON_STORAGETYPE=local in milvus.compose.yaml). Measured 2026-10-04 with strace over 30 acknowledged single-row upserts and one flush: fdatasync ran only on the embedded etcd files (member/wal, member/snap/db), never on rdb_data or the segment files. A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a Milvus process crash alone should not (inferred, not tested). Metadata in embedded etcd is fdatasynced on every commit.", "loadIndexNote": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; flush, index build and load took 14.7 s, then compaction: 1 job(s) accepted by Milvus, layout ready and unchanged for 75 s, 124.1 s for the whole step; search ledger started, later state reads count the searches against the query node\u0027s own counters", "searchSettings": "M=16, ef=100, efConstruction=128", "settled": true, "settleDetail": null, "meanMs": 2.2684608823862993, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 5, "passOrder": [ "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; search ledger since the index finished: the sink sent 0 searches (searchParams.params.ef=100, Strong consistency), the query node counted 0 search requests for this collection, segment searches Sealed \u002B0 and Growing \u002B0, the same sealed segments with the same loaded index builds at both readings, segments compacted away \u002B0" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; search ledger since the index finished: the sink sent 116376 searches (searchParams.params.ef=100, Strong consistency), the query node counted 116376 search requests for this collection, segment searches Sealed \u002B114365 and Growing \u002B0, the same sealed segments with the same loaded index builds at both readings, segments compacted away \u002B0" }, "segmentLayoutAfterLoad": "1 sealed segment covering 524 rows", "segmentLayoutAfterSearch": "1 sealed segment covering 524 rows", "durability": "Writes are not fsynced. Standalone uses the default message queue rocksmq (RocksDB, /var/lib/milvus/rdb_data, mq.type default) and flushed segments go to local-disk object storage (COMMON_STORAGETYPE=local in milvus.compose.yaml). Measured 2026-10-04 with strace over 30 acknowledged single-row upserts and one flush: fdatasync ran only on the embedded etcd files (member/wal, member/snap/db), never on rdb_data or the segment files. A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a Milvus process crash alone should not (inferred, not tested). Metadata in embedded etcd is fdatasynced on every commit.", "loadIndexNote": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; flush, index build and load took 14.7 s, then compaction: 1 job(s) accepted by Milvus, layout ready and unchanged for 75 s, 124.1 s for the whole step; search ledger started, later state reads count the searches against the query node\u0027s own counters", "searchSettings": "M=16, ef=100, efConstruction=128", "settled": true, "settleDetail": null, "meanMs": 2.3251574982562198, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 15, "passOrder": [ "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; search ledger since the index finished: the sink sent 0 searches (searchParams.params.ef=100, Strong consistency), the query node counted 0 search requests for this collection, segment searches Sealed \u002B0 and Growing \u002B0, the same sealed segments with the same loaded index builds at both readings, segments compacted away \u002B0" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; search ledger since the index finished: the sink sent 116349 searches (searchParams.params.ef=100, Strong consistency), the query node counted 116349 search requests for this collection, segment searches Sealed \u002B114333 and Growing \u002B0, the same sealed segments with the same loaded index builds at both readings, segments compacted away \u002B0" }, "segmentLayoutAfterLoad": "1 sealed segment covering 524 rows", "segmentLayoutAfterSearch": "1 sealed segment covering 524 rows", "durability": "Writes are not fsynced. Standalone uses the default message queue rocksmq (RocksDB, /var/lib/milvus/rdb_data, mq.type default) and flushed segments go to local-disk object storage (COMMON_STORAGETYPE=local in milvus.compose.yaml). Measured 2026-10-04 with strace over 30 acknowledged single-row upserts and one flush: fdatasync ran only on the embedded etcd files (member/wal, member/snap/db), never on rdb_data or the segment files. A host power loss or kernel crash can lose acknowledged rows that are still in the OS page cache; a Milvus process crash alone should not (inferred, not tested). Metadata in embedded etcd is fdatasynced on every commit.", "loadIndexNote": "index Finished, type HNSW (COSINE, {\u0022M\u0022:16,\u0022efConstruction\u0022:128}), indexedRows 524 of 524 sealed rows, pendingRows 0; stored rows 524; LoadStateLoaded; query node: 1 sealed segment(s) with the index loaded covering 524 rows against 524 stored rows, 0 sealed without it, 0 rows in growing segments; datacoord: 1 live segment(s), 1 flushed and indexed covering 524 rows, 0 delete-log (L0) segment(s) waiting for a compaction, 3 segment(s) compacted away so far; flush, index build and load took 14.7 s, then compaction: 1 job(s) accepted by Milvus, layout ready and unchanged for 75 s, 124.1 s for the whole step; search ledger started, later state reads count the searches against the query node\u0027s own counters", "searchSettings": "M=16, ef=100, efConstruction=128", "settled": true, "settleDetail": null, "meanMs": 2.33231885494403, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "weaviate", "engines": [ "Weaviate 1.39.8 (single node, REST \u002B GraphQL)" ], "indexes": [ "HNSW maxConnections(M)=16 efConstruction=128, ef=-1 (dynamic: limit x 8 clamped 100..500), cosine, no quantization; approximate only (no exact mode)" ], "hosting": [ "compose" ], "p50Ms": { "median": 5.1855, "min": 5.1845, "max": 5.1918, "n": 3, "perRun": [ 5.1855, 5.1845, 5.1918 ] }, "p95Ms": { "median": 10.40582, "min": 10.370069999999998, "max": 10.421825, "n": 3, "perRun": [ 10.40582, 10.370069999999998, 10.421825 ] }, "qps": { "1": { "median": 171.23961834197834, "min": 171.2197351996061, "max": 171.56599304668825, "n": 3, "perRun": [ 171.2197351996061, 171.56599304668825, 171.23961834197834 ] }, "8": { "median": 312.14389064236065, "min": 312.13130394757167, "max": 312.171953705239, "n": 3, "perRun": [ 312.171953705239, 312.13130394757167, 312.14389064236065 ] } }, "qpsRatio": { "8/1": { "median": 1.8228485537674226, "min": 1.8193075352796249, "max": 1.8232241355899326, "n": 3, "perRun": [ 1.8232241355899326, 1.8193075352796249, 1.8228485537674226 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.5727, "min": 0.5668, "max": 0.5866, "n": 3, "perRun": [ 0.5668, 0.5727, 0.5866 ] }, "8": { "median": 0.6031, "min": 0.5986, "max": 0.6044, "n": 3, "perRun": [ 0.6044, 0.6031, 0.5986 ] } }, "loadRowsPerSecond": { "median": 1007.1261865685884, "min": 997.7075574443454, "max": 1018.3190545179649, "n": 3, "perRun": [ 1007.1261865685884, 997.7075574443454, 1018.3190545179649 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": null, "min": null, "max": null, "n": 0, "perRun": [ null, null, null ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 19, 19, 19 ], "median": 19 }, "p95Ms": { "perRun": [ 19, 19, 19 ], "median": 19 }, "qps@1": { "perRun": [ 19, 19, 19 ], "median": 19 }, "qps@8": { "perRun": [ 19, 19, 19 ], "median": 19 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 4, "passOrder": [ "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "shard JOIqxslXLYy9 vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 0 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard JOIqxslXLYy9 status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "shard JOIqxslXLYy9 vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 524 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard JOIqxslXLYy9 status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Weaviate 1.39.8 defaults, weaviate.compose.yaml sets no persistence variable: every object is appended to the LSM write-ahead log with a plain write and no fsync, and the log is fsynced only when its memtable is flushed, 60 seconds after the last write (PERSISTENCE_MEMTABLES_FLUSH_IDLE_AFTER_SECONDS default; measured with strace: 2,080 writes into the objects log during a 2,000-object load, the first fsync 60 s after the last write), so a power cut can lose the last minute of writes; the HNSW commit log is buffered inside the process (77 writes for 2,000 vectors), so a killed process loses the newest vectors from the vector index while their objects survive (measured with docker kill, which is SIGKILL, one second after the load and a restart, two runs each: 0 to 1 of 50, 264 to 267 of 300 and 1,992 to 1,997 of 2,000 stored objects were still reachable through a vector search)", "loadIndexNote": "HNSW is updated inside every batch, nothing to build; confirmed in 0.0 s: shard JOIqxslXLYy9 vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 0 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard JOIqxslXLYy9 status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported", "searchSettings": "ef=-1, efConstruction=128", "settled": true, "settleDetail": null, "meanMs": 5.840448233576643, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 11, "passOrder": [ "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "shard VyfgMkQG0NE1 vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 0 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard VyfgMkQG0NE1 status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "shard VyfgMkQG0NE1 vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 524 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard VyfgMkQG0NE1 status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Weaviate 1.39.8 defaults, weaviate.compose.yaml sets no persistence variable: every object is appended to the LSM write-ahead log with a plain write and no fsync, and the log is fsynced only when its memtable is flushed, 60 seconds after the last write (PERSISTENCE_MEMTABLES_FLUSH_IDLE_AFTER_SECONDS default; measured with strace: 2,080 writes into the objects log during a 2,000-object load, the first fsync 60 s after the last write), so a power cut can lose the last minute of writes; the HNSW commit log is buffered inside the process (77 writes for 2,000 vectors), so a killed process loses the newest vectors from the vector index while their objects survive (measured with docker kill, which is SIGKILL, one second after the load and a restart, two runs each: 0 to 1 of 50, 264 to 267 of 300 and 1,992 to 1,997 of 2,000 stored objects were still reachable through a vector search)", "loadIndexNote": "HNSW is updated inside every batch, nothing to build; confirmed in 0.0 s: shard VyfgMkQG0NE1 vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 0 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard VyfgMkQG0NE1 status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported", "searchSettings": "ef=-1, efConstruction=128", "settled": true, "settleDetail": null, "meanMs": 5.828660926573427, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 18, "passOrder": [ "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "shard KRx7woimLViV vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 0 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard KRx7woimLViV status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "shard KRx7woimLViV vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 524 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard KRx7woimLViV status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Weaviate 1.39.8 defaults, weaviate.compose.yaml sets no persistence variable: every object is appended to the LSM write-ahead log with a plain write and no fsync, and the log is fsynced only when its memtable is flushed, 60 seconds after the last write (PERSISTENCE_MEMTABLES_FLUSH_IDLE_AFTER_SECONDS default; measured with strace: 2,080 writes into the objects log during a 2,000-object load, the first fsync 60 s after the last write), so a power cut can lose the last minute of writes; the HNSW commit log is buffered inside the process (77 writes for 2,000 vectors), so a killed process loses the newest vectors from the vector index while their objects survive (measured with docker kill, which is SIGKILL, one second after the load and a restart, two runs each: 0 to 1 of 50, 264 to 267 of 300 and 1,992 to 1,997 of 2,000 stored objects were still reachable through a vector search)", "loadIndexNote": "HNSW is updated inside every batch, nothing to build; confirmed in 0.0 s: shard KRx7woimLViV vectorIndexingStatus READY, vectorQueueLength 0, node status objectCount 0 (refreshed only when the memtable is flushed, so it lags a minute and does not decide readiness), schema shard KRx7woimLViV status READY, Aggregate count 524, Aggregate nearVector with objectLimit 524 reached 524, Weaviate reports no count of vectors in the HNSW index, so indexed vectors are not reported", "searchSettings": "ef=-1, efConstruction=128", "settled": true, "settleDetail": null, "meanMs": 5.839770081727963, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "chroma", "engines": [ "Chroma 1.4.4 (single node, REST API v2)" ], "indexes": [ "HNSW M=16 ef_construction=128, ef_search=100 (Chroma default), cosine; approximate only (no exact mode)" ], "hosting": [ "compose" ], "p50Ms": { "median": 2.0266, "min": 2.0172, "max": 2.0317499999999997, "n": 3, "perRun": [ 2.0266, 2.0317499999999997, 2.0172 ] }, "p95Ms": { "median": 2.2970249999999997, "min": 2.28131, "max": 2.32665, "n": 3, "perRun": [ 2.2970249999999997, 2.32665, 2.28131 ] }, "qps": { "1": { "median": 487.59820563860325, "min": 486.2599880968794, "max": 489.9432387833048, "n": 3, "perRun": [ 487.59820563860325, 486.2599880968794, 489.9432387833048 ] }, "8": { "median": 806.8877586440376, "min": 804.399032658061, "max": 807.5345043444976, "n": 3, "perRun": [ 804.399032658061, 806.8877586440376, 807.5345043444976 ] } }, "qpsRatio": { "8/1": { "median": 1.649716966461241, "min": 1.6482205292798398, "max": 1.6593751869283522, "n": 3, "perRun": [ 1.649716966461241, 1.6593751869283522, 1.6482205292798398 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.4561, "min": 0.4543, "max": 0.4663, "n": 3, "perRun": [ 0.4663, 0.4543, 0.4561 ] }, "8": { "median": 0.4557, "min": 0.4525, "max": 0.4579, "n": 3, "perRun": [ 0.4525, 0.4579, 0.4557 ] } }, "loadRowsPerSecond": { "median": 860.4786428872408, "min": 840.9090033751905, "max": 895.3021072233623, "n": 3, "perRun": [ 840.9090033751905, 860.4786428872408, 895.3021072233623 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": null, "min": null, "max": null, "n": 0, "perRun": [ null, null, null ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 12, 12, 12 ], "median": 12 }, "p95Ms": { "perRun": [ 12, 12, 11 ], "median": 12 }, "qps@1": { "perRun": [ 12, 12, 12 ], "median": 12 }, "qps@8": { "perRun": [ 13, 13, 13 ], "median": 13 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 11, "passOrder": [ "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "SQLite rollback journal with its default synchronous=FULL under the data directory (chroma.compose.yaml sets IS_PERSISTENT=1 and PERSIST_DIRECTORY=/data and no sync setting): every write is committed to chroma.sqlite3 with fsync of the journal, the directory and the database file before the call returns (strace: 15 database fsyncs and 38 journal fsyncs for one create, four 500-vector upserts and one delete), and the HNSW files are written every sync_threshold=1000 vectors and rebuilt from the SQLite log after a crash; measured with kill -9 and a restart: all 50, 300 and 2,000 vectors were still stored and searchable", "loadIndexNote": "Chroma has no index build to wait for and exposes no index status; checked in 0.0 s: Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524", "searchSettings": "M=16, ef_construction=128, ef_search=100", "settled": true, "settleDetail": null, "meanMs": 2.0508689089417556, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 13, "passOrder": [ "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "SQLite rollback journal with its default synchronous=FULL under the data directory (chroma.compose.yaml sets IS_PERSISTENT=1 and PERSIST_DIRECTORY=/data and no sync setting): every write is committed to chroma.sqlite3 with fsync of the journal, the directory and the database file before the call returns (strace: 15 database fsyncs and 38 journal fsyncs for one create, four 500-vector upserts and one delete), and the HNSW files are written every sync_threshold=1000 vectors and rebuilt from the SQLite log after a crash; measured with kill -9 and a restart: all 50, 300 and 2,000 vectors were still stored and searchable", "loadIndexNote": "Chroma has no index build to wait for and exposes no index status; checked in 0.0 s: Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524", "searchSettings": "M=16, ef_construction=128, ef_search=100", "settled": true, "settleDetail": null, "meanMs": 2.0565130269381044, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 5, "passOrder": [ "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": null, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "SQLite rollback journal with its default synchronous=FULL under the data directory (chroma.compose.yaml sets IS_PERSISTENT=1 and PERSIST_DIRECTORY=/data and no sync setting): every write is committed to chroma.sqlite3 with fsync of the journal, the directory and the database file before the call returns (strace: 15 database fsyncs and 38 journal fsyncs for one create, four 500-vector upserts and one delete), and the HNSW files are written every sync_threshold=1000 vectors and rebuilt from the SQLite log after a crash; measured with kill -9 and a restart: all 50, 300 and 2,000 vectors were still stored and searchable", "loadIndexNote": "Chroma has no index build to wait for and exposes no index status; checked in 0.0 s: Chroma exposes no index status (indexing_status endpoint: HTTP 500 {\u0022error\u0022:\u0022InternalError\u0022,\u0022message\u0022:\u0022Method scout_logs is not implemented\u0022}), so indexed vectors are not reported and ready only means the two counts agree to within 1 percent: count endpoint 524, a vector search asking for 524 results returned 524", "searchSettings": "M=16, ef_construction=128, ef_search=100", "settled": true, "settleDetail": null, "meanMs": 2.041052760485764, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "elasticsearch", "engines": [ "Elasticsearch 9.5.3 (dense_vector)" ], "indexes": [ "HNSW float32, no quantization, m=16, ef_construction=128, cosine; search k=top, num_candidates=100; 1 shard, 0 replicas; force-merged to one segment after the load (at 1,024 dimensions a segment under 1,043 vectors gets no graph)" ], "hosting": [ "compose" ], "p50Ms": { "median": 1.3092, "min": 1.3044, "max": 1.3123, "n": 3, "perRun": [ 1.3044, 1.3092, 1.3123 ] }, "p95Ms": { "median": 1.4761449999999998, "min": 1.4759, "max": 1.49321, "n": 3, "perRun": [ 1.4759, 1.49321, 1.4761449999999998 ] }, "qps": { "1": { "median": 750.9968157735011, "min": 750.5668699783591, "max": 754.4534351339835, "n": 3, "perRun": [ 754.4534351339835, 750.9968157735011, 750.5668699783591 ] }, "8": { "median": 2706.6457647633156, "min": 2697.4916862045325, "max": 2713.207923167071, "n": 3, "perRun": [ 2706.6457647633156, 2713.207923167071, 2697.4916862045325 ] } }, "qpsRatio": { "8/1": { "median": 3.5939391866341084, "min": 3.5875584081377294, "max": 3.612808824458942, "n": 3, "perRun": [ 3.5875584081377294, 3.612808824458942, 3.5939391866341084 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.4588, "min": 0.4487, "max": 0.4607, "n": 3, "perRun": [ 0.4588, 0.4607, 0.4487 ] }, "8": { "median": 0.4907, "min": 0.4893, "max": 0.4956, "n": 3, "perRun": [ 0.4907, 0.4956, 0.4893 ] } }, "loadRowsPerSecond": { "median": 542.9367094326032, "min": 441.75330876600424, "max": 544.6821249671268, "n": 3, "perRun": [ 441.75330876600424, 542.9367094326032, 544.6821249671268 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 1.2517, "min": 1.2496, "max": 1.2551, "n": 3, "perRun": [ 1.2496, 1.2517, 1.2551 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 7, 7, 7 ], "median": 7 }, "p95Ms": { "perRun": [ 7, 7, 7 ], "median": 7 }, "qps@1": { "perRun": [ 7, 7, 7 ], "median": 7 }, "qps@8": { "perRun": [ 7, 7, 7 ], "median": 7 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 9, "passOrder": [ "default@8", "exact", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": false, "indexedVectors": 0, "totalVectors": 524, "detail": "NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search" }, "afterSearch": { "ready": false, "indexedVectors": 0, "totalVectors": 524, "detail": "NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search" }, "segmentLayoutAfterLoad": "1 segment", "segmentLayoutAfterSearch": "1 segment", "durability": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the Elasticsearch default, which neither this sink nor elasticsearch.compose.yaml overrides (ElasticsearchReadinessTests reads it back from the live index). By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "FAILED after 0.4 s: NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search", "searchSettings": "ef_construction=128, k=top, m=16, num_candidates=100", "settled": true, "settleDetail": null, "meanMs": 1.3254628495692513, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 17, "passOrder": [ "exact", "default@1", "default@8" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": false, "indexedVectors": 0, "totalVectors": 524, "detail": "NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search" }, "afterSearch": { "ready": false, "indexedVectors": 0, "totalVectors": 524, "detail": "NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search" }, "segmentLayoutAfterLoad": "1 segment", "segmentLayoutAfterSearch": "1 segment", "durability": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the Elasticsearch default, which neither this sink nor elasticsearch.compose.yaml overrides (ElasticsearchReadinessTests reads it back from the live index). By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "FAILED after 0.4 s: NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search", "searchSettings": "ef_construction=128, k=top, m=16, num_candidates=100", "settled": true, "settleDetail": null, "meanMs": 1.331563568575233, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 16, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": false, "indexedVectors": 0, "totalVectors": 524, "detail": "NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search" }, "afterSearch": { "ready": false, "indexedVectors": 0, "totalVectors": 524, "detail": "NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search" }, "segmentLayoutAfterLoad": "1 segment", "segmentLayoutAfterSearch": "1 segment", "durability": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the Elasticsearch default, which neither this sink nor elasticsearch.compose.yaml overrides (ElasticsearchReadinessTests reads it back from the live index). By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "FAILED after 0.4 s: NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search", "searchSettings": "ef_construction=128, k=top, m=16, num_candidates=100", "settled": true, "settleDetail": null, "meanMs": 1.3323263256061817, "meanSource": "1000 / QPS@1" } ], "engineNotes": [ { "kind": "scans-all-vectors", "text": "No index graph was built at this size: its searches scanned all 524 vectors, so its default search equals its exact search. The results say: NO HNSW GRAPH, searches scan all 524 vectors.", "source": "results" } ] }, { "name": "opensearch", "engines": [ "OpenSearch 3.9.0 (k-NN plugin, faiss)" ], "indexes": [ "faiss HNSW float32, no compression, m=16, ef_construction=128, cosinesimil; search k=top, ef_search=100; 1 shard, 0 replicas; graph built at any segment size (approximate_threshold=0); force-merged to one segment after the load" ], "hosting": [ "compose" ], "p50Ms": { "median": 1.9906, "min": 1.9638, "max": 1.99795, "n": 3, "perRun": [ 1.9906, 1.99795, 1.9638 ] }, "p95Ms": { "median": 2.241599999999998, "min": 2.2409399999999993, "max": 2.2480349999999985, "n": 3, "perRun": [ 2.2409399999999993, 2.2480349999999985, 2.241599999999998 ] }, "qps": { "1": { "median": 495.2239388402185, "min": 494.06511406229606, "max": 497.21837193936096, "n": 3, "perRun": [ 495.2239388402185, 494.06511406229606, 497.21837193936096 ] }, "8": { "median": 1528.1878981855536, "min": 1511.8597635195188, "max": 1539.3690651456109, "n": 3, "perRun": [ 1528.1878981855536, 1511.8597635195188, 1539.3690651456109 ] } }, "qpsRatio": { "8/1": { "median": 3.085852234373944, "min": 3.060041521832465, "max": 3.095961758495414, "n": 3, "perRun": [ 3.085852234373944, 3.060041521832465, 3.095961758495414 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.4563, "min": 0.451, "max": 0.4583, "n": 3, "perRun": [ 0.4583, 0.4563, 0.451 ] }, "8": { "median": 0.5067, "min": 0.4947, "max": 0.5084, "n": 3, "perRun": [ 0.5084, 0.5067, 0.4947 ] } }, "loadRowsPerSecond": { "median": 405.2590873936582, "min": 387.57063918682354, "max": 421.0775003585989, "n": 3, "perRun": [ 405.2590873936582, 421.0775003585989, 387.57063918682354 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 2.5755, "min": 2.5611, "max": 2.6346499999999997, "n": 3, "perRun": [ 2.5755, 2.5611, 2.6346499999999997 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 11, 11, 11 ], "median": 11 }, "p95Ms": { "perRun": [ 10, 10, 10 ], "median": 10 }, "qps@1": { "perRun": [ 11, 11, 11 ], "median": 11 }, "qps@8": { "perRun": [ 10, 10, 10 ], "median": 10 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 12, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search" }, "segmentLayoutAfterLoad": "1 segment", "segmentLayoutAfterSearch": "1 segment", "durability": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the OpenSearch default, which neither this sink nor opensearch.compose.yaml overrides (OpenSearchReadinessTests reads it back from the live index). By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "force-merged to one segment, graph ready after 0.9 s: every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search", "searchSettings": "approximate_threshold=0, ef_construction=128, ef_search=100, k=top, m=16", "settled": true, "settleDetail": null, "meanMs": 2.0192884906612822, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 2, "passOrder": [ "default@8", "exact", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search" }, "segmentLayoutAfterLoad": "1 segment", "segmentLayoutAfterSearch": "1 segment", "durability": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the OpenSearch default, which neither this sink nor opensearch.compose.yaml overrides (OpenSearchReadinessTests reads it back from the live index). By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "force-merged to one segment, graph ready after 0.9 s: every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search", "searchSettings": "approximate_threshold=0, ef_construction=128, ef_search=100, k=top, m=16", "settled": true, "settleDetail": null, "meanMs": 2.0240247115968426, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 2, "passOrder": [ "default@8", "exact", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search" }, "segmentLayoutAfterLoad": "1 segment", "segmentLayoutAfterSearch": "1 segment", "durability": "Every acknowledged bulk request is fsynced to the translog before the answer: index.translog.durability=request, the OpenSearch default, which neither this sink nor opensearch.compose.yaml overrides (OpenSearchReadinessTests reads it back from the live index). By that setting a process crash or power loss loses no acknowledged write; this is read from the setting, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "force-merged to one segment, graph ready after 0.9 s: every segment searched through its HNSW graph, covering 524 of 524 vectors: 1 segment(s), 0 merge(s) running, profiled probe search: ann_search_count 1 (segments tried through a graph), exact_search_count 0 (of those, scanned instead), approximate_threshold 0; force-merged to one segment on purpose, so one graph answers every search", "searchSettings": "approximate_threshold=0, ef_construction=128, ef_search=100, k=top, m=16", "settled": true, "settleDetail": null, "meanMs": 2.0111887581699346, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "typesense", "engines": [ "Typesense 30.2 (vector search)" ], "indexes": [ "HNSW float32 (hnswlib), m=16, ef_construction=128, cosine; search k=top, ef=100; exact mode = filter ordinal:\u003E=0 with flat_search_cutoff; index held in memory" ], "hosting": [ "compose" ], "p50Ms": { "median": 4.0362, "min": 4.02795, "max": 4.0444, "n": 3, "perRun": [ 4.0362, 4.0444, 4.02795 ] }, "p95Ms": { "median": 4.380799999999999, "min": 4.36485, "max": 4.39011, "n": 3, "perRun": [ 4.39011, 4.380799999999999, 4.36485 ] }, "qps": { "1": { "median": 244.11779842121027, "min": 243.83393378210312, "max": 244.99892200474315, "n": 3, "perRun": [ 244.11779842121027, 243.83393378210312, 244.99892200474315 ] }, "8": { "median": 591.3480901892943, "min": 590.8742796795126, "max": 593.6745929047803, "n": 3, "perRun": [ 591.3480901892943, 590.8742796795126, 593.6745929047803 ] } }, "qpsRatio": { "8/1": { "median": 2.4231722656040375, "min": 2.4223882650660298, "max": 2.423265172793933, "n": 3, "perRun": [ 2.4223882650660298, 2.423265172793933, 2.4231722656040375 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.5073, "min": 0.5046, "max": 0.515, "n": 3, "perRun": [ 0.515, 0.5046, 0.5073 ] }, "8": { "median": 0.5692, "min": 0.5644, "max": 0.577, "n": 3, "perRun": [ 0.577, 0.5644, 0.5692 ] } }, "loadRowsPerSecond": { "median": 1040.383766598241, "min": 838.0925206160364, "max": 1042.7837869409639, "n": 3, "perRun": [ 838.0925206160364, 1040.383766598241, 1042.7837869409639 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 5.4602, "min": 5.4602, "max": 5.4844, "n": 3, "perRun": [ 5.4602, 5.4844, 5.4602 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 17, 17, 17 ], "median": 17 }, "p95Ms": { "perRun": [ 16, 17, 16 ], "median": 16 }, "qps@1": { "perRun": [ 17, 17, 17 ], "median": 17 }, "qps@8": { "perRun": [ 17, 17, 17 ], "median": 17 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 14, "passOrder": [ "default@8", "default@1", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Every acknowledged write is appended to Typesense\u0027s raft log and fsynced before the answer: braft raft_sync=true with raft_sync_policy=0 (sync immediately), read from the running container\u0027s brpc /flags page on 2026-10-04. A restart replays the log from the last snapshot (typesense.compose.yaml sets TYPESENSE_SNAPSHOT_INTERVAL_SECONDS=300) and rebuilds the in-memory HNSW graph, so by those settings a crash loses no acknowledged write, but the restart is slow after a big load. This is read from the settings, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "nothing to build, Typesense inserts into the HNSW graph while writing; ready after 0.0 s: HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step", "searchSettings": "ef=100, ef_construction=128, k=top, m=16", "settled": true, "settleDetail": null, "meanMs": 4.096383002252713, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 8, "passOrder": [ "default@1", "default@8", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Every acknowledged write is appended to Typesense\u0027s raft log and fsynced before the answer: braft raft_sync=true with raft_sync_policy=0 (sync immediately), read from the running container\u0027s brpc /flags page on 2026-10-04. A restart replays the log from the last snapshot (typesense.compose.yaml sets TYPESENSE_SNAPSHOT_INTERVAL_SECONDS=300) and rebuilds the in-memory HNSW graph, so by those settings a crash loses no acknowledged write, but the restart is slow after a big load. This is read from the settings, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "nothing to build, Typesense inserts into the HNSW graph while writing; ready after 0.0 s: HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step", "searchSettings": "ef=100, ef_construction=128, k=top, m=16", "settled": true, "settleDetail": null, "meanMs": 4.101151896657781, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 8, "passOrder": [ "default@8", "default@1", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Every acknowledged write is appended to Typesense\u0027s raft log and fsynced before the answer: braft raft_sync=true with raft_sync_policy=0 (sync immediately), read from the running container\u0027s brpc /flags page on 2026-10-04. A restart replays the log from the last snapshot (typesense.compose.yaml sets TYPESENSE_SNAPSHOT_INTERVAL_SECONDS=300) and rebuilds the in-memory HNSW graph, so by those settings a crash loses no acknowledged write, but the restart is slow after a big load. This is read from the settings, not shown by pulling power. One node and no replicas, so a lost disk loses the data.", "loadIndexNote": "nothing to build, Typesense inserts into the HNSW graph while writing; ready after 0.0 s: HNSW graph returns 524 of 524 vectors, no writes queued: num_documents 524, pending_write_batches 0, health ok, unfiltered vector query with k above the count returned 524; the graph is built in memory during each write, so there is no later build step", "searchSettings": "ef=100, ef_construction=128, k=top, m=16", "settled": true, "settleDetail": null, "meanMs": 4.081650612244898, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "vespa", "engines": [ "Vespa 8.754.14 (tensor attribute \u002B HNSW)" ], "indexes": [ "HNSW float32 tensor, prenormalized-angular (cosine), max-links-per-node=16, neighbors-to-explore-at-insert=128; search targetHits=top, ef=100 via exploreAdditionalHits; exact mode = approximate:false; vectors held in memory" ], "hosting": [ "compose" ], "p50Ms": { "median": 1.8525, "min": 1.8337, "max": 1.8576, "n": 3, "perRun": [ 1.8525, 1.8576, 1.8337 ] }, "p95Ms": { "median": 2.2684, "min": 2.2625999999999995, "max": 2.3116299999999996, "n": 3, "perRun": [ 2.2684, 2.2625999999999995, 2.3116299999999996 ] }, "qps": { "1": { "median": 524.0402161691641, "min": 522.7094743344549, "max": 526.9605411946753, "n": 3, "perRun": [ 524.0402161691641, 522.7094743344549, 526.9605411946753 ] }, "8": { "median": 1559.9602607809638, "min": 1539.8231801628817, "max": 1595.2307888520734, "n": 3, "perRun": [ 1539.8231801628817, 1559.9602607809638, 1595.2307888520734 ] } }, "qpsRatio": { "8/1": { "median": 2.984373418460033, "min": 2.938368340161541, "max": 3.0272300564203847, "n": 3, "perRun": [ 2.938368340161541, 2.984373418460033, 3.0272300564203847 ] } }, "clientCpuMsPerSearch": { "1": { "median": 0.9355, "min": 0.9201, "max": 0.9414, "n": 3, "perRun": [ 0.9414, 0.9355, 0.9201 ] }, "8": { "median": 0.877, "min": 0.8625, "max": 0.8823, "n": 3, "perRun": [ 0.8823, 0.877, 0.8625 ] } }, "loadRowsPerSecond": { "median": 287.9680359876512, "min": 276.03814046379887, "max": 321.76672035585926, "n": 3, "perRun": [ 276.03814046379887, 287.9680359876512, 321.76672035585926 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 1.8444, "min": 1.8378, "max": 1.8481, "n": 3, "perRun": [ 1.8444, 1.8481, 1.8378 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 10, 10, 10 ], "median": 10 }, "p95Ms": { "perRun": [ 11, 11, 12 ], "median": 11 }, "qps@1": { "perRun": [ 10, 10, 10 ], "median": 10 }, "qps@8": { "perRun": [ 9, 9, 9 ], "median": 9 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 13, "passOrder": [ "default@8", "exact", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Vespa\u0027s transaction log server fsyncs after each commit: searchlib.translogserver usefsync=true, and an operation is searchable when it is acknowledged: proton documentdb visibilitydelay=0. Neither is overridden by this sink or by the services.xml it deploys; VespaReadinessTests reads both from the live config server. After a crash the node replays the transaction log over its last flushed data and rebuilds the in-memory HNSW graph. By those settings an acknowledged write survives a crash; this is read from the settings, not shown by pulling power. One node and min-redundancy 1, so a lost disk loses the data.", "loadIndexNote": "nothing to build, Vespa inserts into the HNSW graph while writing; ready after 0.0 s: HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step", "searchSettings": "ef=100, max-links-per-node=16, neighbors-to-explore-at-insert=128, targetHits=top", "settled": true, "settleDetail": null, "meanMs": 1.908250491365328, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 15, "passOrder": [ "default@8", "exact", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Vespa\u0027s transaction log server fsyncs after each commit: searchlib.translogserver usefsync=true, and an operation is searchable when it is acknowledged: proton documentdb visibilitydelay=0. Neither is overridden by this sink or by the services.xml it deploys; VespaReadinessTests reads both from the live config server. After a crash the node replays the transaction log over its last flushed data and rebuilds the in-memory HNSW graph. By those settings an acknowledged write survives a crash; this is read from the settings, not shown by pulling power. One node and min-redundancy 1, so a lost disk loses the data.", "loadIndexNote": "nothing to build, Vespa inserts into the HNSW graph while writing; ready after 0.1 s: HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step", "searchSettings": "ef=100, max-links-per-node=16, neighbors-to-explore-at-insert=128, targetHits=top", "settled": true, "settleDetail": null, "meanMs": 1.913108617886179, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 12, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "Vespa\u0027s transaction log server fsyncs after each commit: searchlib.translogserver usefsync=true, and an operation is searchable when it is acknowledged: proton documentdb visibilitydelay=0. Neither is overridden by this sink or by the services.xml it deploys; VespaReadinessTests reads both from the live config server. After a crash the node replays the transaction log over its last flushed data and rebuilds the in-memory HNSW graph. By those settings an acknowledged write survives a crash; this is read from the settings, not shown by pulling power. One node and min-redundancy 1, so a lost disk loses the data.", "loadIndexNote": "nothing to build, Vespa inserts into the HNSW graph while writing; ready after 0.1 s: HNSW graph returns 524 of 524 vectors: totalCount 524, coverage full True, nearestNeighbor approximate:true with targetHits above the count: has_index True, algorithm \u0022index top k\u0022, top_k_hits 524; the graph is updated while each write is applied, so there is no later build step", "searchSettings": "ef=100, max-links-per-node=16, neighbors-to-explore-at-insert=128, targetHits=top", "settled": true, "settleDetail": null, "meanMs": 1.8976752941176471, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "duckdb", "engines": [ "DuckDB 1.5.6 \u002B vss b833341 (HNSW, embedded, in process)" ], "indexes": [ "HNSW (vss extension) FLOAT[n] metric=cosine m=16 ef_construction=128, ef_search=100 per connection, persistent (hnsw_enable_experimental_persistence=true, checkpoint_threshold=256MB); exact mode = array_cosine_similarity sequential scan; score = 1 - cosine distance; searches run concurrently, one connection per searcher (opened as searchers arrive, at most 32), writes run one at a time and never overlap a search" ], "hosting": [ "embedded" ], "p50Ms": { "median": 3.52215, "min": 3.5138, "max": 3.5273, "n": 3, "perRun": [ 3.5273, 3.52215, 3.5138 ] }, "p95Ms": { "median": 3.9155599999999997, "min": 3.87377, "max": 3.9194899999999997, "n": 3, "perRun": [ 3.9155599999999997, 3.9194899999999997, 3.87377 ] }, "qps": { "1": { "median": 280.87496982706386, "min": 280.59227950342944, "max": 282.11811077934806, "n": 3, "perRun": [ 280.59227950342944, 280.87496982706386, 282.11811077934806 ] }, "8": { "median": 598.5010001834993, "min": 596.6415067084201, "max": 601.8683952397623, "n": 3, "perRun": [ 596.6415067084201, 598.5010001834993, 601.8683952397623 ] } }, "qpsRatio": { "8/1": { "median": 2.1308449113568195, "min": 2.1263646589432548, "max": 2.1333915556754146, "n": 3, "perRun": [ 2.1263646589432548, 2.1308449113568195, 2.1333915556754146 ] } }, "clientCpuMsPerSearch": { "1": { "median": 4.8378, "min": 4.8373, "max": 4.8429, "n": 3, "perRun": [ 4.8378, 4.8373, 4.8429 ] }, "8": { "median": 6.6378, "min": 6.6094, "max": 6.6647, "n": 3, "perRun": [ 6.6647, 6.6378, 6.6094 ] } }, "loadRowsPerSecond": { "median": 1559.1878416674504, "min": 1546.2171384596136, "max": 1574.2633183728246, "n": 3, "perRun": [ 1574.2633183728246, 1559.1878416674504, 1546.2171384596136 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 4.0234, "min": 3.9854, "max": 4.04435, "n": 3, "perRun": [ 4.04435, 3.9854, 4.0234 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 14, 14, 14 ], "median": 14 }, "p95Ms": { "perRun": [ 14, 14, 14 ], "median": 14 }, "qps@1": { "perRun": [ 14, 14, 14 ], "median": 14 }, "qps@8": { "perRun": [ 16, 16, 16 ], "median": 16 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 10, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "DuckDB\u0027s write-ahead log is fsynced at every commit and no DuckDbSink setting changes that (checkpoint_threshold=256MB only spaces out the checkpoints that write the database file; measured with strace: 42 WAL fsyncs for 40 single-record commits plus 2 setup statements), so an operating-system crash or power cut loses no committed row; measured with kill -9 right after the last commit and a reopen: all 50, 300 and 2,000 rows and an HNSW index that counted the same number were recovered from the log; not tested and documented by DuckDB: the HNSW index is file-backed only through hnsw_enable_experimental_persistence = true, which this sink turns on, and WAL recovery for such custom indexes is not complete, so a crash during a checkpoint or a later commit can damage the index while the rows survive", "loadIndexNote": "HNSW is maintained inside every transaction, nothing to build; confirmed in 0.01 s: duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True", "searchSettings": "checkpoint_threshold=256MB, ef_construction=128, ef_search=100, hnsw_enable_experimental_persistence=true, m=16, metric=cosine", "settled": true, "settleDetail": null, "meanMs": 3.563889932287955, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 4, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "DuckDB\u0027s write-ahead log is fsynced at every commit and no DuckDbSink setting changes that (checkpoint_threshold=256MB only spaces out the checkpoints that write the database file; measured with strace: 42 WAL fsyncs for 40 single-record commits plus 2 setup statements), so an operating-system crash or power cut loses no committed row; measured with kill -9 right after the last commit and a reopen: all 50, 300 and 2,000 rows and an HNSW index that counted the same number were recovered from the log; not tested and documented by DuckDB: the HNSW index is file-backed only through hnsw_enable_experimental_persistence = true, which this sink turns on, and WAL recovery for such custom indexes is not complete, so a crash during a checkpoint or a later commit can damage the index while the rows survive", "loadIndexNote": "HNSW is maintained inside every transaction, nothing to build; confirmed in 0.01 s: duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True", "searchSettings": "checkpoint_threshold=256MB, ef_construction=128, ef_search=100, hnsw_enable_experimental_persistence=true, m=16, metric=cosine", "settled": true, "settleDetail": null, "meanMs": 3.5603030081879674, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 3, "passOrder": [ "default@1", "default@8", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True" }, "afterSearch": { "ready": true, "indexedVectors": 524, "totalVectors": 524, "detail": "duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "DuckDB\u0027s write-ahead log is fsynced at every commit and no DuckDbSink setting changes that (checkpoint_threshold=256MB only spaces out the checkpoints that write the database file; measured with strace: 42 WAL fsyncs for 40 single-record commits plus 2 setup statements), so an operating-system crash or power cut loses no committed row; measured with kill -9 right after the last commit and a reopen: all 50, 300 and 2,000 rows and an HNSW index that counted the same number were recovered from the log; not tested and documented by DuckDB: the HNSW index is file-backed only through hnsw_enable_experimental_persistence = true, which this sink turns on, and WAL recovery for such custom indexes is not complete, so a crash during a checkpoint or a later commit can damage the index while the rows survive", "loadIndexNote": "HNSW is maintained inside every transaction, nothing to build; confirmed in 0.01 s: duckdb_indexes() lists gvb_gvbbench_eshoponweb_hnsw, pragma_hnsw_index_info() counts 524 vectors of 524 rows, EXPLAIN of the default search shows HNSW_INDEX_SCAN on it = True", "searchSettings": "checkpoint_threshold=256MB, ef_construction=128, ef_search=100, hnsw_enable_experimental_persistence=true, m=16, metric=cosine", "settled": true, "settleDetail": null, "meanMs": 3.544614690767322, "meanSource": "1000 / QPS@1" } ], "engineNotes": [] }, { "name": "sqlitevec", "engines": [ "SQLite 3.53.3 \u002B sqlite-vec 0.1.7-alpha.2.1 (vec0, embedded, in process)" ], "indexes": [ "vec0 brute-force scan, no ANN index (exact), float32, cosine distance, default chunk_size=1024; score = 1 - cosine distance; searches run concurrently, one WAL reader connection per searcher (opened as searchers arrive, at most 32), writes run one at a time and may overlap searches" ], "hosting": [ "embedded" ], "p50Ms": { "median": 1.8155, "min": 1.8124, "max": 1.8473, "n": 3, "perRun": [ 1.8155, 1.8473, 1.8124 ] }, "p95Ms": { "median": 2.04478, "min": 2.04057, "max": 2.1238599999999996, "n": 3, "perRun": [ 2.04478, 2.1238599999999996, 2.04057 ] }, "qps": { "1": { "median": 542.2305989891681, "min": 529.8183168646515, "max": 543.6880497366668, "n": 3, "perRun": [ 542.2305989891681, 529.8183168646515, 543.6880497366668 ] }, "8": { "median": 1158.9426593635267, "min": 1150.6684245226104, "max": 1161.4104333082294, "n": 3, "perRun": [ 1161.4104333082294, 1150.6684245226104, 1158.9426593635267 ] } }, "qpsRatio": { "8/1": { "median": 2.141912380956262, "min": 2.1316316588618345, "max": 2.1718169944218118, "n": 3, "perRun": [ 2.141912380956262, 2.1718169944218118, 2.1316316588618345 ] } }, "clientCpuMsPerSearch": { "1": { "median": 1.9319, "min": 1.9284, "max": 1.9737, "n": 3, "perRun": [ 1.9319, 1.9737, 1.9284 ] }, "8": { "median": 3.4462, "min": 3.4376, "max": 3.4655, "n": 3, "perRun": [ 3.4376, 3.4655, 3.4462 ] } }, "loadRowsPerSecond": { "median": 4690.133445037275, "min": 4041.264082725601, "max": 5045.539848691577, "n": 3, "perRun": [ 5045.539848691577, 4690.133445037275, 4041.264082725601 ] }, "recall": { "median": 1, "min": 1, "max": 1, "n": 3, "perRun": [ 1, 1, 1 ] }, "ndcg": { "median": 0.5181699774768911, "min": 0.5181699774768911, "max": 0.5181699774768911, "n": 3, "perRun": [ 0.5181699774768911, 0.5181699774768911, 0.5181699774768911 ] }, "exactP50Ms": { "median": 1.8324, "min": 1.8129, "max": 1.8499, "n": 3, "perRun": [ 1.8324, 1.8499, 1.8129 ] }, "errors": 0, "warmupErrors": 0, "ranks": { "p50Ms": { "perRun": [ 9, 9, 9 ], "median": 9 }, "p95Ms": { "perRun": [ 9, 9, 9 ], "median": 9 }, "qps@1": { "perRun": [ 9, 9, 9 ], "median": 9 }, "qps@8": { "perRun": [ 12, 12, 12 ], "median": 12 }, "recall": { "perRun": [ 1, 1, 1 ], "median": 1 }, "ndcg": { "perRun": [ 3, 3, 2 ], "median": 3 } }, "perRun": [ { "run": "20261005-073329-eshoponweb", "orderInRun": 6, "passOrder": [ "default@8", "default@1", "exact" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "PRAGMA journal_mode=WAL and PRAGMA synchronous=NORMAL (set by this sink when it opens the file): each commit is appended to the -wal file and handed to the operating system, but the WAL is fsynced only when SQLite checkpoints it (measured with strace: 40 single-record commits caused 3 WAL fsyncs), so a crash of the process loses nothing (measured with kill -9 and a reopen: all 50, 300 and 2,000 rows were there), while an operating-system crash or power cut can lose the newest commits since the last checkpoint and leaves the database consistent", "loadIndexNote": "nothing to build, every search scans all vectors; checked in 0.00 s: no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)", "searchSettings": "chunk_size=1024", "settled": true, "settleDetail": null, "meanMs": 1.8442338035961274, "meanSource": "1000 / QPS@1" }, { "run": "20261005-085448-eshoponweb", "orderInRun": 6, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "PRAGMA journal_mode=WAL and PRAGMA synchronous=NORMAL (set by this sink when it opens the file): each commit is appended to the -wal file and handed to the operating system, but the WAL is fsynced only when SQLite checkpoints it (measured with strace: 40 single-record commits caused 3 WAL fsyncs), so a crash of the process loses nothing (measured with kill -9 and a reopen: all 50, 300 and 2,000 rows were there), while an operating-system crash or power cut can lose the newest commits since the last checkpoint and leaves the database consistent", "loadIndexNote": "nothing to build, every search scans all vectors; checked in 0.00 s: no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)", "searchSettings": "chunk_size=1024", "settled": true, "settleDetail": null, "meanMs": 1.8874394639992451, "meanSource": "1000 / QPS@1" }, { "run": "20261005-101815-eshoponweb", "orderInRun": 13, "passOrder": [ "exact", "default@8", "default@1" ], "warmupErrors": 0, "errors": 0, "exactRecall": 1, "afterLoad": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)" }, "afterSearch": { "ready": true, "indexedVectors": null, "totalVectors": 524, "detail": "no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)" }, "segmentLayoutAfterLoad": null, "segmentLayoutAfterSearch": null, "durability": "PRAGMA journal_mode=WAL and PRAGMA synchronous=NORMAL (set by this sink when it opens the file): each commit is appended to the -wal file and handed to the operating system, but the WAL is fsynced only when SQLite checkpoints it (measured with strace: 40 single-record commits caused 3 WAL fsyncs), so a crash of the process loses nothing (measured with kill -9 and a reopen: all 50, 300 and 2,000 rows were there), while an operating-system crash or power cut can lose the newest commits since the last checkpoint and leaves the database consistent", "loadIndexNote": "nothing to build, every search scans all vectors; checked in 0.00 s: no index, exact scan by design: gvb_gvbbench_eshoponweb is a vec0 virtual table holding 524 vectors and every search compares all of them (EXPLAIN QUERY PLAN: SCAN gvb_gvbbench_eshoponweb VIRTUAL TABLE INDEX 0:3{___}___ | USE TEMP B-TREE FOR ORDER BY)", "searchSettings": "chunk_size=1024", "settled": true, "settleDetail": null, "meanMs": 1.839290012874747, "meanSource": "1000 / QPS@1" } ], "engineNotes": [ { "kind": "exact-by-design", "text": "Exact by design: every search compares the query with every stored vector, so its time grows with the collection and index quality plays no part in its speed. The results say: vec0 brute-force scan, no ANN index (exact), float32, cosine distance, default chunk_size=1024; score = 1 - cosine distance; searches run concurrently, one WAL reader connection per searcher (opened as searchers arrive, at most 32), writes run one at a time and may overlap searches.", "source": "results" } ] } ], "pairs": [], "exactVsDefault": [ { "target": "sql-diskann", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 3.6036, "exactP50Ms": 3.5483, "differenceMs": -0.05530000000000035, "ratio": 0.9846542346542345, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 3.56355, "exactP50Ms": 3.5627, "differenceMs": -0.0008500000000002395, "ratio": 0.9997614738112275, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 3.6159, "exactP50Ms": 3.577, "differenceMs": -0.038899999999999935, "ratio": 0.9892419591249758, "exactRecall": 1, "exactRanFirst": false } ], "ratio": { "median": 0.9892419591249758, "min": 0.9846542346542345, "max": 0.9997614738112275, "n": 3, "perRun": [ 0.9846542346542345, 0.9997614738112275, 0.9892419591249758 ] }, "differenceMs": { "median": -0.038899999999999935, "min": -0.05530000000000035, "max": -0.0008500000000002395, "n": 3, "perRun": [ -0.05530000000000035, -0.0008500000000002395, -0.038899999999999935 ] }, "runsExactSlower": 0, "declaredPair": null, "note": null }, { "target": "qdrant-hnsw", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 0.9042, "exactP50Ms": 0.8842, "differenceMs": -0.020000000000000018, "ratio": 0.9778809997788099, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 0.9202, "exactP50Ms": 0.8979, "differenceMs": -0.022299999999999986, "ratio": 0.9757661377961313, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 0.9199, "exactP50Ms": 0.8911, "differenceMs": -0.028800000000000048, "ratio": 0.9686922491575171, "exactRecall": 1, "exactRanFirst": true } ], "ratio": { "median": 0.9757661377961313, "min": 0.9686922491575171, "max": 0.9778809997788099, "n": 3, "perRun": [ 0.9778809997788099, 0.9757661377961313, 0.9686922491575171 ] }, "differenceMs": { "median": -0.022299999999999986, "min": -0.028800000000000048, "max": -0.020000000000000018, "n": 3, "perRun": [ -0.020000000000000018, -0.022299999999999986, -0.028800000000000048 ] }, "runsExactSlower": 0, "declaredPair": null, "note": null }, { "target": "pgvector", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 0.8493, "exactP50Ms": 2.1829, "differenceMs": 1.3336000000000001, "ratio": 2.5702343106087366, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 0.8165, "exactP50Ms": 2.1681, "differenceMs": 1.3516, "ratio": 2.65535823637477, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 0.8508, "exactP50Ms": 2.1735, "differenceMs": 1.3227000000000002, "ratio": 2.5546544428772924, "exactRecall": 1, "exactRanFirst": true } ], "ratio": { "median": 2.5702343106087366, "min": 2.5546544428772924, "max": 2.65535823637477, "n": 3, "perRun": [ 2.5702343106087366, 2.65535823637477, 2.5546544428772924 ] }, "differenceMs": { "median": 1.3336000000000001, "min": 1.3227000000000002, "max": 1.3516, "n": 3, "perRun": [ 1.3336000000000001, 1.3516, 1.3227000000000002 ] }, "runsExactSlower": 3, "declaredPair": null, "note": null }, { "target": "mariadb", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 0.625, "exactP50Ms": 1.5776, "differenceMs": 0.9525999999999999, "ratio": 2.5241599999999997, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 0.6219, "exactP50Ms": 1.5836, "differenceMs": 0.9616999999999999, "ratio": 2.546390094870558, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 0.6236, "exactP50Ms": 1.5793, "differenceMs": 0.9556999999999999, "ratio": 2.5325529185375237, "exactRecall": 1, "exactRanFirst": false } ], "ratio": { "median": 2.5325529185375237, "min": 2.5241599999999997, "max": 2.546390094870558, "n": 3, "perRun": [ 2.5241599999999997, 2.546390094870558, 2.5325529185375237 ] }, "differenceMs": { "median": 0.9556999999999999, "min": 0.9525999999999999, "max": 0.9616999999999999, "n": 3, "perRun": [ 0.9525999999999999, 0.9616999999999999, 0.9556999999999999 ] }, "runsExactSlower": 3, "declaredPair": null, "note": null }, { "target": "oracle", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 0.8841, "exactP50Ms": 2.3729, "differenceMs": 1.4888, "ratio": 2.6839724013120687, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 0.8963, "exactP50Ms": 2.3801, "differenceMs": 1.4838, "ratio": 2.655472498047529, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 0.8857, "exactP50Ms": 2.3597, "differenceMs": 1.4740000000000002, "ratio": 2.664220390651462, "exactRecall": 1, "exactRanFirst": true } ], "ratio": { "median": 2.664220390651462, "min": 2.655472498047529, "max": 2.6839724013120687, "n": 3, "perRun": [ 2.6839724013120687, 2.655472498047529, 2.664220390651462 ] }, "differenceMs": { "median": 1.4838, "min": 1.4740000000000002, "max": 1.4888, "n": 3, "perRun": [ 1.4888, 1.4838, 1.4740000000000002 ] }, "runsExactSlower": 3, "declaredPair": null, "note": null }, { "target": "redis", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 0.3582, "exactP50Ms": 0.3503, "differenceMs": -0.007900000000000018, "ratio": 0.9779452819653824, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 0.3725, "exactP50Ms": 0.3495, "differenceMs": -0.02300000000000002, "ratio": 0.938255033557047, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 0.3744, "exactP50Ms": 0.3582, "differenceMs": -0.016199999999999992, "ratio": 0.9567307692307693, "exactRecall": 1, "exactRanFirst": true } ], "ratio": { "median": 0.9567307692307693, "min": 0.938255033557047, "max": 0.9779452819653824, "n": 3, "perRun": [ 0.9779452819653824, 0.938255033557047, 0.9567307692307693 ] }, "differenceMs": { "median": -0.016199999999999992, "min": -0.02300000000000002, "max": -0.007900000000000018, "n": 3, "perRun": [ -0.007900000000000018, -0.02300000000000002, -0.016199999999999992 ] }, "runsExactSlower": 0, "declaredPair": null, "note": null }, { "target": "mongodb", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 1.389, "exactP50Ms": 1.2168, "differenceMs": -0.1721999999999999, "ratio": 0.8760259179265659, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 1.4123, "exactP50Ms": 1.2476, "differenceMs": -0.16470000000000007, "ratio": 0.8833817177653472, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 1.394, "exactP50Ms": 1.2278, "differenceMs": -0.1661999999999999, "ratio": 0.8807747489239599, "exactRecall": 1, "exactRanFirst": false } ], "ratio": { "median": 0.8807747489239599, "min": 0.8760259179265659, "max": 0.8833817177653472, "n": 3, "perRun": [ 0.8760259179265659, 0.8833817177653472, 0.8807747489239599 ] }, "differenceMs": { "median": -0.1661999999999999, "min": -0.1721999999999999, "max": -0.16470000000000007, "n": 3, "perRun": [ -0.1721999999999999, -0.16470000000000007, -0.1661999999999999 ] }, "runsExactSlower": 0, "declaredPair": null, "note": null }, { "target": "clickhouse", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 4.5007, "exactP50Ms": 6.0876, "differenceMs": 1.5869, "ratio": 1.352589597173773, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 4.50825, "exactP50Ms": 6.1026, "differenceMs": 1.5943499999999995, "ratio": 1.353651638662452, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 4.4976, "exactP50Ms": 6.0564, "differenceMs": 1.5587999999999997, "ratio": 1.3465848452508002, "exactRecall": 1, "exactRanFirst": false } ], "ratio": { "median": 1.352589597173773, "min": 1.3465848452508002, "max": 1.353651638662452, "n": 3, "perRun": [ 1.352589597173773, 1.353651638662452, 1.3465848452508002 ] }, "differenceMs": { "median": 1.5869, "min": 1.5587999999999997, "max": 1.5943499999999995, "n": 3, "perRun": [ 1.5869, 1.5943499999999995, 1.5587999999999997 ] }, "runsExactSlower": 3, "declaredPair": null, "note": null }, { "target": "elasticsearch", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 1.3044, "exactP50Ms": 1.2496, "differenceMs": -0.05479999999999996, "ratio": 0.9579883471327814, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 1.3092, "exactP50Ms": 1.2517, "differenceMs": -0.057499999999999885, "ratio": 0.9560800488848152, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 1.3123, "exactP50Ms": 1.2551, "differenceMs": -0.05719999999999992, "ratio": 0.9564124056999163, "exactRecall": 1, "exactRanFirst": true } ], "ratio": { "median": 0.9564124056999163, "min": 0.9560800488848152, "max": 0.9579883471327814, "n": 3, "perRun": [ 0.9579883471327814, 0.9560800488848152, 0.9564124056999163 ] }, "differenceMs": { "median": -0.05719999999999992, "min": -0.057499999999999885, "max": -0.05479999999999996, "n": 3, "perRun": [ -0.05479999999999996, -0.057499999999999885, -0.05719999999999992 ] }, "runsExactSlower": 0, "declaredPair": null, "note": null }, { "target": "opensearch", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 1.9906, "exactP50Ms": 2.5755, "differenceMs": 0.5849, "ratio": 1.2938310057269164, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 1.99795, "exactP50Ms": 2.5611, "differenceMs": 0.5631500000000003, "ratio": 1.2818639105082712, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 1.9638, "exactP50Ms": 2.6346499999999997, "differenceMs": 0.6708499999999997, "ratio": 1.3416081067318464, "exactRecall": 1, "exactRanFirst": true } ], "ratio": { "median": 1.2938310057269164, "min": 1.2818639105082712, "max": 1.3416081067318464, "n": 3, "perRun": [ 1.2938310057269164, 1.2818639105082712, 1.3416081067318464 ] }, "differenceMs": { "median": 0.5849, "min": 0.5631500000000003, "max": 0.6708499999999997, "n": 3, "perRun": [ 0.5849, 0.5631500000000003, 0.6708499999999997 ] }, "runsExactSlower": 3, "declaredPair": null, "note": null }, { "target": "typesense", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 4.0362, "exactP50Ms": 5.4602, "differenceMs": 1.4240000000000004, "ratio": 1.3528070957831624, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 4.0444, "exactP50Ms": 5.4844, "differenceMs": 1.4399999999999995, "ratio": 1.3560478686578972, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 4.02795, "exactP50Ms": 5.4602, "differenceMs": 1.4322500000000007, "ratio": 1.3555778994277488, "exactRecall": 1, "exactRanFirst": false } ], "ratio": { "median": 1.3555778994277488, "min": 1.3528070957831624, "max": 1.3560478686578972, "n": 3, "perRun": [ 1.3528070957831624, 1.3560478686578972, 1.3555778994277488 ] }, "differenceMs": { "median": 1.4322500000000007, "min": 1.4240000000000004, "max": 1.4399999999999995, "n": 3, "perRun": [ 1.4240000000000004, 1.4399999999999995, 1.4322500000000007 ] }, "runsExactSlower": 3, "declaredPair": null, "note": null }, { "target": "vespa", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 1.8525, "exactP50Ms": 1.8444, "differenceMs": -0.008099999999999996, "ratio": 0.9956275303643725, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 1.8576, "exactP50Ms": 1.8481, "differenceMs": -0.009499999999999842, "ratio": 0.9948858742463395, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 1.8337, "exactP50Ms": 1.8378, "differenceMs": 0.0040999999999999925, "ratio": 1.002235916453073, "exactRecall": 1, "exactRanFirst": true } ], "ratio": { "median": 0.9956275303643725, "min": 0.9948858742463395, "max": 1.002235916453073, "n": 3, "perRun": [ 0.9956275303643725, 0.9948858742463395, 1.002235916453073 ] }, "differenceMs": { "median": -0.008099999999999996, "min": -0.009499999999999842, "max": 0.0040999999999999925, "n": 3, "perRun": [ -0.008099999999999996, -0.009499999999999842, 0.0040999999999999925 ] }, "runsExactSlower": 1, "declaredPair": null, "note": null }, { "target": "duckdb", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 3.5273, "exactP50Ms": 4.04435, "differenceMs": 0.5170499999999998, "ratio": 1.1465852068154112, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 3.52215, "exactP50Ms": 3.9854, "differenceMs": 0.46324999999999994, "ratio": 1.131524778899252, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 3.5138, "exactP50Ms": 4.0234, "differenceMs": 0.5095999999999998, "ratio": 1.1450281746257613, "exactRecall": 1, "exactRanFirst": false } ], "ratio": { "median": 1.1450281746257613, "min": 1.131524778899252, "max": 1.1465852068154112, "n": 3, "perRun": [ 1.1465852068154112, 1.131524778899252, 1.1450281746257613 ] }, "differenceMs": { "median": 0.5095999999999998, "min": 0.46324999999999994, "max": 0.5170499999999998, "n": 3, "perRun": [ 0.5170499999999998, 0.46324999999999994, 0.5095999999999998 ] }, "runsExactSlower": 3, "declaredPair": null, "note": null }, { "target": "sqlitevec", "runs": [ { "run": "20261005-073329-eshoponweb", "defaultP50Ms": 1.8155, "exactP50Ms": 1.8324, "differenceMs": 0.016900000000000137, "ratio": 1.0093087303773065, "exactRecall": 1, "exactRanFirst": false }, { "run": "20261005-085448-eshoponweb", "defaultP50Ms": 1.8473, "exactP50Ms": 1.8499, "differenceMs": 0.0026000000000001577, "ratio": 1.0014074595355384, "exactRecall": 1, "exactRanFirst": true }, { "run": "20261005-101815-eshoponweb", "defaultP50Ms": 1.8124, "exactP50Ms": 1.8129, "differenceMs": 0.0004999999999999449, "ratio": 1.0002758772897815, "exactRecall": 1, "exactRanFirst": true } ], "ratio": { "median": 1.0014074595355384, "min": 1.0002758772897815, "max": 1.0093087303773065, "n": 3, "perRun": [ 1.0093087303773065, 1.0014074595355384, 1.0002758772897815 ] }, "differenceMs": { "median": 0.0026000000000001577, "min": 0.0004999999999999449, "max": 0.016900000000000137, "n": 3, "perRun": [ 0.016900000000000137, 0.0026000000000001577, 0.0004999999999999449 ] }, "runsExactSlower": 3, "declaredPair": null, "note": null } ], "flags": [ { "target": "oracle", "kind": "unsettled-target", "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ], "detail": "not settled: latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 6,134 searches (0 failed) at 1 searcher; p50 of the last windows 2.383, 2.353, 2.360 ms (windows of at least 2 s and 100 searches); trial of 1,220 searches in 3.0 s at 1 searcher: p50 2.382 ms against the settled p50 2.360 ms, 1% apart (limit 10%); default@8: warm-up 15.0 s and 43,925 searches (0 failed) at 8 searchers; QPS of the last windows 2,416, 3,480, 2,581 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 9,374 searches in 3.0 s at 8 searchers: 3,124 QPS against the settled 2,581 QPS, 17% apart (limit 10%), so the warm-up was EXTENDED once (120.3 s and 343,792 searches (0 failed) at 8 searchers; QPS of the last windows 3,449, 2,488, 3,050 (windows of at least 2 s and 100 searches), the 120 s cap ran out); second trial of 9,654 searches in 3.0 s at 8 searchers: 3,217 QPS against the settled 3,050 QPS, 5% apart (limit 10%); still disagreeing after the extension; default@1: warm-up 15.0 s and 16,210 searches (0 failed) at 1 searcher; p50 of the last windows 0.894, 0.885, 0.895 ms (windows of at least 2 s and 100 searches); trial of 3,375 searches in 3.0 s at 1 searcher: p50 0.867 ms against the settled p50 0.894 ms, 3% apart (limit 10%). Its numbers may still include warm-up; rerun before quoting them. (1 run); not settled: latency had NOT settled when timing began for default@1, default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 5,949 searches (0 failed) at 1 searcher; p50 of the last windows 2.363, 2.376, 2.356 ms (windows of at least 2 s and 100 searches); trial of 1,229 searches in 3.0 s at 1 searcher: p50 2.387 ms against the settled p50 2.363 ms, 1% apart (limit 10%); default@1: warm-up 15.0 s and 16,397 searches (0 failed) at 1 searcher; p50 of the last windows 0.860, 0.892, 0.913 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 3,246 searches in 3.0 s at 1 searcher: p50 0.900 ms against the settled p50 0.892 ms, 1% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 32,617 searches (0 failed) at 1 searcher; p50 of the last windows 0.883, 0.857, 0.926 ms (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 3,341 searches in 3.0 s at 1 searcher: p50 0.874 ms against the settled p50 0.883 ms, 1% apart (limit 10%); still disagreeing after the extension; default@8: warm-up 15.0 s and 48,930 searches (0 failed) at 8 searchers; QPS of the last windows 3,514, 2,364, 3,100 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 7,829 searches in 3.1 s at 8 searchers: 2,519 QPS against the settled 3,100 QPS, 23% apart (limit 10%), so the warm-up was EXTENDED once (120.0 s and 315,148 searches (0 failed) at 8 searchers; QPS of the last windows 2,207, 2,452, 2,930 (windows of at least 2 s and 100 searches), the 120 s cap ran out); second trial of 7,269 searches in 3.3 s at 8 searchers: 2,230 QPS against the settled 2,452 QPS, 10% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them. (1 run); not settled: latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 6,064 searches (0 failed) at 1 searcher; p50 of the last windows 2.371, 2.350, 2.331 ms (windows of at least 2 s and 100 searches); trial of 1,244 searches in 3.0 s at 1 searcher: p50 2.356 ms against the settled p50 2.350 ms, 0% apart (limit 10%); default@1: warm-up 15.0 s and 16,495 searches (0 failed) at 1 searcher; p50 of the last windows 0.888, 0.870, 0.904 ms (windows of at least 2 s and 100 searches); trial of 3,251 searches in 3.0 s at 1 searcher: p50 0.896 ms against the settled p50 0.888 ms, 1% apart (limit 10%); default@8: warm-up 15.3 s and 44,258 searches (0 failed) at 8 searchers; QPS of the last windows 3,062, 2,834, 2,622 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 9,357 searches in 3.0 s at 8 searchers: 3,118 QPS against the settled 2,834 QPS, 9% apart (limit 10%), so the warm-up was EXTENDED once (120.2 s and 344,269 searches (0 failed) at 8 searchers; QPS of the last windows 3,394, 2,474, 2,998 (windows of at least 2 s and 100 searches), the 120 s cap ran out); second trial of 9,022 searches in 3.0 s at 8 searchers: 3,006 QPS against the settled 2,998 QPS, 0% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them. (1 run)" }, { "target": "redis", "kind": "unsettled-target", "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ], "detail": "not settled: latency had NOT settled when timing began for exact (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). default@1: warm-up 15.0 s and 39,275 searches (0 failed) at 1 searcher; p50 of the last windows 0.383, 0.380, 0.382 ms (windows of at least 2 s and 100 searches); trial of 8,168 searches in 3.0 s at 1 searcher: p50 0.349 ms against the settled p50 0.382 ms, 10% apart (limit 10%); default@8: warm-up 15.0 s and 76,531 searches (0 failed) at 8 searchers; QPS of the last windows 5,217, 5,302, 5,056 (windows of at least 2 s and 100 searches); trial of 15,028 searches in 3.0 s at 8 searchers: 5,007 QPS against the settled 5,217 QPS, 4% apart (limit 10%); exact: warm-up 15.0 s and 41,741 searches (0 failed) at 1 searcher; p50 of the last windows 0.364, 0.362, 0.324 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 7,959 searches in 3.0 s at 1 searcher: p50 0.380 ms against the settled p50 0.362 ms, 5% apart (limit 10%), so the warm-up was EXTENDED once (36.0 s and 99,849 searches (0 failed) at 1 searcher; p50 of the last windows 0.362, 0.353, 0.342 ms (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 8,351 searches in 3.0 s at 1 searcher: p50 0.359 ms against the settled p50 0.353 ms, 2% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them. (1 run); not settled: latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). default@1: warm-up 15.0 s and 39,692 searches (0 failed) at 1 searcher; p50 of the last windows 0.352, 0.350, 0.369 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 8,035 searches in 3.0 s at 1 searcher: p50 0.355 ms against the settled p50 0.352 ms, 1% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 77,908 searches (0 failed) at 1 searcher; p50 of the last windows 0.379, 0.384, 0.376 ms (windows of at least 2 s and 100 searches)); second trial of 7,691 searches in 3.0 s at 1 searcher: p50 0.388 ms against the settled p50 0.379 ms, 3% apart (limit 10%); settled after the extension; exact: warm-up 15.0 s and 41,325 searches (0 failed) at 1 searcher; p50 of the last windows 0.367, 0.376, 0.359 ms (windows of at least 2 s and 100 searches); trial of 8,543 searches in 3.0 s at 1 searcher: p50 0.342 ms against the settled p50 0.367 ms, 7% apart (limit 10%); default@8: warm-up 15.0 s and 75,695 searches (0 failed) at 8 searchers; QPS of the last windows 4,921, 5,016, 5,277 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 15,098 searches in 3.0 s at 8 searchers: 5,030 QPS against the settled 5,016 QPS, 0% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 153,806 searches (0 failed) at 8 searchers; QPS of the last windows 5,068, 5,484, 4,940 (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 15,130 searches in 3.0 s at 8 searchers: 5,041 QPS against the settled 5,068 QPS, 1% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them. (1 run); not settled: latency had NOT settled when timing began for exact (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 41,314 searches (0 failed) at 1 searcher; p50 of the last windows 0.336, 0.374, 0.372 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 8,416 searches in 3.0 s at 1 searcher: p50 0.335 ms against the settled p50 0.372 ms, 11% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 81,883 searches (0 failed) at 1 searcher; p50 of the last windows 0.363, 0.369, 0.338 ms (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 8,081 searches in 3.0 s at 1 searcher: p50 0.369 ms against the settled p50 0.363 ms, 2% apart (limit 10%); still disagreeing after the extension; default@1: warm-up 15.0 s and 38,516 searches (0 failed) at 1 searcher; p50 of the last windows 0.372, 0.407, 0.378 ms (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 7,842 searches in 3.0 s at 1 searcher: p50 0.377 ms against the settled p50 0.378 ms, 0% apart (limit 10%), so the warm-up was EXTENDED once (32.0 s and 82,511 searches (0 failed) at 1 searcher; p50 of the last windows 0.385, 0.380, 0.391 ms (windows of at least 2 s and 100 searches)); second trial of 7,681 searches in 3.0 s at 1 searcher: p50 0.390 ms against the settled p50 0.385 ms, 1% apart (limit 10%); settled after the extension; default@8: warm-up 15.0 s and 75,402 searches (0 failed) at 8 searchers; QPS of the last windows 4,844, 5,008, 5,010 (windows of at least 2 s and 100 searches); trial of 14,794 searches in 3.0 s at 8 searchers: 4,929 QPS against the settled 5,008 QPS, 2% apart (limit 10%). Its numbers may still include warm-up; rerun before quoting them. (1 run)" }, { "target": "mongodb", "kind": "segment-layout-differs-between-runs", "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ], "detail": "after the load: 2 segments (20261005-073329-eshoponweb, 20261005-101815-eshoponweb); 4 segments (20261005-085448-eshoponweb); after the searches: 2 segments (20261005-073329-eshoponweb, 20261005-101815-eshoponweb); 4 segments (20261005-085448-eshoponweb). Searches ran over different layouts, so the medians mix them." }, { "target": "clickhouse", "kind": "unsettled-target", "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ], "detail": "not settled: latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). exact: warm-up 15.0 s and 2,395 searches (0 failed) at 1 searcher; p50 of the last windows 6.058, 6.063, 6.057 ms (windows of at least 2 s and 100 searches); trial of 488 searches in 3.0 s at 1 searcher: p50 6.032 ms against the settled p50 6.058 ms, 0% apart (limit 10%); default@1: warm-up 15.0 s and 3,279 searches (0 failed) at 1 searcher; p50 of the last windows 4.472, 4.478, 4.471 ms (windows of at least 2 s and 100 searches); trial of 654 searches in 3.0 s at 1 searcher: p50 4.486 ms against the settled p50 4.472 ms, 0% apart (limit 10%); default@8: warm-up 15.0 s and 7,362 searches (0 failed) at 8 searchers; QPS of the last windows 434, 504, 522 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,512 searches in 3.0 s at 8 searchers: 502 QPS against the settled 504 QPS, 0% apart (limit 10%), so the warm-up was EXTENDED once (38.1 s and 18,460 searches (0 failed) at 8 searchers; QPS of the last windows 525, 524, 488 (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 1,302 searches in 3.0 s at 8 searchers: 432 QPS against the settled 524 QPS, 21% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them. (1 run); not settled: latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). default@8: warm-up 15.0 s and 7,401 searches (0 failed) at 8 searchers; QPS of the last windows 458, 442, 444 (windows of at least 2 s and 100 searches); trial of 1,581 searches in 3.0 s at 8 searchers: 525 QPS against the settled 444 QPS, 16% apart (limit 10%), so the warm-up was EXTENDED once (30.0 s and 14,706 searches (0 failed) at 8 searchers; QPS of the last windows 530, 525, 452 (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 1,311 searches in 3.0 s at 8 searchers: 435 QPS against the settled 525 QPS, 21% apart (limit 10%); still disagreeing after the extension; exact: warm-up 15.0 s and 2,370 searches (0 failed) at 1 searcher; p50 of the last windows 6.089, 6.058, 6.067 ms (windows of at least 2 s and 100 searches); trial of 487 searches in 3.0 s at 1 searcher: p50 6.077 ms against the settled p50 6.067 ms, 0% apart (limit 10%); default@1: warm-up 15.0 s and 3,267 searches (0 failed) at 1 searcher; p50 of the last windows 4.489, 4.504, 4.493 ms (windows of at least 2 s and 100 searches); trial of 657 searches in 3.0 s at 1 searcher: p50 4.468 ms against the settled p50 4.493 ms, 1% apart (limit 10%). Its numbers may still include warm-up; rerun before quoting them. (1 run); not settled: latency had NOT settled when timing began for default@8 (each by its own warm-up at its own concurrency and a trial of the same pass within 10% of the warm-up\u0027s settled figure). default@1: warm-up 15.0 s and 3,232 searches (0 failed) at 1 searcher; p50 of the last windows 4.436, 4.452, 4.476 ms (windows of at least 2 s and 100 searches); trial of 655 searches in 3.0 s at 1 searcher: p50 4.479 ms against the settled p50 4.452 ms, 1% apart (limit 10%); exact: warm-up 15.0 s and 2,442 searches (0 failed) at 1 searcher; p50 of the last windows 6.029, 6.056, 6.035 ms (windows of at least 2 s and 100 searches); trial of 457 searches in 3.0 s at 1 searcher: p50 6.120 ms against the settled p50 6.035 ms, 1% apart (limit 10%); default@8: warm-up 15.0 s and 7,381 searches (0 failed) at 8 searchers; QPS of the last windows 435, 510, 524 (windows of at least 2 s and 100 searches) (its last 3 windows differed by more than 5%); trial of 1,509 searches in 3.0 s at 8 searchers: 501 QPS against the settled 510 QPS, 2% apart (limit 10%), so the warm-up was EXTENDED once (32.0 s and 15,463 searches (0 failed) at 8 searchers; QPS of the last windows 440, 445, 524 (windows of at least 2 s and 100 searches), its last 3 windows differed by more than 5%); second trial of 1,583 searches in 3.0 s at 8 searchers: 526 QPS against the settled 445 QPS, 15% apart (limit 10%); still disagreeing after the extension. Its numbers may still include warm-up; rerun before quoting them. (1 run)" }, { "target": "elasticsearch", "kind": "index-not-ready-after-load", "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ], "detail": "ready false, 0 of 524: NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search" }, { "target": "elasticsearch", "kind": "index-not-ready-after-search", "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ], "detail": "ready false, 0 of 524: NO HNSW GRAPH, searches scan all 524 vectors: 1 segment(s), 524 vectors, total_vex_size_bytes 0 (_stats dense_vector), index_options hnsw m=16 ef_construction=128; Lucene builds no graph for a segment this small (measured at 1,024 dimensions: 1,042 vectors none, 1,043 vectors a graph), so default search equals exact search" }, { "target": "duckdb", "kind": "client-engine-share-cores", "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ], "detail": "embedded engine runs inside the client process on the client CPUs 0-1,4-5" }, { "target": "sqlitevec", "kind": "client-engine-share-cores", "runs": [ "20261005-073329-eshoponweb", "20261005-085448-eshoponweb", "20261005-101815-eshoponweb" ], "detail": "embedded engine runs inside the client process on the client CPUs 0-1,4-5" } ], "notes": [ "Load rows/s is reported but not ranked and does not feed any comparison: each load wrote 524 rows, which is too few to separate engines from connection set-up and first-call cost.", "Runs were used together only when their build configuration, CPU governor, CPU partition, warm-up count, exact-mode seconds, seconds per level and every listed target\u0027s search settings matched; runs that differed are listed under Runs dropped. Spread is flagged when the largest value is more than 1.15 times the smallest across runs.", "Engines in different bands never overlap: every run of an engine in a faster band beat every run of an engine in a slower band, and the medians on either side of a band boundary are at least 3% apart. Engines in one band are linked by overlapping slowest-to-fastest ranges or by neighboring medians less than 3% apart (an engine varies about 2% from run to run), so these runs do not separate them cleanly. Inside a band they are listed by median, and that order is not a ranking. Bands are drawn only from two runs or more.", "Runs of different commands (run-all against bench) are refused, never merged, and nothing is written. A target whose engine hosting differs between the runs (container against native) is refused for that target alone, never merged: it has no row and is named under \u0022Targets not in this report\u0022 with the reason. The largest-group rule does not apply to either.", "A segment layout that differs between the runs (the engine\u0027s own count after the load or after the searches) does not refuse or drop anything: the target stays in every table and carries the segment-layout-differs-between-runs flag, which names each layout and the runs that had it.", "Every target the runs hold that has no row in this report (left out of --targets, or refused for a different hosting) is named under \u0022Targets not in this report\u0022 with its reason (0 in this report), so no measured engine is left out unsaid." ], "bands": [ { "metric": "p50Ms", "title": "p50 latency, one search at a time (lower is faster)", "higherIsBetter": false, "runs": 3, "banded": true, "entries": [ { "target": "redis", "band": 1, "orderInBand": 1, "median": 0.3725, "min": 0.3582, "max": 0.3744, "n": 3 }, { "target": "mariadb", "band": 2, "orderInBand": 1, "median": 0.6236, "min": 0.6219, "max": 0.625, "n": 3 }, { "target": "pgvector", "band": 3, "orderInBand": 1, "median": 0.8493, "min": 0.8165, "max": 0.8508, "n": 3 }, { "target": "qdrant", "band": 4, "orderInBand": 1, "median": 0.8786, "min": 0.8765, "max": 0.8794, "n": 3 }, { "target": "oracle", "band": 4, "orderInBand": 2, "median": 0.8857, "min": 0.8841, "max": 0.8963, "n": 3 }, { "target": "qdrant-hnsw", "band": 5, "orderInBand": 1, "median": 0.9199, "min": 0.9042, "max": 0.9202, "n": 3 }, { "target": "elasticsearch", "band": 6, "orderInBand": 1, "median": 1.3092, "min": 1.3044, "max": 1.3123, "n": 3 }, { "target": "mongodb", "band": 7, "orderInBand": 1, "median": 1.394, "min": 1.389, "max": 1.4123, "n": 3 }, { "target": "sqlitevec", "band": 8, "orderInBand": 1, "median": 1.8155, "min": 1.8124, "max": 1.8473, "n": 3 }, { "target": "vespa", "band": 8, "orderInBand": 2, "median": 1.8525, "min": 1.8337, "max": 1.8576, "n": 3 }, { "target": "opensearch", "band": 9, "orderInBand": 1, "median": 1.9906, "min": 1.9638, "max": 1.99795, "n": 3 }, { "target": "chroma", "band": 9, "orderInBand": 2, "median": 2.0266, "min": 2.0172, "max": 2.0317499999999997, "n": 3 }, { "target": "milvus", "band": 10, "orderInBand": 1, "median": 2.2078, "min": 2.1592, "max": 2.2176, "n": 3 }, { "target": "duckdb", "band": 11, "orderInBand": 1, "median": 3.52215, "min": 3.5138, "max": 3.5273, "n": 3 }, { "target": "sql-diskann", "band": 11, "orderInBand": 2, "median": 3.6036, "min": 3.56355, "max": 3.6159, "n": 3 }, { "target": "sql", "band": 12, "orderInBand": 1, "median": 3.89, "min": 3.8233, "max": 3.8929, "n": 3 }, { "target": "typesense", "band": 13, "orderInBand": 1, "median": 4.0362, "min": 4.02795, "max": 4.0444, "n": 3 }, { "target": "clickhouse", "band": 14, "orderInBand": 1, "median": 4.5007, "min": 4.4976, "max": 4.50825, "n": 3 }, { "target": "weaviate", "band": 15, "orderInBand": 1, "median": 5.1855, "min": 5.1845, "max": 5.1918, "n": 3 } ] }, { "metric": "qps@8", "title": "QPS with 8 searchers at once (higher is faster)", "higherIsBetter": true, "runs": 3, "banded": true, "entries": [ { "target": "mariadb", "band": 1, "orderInBand": 1, "median": 5940.1061883415, "min": 5930.118349029013, "max": 5944.1738027383435, "n": 3 }, { "target": "redis", "band": 2, "orderInBand": 1, "median": 5074.549465810664, "min": 5039.114772889733, "max": 5132.6717990784855, "n": 3 }, { "target": "pgvector", "band": 3, "orderInBand": 1, "median": 4049.9862214772998, "min": 4007.5671515102313, "max": 4142.489789207402, "n": 3 }, { "target": "qdrant-hnsw", "band": 4, "orderInBand": 1, "median": 3465.8944897862284, "min": 3453.7536264850546, "max": 3473.1995748210193, "n": 3 }, { "target": "qdrant", "band": 5, "orderInBand": 1, "median": 3295.513777572424, "min": 3291.3846853471437, "max": 3303.0483656232514, "n": 3 }, { "target": "oracle", "band": 6, "orderInBand": 1, "median": 2879.2116939366606, "min": 2826.271911110787, "max": 2896.5483183004194, "n": 3 }, { "target": "elasticsearch", "band": 7, "orderInBand": 1, "median": 2706.6457647633156, "min": 2697.4916862045325, "max": 2713.207923167071, "n": 3 }, { "target": "mongodb", "band": 8, "orderInBand": 1, "median": 2088.8625589929193, "min": 2088.0560852254434, "max": 2102.412694793193, "n": 3 }, { "target": "vespa", "band": 9, "orderInBand": 1, "median": 1559.9602607809638, "min": 1539.8231801628817, "max": 1595.2307888520734, "n": 3 }, { "target": "opensearch", "band": 9, "orderInBand": 2, "median": 1528.1878981855536, "min": 1511.8597635195188, "max": 1539.3690651456109, "n": 3 }, { "target": "milvus", "band": 10, "orderInBand": 1, "median": 1285.0924096519475, "min": 1281.516014398511, "max": 1290.5351276847628, "n": 3 }, { "target": "sqlitevec", "band": 11, "orderInBand": 1, "median": 1158.9426593635267, "min": 1150.6684245226104, "max": 1161.4104333082294, "n": 3 }, { "target": "chroma", "band": 12, "orderInBand": 1, "median": 806.8877586440376, "min": 804.399032658061, "max": 807.5345043444976, "n": 3 }, { "target": "sql-diskann", "band": 13, "orderInBand": 1, "median": 690.4279825257992, "min": 690.3636272262703, "max": 690.7239886382225, "n": 3 }, { "target": "sql", "band": 14, "orderInBand": 1, "median": 613.8945213079639, "min": 612.512633572833, "max": 621.4294884199052, "n": 3 }, { "target": "duckdb", "band": 14, "orderInBand": 2, "median": 598.5010001834993, "min": 596.6415067084201, "max": 601.8683952397623, "n": 3 }, { "target": "typesense", "band": 14, "orderInBand": 3, "median": 591.3480901892943, "min": 590.8742796795126, "max": 593.6745929047803, "n": 3 }, { "target": "clickhouse", "band": 15, "orderInBand": 1, "median": 492.78898362481993, "min": 479.84989575485946, "max": 495.78019393955714, "n": 3 }, { "target": "weaviate", "band": 16, "orderInBand": 1, "median": 312.14389064236065, "min": 312.13130394757167, "max": 312.171953705239, "n": 3 } ] }, { "metric": "qps@1", "title": "QPS with one searcher (higher is faster)", "higherIsBetter": true, "runs": 3, "banded": true, "entries": [ { "target": "redis", "band": 1, "orderInBand": 1, "median": 2620.4265995904652, "min": 2594.784937273439, "max": 2653.8854965157616, "n": 3 }, { "target": "mariadb", "band": 2, "orderInBand": 1, "median": 1573.403230588971, "min": 1569.4076887687108, "max": 1581.5353075369928, "n": 3 }, { "target": "pgvector", "band": 3, "orderInBand": 1, "median": 1153.3591710853436, "min": 1150.8681669865011, "max": 1200.4241008500242, "n": 3 }, { "target": "qdrant", "band": 3, "orderInBand": 2, "median": 1122.7265630829957, "min": 1122.0466731316142, "max": 1125.4144425306883, "n": 3 }, { "target": "oracle", "band": 3, "orderInBand": 3, "median": 1096.7649802941794, "min": 1089.134321911436, "max": 1103.8255668210784, "n": 3 }, { "target": "qdrant-hnsw", "band": 3, "orderInBand": 4, "median": 1074.8272888993856, "min": 1073.0969363082468, "max": 1092.976987369531, "n": 3 }, { "target": "elasticsearch", "band": 4, "orderInBand": 1, "median": 750.9968157735011, "min": 750.5668699783591, "max": 754.4534351339835, "n": 3 }, { "target": "mongodb", "band": 5, "orderInBand": 1, "median": 703.9109540593784, "min": 693.6669606426645, "max": 704.9536880674624, "n": 3 }, { "target": "sqlitevec", "band": 6, "orderInBand": 1, "median": 542.2305989891681, "min": 529.8183168646515, "max": 543.6880497366668, "n": 3 }, { "target": "vespa", "band": 7, "orderInBand": 1, "median": 524.0402161691641, "min": 522.7094743344549, "max": 526.9605411946753, "n": 3 }, { "target": "opensearch", "band": 8, "orderInBand": 1, "median": 495.2239388402185, "min": 494.06511406229606, "max": 497.21837193936096, "n": 3 }, { "target": "chroma", "band": 8, "orderInBand": 2, "median": 487.59820563860325, "min": 486.2599880968794, "max": 489.9432387833048, "n": 3 }, { "target": "milvus", "band": 9, "orderInBand": 1, "median": 430.0783928615426, "min": 428.75784238514746, "max": 440.8275266126933, "n": 3 }, { "target": "duckdb", "band": 10, "orderInBand": 1, "median": 280.87496982706386, "min": 280.59227950342944, "max": 282.11811077934806, "n": 3 }, { "target": "sql-diskann", "band": 10, "orderInBand": 2, "median": 273.3524626399846, "min": 272.4901658299152, "max": 276.38487760142203, "n": 3 }, { "target": "sql", "band": 11, "orderInBand": 1, "median": 250.7237818141357, "min": 249.9475055238949, "max": 255.63820101883195, "n": 3 }, { "target": "typesense", "band": 11, "orderInBand": 2, "median": 244.11779842121027, "min": 243.83393378210312, "max": 244.99892200474315, "n": 3 }, { "target": "clickhouse", "band": 12, "orderInBand": 1, "median": 214.68461033370824, "min": 214.6841724093891, "max": 215.15501646493257, "n": 3 }, { "target": "weaviate", "band": 13, "orderInBand": 1, "median": 171.23961834197834, "min": 171.2197351996061, "max": 171.56599304668825, "n": 3 } ] } ] }