mirror of
https://github.com/hanzoai/sql-vector.git
synced 2026-10-01 00:37:19 +00:00
Merge remote-tracking branch 'upstream/master'
This commit is contained in:
@@ -72,7 +72,7 @@ jobs:
|
||||
tar xf $TAG.tar.gz
|
||||
mv postgres-$TAG postgres
|
||||
env:
|
||||
TAG: ${{ matrix.postgres == 18 && 'REL_18_0' || 'REL_14_19' }}
|
||||
TAG: ${{ matrix.postgres == 18 && 'REL_18_2' || 'REL_14_21' }}
|
||||
- run: make prove_installcheck PROVE_FLAGS="-I ./postgres/src/test/perl -I ./test/perl"
|
||||
env:
|
||||
PERL5LIB: /Users/runner/perl5/lib/perl5
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
## 0.8.2 (unreleased)
|
||||
## 0.8.2 (2026-02-25)
|
||||
|
||||
- Fixed buffer overflow with parallel HNSW index build - [more info](https://github.com/pgvector/pgvector/issues/959)
|
||||
- Improved `install` target on Windows
|
||||
- Fixed `Index Searches` in `EXPLAIN` output for Postgres 18
|
||||
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ ARG DEBIAN_CODENAME=bookworm
|
||||
FROM postgres:$PG_MAJOR-$DEBIAN_CODENAME
|
||||
ARG PG_MAJOR
|
||||
|
||||
ADD https://github.com/pgvector/pgvector.git#v0.8.1 /tmp/pgvector
|
||||
ADD https://github.com/pgvector/pgvector.git#v0.8.2 /tmp/pgvector
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-mark hold locales && \
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"name": "vector",
|
||||
"abstract": "Open-source vector similarity search for Postgres",
|
||||
"description": "Supports L2 distance, inner product, and cosine distance",
|
||||
"version": "0.8.1",
|
||||
"version": "0.8.2",
|
||||
"maintainer": [
|
||||
"Andrew Kane <[email protected]>"
|
||||
],
|
||||
@@ -20,7 +20,7 @@
|
||||
"vector": {
|
||||
"file": "sql/vector.sql",
|
||||
"docfile": "README.md",
|
||||
"version": "0.8.1",
|
||||
"version": "0.8.2",
|
||||
"abstract": "Open-source vector similarity search for Postgres"
|
||||
}
|
||||
},
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.8.1
|
||||
EXTVERSION = 0.8.2
|
||||
|
||||
MODULE_big = vector
|
||||
DATA = $(wildcard sql/*--*--*.sql)
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
EXTENSION = vector
|
||||
EXTVERSION = 0.8.1
|
||||
EXTVERSION = 0.8.2
|
||||
|
||||
DATA_built = sql\$(EXTENSION)--$(EXTVERSION).sql
|
||||
OBJS = src\bitutils.obj src\bitvec.obj src\halfutils.obj src\halfvec.obj src\hnsw.obj src\hnswbuild.obj src\hnswinsert.obj src\hnswscan.obj src\hnswutils.obj src\hnswvacuum.obj src\ivfbuild.obj src\ivfflat.obj src\ivfinsert.obj src\ivfkmeans.obj src\ivfscan.obj src\ivfutils.obj src\ivfvacuum.obj src\sparsevec.obj src\vector.obj
|
||||
|
||||
@@ -11,6 +11,8 @@ Store your vectors with the rest of your data. Supports:
|
||||
|
||||
Plus [ACID](https://en.wikipedia.org/wiki/ACID) compliance, point-in-time recovery, JOINs, and all of the other [great features](https://www.postgresql.org/about/) of Postgres
|
||||
|
||||
Have a lot of vectors? Use [quantization](#scaling) to scale
|
||||
|
||||
[](https://github.com/pgvector/pgvector/actions)
|
||||
|
||||
## Installation
|
||||
@@ -21,7 +23,7 @@ Compile and install the extension (supports Postgres 13+)
|
||||
|
||||
```sh
|
||||
cd /tmp
|
||||
git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
make
|
||||
make install # may need sudo
|
||||
@@ -38,7 +40,7 @@ Ensure [C++ support in Visual Studio](https://learn.microsoft.com/en-us/cpp/buil
|
||||
```cmd
|
||||
set "PGROOT=C:\Program Files\PostgreSQL\18"
|
||||
cd %TEMP%
|
||||
git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
nmake /F Makefile.win
|
||||
nmake /F Makefile.win install
|
||||
@@ -314,6 +316,8 @@ For a large number of workers, you may need to increase `max_parallel_workers` (
|
||||
|
||||
The [index options](#index-options) also have a significant impact on build time (use the defaults unless seeing low recall)
|
||||
|
||||
Use [binary quantization](#binary-quantization) for faster build times at scale
|
||||
|
||||
### Indexing Progress
|
||||
|
||||
Check [indexing progress](https://www.postgresql.org/docs/current/progress-reporting.html#CREATE-INDEX-PROGRESS-REPORTING)
|
||||
@@ -673,6 +677,10 @@ SHOW shared_buffers;
|
||||
|
||||
Be sure to restart Postgres for changes to take effect.
|
||||
|
||||
### Storing
|
||||
|
||||
Use the `halfvec` type instead of `vector` for a smaller working set.
|
||||
|
||||
### Loading
|
||||
|
||||
Use `COPY` for bulk loading data ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/loading/example.py)).
|
||||
@@ -687,6 +695,8 @@ Add any indexes *after* loading the initial data for best performance.
|
||||
|
||||
See index build time for [HNSW](#index-build-time) and [IVFFlat](#index-build-time-1).
|
||||
|
||||
Use [binary quantization](#binary-quantization) for smaller indexes and faster build times at scale.
|
||||
|
||||
In production environments, create indexes concurrently to avoid blocking writes.
|
||||
|
||||
```sql
|
||||
@@ -717,6 +727,8 @@ SELECT * FROM items ORDER BY embedding <#> '[3,1,2]' LIMIT 5;
|
||||
|
||||
#### Approximate Search
|
||||
|
||||
Use [binary quantization](#binary-quantization) with re-ranking to keep indexes in-memory at scale.
|
||||
|
||||
To speed up queries with an IVFFlat index, increase the number of inverted lists (at the expense of recall).
|
||||
|
||||
```sql
|
||||
@@ -732,21 +744,20 @@ REINDEX INDEX CONCURRENTLY index_name;
|
||||
VACUUM table_name;
|
||||
```
|
||||
|
||||
## Scaling
|
||||
|
||||
For a smaller working set:
|
||||
|
||||
1. Use the `halfvec` type instead of `vector` for tables
|
||||
2. Use [binary quantization](#binary-quantization) for indexes (with re-ranking for search)
|
||||
|
||||
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
|
||||
|
||||
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus), [PgDog](https://github.com/pgdogdev/pgdog), or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
|
||||
|
||||
## Monitoring
|
||||
|
||||
Monitor performance with [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) (be sure to add it to `shared_preload_libraries`).
|
||||
|
||||
```sql
|
||||
CREATE EXTENSION pg_stat_statements;
|
||||
```
|
||||
|
||||
Get the most time-consuming queries with:
|
||||
|
||||
```sql
|
||||
SELECT query, calls, ROUND((total_plan_time + total_exec_time) / calls) AS avg_time_ms,
|
||||
ROUND((total_plan_time + total_exec_time) / 60000) AS total_time_min
|
||||
FROM pg_stat_statements ORDER BY total_plan_time + total_exec_time DESC LIMIT 20;
|
||||
```
|
||||
Use existing tools like [pg_stat_statements](https://www.postgresql.org/docs/current/pgstatstatements.html) or [PgHero](https://github.com/ankane/pghero) to monitor performance.
|
||||
|
||||
Monitor recall by comparing results from approximate search with exact search.
|
||||
|
||||
@@ -757,14 +768,6 @@ SELECT ...
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
## Scaling
|
||||
|
||||
Scale pgvector the same way you scale Postgres.
|
||||
|
||||
Scale vertically by increasing memory, CPU, and storage on a single instance. Use existing tools to [tune parameters](#tuning) and [monitor performance](#monitoring).
|
||||
|
||||
Scale horizontally with [replicas](https://www.postgresql.org/docs/current/hot-standby.html), or use [Citus](https://github.com/citusdata/citus) or another approach for sharding ([example](https://github.com/pgvector/pgvector-python/blob/master/examples/citus/example.py)).
|
||||
|
||||
## Languages
|
||||
|
||||
Use pgvector from any language with a Postgres client. You can even generate and store vectors in one language and query them in another.
|
||||
@@ -878,6 +881,8 @@ No, but like other index types, you’ll likely see better performance if they d
|
||||
SELECT pg_size_pretty(pg_relation_size('index_name'));
|
||||
```
|
||||
|
||||
Use [half-precision indexing](#half-precision-indexing) or [binary quantization](#binary-quantization) for smaller indexes.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
#### Why isn’t a query using an index?
|
||||
@@ -1152,23 +1157,23 @@ This adds pgvector to the [Postgres image](https://hub.docker.com/_/postgres) (r
|
||||
|
||||
Supported tags are:
|
||||
|
||||
- `pg18-trixie`, `0.8.1-pg18-trixie`
|
||||
- `pg18-bookworm`, `0.8.1-pg18-bookworm`, `pg18`, `0.8.1-pg18`
|
||||
- `pg17-trixie`, `0.8.1-pg17-trixie`
|
||||
- `pg17-bookworm`, `0.8.1-pg17-bookworm`, `pg17`, `0.8.1-pg17`
|
||||
- `pg16-trixie`, `0.8.1-pg16-trixie`
|
||||
- `pg16-bookworm`, `0.8.1-pg16-bookworm`, `pg16`, `0.8.1-pg16`
|
||||
- `pg15-trixie`, `0.8.1-pg15-trixie`
|
||||
- `pg15-bookworm`, `0.8.1-pg15-bookworm`, `pg15`, `0.8.1-pg15`
|
||||
- `pg14-trixie`, `0.8.1-pg14-trixie`
|
||||
- `pg14-bookworm`, `0.8.1-pg14-bookworm`, `pg14`, `0.8.1-pg14`
|
||||
- `pg13-trixie`, `0.8.1-pg13-trixie`
|
||||
- `pg13-bookworm`, `0.8.1-pg13-bookworm`, `pg13`, `0.8.1-pg13`
|
||||
- `pg18-trixie`, `0.8.2-pg18-trixie`
|
||||
- `pg18-bookworm`, `0.8.2-pg18-bookworm`, `pg18`, `0.8.2-pg18`
|
||||
- `pg17-trixie`, `0.8.2-pg17-trixie`
|
||||
- `pg17-bookworm`, `0.8.2-pg17-bookworm`, `pg17`, `0.8.2-pg17`
|
||||
- `pg16-trixie`, `0.8.2-pg16-trixie`
|
||||
- `pg16-bookworm`, `0.8.2-pg16-bookworm`, `pg16`, `0.8.2-pg16`
|
||||
- `pg15-trixie`, `0.8.2-pg15-trixie`
|
||||
- `pg15-bookworm`, `0.8.2-pg15-bookworm`, `pg15`, `0.8.2-pg15`
|
||||
- `pg14-trixie`, `0.8.2-pg14-trixie`
|
||||
- `pg14-bookworm`, `0.8.2-pg14-bookworm`, `pg14`, `0.8.2-pg14`
|
||||
- `pg13-trixie`, `0.8.2-pg13-trixie`
|
||||
- `pg13-bookworm`, `0.8.2-pg13-bookworm`, `pg13`, `0.8.2-pg13`
|
||||
|
||||
You can also build the image manually:
|
||||
|
||||
```sh
|
||||
git clone --branch v0.8.1 https://github.com/pgvector/pgvector.git
|
||||
git clone --branch v0.8.2 https://github.com/pgvector/pgvector.git
|
||||
cd pgvector
|
||||
docker build --pull --build-arg PG_MAJOR=18 -t myuser/pgvector .
|
||||
```
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION vector UPDATE TO '0.8.2'" to load this file. \quit
|
||||
+1
-1
@@ -169,7 +169,7 @@ BitJaccardDistanceAvx512Popcount(uint32 bytes, unsigned char *ax, unsigned char
|
||||
#endif
|
||||
|
||||
TARGET_XSAVE static bool
|
||||
SupportsAvx512Popcount()
|
||||
SupportsAvx512Popcount(void)
|
||||
{
|
||||
unsigned int exx[4] = {0, 0, 0, 0};
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include "hnsw.h"
|
||||
#include "miscadmin.h"
|
||||
#include "nodes/pg_list.h"
|
||||
#include "storage/lwlock.h"
|
||||
#include "utils/float.h"
|
||||
#include "utils/guc.h"
|
||||
#include "utils/relcache.h"
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
#include "lib/pairingheap.h"
|
||||
#include "nodes/execnodes.h"
|
||||
#include "port.h" /* for random() */
|
||||
#include "storage/bufpage.h"
|
||||
#include "storage/condition_variable.h"
|
||||
#include "storage/lwlock.h"
|
||||
#include "storage/s_lock.h"
|
||||
#include "utils/relptr.h"
|
||||
#include "utils/sampling.h"
|
||||
#include "vector.h"
|
||||
|
||||
+29
-7
@@ -36,6 +36,8 @@
|
||||
*/
|
||||
#include "postgres.h"
|
||||
|
||||
#include <limits.h>
|
||||
|
||||
#include "access/genam.h"
|
||||
#include "access/parallel.h"
|
||||
#include "access/relscan.h"
|
||||
@@ -52,6 +54,7 @@
|
||||
#include "nodes/execnodes.h"
|
||||
#include "optimizer/optimizer.h"
|
||||
#include "storage/bufmgr.h"
|
||||
#include "storage/condition_variable.h"
|
||||
#include "tcop/tcopprot.h"
|
||||
#include "utils/datum.h"
|
||||
#include "utils/memutils.h"
|
||||
@@ -77,6 +80,8 @@
|
||||
#define PARALLEL_KEY_HNSW_AREA UINT64CONST(0xA000000000000002)
|
||||
#define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xA000000000000003)
|
||||
|
||||
#define HNSW_MAX_GRAPH_MEMORY (SIZE_MAX / 2)
|
||||
|
||||
/*
|
||||
* Create the metapage
|
||||
*/
|
||||
@@ -489,6 +494,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
|
||||
LWLock *flushLock = &graph->flushLock;
|
||||
char *base = buildstate->hnswarea;
|
||||
Datum value;
|
||||
Size memoryMargin;
|
||||
|
||||
/* Form index value */
|
||||
if (!HnswFormIndexValue(&value, values, isnull, buildstate->typeInfo, support))
|
||||
@@ -497,6 +503,9 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
|
||||
/* Get datum size */
|
||||
valueSize = VARSIZE_ANY(DatumGetPointer(value));
|
||||
|
||||
/* In a parallel build, add a margin so allocations never fail */
|
||||
memoryMargin = base == NULL ? 0 : 1024 * 1024;
|
||||
|
||||
/* Ensure graph not flushed when inserting */
|
||||
LWLockAcquire(flushLock, LW_SHARED);
|
||||
|
||||
@@ -518,7 +527,7 @@ InsertTuple(Relation index, Datum *values, bool *isnull, ItemPointer heaptid, Hn
|
||||
* Check that we have enough memory available for the new element now that
|
||||
* we have the allocator lock, and flush pages if needed.
|
||||
*/
|
||||
if (graph->memoryUsed >= graph->memoryTotal)
|
||||
if (graph->memoryUsed + memoryMargin >= graph->memoryTotal)
|
||||
{
|
||||
LWLockRelease(&graph->allocatorLock);
|
||||
|
||||
@@ -611,7 +620,7 @@ InitGraph(HnswGraph * graph, char *base, Size memoryTotal)
|
||||
HnswPtrStore(base, graph->head, (HnswElement) NULL);
|
||||
HnswPtrStore(base, graph->entryPoint, (HnswElement) NULL);
|
||||
graph->memoryUsed = 0;
|
||||
graph->memoryTotal = memoryTotal;
|
||||
graph->memoryTotal = Min(memoryTotal, HNSW_MAX_GRAPH_MEMORY);
|
||||
graph->flushed = false;
|
||||
graph->indtuples = 0;
|
||||
SpinLockInit(&graph->lock);
|
||||
@@ -652,9 +661,17 @@ static void *
|
||||
HnswSharedMemoryAlloc(Size size, void *state)
|
||||
{
|
||||
HnswBuildState *buildstate = (HnswBuildState *) state;
|
||||
void *chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
|
||||
Size alignedSize = MAXALIGN(size);
|
||||
void *chunk;
|
||||
|
||||
buildstate->graph->memoryUsed += MAXALIGN(size);
|
||||
if (alignedSize > 1024 * 1024)
|
||||
elog(ERROR, "hnsw allocation too large");
|
||||
|
||||
if (buildstate->graph->memoryUsed + alignedSize > buildstate->graph->memoryTotal)
|
||||
elog(ERROR, "hnsw allocator out of memory");
|
||||
|
||||
chunk = buildstate->hnswarea + buildstate->graph->memoryUsed;
|
||||
buildstate->graph->memoryUsed += alignedSize;
|
||||
return chunk;
|
||||
}
|
||||
|
||||
@@ -786,7 +803,11 @@ HnswParallelScanAndInsert(Relation heapRel, Relation indexRel, HnswShared * hnsw
|
||||
buildstate.hnswarea = hnswarea;
|
||||
InitAllocator(&buildstate.allocator, &HnswSharedMemoryAlloc, &buildstate);
|
||||
scan = table_beginscan_parallel(heapRel,
|
||||
ParallelTableScanFromHnswShared(hnswshared));
|
||||
ParallelTableScanFromHnswShared(hnswshared)
|
||||
#if PG_VERSION_NUM >= 190000
|
||||
,SO_NONE
|
||||
#endif
|
||||
);
|
||||
reltuples = table_index_build_scan(heapRel, indexRel, indexInfo,
|
||||
true, progress, BuildCallback,
|
||||
(void *) &buildstate, scan);
|
||||
@@ -940,6 +961,8 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
|
||||
if (esthnswarea > estother)
|
||||
esthnswarea -= estother;
|
||||
|
||||
esthnswarea = Min(esthnswarea, HNSW_MAX_GRAPH_MEMORY);
|
||||
|
||||
shm_toc_estimate_chunk(&pcxt->estimator, esthnswarea);
|
||||
shm_toc_estimate_keys(&pcxt->estimator, 2);
|
||||
|
||||
@@ -982,8 +1005,7 @@ HnswBeginParallel(HnswBuildState * buildstate, bool isconcurrent, int request)
|
||||
snapshot);
|
||||
|
||||
hnswarea = (char *) shm_toc_allocate(pcxt->toc, esthnswarea);
|
||||
/* Report less than allocated so never fails */
|
||||
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea - 1024 * 1024);
|
||||
InitGraph(&hnswshared->graphData, hnswarea, esthnswarea);
|
||||
|
||||
/*
|
||||
* Avoid base address for relptr for Postgres < 14.5
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include "nodes/execnodes.h"
|
||||
#include "storage/bufmgr.h"
|
||||
#include "storage/lmgr.h"
|
||||
#include "storage/lwlock.h"
|
||||
#include "utils/datum.h"
|
||||
#include "utils/memutils.h"
|
||||
#include "utils/rel.h"
|
||||
|
||||
+3
-1
@@ -1,5 +1,7 @@
|
||||
#include "postgres.h"
|
||||
|
||||
#include <limits.h>
|
||||
|
||||
#include "access/genam.h"
|
||||
#include "access/relscan.h"
|
||||
#include "hnsw.h"
|
||||
@@ -151,7 +153,7 @@ hnswbeginscan(Relation index, int nkeys, int norderbys)
|
||||
/* Calculate max memory */
|
||||
/* Add 256 extra bytes to fill last block when close */
|
||||
maxMemory = (double) work_mem * hnsw_scan_mem_multiplier * 1024.0 + 256;
|
||||
so->maxMemory = Min(maxMemory, (double) SIZE_MAX);
|
||||
so->maxMemory = Min(maxMemory, (double) (SIZE_MAX / 2));
|
||||
|
||||
scan->opaque = so;
|
||||
|
||||
|
||||
+39
-8
@@ -22,6 +22,7 @@
|
||||
#include "nodes/execnodes.h"
|
||||
#include "optimizer/optimizer.h"
|
||||
#include "storage/bufmgr.h"
|
||||
#include "storage/condition_variable.h"
|
||||
#include "tcop/tcopprot.h"
|
||||
#include "utils/memutils.h"
|
||||
#include "utils/rel.h"
|
||||
@@ -62,15 +63,13 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
|
||||
Datum value = PointerGetDatum(PG_DETOAST_DATUM(values[0]));
|
||||
|
||||
/*
|
||||
* Normalize with KMEANS_NORM_PROC since spherical distance function
|
||||
* expects unit vectors
|
||||
* Check with KMEANS_NORM_PROC that the value can be normalized since
|
||||
* spherical distance function expects unit vectors
|
||||
*/
|
||||
if (buildstate->kmeansnormprocinfo != NULL)
|
||||
{
|
||||
if (!IvfflatCheckNorm(buildstate->kmeansnormprocinfo, buildstate->collation, value))
|
||||
return;
|
||||
|
||||
value = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
|
||||
}
|
||||
|
||||
if (samples->length < targsamples)
|
||||
@@ -81,7 +80,7 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
|
||||
else
|
||||
{
|
||||
if (buildstate->rowstoskip < 0)
|
||||
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, samples->length, targsamples);
|
||||
buildstate->rowstoskip = reservoir_get_next_S(&buildstate->rstate, buildstate->samplerows, targsamples);
|
||||
|
||||
if (buildstate->rowstoskip <= 0)
|
||||
{
|
||||
@@ -97,6 +96,9 @@ AddSample(Datum *values, IvfflatBuildState * buildstate)
|
||||
|
||||
buildstate->rowstoskip -= 1;
|
||||
}
|
||||
|
||||
/* Increment after reservoir_get_next_S */
|
||||
buildstate->samplerows += 1;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -133,6 +135,7 @@ SampleRows(IvfflatBuildState * buildstate)
|
||||
int targsamples = buildstate->samples->maxlen;
|
||||
BlockNumber totalblocks = RelationGetNumberOfBlocks(buildstate->heap);
|
||||
|
||||
buildstate->samplerows = 0;
|
||||
buildstate->rowstoskip = -1;
|
||||
|
||||
BlockSampler_Init(&buildstate->bs, totalblocks, targsamples, RandomInt());
|
||||
@@ -142,8 +145,29 @@ SampleRows(IvfflatBuildState * buildstate)
|
||||
{
|
||||
BlockNumber targblock = BlockSampler_Next(&buildstate->bs);
|
||||
|
||||
/*
|
||||
* anyvisible must be false when the scan uses an MVCC snapshot
|
||||
* (i.e. CREATE INDEX CONCURRENTLY), otherwise PG will assert in
|
||||
* heapam_index_build_range_scan. Always pass false here, matching
|
||||
* what table_index_build_scan does internally.
|
||||
*/
|
||||
table_index_build_range_scan(buildstate->heap, buildstate->index, buildstate->indexInfo,
|
||||
false, true, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
|
||||
false, false, false, targblock, 1, SampleCallback, (void *) buildstate, NULL);
|
||||
}
|
||||
|
||||
/* Normalize if needed */
|
||||
if (buildstate->kmeansnormprocinfo != NULL)
|
||||
{
|
||||
VectorArray samples = buildstate->samples;
|
||||
|
||||
for (int i = 0; i < samples->length; i++)
|
||||
{
|
||||
Datum value = PointerGetDatum(VectorArrayGet(samples, i));
|
||||
Datum normValue = IvfflatNormValue(buildstate->typeInfo, buildstate->collation, value);
|
||||
|
||||
VectorArraySet(samples, i, DatumGetPointer(normValue));
|
||||
pfree(DatumGetPointer(normValue));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -374,6 +398,9 @@ InitBuildState(IvfflatBuildState * buildstate, Relation heap, Relation index, In
|
||||
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 1, "list", INT4OID, -1, 0);
|
||||
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 2, "tid", TIDOID, -1, 0);
|
||||
TupleDescInitEntry(buildstate->sortdesc, (AttrNumber) 3, "vector", TupleDescAttr(buildstate->tupdesc, 0)->atttypid, -1, 0);
|
||||
#if PG_VERSION_NUM >= 190000
|
||||
TupleDescFinalize(buildstate->sortdesc);
|
||||
#endif
|
||||
|
||||
buildstate->slot = MakeSingleTupleTableSlot(buildstate->sortdesc, &TTSOpsVirtual);
|
||||
|
||||
@@ -435,7 +462,7 @@ ComputeCenters(IvfflatBuildState * buildstate)
|
||||
buildstate->samples = VectorArrayInit(numSamples, buildstate->dimensions, buildstate->centers->itemsize);
|
||||
if (buildstate->heap != NULL)
|
||||
{
|
||||
SampleRows(buildstate);
|
||||
IvfflatBench("sample rows", SampleRows(buildstate));
|
||||
|
||||
if (buildstate->samples->length < buildstate->lists)
|
||||
{
|
||||
@@ -650,7 +677,11 @@ IvfflatParallelScanAndSort(IvfflatSpool * ivfspool, IvfflatShared * ivfshared, S
|
||||
ivfspool->sortstate = InitBuildSortState(buildstate.sortdesc, sortmem, coordinate);
|
||||
buildstate.sortstate = ivfspool->sortstate;
|
||||
scan = table_beginscan_parallel(ivfspool->heap,
|
||||
ParallelTableScanFromIvfflatShared(ivfshared));
|
||||
ParallelTableScanFromIvfflatShared(ivfshared)
|
||||
#if PG_VERSION_NUM >= 190000
|
||||
,SO_NONE
|
||||
#endif
|
||||
);
|
||||
reltuples = table_index_build_scan(ivfspool->heap, ivfspool->index, indexInfo,
|
||||
true, progress, BuildCallback,
|
||||
(void *) &buildstate, scan);
|
||||
|
||||
+3
-1
@@ -9,6 +9,7 @@
|
||||
#include "lib/pairingheap.h"
|
||||
#include "nodes/execnodes.h"
|
||||
#include "port.h" /* for random() */
|
||||
#include "storage/condition_variable.h"
|
||||
#include "utils/sampling.h"
|
||||
#include "utils/tuplesort.h"
|
||||
#include "vector.h"
|
||||
@@ -213,7 +214,8 @@ typedef struct IvfflatBuildState
|
||||
/* Sampling */
|
||||
BlockSamplerData bs;
|
||||
ReservoirStateData rstate;
|
||||
int rowstoskip;
|
||||
double samplerows;
|
||||
double rowstoskip;
|
||||
|
||||
/* Sorting */
|
||||
Tuplesortstate *sortstate;
|
||||
|
||||
@@ -298,6 +298,9 @@ ivfflatbeginscan(Relation index, int nkeys, int norderbys)
|
||||
so->tupdesc = CreateTemplateTupleDesc(2);
|
||||
TupleDescInitEntry(so->tupdesc, (AttrNumber) 1, "distance", FLOAT8OID, -1, 0);
|
||||
TupleDescInitEntry(so->tupdesc, (AttrNumber) 2, "heaptid", TIDOID, -1, 0);
|
||||
#if PG_VERSION_NUM >= 190000
|
||||
TupleDescFinalize(so->tupdesc);
|
||||
#endif
|
||||
|
||||
/* Prep sort */
|
||||
so->sortstate = InitScanSortState(so->tupdesc);
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@
|
||||
#endif
|
||||
|
||||
#if PG_VERSION_NUM >= 180000
|
||||
PG_MODULE_MAGIC_EXT(.name = "vector",.version = "0.8.1");
|
||||
PG_MODULE_MAGIC_EXT(.name = "vector",.version = "0.8.2");
|
||||
#else
|
||||
PG_MODULE_MAGIC;
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
# Initialize node
|
||||
my $node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init;
|
||||
$node->start;
|
||||
|
||||
# Create table
|
||||
$node->safe_psql("postgres", "CREATE EXTENSION vector;");
|
||||
$node->safe_psql("postgres", "CREATE TABLE tst (v vector(3));");
|
||||
$node->safe_psql("postgres",
|
||||
"INSERT INTO tst SELECT ARRAY[random(), random(), random()] FROM generate_series(1, 1000) i;"
|
||||
);
|
||||
|
||||
my ($ret, $stdout, $stderr) = $node->psql("postgres", qq(
|
||||
SET client_min_messages = DEBUG;
|
||||
SET maintenance_work_mem = '3073kB';
|
||||
ALTER TABLE tst SET (parallel_workers = 1);
|
||||
CREATE INDEX ON tst USING hnsw (v vector_l2_ops);
|
||||
));
|
||||
is($ret, 0, $stderr);
|
||||
like($stderr, qr/using \d+ parallel workers/);
|
||||
like($stderr, qr/hnsw graph no longer fits into maintenance_work_mem after 0 tuples/);
|
||||
|
||||
done_testing();
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
comment = 'vector data type and ivfflat and hnsw access methods'
|
||||
default_version = '0.8.1'
|
||||
default_version = '0.8.2'
|
||||
module_pathname = '$libdir/vector'
|
||||
relocatable = true
|
||||
|
||||
Reference in New Issue
Block a user