Vector Search

tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()

const results2 = await (
  tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
)
  .distanceType("cosine")
  .limit(10)
  .toArray();

// Use the same distance metric the index was trained with.
let mut results = table
    .vector_search(&query_vector)?
    .distance_type(DistanceType::Cosine)
    .limit(10)
    .execute()
    .await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

import pyarrow as pa

schema = pa.schema([
    pa.field("id", pa.int32()),
    pa.field(
        "image",
        pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
    ),
])
table = db.create_table(
    "nested",
    data=[{"id": 0, "image": {"embedding": [0.0, 1.0]}}],
    schema=schema,
)

# Inferred: the only vector leaf is `image.embedding`.
table.search([0.0, 1.0]).limit(1).to_list()

# Explicit: required when more than one vector column matches.
table.search([0.0, 1.0], vector_column_name="image.embedding").limit(1).to_list()

const table = await db.openTable("nested");

// Inferred: LanceDB finds the single nested vector leaf automatically.
await table.query().nearestTo([0.0, 1.0]).limit(1).toArray();

// Explicit: required when more than one vector column matches.
await table
  .query()
  .nearestTo([0.0, 1.0])
  .column("image.embedding")
  .limit(1)
  .toArray();

table.create_index(vector_column_name="image.embedding")

await table.createIndex("image.embedding");

# Indexed ANN search without refinement (fast, approximate `_distance`)
fast_results = (
    table.search(embedding)
    .limit(10)
    .to_pandas()
)

# Recompute distances on full vectors for reranked candidates
exact_distance_results = (
    table.search(embedding)
    .limit(10)
    .refine_factor(1)
    .to_pandas()
)

# Rerank a larger candidate set for better recall (higher latency)
higher_recall_results = (
    table.search(embedding)
    .limit(10)
    .refine_factor(20)
    .to_pandas()
)

// Indexed ANN search without refinement (fast, approximate `_distance`)
const fastResults = await (table.search(embedding) as lancedb.VectorQuery)
  .limit(10)
  .toArray();

// Recompute distances on full vectors for reranked candidates
const exactDistanceResults = await (
  table.search(embedding) as lancedb.VectorQuery
)
  .limit(10)
  .refineFactor(1)
  .toArray();

// Rerank a larger candidate set for better recall (higher latency)
const higherRecallResults = await (
  table.search(embedding) as lancedb.VectorQuery
)
  .limit(10)
  .refineFactor(20)
  .toArray();

// Approximate ANN search (fast, distances may come from the index representation)
let mut fast_results = table.vector_search(&query_vector)?.limit(10).execute().await?;
while let Some(batch) = fast_results.try_next().await? {
    println!("{:?}", batch);
}

// Rerank a larger candidate set on full vectors for better recall
let mut refined_results = table
    .vector_search(&query_vector)?
    .limit(10)
    .refine_factor(20)
    .execute()
    .await?;
while let Some(batch) = refined_results.try_next().await? {
    println!("{:?}", batch);
}

from datasets import load_dataset

# Load query vector from dataset
query_dataset = load_dataset("sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5001]")
print(f"Query keywords: {query_dataset[0]['keywords']}")
query_embed = query_dataset["keywords_embeddings"][0]

# Open table and perform search
table_name = "lancedb-enterprise-quickstart"
table = db.open_table(table_name)

# Vector search with filters (pre-filtering is the default)
search_results = (
    table.search(query_embed)
    .where("label > 2")
    .select(["text", "keywords", "label"])
    .limit(5)
    .to_pandas()
)

print("Search results (with pre-filtering):")
print(search_results)

// Generate a sample 768-dimension embedding vector (typical for BERT-based models)
// In real applications, you would get this from an embedding model
const dimensions = 768;
const queryEmbed = Array.from(
  { length: dimensions },
  () => Math.random() * 2 - 1,
);

// Open table and perform search
const tableName = "lancedb-enterprise-quickstart";
const table = await db.openTable(tableName);

// Vector search with filters (pre-filtering is the default)
const vectorResults = await table
  .search(queryEmbed)
  .where("label > 2")
  .select(["text", "keywords", "label"])
  .limit(5)
  .toArray();

console.log("Search results (with pre-filtering):");
console.log(vectorResults);

// Prefiltering is the default: the filter is applied before vector search.
let mut results = table
    .vector_search(&query_vector)?
    .only_if("id > 100")
    .select(Select::columns(&["id"]))
    .limit(5)
    .execute()
    .await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

results_post_filtered = (
    table.search(query_embed)
    .where("label > 1", prefilter=False)
    .select(["text", "keywords", "label"])
    .limit(5)
    .to_pandas()
)

print("Vector search results with post-filter:")
print(results_post_filtered)

const vectorResultsWithPostFilter = await (
  table.search(queryEmbed) as lancedb.VectorQuery
)
  .where("label > 2")
  .postfilter()
  .select(["text", "keywords", "label"])
  .limit(5)
  .toArray();

console.log("Vector search results with post-filter:");
console.log(vectorResultsWithPostFilter);

// Apply the filter after vector search by calling postfilter().
let mut results = table
    .vector_search(&query_vector)?
    .only_if("id > 100")
    .postfilter()
    .select(Select::columns(&["id"]))
    .limit(5)
    .execute()
    .await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

query_multi = np.random.random(size=(2, 256))
results_multi = tbl.search(query_multi).limit(5).to_pandas()

query = np.random.random(256)

# Search for the vectors within the range of [0.1, 0.5)
tbl.search(query).distance_range(0.1, 0.5).to_arrow()

# Search for the vectors with the distance less than 0.5
tbl.search(query).distance_range(upper_bound=0.5).to_arrow()

# Search for the vectors with the distance greater or equal to 0.1
tbl.search(query).distance_range(lower_bound=0.1).to_arrow()

const results3 = await (
  tbl.search(Array(128).fill(1.2)) as lancedb.VectorQuery
)
  .distanceType("cosine")
  .distanceRange(0.1, 0.2)
  .limit(10)
  .toArray();

// Only return rows whose distance falls within [0.1, 0.5).
let mut results = table
    .vector_search(&query_vector)?
    .distance_range(Some(0.1), Some(0.5))
    .execute()
    .await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

import numpy as np
import pyarrow as pa

schema = pa.schema(
    [
        pa.field("id", pa.int64()),
        # for dim=256, lance stores every 8 bits in a byte
        # so the vector field should be a list of 256 / 8 = 32 bytes
        pa.field("vector", pa.list_(pa.uint8(), 32)),
    ]
)
tbl = db.create_table("my_binary_vectors", schema=schema)

data = []
for i in range(1024):
    vector = np.random.randint(0, 2, size=256)
    # pack the binary vector into bytes to save space
    packed_vector = np.packbits(vector)
    data.append(
        {
            "id": i,
            "vector": packed_vector,
        }
    )
tbl.add(data)

query = np.random.randint(0, 2, size=256)
packed_query = np.packbits(query)
tbl.search(packed_query).distance_type("hamming").to_arrow()

const tbl = await db.createTable("binary_vectors", data, {
  mode: "overwrite",
});
await tbl.createIndex("vector", {
  config: lancedb.Index.ivfFlat({
    numPartitions: 10,
    distanceType: "hamming",
  }),
});

const query = Array(32)
  .fill(1)
  .map(() => Math.floor(Math.random() * 255));
const results = await tbl.query().nearestTo(query).limit(10).toArray();

// Binary vectors use `hamming` distance over the packed uint8 bytes.
let query: Arc<dyn Array> = Arc::new(UInt8Array::from(vec![1u8; NUM_BYTES as usize]));
let mut results = tbl
    .vector_search(query)?
    .distance_type(DistanceType::Hamming)
    .limit(10)
    .execute()
    .await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

# Load a batch of query embeddings
query_dataset = load_dataset(
    "sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5005]"
)
query_embeds = query_dataset["keywords_embeddings"]
batch_results = table.search(query_embeds).limit(5).to_pandas()
print(batch_results)

// Batch query
console.log("Performing batch vector search...");
const batchSize = 5;
const queryVectors = Array.from({ length: batchSize }, () =>
  Array.from({ length: dimensions }, () => Math.random() * 2 - 1),
);
let batchQuery = table.search(queryVectors[0]) as lancedb.VectorQuery;
for (let i = 1; i < batchSize; i++) {
  batchQuery = batchQuery.addQueryVector(queryVectors[i]);
}
const batchResults = await batchQuery
  .select(["text", "keywords", "label"])
  .limit(5)
  .toArray();
console.log("Batch vector search results:");
console.log(batchResults);

// Search multiple query vectors in one call. Each result row carries a
// `query_index` mapping it back to the query it matched.
let mut results = table
    .vector_search(&query_1)?
    .add_query_vector(&query_2)?
    .limit(5)
    .execute()
    .await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

table.search(embedding, fast_search=True).limit(5).to_pandas()

await table
  .query()
  .nearestTo(embedding)
  .fastSearch()
  .limit(5)
  .toArray();

// Skip unindexed data for lower latency.
let mut results = table
    .vector_search(&query_vector)?
    .fast_search()
    .limit(5)
    .execute()
    .await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

tbl.search(np.random.random((1536))).limit(3).to_list()

const tbl = await db.openTable("my_vectors");

const results1 = await tbl.search(Array(128).fill(1.2)).limit(3).toArray();

// A plain vector search returns the top-k closest rows.
let mut results = table.vector_search(&query_vector)?.limit(3).execute().await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

table.search(embedding).bypass_vector_index().limit(5).to_pandas()

await table
  .query()
  .nearestTo(embedding)
  .bypassVectorIndex()
  .limit(5)
  .toArray();

// Force an exhaustive (flat) scan for exact, ground-truth results.
let mut results = table
    .vector_search(&query_vector)?
    .bypass_vector_index()
    .limit(5)
    .execute()
    .await?;
while let Some(batch) = results.try_next().await? {
    println!("{:?}", batch);
}

import numpy as np
import pyarrow as pa

db = lancedb.connect("data/multivector_demo")

schema = pa.schema(
    [
        pa.field("id", pa.int64()),
        # float16, float32, and float64 are supported
        pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
    ]
)

data = [
    {
        "id": i,
        "vector": np.random.random(size=(2, 256)).tolist(),  # Each document has 2 vectors
    }
    for i in range(1024)
]

tbl = db.create_table("multivector_example", data=data, schema=schema)

tbl.create_index(metric="cosine", vector_column_name="vector")

query = np.random.random(256)
results_single = tbl.search(query).limit(5).to_pandas()

query_multi = np.random.random(size=(2, 256))
results_multi = tbl.search(query_multi).limit(5).to_pandas()

pip install pylate lancedb pandas

import numpy as np
import pyarrow as pa
from pylate import models

## 1) Load a late-interaction model via PyLate
## PyLate docs show ColBERT() + encode(..., is_query=...)
model = models.ColBERT(model_name_or_path="lightonai/GTE-ModernColBERT-v1")

## You can discover dim from one embedding (avoid guessing)
dim = model.encode(["hello"], is_query=True)[0].shape[1]

## 2) Create a LanceDB table with a multivector column
db = lancedb.connect("./pylate_lancedb")
schema = pa.schema([
    pa.field("doc_id", pa.string()),
    pa.field("text", pa.string()),
    # multivector: list<list<float32, dim>>
    pa.field("mv", pa.list_(pa.list_(pa.float32(), dim))),
])

docs = [
    {"doc_id": "1", "text": "The train to Tokyo leaves at 5pm."},
    {"doc_id": "2", "text": "That Pho restaurant in Hanoi is highly rated."},
    {"doc_id": "3", "text": "This is a noodle bar in Osaka, Japan."},
]

## 3) Encode documents with PyLate (token vectors per doc)
doc_texts = [d["text"] for d in docs]
doc_embs = model.encode(doc_texts, is_query=False)  # list/array of (T, dim) per doc

rows = []
for d, emb in zip(docs, doc_embs):
    emb = np.asarray(emb, dtype=np.float32)
    rows.append({**d, "mv": emb.tolist()})

tbl = db.create_table("docs", data=rows, schema=schema, mode="overwrite")

## 4) Build an index + query using a query matrix.
## Multivector brute-force scales with rows × vectors-per-row, so build the index
## at much smaller dataset sizes than you would for single-vector search; and
## always before exposing the table to remote traffic.
tbl.create_index(vector_column_name="mv", metric="cosine")

query = "Tell me about ramen in Japan"
q_emb = np.asarray(model.encode([query], is_query=True)[0], dtype=np.float32)  # (Tq, dim)

out = tbl.search(q_emb).limit(5).to_pandas()  # multivector search accepts a matrix
print(out[["doc_id", "text"]])