> ## Documentation Index
> Fetch the complete documentation index at: https://docs.lancedb.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Schema and Data Evolution

> Learn how to manage table schemas in LanceDB, including adding, altering, and dropping columns.

export const RsAlterVectorColumn = "let old_dim = 384;\nlet new_dim = 1024;\nlet vector_schema = Arc::new(Schema::new(vec![\n    Field::new(\"id\", DataType::Int64, false),\n    Field::new(\n        \"embedding\",\n        DataType::FixedSizeList(\n            Arc::new(Field::new(\"item\", DataType::Float32, true)),\n            old_dim,\n        ),\n        true,\n    ),\n]));\nlet vector_batch = RecordBatch::try_new(\n    vector_schema.clone(),\n    vec![\n        Arc::new(Int64Array::from(vec![1])),\n        Arc::new(\n            FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(\n                vec![Some(vec![Some(0.1_f32); old_dim as usize])],\n                old_dim,\n            ),\n        ),\n    ],\n)\n.unwrap();\nlet vector_reader: Box<dyn RecordBatchReader + Send> =\n    Box::new(RecordBatchIterator::new(vec![Ok(vector_batch)].into_iter(), vector_schema.clone()));\nlet vector_table = db\n    .create_table(\"vector_alter_example\", vector_reader)\n    .mode(CreateTableMode::Overwrite)\n    .execute()\n    .await\n    .unwrap();\n\n// Changing FixedSizeList dimensions (384 -> 1024) is not supported via alter_columns.\n// Use add_columns + drop_columns + alter_columns(rename) to replace the column.\nvector_table\n    .add_columns(\n        NewColumnTransform::SqlExpressions(vec![(\n            \"embedding_v2\".to_string(),\n            format!(\"arrow_cast(NULL, 'FixedSizeList({}, Float32)')\", new_dim),\n        )]),\n        None,\n    )\n    .await\n    .unwrap();\nvector_table.drop_columns(&[\"embedding\"]).await.unwrap();\nvector_table\n    .alter_columns(&[ColumnAlteration::new(\"embedding_v2\".to_string())\n        .rename(\"embedding\".to_string())])\n    .await\n    .unwrap();\n";

export const TsAlterVectorColumn = "const oldDim = 384;\nconst newDim = 1024;\nconst vectorSchema = new arrow.Schema([\n  new arrow.Field(\"id\", new arrow.Int64()),\n  new arrow.Field(\n    \"embedding\",\n    new arrow.FixedSizeList(\n      oldDim,\n      new arrow.Field(\"item\", new arrow.Float16(), true),\n    ),\n    true,\n  ),\n]);\nconst vectorData = lancedb.makeArrowTable(\n  [{ id: 1, embedding: Array.from({ length: oldDim }, () => Math.random()) }],\n  { schema: vectorSchema },\n);\nconst vectorTable = await db.createTable(\"vector_alter_example\", vectorData, {\n  mode: \"overwrite\",\n});\n\n// Changing FixedSizeList dimensions (384 -> 1024) is not supported via alterColumns.\n// Use addColumns + dropColumns + alterColumns(rename) to replace the column.\nawait vectorTable.addColumns([\n  {\n    name: \"embedding_v2\",\n    valueSql: `arrow_cast(NULL, 'FixedSizeList(${newDim}, Float16)')`,\n  },\n]);\nawait vectorTable.dropColumns([\"embedding\"]);\nawait vectorTable.alterColumns([{ path: \"embedding_v2\", rename: \"embedding\" }]);\n";

export const AlterVectorColumn = "vector_dim = 768  # Your embedding dimension\ntable_name = \"vector_alter_example\"\ndb = tmp_db\ndata = [\n    {\n        \"id\": 1,\n        \"embedding\": np.random.random(vector_dim).tolist(),\n    },\n]\ntable = db.create_table(table_name, data, mode=\"overwrite\")\n\ntable.alter_columns(\n    dict(path=\"embedding\", data_type=pa.list_(pa.float32(), vector_dim))\n)\n";

export const RsDropColumnsMultiple = "// Remove the second temporary column\nschema_drop_table.drop_columns(&[\"temp_col2\"]).await.unwrap();\n";

export const TsDropColumnsMultiple = "// Remove the second temporary column\nawait schemaDropTable.dropColumns([\"temp_col2\"]);\n";

export const DropColumnsMultiple = "# Remove the second temporary column\ntable.drop_columns([\"temp_col2\"])\n";

export const RsDropColumnsSingle = "// Remove the first temporary column\nschema_drop_table.drop_columns(&[\"temp_col1\"]).await.unwrap();\n";

export const TsDropColumnsSingle = "// Remove the first temporary column\nawait schemaDropTable.dropColumns([\"temp_col1\"]);\n";

export const DropColumnsSingle = "# Remove the first temporary column\ntable.drop_columns([\"temp_col1\"])\n";

export const RsSchemaDropSetup = "let schema_drop_schema = Arc::new(Schema::new(vec![\n    Field::new(\"id\", DataType::Int64, false),\n    Field::new(\"name\", DataType::Utf8, false),\n    Field::new(\"price\", DataType::Float64, false),\n    Field::new(\"temp_col1\", DataType::Utf8, false),\n    Field::new(\"temp_col2\", DataType::Int32, false),\n    Field::new(\n        \"vector\",\n        DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n        false,\n    ),\n]));\nlet schema_drop_batch = RecordBatch::try_new(\n    schema_drop_schema.clone(),\n    vec![\n        Arc::new(Int64Array::from(vec![1, 2, 3])),\n        Arc::new(StringArray::from(vec![\"Laptop\", \"Smartphone\", \"Headphones\"])),\n        Arc::new(Float64Array::from(vec![1200.0, 800.0, 150.0])),\n        Arc::new(StringArray::from(vec![\"X\", \"Y\", \"Z\"])),\n        Arc::new(Int32Array::from(vec![100, 200, 300])),\n        Arc::new(\n            FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(\n                vec![\n                    Some(vec![Some(0.1_f32); 128]),\n                    Some(vec![Some(0.2_f32); 128]),\n                    Some(vec![Some(0.3_f32); 128]),\n                ],\n                128,\n            ),\n        ),\n    ],\n)\n.unwrap();\nlet schema_drop_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(\n    vec![Ok(schema_drop_batch)].into_iter(),\n    schema_drop_schema.clone(),\n));\nlet schema_drop_table = db\n    .create_table(\"schema_evolution_drop_example\", schema_drop_reader)\n    .mode(CreateTableMode::Overwrite)\n    .execute()\n    .await\n    .unwrap();\n";

export const TsSchemaDropSetup = "const schemaDropData = [\n  {\n    id: 1,\n    name: \"Laptop\",\n    price: 1200.0,\n    temp_col1: \"X\",\n    temp_col2: 100,\n    vector: Array.from({ length: 128 }, () => Math.random()),\n  },\n  {\n    id: 2,\n    name: \"Smartphone\",\n    price: 800.0,\n    temp_col1: \"Y\",\n    temp_col2: 200,\n    vector: Array.from({ length: 128 }, () => Math.random()),\n  },\n  {\n    id: 3,\n    name: \"Headphones\",\n    price: 150.0,\n    temp_col1: \"Z\",\n    temp_col2: 300,\n    vector: Array.from({ length: 128 }, () => Math.random()),\n  },\n];\nconst schemaDropTable = await db.createTable(\n  \"schema_evolution_drop_example\",\n  schemaDropData,\n  { mode: \"overwrite\" },\n);\n";

export const SchemaDropSetup = "if data is None:\n    data = [\n        {\n            \"id\": 1,\n            \"name\": \"Laptop\",\n            \"price\": 1200.00,\n            \"temp_col1\": \"X\",\n            \"temp_col2\": 100,\n            \"vector\": np.random.random(128).tolist(),\n        },\n        {\n            \"id\": 2,\n            \"name\": \"Smartphone\",\n            \"price\": 800.00,\n            \"temp_col1\": \"Y\",\n            \"temp_col2\": 200,\n            \"vector\": np.random.random(128).tolist(),\n        },\n        {\n            \"id\": 3,\n            \"name\": \"Headphones\",\n            \"price\": 150.00,\n            \"temp_col1\": \"Z\",\n            \"temp_col2\": 300,\n            \"vector\": np.random.random(128).tolist(),\n        },\n    ]\ntable = tmp_db.create_table(\"schema_evolution_drop_example\", data, mode=\"overwrite\")\n";

export const RsAlterColumnsWithExpression = "// For custom transforms, create a new column from a SQL expression.\nlet expression_schema = Arc::new(Schema::new(vec![\n    Field::new(\"id\", DataType::Int64, false),\n    Field::new(\"price_text\", DataType::Utf8, false),\n]));\nlet expression_batch = RecordBatch::try_new(\n    expression_schema.clone(),\n    vec![\n        Arc::new(Int64Array::from(vec![1])),\n        Arc::new(StringArray::from(vec![\"$100\"])),\n    ],\n)\n.unwrap();\nlet expression_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(\n    vec![Ok(expression_batch)].into_iter(),\n    expression_schema.clone(),\n));\nlet expression_table = db\n    .create_table(\"schema_evolution_expression_example\", expression_reader)\n    .mode(CreateTableMode::Overwrite)\n    .execute()\n    .await\n    .unwrap();\n\nexpression_table\n    .add_columns(\n        NewColumnTransform::SqlExpressions(vec![(\n            \"price_numeric\".to_string(),\n            \"cast(replace(price_text, '$', '') as int)\".to_string(),\n        )]),\n        None,\n    )\n    .await\n    .unwrap();\nexpression_table.drop_columns(&[\"price_text\"]).await.unwrap();\nexpression_table\n    .alter_columns(&[ColumnAlteration::new(\"price_numeric\".to_string())\n        .rename(\"price\".to_string())])\n    .await\n    .unwrap();\n";

export const RsAlterColumnsMultiple = "// Rename, change type, and make nullable in one operation\nschema_alter_table\n    .alter_columns(&[ColumnAlteration::new(\"sale_price\".to_string())\n        .rename(\"final_price\".to_string())\n        .cast_to(DataType::Float64)\n        .set_nullable(true)])\n    .await\n    .unwrap();\n";

export const TsAlterColumnsWithExpression = "// For custom transforms, create a new column from a SQL expression.\nconst expressionTable = await db.createTable(\n  \"schema_evolution_expression_example\",\n  [{ id: 1, price_text: \"$100\" }],\n  { mode: \"overwrite\" },\n);\n\nawait expressionTable.addColumns([\n  {\n    name: \"price_numeric\",\n    valueSql: \"cast(replace(price_text, '$', '') as int)\",\n  },\n]);\nawait expressionTable.dropColumns([\"price_text\"]);\nawait expressionTable.alterColumns([\n  { path: \"price_numeric\", rename: \"price\" },\n]);\n";

export const AlterColumnsWithExpression = "# For custom transforms, create a new column from a SQL expression.\nexpression_table = tmp_db.create_table(\n    \"schema_evolution_expression_example\",\n    [{\"id\": 1, \"price_text\": \"$100\"}],\n    mode=\"overwrite\",\n)\n\nexpression_table.add_columns(\n    {\"price_numeric\": \"cast(replace(price_text, '$', '') as int)\"}\n)\nexpression_table.drop_columns([\"price_text\"])\nexpression_table.alter_columns({\"path\": \"price_numeric\", \"rename\": \"price\"})\n";

export const TsAlterColumnsMultiple = "// Rename, change type, and make nullable in one operation\nawait schemaAlterTable.alterColumns([\n  {\n    path: \"sale_price\",\n    rename: \"final_price\",\n    dataType: new arrow.Float64(),\n    nullable: true,\n  },\n]);\n";

export const AlterColumnsMultiple = "# Rename, change type, and make nullable in one operation\ntable.alter_columns(\n    {\n        \"path\": \"sale_price\",\n        \"rename\": \"final_price\",\n        \"data_type\": pa.float64(),\n        \"nullable\": True,\n    }\n)\n";

export const RsAlterColumnsNullable = "// Make the name column nullable\nschema_alter_table\n    .alter_columns(&[ColumnAlteration::new(\"name\".to_string()).set_nullable(true)])\n    .await\n    .unwrap();\n";

export const TsAlterColumnsNullable = "// Make the name column nullable\nawait schemaAlterTable.alterColumns([{ path: \"name\", nullable: true }]);\n";

export const AlterColumnsNullable = "# Make the name column nullable\ntable.alter_columns({\"path\": \"name\", \"nullable\": True})\n";

export const RsAlterColumnsDataType = "// Change price from int32 to int64 for larger numbers\nschema_alter_table\n    .alter_columns(&[ColumnAlteration::new(\"price\".to_string()).cast_to(DataType::Int64)])\n    .await\n    .unwrap();\n";

export const TsAlterColumnsDataType = "// Change price from int32 to int64 for larger numbers\nawait schemaAlterTable.alterColumns([\n  { path: \"price\", dataType: new arrow.Int64() },\n]);\n";

export const AlterColumnsDataType = "# Change price from int32 to int64 for larger numbers\ntable.alter_columns({\"path\": \"price\", \"data_type\": pa.int64()})\n";

export const RsAlterColumnsRename = "// Rename discount_price to sale_price\nschema_alter_table\n    .alter_columns(&[ColumnAlteration::new(\"discount_price\".to_string())\n        .rename(\"sale_price\".to_string())])\n    .await\n    .unwrap();\n";

export const TsAlterColumnsRename = "// Rename discount_price to sale_price\nawait schemaAlterTable.alterColumns([\n  { path: \"discount_price\", rename: \"sale_price\" },\n]);\n";

export const AlterColumnsRename = "# Rename discount_price to sale_price\ntable.alter_columns({\"path\": \"discount_price\", \"rename\": \"sale_price\"})\n";

export const RsSchemaAlterSetup = "let schema_alter_schema = Arc::new(Schema::new(vec![\n    Field::new(\"id\", DataType::Int64, false),\n    Field::new(\"name\", DataType::Utf8, false),\n    Field::new(\"price\", DataType::Int32, false),\n    Field::new(\"discount_price\", DataType::Float64, false),\n    Field::new(\n        \"vector\",\n        DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n        false,\n    ),\n]));\nlet schema_alter_batch = RecordBatch::try_new(\n    schema_alter_schema.clone(),\n    vec![\n        Arc::new(Int64Array::from(vec![1, 2])),\n        Arc::new(StringArray::from(vec![\"Laptop\", \"Smartphone\"])),\n        Arc::new(Int32Array::from(vec![1200, 800])),\n        Arc::new(Float64Array::from(vec![1080.0, 720.0])),\n        Arc::new(\n            FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(\n                vec![Some(vec![Some(0.1_f32); 128]), Some(vec![Some(0.2_f32); 128])],\n                128,\n            ),\n        ),\n    ],\n)\n.unwrap();\nlet schema_alter_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(\n    vec![Ok(schema_alter_batch)].into_iter(),\n    schema_alter_schema.clone(),\n));\nlet schema_alter_table = db\n    .create_table(\"schema_evolution_alter_example\", schema_alter_reader)\n    .mode(CreateTableMode::Overwrite)\n    .execute()\n    .await\n    .unwrap();\n";

export const TsSchemaAlterSetup = "const schemaAlter = new arrow.Schema([\n  new arrow.Field(\"id\", new arrow.Int64()),\n  new arrow.Field(\"name\", new arrow.Utf8()),\n  new arrow.Field(\"price\", new arrow.Int32()),\n  new arrow.Field(\"discount_price\", new arrow.Float64()),\n  new arrow.Field(\n    \"vector\",\n    new arrow.FixedSizeList(\n      128,\n      new arrow.Field(\"item\", new arrow.Float32(), true),\n    ),\n  ),\n]);\nconst schemaAlterData = lancedb.makeArrowTable(\n  [\n    {\n      id: 1,\n      name: \"Laptop\",\n      price: 1200,\n      discount_price: 1080.0,\n      vector: Array.from({ length: 128 }, () => Math.random()),\n    },\n    {\n      id: 2,\n      name: \"Smartphone\",\n      price: 800,\n      discount_price: 720.0,\n      vector: Array.from({ length: 128 }, () => Math.random()),\n    },\n  ],\n  { schema: schemaAlter },\n);\nconst schemaAlterTable = await db.createTable(\n  \"schema_evolution_alter_example\",\n  schemaAlterData,\n  { mode: \"overwrite\" },\n);\n";

export const SchemaAlterSetup = "table_name = \"schema_evolution_alter_example\"\nif data is None:\n    data = [\n        {\n            \"id\": 1,\n            \"name\": \"Laptop\",\n            \"price\": 1200,\n            \"discount_price\": 1080.0,\n            \"vector\": np.random.random(128).tolist(),\n        },\n        {\n            \"id\": 2,\n            \"name\": \"Smartphone\",\n            \"price\": 800,\n            \"discount_price\": 720.0,\n            \"vector\": np.random.random(128).tolist(),\n        },\n    ]\nschema = pa.schema(\n    {\n        \"id\": pa.int64(),\n        \"name\": pa.string(),\n        \"price\": pa.int32(),\n        \"discount_price\": pa.float64(),\n        \"vector\": pa.list_(pa.float32(), 128),\n    }\n)\ntable = tmp_db.create_table(table_name, data, schema=schema, mode=\"overwrite\")\n";

export const RsAddFeatureColumnsSql = "schema_add_table\n    .add_columns(\n        NewColumnTransform::SqlExpressions(vec![\n            (\n                \"price_per_id\".to_string(),\n                \"cast(price / id as float)\".to_string(),\n            ),\n            (\"price_log\".to_string(), \"ln(price)\".to_string()),\n            (\n                \"price_score\".to_string(),\n                \"cast(price / (price + 100.0) as float)\".to_string(),\n            ),\n        ]),\n        None,\n    )\n    .await\n    .unwrap();\n";

export const TsAddFeatureColumnsSql = "await schemaAddTable.addColumns([\n  {\n    name: \"price_per_id\",\n    valueSql: \"cast(price / id as float)\",\n  },\n  {\n    name: \"price_log\",\n    valueSql: \"ln(price)\",\n  },\n  {\n    name: \"price_score\",\n    valueSql: \"cast(price / (price + 100.0) as float)\",\n  },\n]);\n";

export const AddFeatureColumnsSql = "table.add_columns(\n    {\n        \"price_per_id\": \"cast(price / id as float)\",\n        \"price_log\": \"ln(price)\",\n        \"price_score\": \"cast(price / (price + 100.0) as float)\",\n    }\n)\n";

export const RsAddColumnsNullable = "// Add a nullable timestamp column\nschema_add_table\n    .add_columns(\n        NewColumnTransform::SqlExpressions(vec![(\n            \"last_ordered\".to_string(),\n            \"cast(NULL as timestamp)\".to_string(),\n        )]),\n        None,\n    )\n    .await\n    .unwrap();\n";

export const TsAddColumnsNullable = "// Add a nullable timestamp column\nawait schemaAddTable.addColumns([\n  {\n    name: \"last_ordered\",\n    valueSql: \"cast(NULL as timestamp)\",\n  },\n]);\n";

export const AddColumnsNullable = "# Add a nullable timestamp column\ntable.add_columns({\"last_ordered\": \"cast(NULL as timestamp)\"})\n";

export const RsAddColumnsDefaultValues = "// Add a stock status column with default value\nschema_add_table\n    .add_columns(\n        NewColumnTransform::SqlExpressions(vec![(\n            \"in_stock\".to_string(),\n            \"cast(true as boolean)\".to_string(),\n        )]),\n        None,\n    )\n    .await\n    .unwrap();\n";

export const TsAddColumnsDefaultValues = "// Add a stock status column with default value\nawait schemaAddTable.addColumns([\n  {\n    name: \"in_stock\",\n    valueSql: \"cast(true as boolean)\",\n  },\n]);\n";

export const AddColumnsDefaultValues = "# Add a stock status column with default value\ntable.add_columns({\"in_stock\": \"cast(true as boolean)\"})\n";

export const RsAddColumnsCalculated = "// Add a discounted price column (10% discount)\nschema_add_table\n    .add_columns(\n        NewColumnTransform::SqlExpressions(vec![(\n            \"discounted_price\".to_string(),\n            \"cast((price * 0.9) as float)\".to_string(),\n        )]),\n        None,\n    )\n    .await\n    .unwrap();\n";

export const TsAddColumnsCalculated = "// Add a discounted price column (10% discount)\nawait schemaAddTable.addColumns([\n  {\n    name: \"discounted_price\",\n    valueSql: \"cast((price * 0.9) as float)\",\n  },\n]);\n";

export const AddColumnsCalculated = "# Add a discounted price column (10% discount)\ntable.add_columns({\"discounted_price\": \"cast((price * 0.9) as float)\"})\n";

export const RsSchemaAddSetup = "let schema_add_schema = Arc::new(Schema::new(vec![\n    Field::new(\"id\", DataType::Int64, false),\n    Field::new(\"name\", DataType::Utf8, false),\n    Field::new(\"price\", DataType::Float64, false),\n    Field::new(\n        \"vector\",\n        DataType::FixedSizeList(Arc::new(Field::new(\"item\", DataType::Float32, true)), 128),\n        false,\n    ),\n]));\nlet schema_add_batch = RecordBatch::try_new(\n    schema_add_schema.clone(),\n    vec![\n        Arc::new(Int64Array::from(vec![1, 2, 3])),\n        Arc::new(StringArray::from(vec![\"Laptop\", \"Smartphone\", \"Headphones\"])),\n        Arc::new(Float64Array::from(vec![1200.0, 800.0, 150.0])),\n        Arc::new(\n            FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(\n                vec![\n                    Some(vec![Some(0.1_f32); 128]),\n                    Some(vec![Some(0.2_f32); 128]),\n                    Some(vec![Some(0.3_f32); 128]),\n                ],\n                128,\n            ),\n        ),\n    ],\n)\n.unwrap();\nlet schema_add_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(\n    vec![Ok(schema_add_batch)].into_iter(),\n    schema_add_schema.clone(),\n));\nlet schema_add_table = db\n    .create_table(\"schema_evolution_add_example\", schema_add_reader)\n    .mode(CreateTableMode::Overwrite)\n    .execute()\n    .await\n    .unwrap();\n";

export const TsSchemaAddSetup = "const schemaAddData = [\n  {\n    id: 1,\n    name: \"Laptop\",\n    price: 1200.0,\n    vector: Array.from({ length: 128 }, () => Math.random()),\n  },\n  {\n    id: 2,\n    name: \"Smartphone\",\n    price: 800.0,\n    vector: Array.from({ length: 128 }, () => Math.random()),\n  },\n  {\n    id: 3,\n    name: \"Headphones\",\n    price: 150.0,\n    vector: Array.from({ length: 128 }, () => Math.random()),\n  },\n];\nconst schemaAddTable = await db.createTable(\n  \"schema_evolution_add_example\",\n  schemaAddData,\n  { mode: \"overwrite\" },\n);\n";

export const SchemaAddSetup = "table_name = \"schema_evolution_add_example\"\nif data is None:\n    data = [\n        {\n            \"id\": 1,\n            \"name\": \"Laptop\",\n            \"price\": 1200.00,\n            \"vector\": np.random.random(128).tolist(),\n        },\n        {\n            \"id\": 2,\n            \"name\": \"Smartphone\",\n            \"price\": 800.00,\n            \"vector\": np.random.random(128).tolist(),\n        },\n        {\n            \"id\": 3,\n            \"name\": \"Headphones\",\n            \"price\": 150.00,\n            \"vector\": np.random.random(128).tolist(),\n        },\n    ]\ntable = tmp_db.create_table(table_name, data, mode=\"overwrite\")\n";

Schema evolution enables non-breaking modifications to a database table's structure — such as adding columns, altering data types, or dropping fields — to adapt to evolving data requirements without service interruptions.
LanceDB supports ACID-compliant schema evolution through granular operations (add/alter/drop columns), allowing you to:

* Iterate Safely: Modify schemas in production with versioned datasets and backward compatibility
* Scale Seamlessly: Handle ML model iterations, regulatory changes, or feature additions
* Optimize Continuously: Remove unused fields or enforce new constraints without downtime

## Schema evolution operations

LanceDB supports three primary schema evolution operations:

1. **Adding new columns**: Extend your table with additional attributes
2. **Altering existing columns**: Change column names, data types, or nullability
3. **Dropping columns**: Remove unnecessary columns from your schema

<Tip title="Schema Evolution Performance">
  Schema evolution operations are applied immediately but do not typically require rewriting all data. However, data type changes may involve more substantial operations.
</Tip>

Each schema evolution operation commits a new table version and returns status metadata such as
the committed `version`. Run these operations from a mutable table handle; if you checked out an
older version for reads, call `checkout_latest` / `checkoutLatest` before modifying the schema.

## Add new columns

You can add new columns to a table with the [`add_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.add_columns)
method in Python, [`addColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#addcolumns) in TypeScript/JavaScript, or `add_columns` in Rust.
New columns are populated based on SQL expressions you provide.

### Set up the example table

First, let's create a sample table with product data to demonstrate schema evolution:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {SchemaAddSetup}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsSchemaAddSetup}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsSchemaAddSetup}
  </CodeBlock>
</CodeGroup>

### Add derived columns

You can add new columns that are derived from existing data using SQL expressions.
For feature engineering on large existing tables, group related derived features into
one `add_columns` operation instead of running many separate writes. This creates one
new table version for the schema change and computes the new columns from the existing
rows, which avoids growing the table's version history with many small updates.

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AddColumnsCalculated}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAddColumnsCalculated}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAddColumnsCalculated}
  </CodeBlock>
</CodeGroup>

The same call can add multiple derived columns at once. For example, if you are
building several lightweight features from existing product fields, pass all of the
new column expressions together:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AddFeatureColumnsSql}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAddFeatureColumnsSql}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAddFeatureColumnsSql}
  </CodeBlock>
</CodeGroup>

<Info>
  LanceDB `add_columns` does not currently accept Python callables, batch UDFs, or
  PyArrow `RecordBatch` iterators for populating new columns. New column values must be
  defined with SQL expressions, or added as NULL columns from an Arrow field or schema.
  If your transformation cannot be expressed in SQL, compute the values outside
  `add_columns` before writing them back through another workflow.
</Info>

### Add columns with default values

Add boolean columns with default values for status tracking:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AddColumnsDefaultValues}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAddColumnsDefaultValues}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAddColumnsDefaultValues}
  </CodeBlock>
</CodeGroup>

### Add nullable columns

Add timestamp columns that can contain NULL values:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AddColumnsNullable}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAddColumnsNullable}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAddColumnsNullable}
  </CodeBlock>
</CodeGroup>

<Warning title="NULL Values in New Columns">
  When adding columns that should contain NULL values, be sure to cast the NULL to the appropriate type, e.g., `cast(NULL as timestamp)`.
</Warning>

## Alter existing columns

You can alter columns using the [`alter_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.alter_columns)
method in Python, [`alterColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#altercolumns) in TypeScript/JavaScript, or `alter_columns` in Rust. This allows you to:

* Rename a column
* Change a column's data type
* Modify nullability (whether a column can contain NULL values)

### Set up the example table

Create a table with a custom schema to demonstrate column alterations:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {SchemaAlterSetup}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsSchemaAlterSetup}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsSchemaAlterSetup}
  </CodeBlock>
</CodeGroup>

### Rename columns

Change column names to better reflect their purpose:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AlterColumnsRename}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAlterColumnsRename}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAlterColumnsRename}
  </CodeBlock>
</CodeGroup>

### Change data types

Convert column data types for better performance or compatibility:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AlterColumnsDataType}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAlterColumnsDataType}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAlterColumnsDataType}
  </CodeBlock>
</CodeGroup>

### Make columns nullable

You can alter columns to contain NULL values:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AlterColumnsNullable}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAlterColumnsNullable}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAlterColumnsNullable}
  </CodeBlock>
</CodeGroup>

Changing a column to nullable affects future writes and merges too: missing values are accepted
only when the target column is nullable.

### Multiple changes at once

Apply several alterations in a single operation:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AlterColumnsMultiple}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAlterColumnsMultiple}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAlterColumnsMultiple}
  </CodeBlock>
</CodeGroup>

### Expression-based type changes

For transformations that are not simple casts (for example, converting `"$100"` to an integer), use a SQL-expression column add, then drop and rename:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AlterColumnsWithExpression}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAlterColumnsWithExpression}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAlterColumnsWithExpression}
  </CodeBlock>
</CodeGroup>

### Alter embedding types and dimensions

It's quite common to need to change an embedding column's schema, in case a new model becomes available with a different embedding dimension.

* In Python, the example shows an in-place type update when the cast is compatible.
* In TypeScript and Rust, the example shows a dimension change (`384 -> 1024`), which cannot be cast in-place.

For dimension changes, use this 3-step pattern: add a new column with the target type, drop the old column, then rename the new column to the original name.

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {AlterVectorColumn}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsAlterVectorColumn}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsAlterVectorColumn}
  </CodeBlock>
</CodeGroup>

<Warning>
  **`FixedSizeList` Dimension Changes in TypeScript and Rust**

  `alterColumns` / `alter_columns` can cast between compatible types, but changing `FixedSizeList` dimensions (for example `384 -> 1024`) is not a compatible cast.
  For such cases, use `addColumns` / `add_columns` (with `arrow_cast`), then `dropColumns` / `drop_columns`, then rename the replacement column.
</Warning>

<Warning title="Data Type Changes">
  Changing data types requires rewriting the column data and may be resource-intensive for large tables. Renaming columns or changing nullability is more efficient as it only updates metadata.
</Warning>

## Drop columns

You can remove columns using the [`drop_columns`](https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.drop_columns)
method in Python, [`dropColumns`](https://lancedb.github.io/lancedb/js/classes/Table/#dropcolumns) in TypeScript/JavaScript, or `drop_columns` in Rust.

### Set Up the example table

Create a table with temporary columns that we'll remove:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {SchemaDropSetup}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsSchemaDropSetup}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsSchemaDropSetup}
  </CodeBlock>
</CodeGroup>

### Drop single columns

Remove individual columns that are no longer needed:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {DropColumnsSingle}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsDropColumnsSingle}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsDropColumnsSingle}
  </CodeBlock>
</CodeGroup>

### Drop multiple columns

Remove several columns at once for efficiency:

<CodeGroup>
  <CodeBlock filename="Python" language="Python" icon="python">
    {DropColumnsMultiple}
  </CodeBlock>

  <CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
    {TsDropColumnsMultiple}
  </CodeBlock>

  <CodeBlock filename="Rust" language="Rust" icon="rust">
    {RsDropColumnsMultiple}
  </CodeBlock>
</CodeGroup>

<Warning title="Irreversible Column Deletion">
  Dropping columns cannot be undone. Make sure you have backups or are certain before removing columns.
</Warning>