Skip to content

Commit 4e51e4c

Browse files
mrdrivingduckclaude
andcommitted
feat(write): support INSERT INTO for Paimon tables
Implement PlanInsert in the Paimon catalog to enable appending data to existing Paimon tables via INSERT INTO ... SELECT. The implementation reuses the existing parallel sink operator in insert-only mode (no table creation), supporting append-only tables. Also fix a bug where primary key constraints were read from the empty BoundCreateTableInfo.constraints instead of the populated CreateTableInfo.constraints, causing PRIMARY KEY clauses to be silently ignored during table creation. Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 959ddf2 commit 4e51e4c

4 files changed

Lines changed: 124 additions & 5 deletions

File tree

README.md

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ This extension is built on top of [paimon-cpp](https://github.com/alibaba/paimon
2020
## Features
2121

2222
- Read Paimon table data (local and remote OSS)
23+
- Write Paimon tables (CREATE TABLE AS, INSERT INTO; append-only tables only)
24+
- DDL support (CREATE/DROP SCHEMA, CREATE/DROP TABLE)
2325
- Projection pushdown optimization
2426
- Predicate pushdown optimization
2527
- Multiple file format support (manifest / data)
@@ -151,6 +153,35 @@ SELECT * FROM my_catalog.testdb.testtbl;
151153
-- ATTACH 'oss://my-bucket/warehouse' AS my_catalog (TYPE paimon);
152154
```
153155

156+
### Write Data
157+
158+
With an ATTACHed catalog, you can create schemas, create tables, and insert data using standard DuckDB SQL:
159+
160+
```sql
161+
ATTACH './data' AS my_catalog (TYPE paimon);
162+
163+
CREATE SCHEMA my_catalog.my_db;
164+
165+
CREATE TABLE my_catalog.my_db.orders AS
166+
SELECT 1 AS order_id, 99.9::DECIMAL(18,2) AS amount, 'Alice' AS customer;
167+
168+
INSERT INTO my_catalog.my_db.orders
169+
SELECT 2, 49.5, 'Bob'
170+
UNION ALL
171+
SELECT 3, 150.0, 'Charlie';
172+
173+
SELECT * FROM my_catalog.my_db.orders ORDER BY order_id;
174+
175+
DROP TABLE my_catalog.my_db.orders;
176+
DROP SCHEMA my_catalog.my_db;
177+
```
178+
179+
To prevent accidental writes, attach the catalog in read-only mode:
180+
181+
```sql
182+
ATTACH './data' AS my_catalog (TYPE paimon, READ_ONLY);
183+
```
184+
154185
### Inspect Snapshot History
155186

156187
Use `paimon_snapshots` to list all snapshots of a Paimon table — useful for auditing commit history, diagnosing data issues, or identifying a snapshot ID for time-travel queries:

src/paimon_storage/paimon_catalog.cpp

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
*/
2424

2525
#include "duckdb/catalog/catalog.hpp"
26+
#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
2627
#include "duckdb/common/string_util.hpp"
2728
#include "duckdb/common/types/timestamp.hpp"
2829
#include "duckdb/main/secret/secret_manager.hpp"
@@ -31,6 +32,7 @@
3132
#include "duckdb/parser/parsed_data/attach_info.hpp"
3233
#include "duckdb/parser/parsed_data/create_table_info.hpp"
3334
#include "duckdb/planner/operator/logical_create_table.hpp"
35+
#include "duckdb/planner/operator/logical_insert.hpp"
3436
#include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
3537

3638
#include "paimon/catalog/catalog.h"
@@ -240,9 +242,26 @@ PhysicalOperator &PaimonCatalog::PlanCreateTableAs(ClientContext &context, Physi
240242
return insert;
241243
}
242244

243-
PhysicalOperator &PaimonCatalog::PlanInsert(ClientContext &, PhysicalPlanGenerator &, LogicalInsert &,
244-
optional_ptr<PhysicalOperator>) {
245-
throw NotImplementedException("PlanInsert not supported yet");
245+
PhysicalOperator &PaimonCatalog::PlanInsert(ClientContext &context, PhysicalPlanGenerator &planner, LogicalInsert &op,
246+
optional_ptr<PhysicalOperator> plan) {
247+
if (access_mode == AccessMode::READ_ONLY) {
248+
throw PermissionException("Cannot write to a read-only Paimon catalog");
249+
}
250+
251+
auto &table = op.table;
252+
auto table_path = path + "/" + table.schema.name + paimon::Catalog::DB_SUFFIX + "/" + table.name;
253+
auto paimon_options = GetPaimonOptions(context, path, attached_options);
254+
255+
if (plan && !op.column_index_map.empty()) {
256+
plan = planner.ResolveDefaultsProjection(op, *plan);
257+
}
258+
259+
auto &insert = planner.Make<PhysicalPaimonInsert>(op, table.schema, nullptr, std::move(table_path),
260+
std::move(paimon_options), 0U);
261+
if (plan) {
262+
insert.children.push_back(*plan);
263+
}
264+
return insert;
246265
}
247266

248267
PhysicalOperator &PaimonCatalog::PlanDelete(ClientContext &, PhysicalPlanGenerator &, LogicalDelete &,

src/paimon_storage/paimon_schema_entry.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ optional_ptr<CatalogEntry> PaimonSchemaEntry::CreateTable(CatalogTransaction tra
111111
} arrow_guard {arrow_schema};
112112

113113
vector<string> primary_keys;
114-
for (auto &constraint : info.constraints) {
114+
for (auto &constraint : base.constraints) {
115115
if (constraint->type != ConstraintType::UNIQUE) {
116116
continue;
117117
}

test/sql/paimon_write.test

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,71 @@ SELECT min(id), max(id) FROM pm.__test_write.t_ctas_large;
7171
----
7272
1 100000
7373

74-
# --- CTAS into read-only catalog should fail ---
74+
# --- INSERT INTO existing table ---
75+
76+
statement ok
77+
INSERT INTO pm.__test_write.t_ctas_small
78+
SELECT * FROM (VALUES
79+
(4, 400000000000::BIGINT, 'duck', 6.28::FLOAT, 1.414213562::DOUBLE, FALSE, '2025-12-25'::DATE, '2025-12-25 00:00:00'::TIMESTAMP),
80+
(5, 500000000000::BIGINT, 'paimon', 9.81::FLOAT, 2.236067977::DOUBLE, TRUE, '2025-07-04'::DATE, '2025-07-04 12:00:00'::TIMESTAMP)
81+
) t(id, big_id, name, score_f, score_d, flag, dt, ts);
82+
83+
query I
84+
SELECT count(*) FROM pm.__test_write.t_ctas_small;
85+
----
86+
5
87+
88+
query IITRRTTT
89+
SELECT * FROM pm.__test_write.t_ctas_small ORDER BY id;
90+
----
91+
1 100000000000 hello 3.14 2.718281828 true 2025-01-15 2025-01-15 10:30:00
92+
2 200000000000 world 1.41 3.141592653 false 2025-06-30 2025-06-30 23:59:59
93+
3 NULL NULL NULL NULL true NULL NULL
94+
4 400000000000 duck 6.28 1.414213562 false 2025-12-25 2025-12-25 00:00:00
95+
5 500000000000 paimon 9.81 2.236067977 true 2025-07-04 2025-07-04 12:00:00
96+
97+
# --- INSERT INTO with large dataset ---
98+
99+
statement ok
100+
INSERT INTO pm.__test_write.t_ctas_large
101+
SELECT
102+
i AS id,
103+
i * 1000000000::BIGINT AS big_id,
104+
'ins_' || i::VARCHAR AS name,
105+
(i * 0.2)::FLOAT AS score_f,
106+
i * 0.002 AS score_d,
107+
(i % 2 = 1) AS flag,
108+
('2021-01-01'::DATE + (i % 500)::INTEGER)::DATE AS dt,
109+
('2021-01-01 00:00:00'::TIMESTAMP + INTERVAL (i) SECOND) AS ts
110+
FROM generate_series(100001, 150000) t(i);
111+
112+
query I
113+
SELECT count(*) FROM pm.__test_write.t_ctas_large;
114+
----
115+
150000
116+
117+
query II
118+
SELECT min(id), max(id) FROM pm.__test_write.t_ctas_large;
119+
----
120+
1 150000
121+
122+
# --- INSERT INTO with reordered / partial columns ---
123+
124+
statement ok
125+
INSERT INTO pm.__test_write.t_ctas_small (name, id, flag, big_id, score_f, score_d, dt, ts)
126+
VALUES ('reordered', 6, FALSE, 600000000000, 2.22, 0.123456789, '2025-03-15', '2025-03-15 08:00:00');
127+
128+
statement ok
129+
INSERT INTO pm.__test_write.t_ctas_small (id, name)
130+
VALUES (7, 'partial');
131+
132+
query IITRRTTT
133+
SELECT * FROM pm.__test_write.t_ctas_small WHERE id >= 6 ORDER BY id;
134+
----
135+
6 600000000000 reordered 2.22 0.123456789 false 2025-03-15 2025-03-15 08:00:00
136+
7 NULL partial NULL NULL NULL NULL NULL
137+
138+
# --- INSERT/CTAS into read-only catalog should fail ---
75139

76140
statement ok
77141
DETACH pm;
@@ -84,6 +148,11 @@ CREATE TABLE pm_ro.__test_write.t_fail AS SELECT 1 AS id;
84148
----
85149
read-only
86150

151+
statement error
152+
INSERT INTO pm_ro.__test_write.t_ctas_small SELECT 1, 1::BIGINT, 'x', 1.0::FLOAT, 1.0, true, '2025-01-01'::DATE, '2025-01-01 00:00:00'::TIMESTAMP;
153+
----
154+
read-only
155+
87156
statement ok
88157
DETACH pm_ro;
89158

0 commit comments

Comments
 (0)