Skip to content

feat(ext/node): (partial) impl sqlite 'backup' function #29842

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ rand = "=0.8.5"
rayon = "1.8.0"
regex = "^1.7.0"
reqwest = { version = "=0.12.5", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json", "http2"] } # pinned because of https://github.com/seanmonstar/reqwest/pull/1955
rusqlite = { version = "0.34.0", features = ["unlock_notify", "bundled", "session", "modern_sqlite", "limits"] } # "modern_sqlite": need sqlite >= 3.49.0 for some db configs
rusqlite = { version = "0.34.0", features = ["unlock_notify", "bundled", "session", "modern_sqlite", "limits", "backup"] } # "modern_sqlite": need sqlite >= 3.49.0 for some db configs
rustls = { version = "0.23.11", default-features = false, features = ["logging", "std", "tls12", "aws_lc_rs"] }
rustls-pemfile = "2"
rustls-tokio-stream = "=0.5.0"
Expand Down
2 changes: 2 additions & 0 deletions ext/node/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,8 @@ deno_core::extension!(deno_node,
ops::inspector::op_inspector_disconnect,
ops::inspector::op_inspector_emit_protocol_event,
ops::inspector::op_inspector_enabled,

ops::sqlite::op_node_database_backup,
],
objects = [
ops::perf_hooks::EldHistogram,
Expand Down
41 changes: 41 additions & 0 deletions ext/node/ops/sqlite/backup.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Copyright 2018-2025 the Deno authors. MIT license.

use std::ffi::c_int;
use std::time;

use deno_core::op2;
use rusqlite::backup;
use rusqlite::Connection;
use serde::Deserialize;
use serde::Serialize;

use super::DatabaseSync;
use super::SqliteError;

const DEFAULT_BACKUP_RATE: c_int = 5;

#[derive(Serialize, Deserialize)]
struct BackupOptions {
source: Option<String>,
target: Option<String>,
rate: Option<c_int>,
// progress: fn(backup::Progress),
}

#[op2]
#[serde]
pub fn op_node_database_backup(
#[cppgc] source_db: &DatabaseSync,
#[string] path: String,
#[serde] options: Option<BackupOptions>,
) -> std::result::Result<(), SqliteError> {
let src_conn_ref = source_db.conn.borrow();
let src_conn = src_conn_ref.as_ref().ok_or(SqliteError::SessionClosed)?;
let path = std::path::Path::new(&path);
let mut dst_conn = Connection::open(path)?;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we'll need to add write permission checks here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

judging by the signature

function backup(
  sourceDb: DatabaseSync,
  path: string | Buffer | URL,
  options?: BackupOptions,
): Promise<void>;

Since path can have different forms, we'll need to check diff permissions accordingly.

let rate = options
.and_then(|opts| opts.rate)
.unwrap_or(DEFAULT_BACKUP_RATE);
let backup = backup::Backup::new(src_conn, &mut dst_conn)?;
Ok(backup.run_to_completion(rate, time::Duration::from_millis(250), None)?)
}
2 changes: 1 addition & 1 deletion ext/node/ops/sqlite/database.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ impl<'a> ApplyChangesetOptions<'a> {
}

pub struct DatabaseSync {
conn: Rc<RefCell<Option<rusqlite::Connection>>>,
pub conn: Rc<RefCell<Option<rusqlite::Connection>>>,
statements: Rc<RefCell<Vec<*mut libsqlite3_sys::sqlite3_stmt>>>,
options: DatabaseSyncOptions,
location: String,
Expand Down
2 changes: 2 additions & 0 deletions ext/node/ops/sqlite/mod.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
// Copyright 2018-2025 the Deno authors. MIT license.

mod backup;
mod database;
mod session;
mod statement;
mod validators;

pub use backup::op_node_database_backup;
pub use database::DatabaseSync;
pub use session::Session;
pub use statement::StatementSync;
Expand Down
79 changes: 77 additions & 2 deletions ext/node/polyfills/sqlite.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,80 @@
// Copyright 2018-2025 the Deno authors. MIT license.

import { DatabaseSync, StatementSync } from "ext:core/ops";
import {
DatabaseSync,
op_node_database_backup,
StatementSync,
} from "ext:core/ops";

import { Buffer } from "node:buffer";

interface BackupOptions {
/**
* Name of the source database. This can be `'main'` (the default primary database) or any other
* database that have been added with [`ATTACH DATABASE`](https://www.sqlite.org/lang_attach.html)
* @default 'main'
*/
source?: string | undefined;
/**
* Name of the target database. This can be `'main'` (the default primary database) or any other
* database that have been added with [`ATTACH DATABASE`](https://www.sqlite.org/lang_attach.html)
* @default 'main'
*/
target?: string | undefined;
/**
* Number of pages to be transmitted in each batch of the backup.
* @default 100
*/
rate?: number | undefined;
/**
* Callback function that will be called with the number of pages copied and the total number of
* pages.
*/
progress?: ((progressInfo: BackupProgressInfo) => void) | undefined;
}

interface BackupProgressInfo {
totalPages: number;
remainingPages: number;
}

/**
* This method makes a database backup. This method abstracts the
* [`sqlite3_backup_init()`](https://www.sqlite.org/c3ref/backup_finish.html#sqlite3backupinit),
* [`sqlite3_backup_step()`](https://www.sqlite.org/c3ref/backup_finish.html#sqlite3backupstep)
* and [`sqlite3_backup_finish()`](https://www.sqlite.org/c3ref/backup_finish.html#sqlite3backupfinish) functions.
*
* The backed-up database can be used normally during the backup process. Mutations coming from the same connection - same
* `DatabaseSync` - object will be reflected in the backup right away. However, mutations from other connections will cause
* the backup process to restart.
*
* ```js
* import { backup, DatabaseSync } from 'node:sqlite';
*
* const sourceDb = new DatabaseSync('source.db');
* const totalPagesTransferred = await backup(sourceDb, 'backup.db', {
* rate: 1, // Copy one page at a time.
* progress: ({ totalPages, remainingPages }) => {
* console.log('Backup in progress', { totalPages, remainingPages });
* },
* });
*
* console.log('Backup completed', totalPagesTransferred);
* ```
* @param sourceDb The database to backup. The source database must be open.
* @param path The path where the backup will be created. If the file already exists,
* the contents will be overwritten.
* @param options Optional configuration for the backup. The
* following properties are supported:
* @returns A promise that resolves when the backup is completed and rejects if an error occurs.
*/
function backup(
sourceDb: DatabaseSync,
path: string | Buffer | URL,
options?: BackupOptions,
) {
op_node_database_backup(sourceDb, path, options);
}

export const constants = {
SQLITE_CHANGESET_OMIT: 0,
Expand All @@ -14,9 +88,10 @@ export const constants = {
SQLITE_CHANGESET_FOREIGN_KEY: 5,
};

export { DatabaseSync, StatementSync };
export { backup, DatabaseSync, StatementSync };

export default {
backup,
constants,
DatabaseSync,
StatementSync,
Expand Down
Loading