Merge pull request #166 from jump-cellpainting/sqlite-export

feat: Add SQLite export script and documentation

Author: shntnu

.gitignore

@@ -135,3 +135,4 @@ dmypy.json
 CLAUDE.md
 .mcp.json
 *.duckdb
+*.sqlite

.markdownlint.yaml

@@ -0,0 +1,9 @@
+# Markdown style configuration
+MD007:
+  indent: 4          # List indent
+MD013: false         # Line length
+MD024: false         # Multiple headers with same content
+MD029:
+  style: ordered     # Ordered list style
+MD033: false         # Inline HTML
+MD046: false         # Code block style

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+  - id: trailing-whitespace
+    exclude: \.dvc$|dvc\.lock|\.local\.json$
+  - id: check-added-large-files
+    args: [--maxkb=10240]
+  - id: check-yaml
+  - id: end-of-file-fixer
+    exclude: \.dvc$|dvc\.lock|\.local\.json$
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.9.1
+  hooks:
+  - id: ruff
+    args: [--fix]
+  - id: ruff-format

.zenodo.json

@@ -2,12 +2,12 @@
     "creators": [
         {
             "name": "The JUMP Cell Painting Consortium"
-            
+
         }
     ],
     "title": "The Joint Undertaking for Morphological Profiling (JUMP) Consortium Datasets",
-    "upload_type": "dataset", 
-    "access_right": "open", 
+    "upload_type": "dataset",
+    "access_right": "open",
     "related_identifiers": [
         {
             "scheme": "doi",
@@ -21,6 +21,6 @@
             "relation": "describes",
             "resource_type": "publication-preprint"
         }
-        
+
         ]
 }

LICENCE.md

@@ -26,4 +26,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

metadata/README.md

@@ -14,7 +14,7 @@ erDiagram
         string Metadata_JCP2022 FK
         string Metadata_Source
     }
-    
+
     PLATE }|--|| MICROSCOPE-CONFIG : "imaged with"
     PLATE }|--|| CELLPROFILER-VERSION : "analyzed with"
     PLATE {
@@ -23,58 +23,58 @@ erDiagram
         string Metadata_Batch
         string Metadata_PlateType
     }
-    
+
     PERTURBATION ||--o{ COMPOUND : "if compound"
     PERTURBATION ||--o{ ORF : "if ORF"
     PERTURBATION ||--o{ CRISPR : "if CRISPR"
     PERTURBATION {
         string Metadata_JCP2022 PK
         string Metadata_perturbation_modality "compound/orf/crispr/unknown"
     }
-    
+
     PERTURBATION-CONTROL }o--|| PERTURBATION : "describes"
     PERTURBATION-CONTROL {
         string Metadata_JCP2022 PK,FK
         string Metadata_pert_type "poscon/negcon/empty"
         string Metadata_Name "Human-readable name"
     }
-    
+
     COMPOUND ||--o{ COMPOUND-SOURCE : "sourced from"
     COMPOUND {
         string Metadata_JCP2022 PK
         string Metadata_InChIKey
         string Metadata_SMILES
     }
-    
+
     COMPOUND-SOURCE {
         string Metadata_JCP2022 PK,FK
         string Metadata_Compound_Source PK
     }
-    
+
     ORF {
         string Metadata_JCP2022 PK
         string Metadata_Symbol "Gene symbol"
         string Metadata_NCBI_Gene_ID
     }
-    
+
     CRISPR {
         string Metadata_JCP2022 PK
         string Metadata_Symbol "Gene symbol"
         string Metadata_NCBI_Gene_ID
     }
-    
+
     MICROSCOPE-CONFIG }o--|| MICROSCOPE-FILTER : "uses"
     MICROSCOPE-CONFIG {
         string Metadata_Source PK
         string Metadata_Microscope_Name
         string Metadata_Filter_Configuration FK
     }
-    
+
     MICROSCOPE-FILTER {
         string Metadata_Filter_Configuration PK
         string wavelength_configs "DNA/ER/RNA/AGP/Mito channels"
     }
-    
+
     CELLPROFILER-VERSION {
         string Metadata_Source PK
         string Metadata_CellProfiler_Version
@@ -101,6 +101,14 @@ This creates a database with:
 - All CSV data imported as tables with data validation
 - Documentation for all tables and columns embedded in the schema
 
+## SQLite Export
+
+To export the DuckDB database to SQLite format:
+
+```bash
+bash db/export_sqlite.sh
+```
+
 ## Querying the Database
 
 ```bash
@@ -121,13 +129,13 @@ Full schema documentation is embedded in the database. To view:
 SELECT table_name, comment FROM duckdb_tables();
 
 -- View column descriptions
-SELECT table_name, column_name, comment 
-FROM duckdb_columns() 
+SELECT table_name, column_name, comment
+FROM duckdb_columns()
 WHERE comment IS NOT NULL;
 
 -- View all foreign key relationships
-SELECT table_name, constraint_text 
-FROM duckdb_constraints() 
+SELECT table_name, constraint_text
+FROM duckdb_constraints()
 WHERE constraint_type = 'FOREIGN KEY';
 ```
 
@@ -136,7 +144,7 @@ WHERE constraint_type = 'FOREIGN KEY';
 When adding or modifying tables:
 
 1. **Add data file**: Use `.csv` for small tables (<1MB) or `.csv.gz` for larger ones
-2. **Update `db/setup.sql`**: 
+2. **Update `db/setup.sql`**:
    - Define table with PRIMARY KEY and FOREIGN KEY constraints
    - Add COMMENT statements for table and columns
    - Update the import section with correct file extension

metadata/db/export_sqlite.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Export DuckDB database to SQLite format
+# Usage: bash db/export_sqlite.sh
+#
+# This script exports the DuckDB database to SQLite format rather than maintaining
+# a separate SQLite setup script. DuckDB is the primary interaction mode, and this
+# export approach ensures schema consistency between formats while avoiding dual
+# maintenance of separate setup scripts.
+
+set -euo pipefail
+
+DUCKDB_FILE="db/jump_metadata.duckdb"
+SQLITE_FILE="db/jump_metadata.sqlite"
+
+if [[ ! -f "$DUCKDB_FILE" ]]; then
+    echo "Error: DuckDB file $DUCKDB_FILE not found. Run setup first:"
+    echo "  duckdb $DUCKDB_FILE < db/setup.sql"
+    exit 1
+fi
+
+echo "Exporting DuckDB to SQLite..."
+
+duckdb "$DUCKDB_FILE" -c "
+ATTACH '$SQLITE_FILE' AS sqlite_db (TYPE SQLITE);
+CREATE TABLE sqlite_db.microscope_filter AS SELECT * FROM microscope_filter;
+CREATE TABLE sqlite_db.microscope_config AS SELECT * FROM microscope_config;
+CREATE TABLE sqlite_db.cellprofiler_version AS SELECT * FROM cellprofiler_version;
+CREATE TABLE sqlite_db.compound AS SELECT * FROM compound;
+CREATE TABLE sqlite_db.orf AS SELECT * FROM orf;
+CREATE TABLE sqlite_db.crispr AS SELECT * FROM crispr;
+CREATE TABLE sqlite_db.perturbation AS SELECT * FROM perturbation;
+CREATE TABLE sqlite_db.perturbation_control AS SELECT * FROM perturbation_control;
+CREATE TABLE sqlite_db.plate AS SELECT * FROM plate;
+CREATE TABLE sqlite_db.well AS SELECT * FROM well;
+CREATE TABLE sqlite_db.compound_source AS SELECT * FROM compound_source;
+"
+
+echo "SQLite database created: $SQLITE_FILE"

metadata/db/setup.sql

@@ -236,7 +236,7 @@ INSERT INTO orf SELECT * FROM read_csv_auto('orf.csv.gz');
 INSERT INTO crispr SELECT * FROM read_csv_auto('crispr.csv.gz');
 
 -- Populate the perturbation union table
-INSERT INTO perturbation 
+INSERT INTO perturbation
 SELECT Metadata_JCP2022, 'compound' FROM compound
 UNION ALL
 SELECT Metadata_JCP2022, 'orf' FROM orf
@@ -248,14 +248,14 @@ INSERT INTO perturbation VALUES ('JCP2022_UNKNOWN', 'unknown');
 
 -- Load perturbation control information
 INSERT INTO perturbation_control (Metadata_JCP2022, Metadata_pert_type, Metadata_Name)
-SELECT Metadata_JCP2022, Metadata_pert_type, Metadata_Name 
+SELECT Metadata_JCP2022, Metadata_pert_type, Metadata_Name
 FROM read_csv_auto('perturbation_control.csv');
 
 -- Then load tables with foreign keys
 INSERT INTO plate SELECT * FROM read_csv_auto('plate.csv.gz');
 
 -- Only insert compound_source records that have matching compounds (deduplicated)
-INSERT INTO compound_source 
+INSERT INTO compound_source
 SELECT DISTINCT cs.* FROM read_csv_auto('compound_source.csv.gz') cs
 WHERE cs.Metadata_JCP2022 IN (SELECT Metadata_JCP2022 FROM compound);
 
@@ -267,4 +267,4 @@ INSERT INTO well SELECT * FROM read_csv_auto('well.csv.gz');
 -- Create additional indexes for query performance
 -- (Primary keys already create indexes automatically)
 
-CREATE INDEX idx_well_jcp ON well(Metadata_JCP2022);
+CREATE INDEX idx_well_jcp ON well(Metadata_JCP2022);