Merge branch 'architectural_overhaul' into tables_list

config/settings/common_settings.py

@@ -66,7 +66,8 @@ def pipe_delim(pipe_string):
 
 MODERNRPC_METHODS_MODULES = [
  'mathesar.rpc.connections',
- 'mathesar.rpc.columns'
+ 'mathesar.rpc.columns',
+ 'mathesar.rpc.schemas'
 ]
 
 TEMPLATES = [

db/columns/operations/drop.py

@@ -1,14 +1,14 @@
-"""The function in this module wraps SQL functions that drop columns."""
+"""The functions in this module wrap SQL functions that drop columns."""
 from db import connection as db_conn
 
 
 def drop_column(table_oid, column_attnum, engine):
  """
- Drop the given columns from the given table.
+ Drop the given column from the given table.
 
  Args:
- table_oid: OID of the table whose columns we'll drop.
- column_attnum: The attnums of the columns to drop.
+ table_oid: OID of the table whose column we'll drop.
+ column_attnum: The attnum of the column to drop.
  engine: SQLAlchemy engine object for connecting.
 
  Returns:
@@ -17,3 +17,17 @@ def drop_column(table_oid, column_attnum, engine):
  return db_conn.execute_msar_func_with_engine(
  engine, 'drop_columns', table_oid, column_attnum
  ).fetchone()[0]
+
+
+def drop_columns_from_table(table_oid, column_attnums, conn):
+ """
+ Drop the given columns from the given table.
+
+ Args:
+ table_oid: OID of the table whose columns we'll drop.
+ column_attnums: The attnums of the columns to drop.
+ conn: A psycopg connection to the relevant database.
+ """
+ return db_conn.exec_msar_func(
+ conn, 'drop_columns', table_oid, *column_attnums
+ ).fetchone()[0]

db/constants.py

@@ -1,8 +1,18 @@
-MATHESAR_PREFIX = "mathesar_"
 ID = "id"
 ID_ORIGINAL = "id_original"
-INFERENCE_SCHEMA = f"{MATHESAR_PREFIX}inference_schema"
 COLUMN_NAME_TEMPLATE = 'Column ' # auto generated column name 'Column 1' (no undescore)
-MSAR_PUBLIC = 'msar'
-MSAR_PRIVAT = f"__{MSAR_PUBLIC}"
-MSAR_VIEWS = f"{MSAR_PUBLIC}_views"
+
+MATHESAR_PREFIX = "mathesar_"
+MSAR_PUBLIC_SCHEMA = 'msar'
+MSAR_PRIVATE_SCHEMA = f"__{MSAR_PUBLIC_SCHEMA}"
+TYPES_SCHEMA = f"{MATHESAR_PREFIX}types"
+INFERENCE_SCHEMA = f"{MATHESAR_PREFIX}inference_schema"
+VIEWS_SCHEMA = f"{MSAR_PUBLIC_SCHEMA}_views"
+
+INTERNAL_SCHEMAS = {
+ TYPES_SCHEMA,
+ MSAR_PUBLIC_SCHEMA,
+ MSAR_PRIVATE_SCHEMA,
+ VIEWS_SCHEMA,
+ INFERENCE_SCHEMA
+}

db/schemas/operations/select.py

@@ -1,23 +1,13 @@
 from sqlalchemy import select, and_, not_, or_, func
 
-from db import constants
-from db import types
+from db.constants import INTERNAL_SCHEMAS
 from db.utils import get_pg_catalog_table
 from db.metadata import get_empty_metadata
+from db.connection import exec_msar_func
 
-TYPES_SCHEMA = types.base.SCHEMA
-TEMP_INFER_SCHEMA = constants.INFERENCE_SCHEMA
-MSAR_PUBLIC = constants.MSAR_PUBLIC
-MSAR_PRIVAT = constants.MSAR_PRIVAT
-MSAR_VIEWS = constants.MSAR_VIEWS
-EXCLUDED_SCHEMATA = [
- "information_schema",
- MSAR_PRIVAT,
- MSAR_PUBLIC,
- MSAR_VIEWS,
- TEMP_INFER_SCHEMA,
- TYPES_SCHEMA,
-]
+
+def get_schemas(conn):
+ return exec_msar_func(conn, 'get_schemas').fetchone()[0]
 
 
 def reflect_schema(engine, name=None, oid=None, metadata=None):
@@ -46,7 +36,8 @@ def get_mathesar_schemas_with_oids(engine):
  select(pg_namespace.c.nspname.label('schema'), pg_namespace.c.oid)
  .where(
  and_(
- *[pg_namespace.c.nspname != schema for schema in EXCLUDED_SCHEMATA],
+ *[pg_namespace.c.nspname != schema for schema in INTERNAL_SCHEMAS],
+ pg_namespace.c.nspname != "information_schema",
  not_(pg_namespace.c.nspname.like("pg_%"))
  )
  )

db/sql/0_msar.sql → db/sql/00_msar.sql

@@ -219,6 +219,26 @@ END;
 $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT;
 
 
+CREATE OR REPLACE FUNCTION
+msar.get_relation_name_or_null(rel_id oid) RETURNS text AS $$/*
+Return the name for a given relation (e.g., table), qualified or quoted as appropriate.
+
+In cases where the relation is already included in the search path, the returned name will not be
+fully-qualified.
+
+The relation *must* be in the pg_class table to use this function. This function will return NULL if
+no corresponding relation can be found.
+
+Args:
+ rel_id: The OID of the relation.
+*/
+SELECT CASE
+ WHEN EXISTS (SELECT oid FROM pg_catalog.pg_class WHERE oid=rel_id) THEN rel_id::regclass::text
+END
+$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT;
+
+
+
 DROP FUNCTION IF EXISTS msar.get_relation_oid(text, text) CASCADE;
 CREATE OR REPLACE FUNCTION
 msar.get_relation_oid(sch_name text, rel_name text) RETURNS oid AS $$/*
@@ -701,6 +721,46 @@ WHERE pgc.relnamespace = sch_id AND pgc.relkind = 'r';
 $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT;
 
 
+CREATE OR REPLACE FUNCTION msar.get_schemas() RETURNS jsonb AS $$/*
+Return a json array of objects describing the user-defined schemas in the database.
+
+PostgreSQL system schemas are ignored.
+
+Internal Mathesar-specifc schemas are INCLUDED. These should be filtered out by the caller. This
+behavior is to avoid tight coupling between this function and other SQL files that might need to
+define additional Mathesar-specific schemas as our codebase grows.
+
+Each returned JSON object in the array will have the form:
+ {
+ "oid": <int>
+ "name": <str>
+ "description": <str|null>
+ "table_count": <int>
+ }
+*/
+SELECT jsonb_agg(schema_data)
+FROM (
+ SELECT 
+ s.oid AS oid,
+ s.nspname AS name,
+ pg_catalog.obj_description(s.oid) AS description,
+ COALESCE(count(c.oid), 0) AS table_count
+ FROM pg_catalog.pg_namespace s
+ LEFT JOIN pg_catalog.pg_class c ON
+ c.relnamespace = s.oid AND
+ -- Filter on relkind so that we only count tables. This must be done in the ON clause so that
+ -- we still get a row for schemas with no tables.
+ c.relkind = 'r'
+ WHERE
+ s.nspname <> 'information_schema' AND
+ s.nspname NOT LIKE 'pg_%'
+ GROUP BY
+ s.oid,
+ s.nspname
+) AS schema_data;
+$$ LANGUAGE sql;
+
+
 ----------------------------------------------------------------------------------------------------
 ----------------------------------------------------------------------------------------------------
 -- ROLE MANIPULATION FUNCTIONS
@@ -1155,10 +1215,11 @@ Args:
 DECLARE col_names text[];
 BEGIN
  SELECT array_agg(quote_ident(attname))
- FROM pg_attribute
- WHERE attrelid=tab_id AND ARRAY[attnum::integer] <@ col_ids
+ FROM pg_catalog.pg_attribute
+ WHERE attrelid=tab_id AND NOT attisdropped AND ARRAY[attnum::integer] <@ col_ids
  INTO col_names;
- RETURN __msar.drop_columns(__msar.get_relation_name(tab_id), variadic col_names);
+ PERFORM __msar.drop_columns(msar.get_relation_name_or_null(tab_id), variadic col_names);
+ RETURN array_length(col_names, 1);
 END;
 $$ LANGUAGE plpgsql RETURNS NULL ON NULL INPUT;
 

db/sql/2_msar_views.sql → db/sql/20_msar_views.sql

@@ -18,7 +18,7 @@
 --
 -- This file creates a schema `msar_views` where internal mathesar views will be stored.
 --
--- For naming conventions, see 0_msar.sql
+-- For naming conventions, see 00_msar.sql
 ----------------------------------------------------------------------------------------------------
 ----------------------------------------------------------------------------------------------------
 CREATE SCHEMA IF NOT EXISTS msar_views;

db/sql/install.py

@@ -2,8 +2,8 @@
 from db.connection import load_file_with_engine
 
 FILE_DIR = os.path.abspath(os.path.dirname(__file__))
-MSAR_SQL = os.path.join(FILE_DIR, '0_msar.sql')
-MSAR_AGGREGATE_SQL = os.path.join(FILE_DIR, '3_msar_custom_aggregates.sql')
+MSAR_SQL = os.path.join(FILE_DIR, '00_msar.sql')
+MSAR_AGGREGATE_SQL = os.path.join(FILE_DIR, '30_msar_custom_aggregates.sql')
 
 
 def install(engine):

db/sql/test_0_msar.sql → db/sql/test_00_msar.sql

@@ -30,6 +30,17 @@ END;
 $$ LANGUAGE plpgsql;
 
 
+CREATE OR REPLACE FUNCTION test_drop_columns_ne_oid() RETURNS SETOF TEXT AS $$
+BEGIN
+ CREATE TABLE "12345" (bleh text, bleh2 numeric);
+ PERFORM msar.drop_columns(12345, 1);
+ RETURN NEXT has_column(
+ '12345', 'bleh', 'Doesn''t drop columns of stupidly-named table'
+ );
+END;
+$$ LANGUAGE plpgsql;
+
+
 CREATE OR REPLACE FUNCTION test_drop_columns_names() RETURNS SETOF TEXT AS $$
 BEGIN
  PERFORM __setup_drop_columns();
@@ -1443,7 +1454,7 @@ $f$ LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION test_add_mathesar_table_badname() RETURNS SETOF TEXT AS $f$
 DECLARE
- badname text := $b$M"new"'dsf' \t"$b$;
+ badname text := '"new"''dsf'' \t"';
 BEGIN
  PERFORM __setup_create_table();
  PERFORM msar.add_mathesar_table(
@@ -2557,3 +2568,54 @@ BEGIN
  );
 END;
 $$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION test_get_schemas() RETURNS SETOF TEXT AS $$
+DECLARE
+ initial_schema_count int;
+ foo_schema jsonb;
+BEGIN
+ -- Get the initial schema count
+ SELECT jsonb_array_length(msar.get_schemas()) INTO initial_schema_count;
+
+ -- Create a schema
+ CREATE SCHEMA foo;
+ -- We should now have one additional schema
+ RETURN NEXT is(jsonb_array_length(msar.get_schemas()), initial_schema_count + 1);
+ -- Reflect the "foo" schema
+ SELECT jsonb_path_query(msar.get_schemas(), '$[*] ? (@.name == "foo")') INTO foo_schema;
+ -- We should have a foo schema object
+ RETURN NEXT is(jsonb_typeof(foo_schema), 'object');
+ -- It should have no description
+ RETURN NEXT is(jsonb_typeof(foo_schema->'description'), 'null');
+ -- It should have no tables
+ RETURN NEXT is((foo_schema->'table_count')::int, 0);
+
+ -- And comment
+ COMMENT ON SCHEMA foo IS 'A test schema';
+ -- Create two tables
+ CREATE TABLE foo.test_table_1 (id serial PRIMARY KEY);
+ CREATE TABLE foo.test_table_2 (id serial PRIMARY KEY);
+ -- Reflect again
+ SELECT jsonb_path_query(msar.get_schemas(), '$[*] ? (@.name == "foo")') INTO foo_schema;
+ -- We should see the description we set
+ RETURN NEXT is(foo_schema->'description'#>>'{}', 'A test schema');
+ -- We should see two tables
+ RETURN NEXT is((foo_schema->'table_count')::int, 2);
+
+ -- Drop the tables we created
+ DROP TABLE foo.test_table_1;
+ DROP TABLE foo.test_table_2;
+ -- Reflect the "foo" schema
+ SELECT jsonb_path_query(msar.get_schemas(), '$[*] ? (@.name == "foo")') INTO foo_schema;
+ -- The "foo" schema should now have no tables
+ RETURN NEXT is((foo_schema->'table_count')::int, 0);
+
+ -- Drop the "foo" schema
+ DROP SCHEMA foo;
+ -- We should now have no "foo" schema
+ RETURN NEXT ok(NOT jsonb_path_exists(msar.get_schemas(), '$[*] ? (@.name == "foo")'));
+ -- We should see the initial schema count again
+ RETURN NEXT is(jsonb_array_length(msar.get_schemas()), initial_schema_count);
+END;
+$$ LANGUAGE plpgsql;

db/sql/test_startup.sql

@@ -8,5 +8,5 @@ $$ LANGUAGE plpgsql;
 CALL raise_notice('Creating testing DB');
 CREATE DATABASE mathesar_testing;
 \c mathesar_testing
-\ir 0_msar.sql
-\ir test_0_msar.sql
+\ir 00_msar.sql
+\ir test_00_msar.sql

db/tests/columns/operations/test_drop.py

@@ -0,0 +1,18 @@
+from unittest.mock import patch
+from db.columns.operations import drop as col_drop
+
+
+def test_drop_columns():
+ with patch.object(col_drop.db_conn, 'exec_msar_func') as mock_exec:
+ mock_exec.return_value.fetchone = lambda: (3,)
+ result = col_drop.drop_columns_from_table(123, [1, 3, 5], 'conn')
+ mock_exec.assert_called_once_with('conn', 'drop_columns', 123, 1, 3, 5)
+ assert result == 3
+
+
+def test_drop_columns_single():
+ with patch.object(col_drop.db_conn, 'exec_msar_func') as mock_exec:
+ mock_exec.return_value.fetchone = lambda: (1,)
+ result = col_drop.drop_columns_from_table(123, [1], 'conn')
+ mock_exec.assert_called_once_with('conn', 'drop_columns', 123, 1)
+ assert result == 1

db/tests/schemas/operations/test_select.py

@@ -1,7 +1,7 @@
 import warnings
 from sqlalchemy import select, Table, MetaData, text
 
-from db import types
+from db.constants import TYPES_SCHEMA
 from db.tables.operations import infer_types
 from db.schemas.operations import select as ssel
 
@@ -27,7 +27,7 @@ def test_get_mathesar_schemas_with_oids_avoids_information_schema(engine_with_sc
 def test_get_mathesar_schemas_with_oids_avoids_types_schema(engine_with_schema):
  engine, schema = engine_with_schema
  actual_schemas = ssel.get_mathesar_schemas_with_oids(engine)
- assert all([schema != types.base.SCHEMA for schema, _ in actual_schemas])
+ assert all([schema != TYPES_SCHEMA for schema, _ in actual_schemas])
 
 
 def test_get_mathesar_schemas_with_oids_avoids_temp_schema(engine_with_schema):

db/tests/types/test_install.py

@@ -1,6 +1,6 @@
 from sqlalchemy import text
+from db.constants import TYPES_SCHEMA
 from db.types import install
-from db.types import base
 from db.types.custom import email
 from db.types.install import install_mathesar_on_database
 
@@ -10,7 +10,7 @@ def test_create_type_schema(engine):
  with engine.connect() as conn:
  res = conn.execute(text("SELECT * FROM information_schema.schemata"))
  schemata = {row['schema_name'] for row in res.fetchall()}
- assert base.SCHEMA in schemata
+ assert TYPES_SCHEMA in schemata
 
 
 def test_create_type_schema_when_exists(engine):

db/types/base.py

@@ -2,7 +2,7 @@
 
 from sqlalchemy import text, create_engine as sa_create_engine
 
-from db import constants
+from db.constants import TYPES_SCHEMA
 from db.utils import OrderByIds
 
 
@@ -143,8 +143,6 @@ class PostgresType(DatabaseType, Enum):
  UUID = 'uuid'
 
 
-SCHEMA = f"{constants.MATHESAR_PREFIX}types"
-
 # Since we want to have our identifiers quoted appropriately for use in
 # PostgreSQL, we want to use the postgres dialect preparer to set this up.
 _preparer = sa_create_engine("postgresql://").dialect.identifier_preparer
@@ -154,7 +152,7 @@ def get_ma_qualified_schema():
  """
  Should usually return `mathesar_types`
  """
- return _preparer.quote_schema(SCHEMA)
+ return _preparer.quote_schema(TYPES_SCHEMA)
 
 
 # TODO rename to get_qualified_mathesar_obj_name