Merge branch 'architectural_overhaul' into col_delete_func

config/settings/common_settings.py

@@ -66,7 +66,8 @@ def pipe_delim(pipe_string):
 
 MODERNRPC_METHODS_MODULES = [
  'mathesar.rpc.connections',
- 'mathesar.rpc.columns'
+ 'mathesar.rpc.columns',
+ 'mathesar.rpc.schemas'
 ]
 
 TEMPLATES = [

db/constants.py

@@ -1,8 +1,18 @@
-MATHESAR_PREFIX = "mathesar_"
 ID = "id"
 ID_ORIGINAL = "id_original"
-INFERENCE_SCHEMA = f"{MATHESAR_PREFIX}inference_schema"
 COLUMN_NAME_TEMPLATE = 'Column ' # auto generated column name 'Column 1' (no undescore)
-MSAR_PUBLIC = 'msar'
-MSAR_PRIVAT = f"__{MSAR_PUBLIC}"
-MSAR_VIEWS = f"{MSAR_PUBLIC}_views"
+
+MATHESAR_PREFIX = "mathesar_"
+MSAR_PUBLIC_SCHEMA = 'msar'
+MSAR_PRIVATE_SCHEMA = f"__{MSAR_PUBLIC_SCHEMA}"
+TYPES_SCHEMA = f"{MATHESAR_PREFIX}types"
+INFERENCE_SCHEMA = f"{MATHESAR_PREFIX}inference_schema"
+VIEWS_SCHEMA = f"{MSAR_PUBLIC_SCHEMA}_views"
+
+INTERNAL_SCHEMAS = {
+ TYPES_SCHEMA,
+ MSAR_PUBLIC_SCHEMA,
+ MSAR_PRIVATE_SCHEMA,
+ VIEWS_SCHEMA,
+ INFERENCE_SCHEMA
+}

db/schemas/operations/select.py

@@ -1,23 +1,13 @@
 from sqlalchemy import select, and_, not_, or_, func
 
-from db import constants
-from db import types
+from db.constants import INTERNAL_SCHEMAS
 from db.utils import get_pg_catalog_table
 from db.metadata import get_empty_metadata
+from db.connection import exec_msar_func
 
-TYPES_SCHEMA = types.base.SCHEMA
-TEMP_INFER_SCHEMA = constants.INFERENCE_SCHEMA
-MSAR_PUBLIC = constants.MSAR_PUBLIC
-MSAR_PRIVAT = constants.MSAR_PRIVAT
-MSAR_VIEWS = constants.MSAR_VIEWS
-EXCLUDED_SCHEMATA = [
- "information_schema",
- MSAR_PRIVAT,
- MSAR_PUBLIC,
- MSAR_VIEWS,
- TEMP_INFER_SCHEMA,
- TYPES_SCHEMA,
-]
+
+def get_schemas(conn):
+ return exec_msar_func(conn, 'get_schemas').fetchone()[0]
 
 
 def reflect_schema(engine, name=None, oid=None, metadata=None):
@@ -46,7 +36,8 @@ def get_mathesar_schemas_with_oids(engine):
  select(pg_namespace.c.nspname.label('schema'), pg_namespace.c.oid)
  .where(
  and_(
- *[pg_namespace.c.nspname != schema for schema in EXCLUDED_SCHEMATA],
+ *[pg_namespace.c.nspname != schema for schema in INTERNAL_SCHEMAS],
+ pg_namespace.c.nspname != "information_schema",
  not_(pg_namespace.c.nspname.like("pg_%"))
  )
  )

db/sql/00_msar.sql

@@ -682,6 +682,46 @@ SELECT EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid=tab_id AND attname=col_
 $$ LANGUAGE SQL RETURNS NULL ON NULL INPUT;
 
 
+CREATE OR REPLACE FUNCTION msar.get_schemas() RETURNS jsonb AS $$/*
+Return a json array of objects describing the user-defined schemas in the database.
+
+PostgreSQL system schemas are ignored.
+
+Internal Mathesar-specifc schemas are INCLUDED. These should be filtered out by the caller. This
+behavior is to avoid tight coupling between this function and other SQL files that might need to
+define additional Mathesar-specific schemas as our codebase grows.
+
+Each returned JSON object in the array will have the form:
+ {
+ "oid": <int>
+ "name": <str>
+ "description": <str|null>
+ "table_count": <int>
+ }
+*/
+SELECT jsonb_agg(schema_data)
+FROM (
+ SELECT 
+ s.oid AS oid,
+ s.nspname AS name,
+ pg_catalog.obj_description(s.oid) AS description,
+ COALESCE(count(c.oid), 0) AS table_count
+ FROM pg_catalog.pg_namespace s
+ LEFT JOIN pg_catalog.pg_class c ON
+ c.relnamespace = s.oid AND
+ -- Filter on relkind so that we only count tables. This must be done in the ON clause so that
+ -- we still get a row for schemas with no tables.
+ c.relkind = 'r'
+ WHERE
+ s.nspname <> 'information_schema' AND
+ s.nspname NOT LIKE 'pg_%'
+ GROUP BY
+ s.oid,
+ s.nspname
+) AS schema_data;
+$$ LANGUAGE sql;
+
+
 ----------------------------------------------------------------------------------------------------
 ----------------------------------------------------------------------------------------------------
 -- ROLE MANIPULATION FUNCTIONS

db/sql/test_00_msar.sql

@@ -2568,3 +2568,54 @@ BEGIN
  );
 END;
 $$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION test_get_schemas() RETURNS SETOF TEXT AS $$
+DECLARE
+ initial_schema_count int;
+ foo_schema jsonb;
+BEGIN
+ -- Get the initial schema count
+ SELECT jsonb_array_length(msar.get_schemas()) INTO initial_schema_count;
+
+ -- Create a schema
+ CREATE SCHEMA foo;
+ -- We should now have one additional schema
+ RETURN NEXT is(jsonb_array_length(msar.get_schemas()), initial_schema_count + 1);
+ -- Reflect the "foo" schema
+ SELECT jsonb_path_query(msar.get_schemas(), '$[*] ? (@.name == "foo")') INTO foo_schema;
+ -- We should have a foo schema object
+ RETURN NEXT is(jsonb_typeof(foo_schema), 'object');
+ -- It should have no description
+ RETURN NEXT is(jsonb_typeof(foo_schema->'description'), 'null');
+ -- It should have no tables
+ RETURN NEXT is((foo_schema->'table_count')::int, 0);
+
+ -- And comment
+ COMMENT ON SCHEMA foo IS 'A test schema';
+ -- Create two tables
+ CREATE TABLE foo.test_table_1 (id serial PRIMARY KEY);
+ CREATE TABLE foo.test_table_2 (id serial PRIMARY KEY);
+ -- Reflect again
+ SELECT jsonb_path_query(msar.get_schemas(), '$[*] ? (@.name == "foo")') INTO foo_schema;
+ -- We should see the description we set
+ RETURN NEXT is(foo_schema->'description'#>>'{}', 'A test schema');
+ -- We should see two tables
+ RETURN NEXT is((foo_schema->'table_count')::int, 2);
+
+ -- Drop the tables we created
+ DROP TABLE foo.test_table_1;
+ DROP TABLE foo.test_table_2;
+ -- Reflect the "foo" schema
+ SELECT jsonb_path_query(msar.get_schemas(), '$[*] ? (@.name == "foo")') INTO foo_schema;
+ -- The "foo" schema should now have no tables
+ RETURN NEXT is((foo_schema->'table_count')::int, 0);
+
+ -- Drop the "foo" schema
+ DROP SCHEMA foo;
+ -- We should now have no "foo" schema
+ RETURN NEXT ok(NOT jsonb_path_exists(msar.get_schemas(), '$[*] ? (@.name == "foo")'));
+ -- We should see the initial schema count again
+ RETURN NEXT is(jsonb_array_length(msar.get_schemas()), initial_schema_count);
+END;
+$$ LANGUAGE plpgsql;

db/tests/schemas/operations/test_select.py

@@ -1,7 +1,7 @@
 import warnings
 from sqlalchemy import select, Table, MetaData, text
 
-from db import types
+from db.constants import TYPES_SCHEMA
 from db.tables.operations import infer_types
 from db.schemas.operations import select as ssel
 
@@ -27,7 +27,7 @@ def test_get_mathesar_schemas_with_oids_avoids_information_schema(engine_with_sc
 def test_get_mathesar_schemas_with_oids_avoids_types_schema(engine_with_schema):
  engine, schema = engine_with_schema
  actual_schemas = ssel.get_mathesar_schemas_with_oids(engine)
- assert all([schema != types.base.SCHEMA for schema, _ in actual_schemas])
+ assert all([schema != TYPES_SCHEMA for schema, _ in actual_schemas])
 
 
 def test_get_mathesar_schemas_with_oids_avoids_temp_schema(engine_with_schema):

db/tests/types/test_install.py

@@ -1,6 +1,6 @@
 from sqlalchemy import text
+from db.constants import TYPES_SCHEMA
 from db.types import install
-from db.types import base
 from db.types.custom import email
 from db.types.install import install_mathesar_on_database
 
@@ -10,7 +10,7 @@ def test_create_type_schema(engine):
  with engine.connect() as conn:
  res = conn.execute(text("SELECT * FROM information_schema.schemata"))
  schemata = {row['schema_name'] for row in res.fetchall()}
- assert base.SCHEMA in schemata
+ assert TYPES_SCHEMA in schemata
 
 
 def test_create_type_schema_when_exists(engine):

db/types/base.py

@@ -2,7 +2,7 @@
 
 from sqlalchemy import text, create_engine as sa_create_engine
 
-from db import constants
+from db.constants import TYPES_SCHEMA
 from db.utils import OrderByIds
 
 
@@ -143,8 +143,6 @@ class PostgresType(DatabaseType, Enum):
  UUID = 'uuid'
 
 
-SCHEMA = f"{constants.MATHESAR_PREFIX}types"
-
 # Since we want to have our identifiers quoted appropriately for use in
 # PostgreSQL, we want to use the postgres dialect preparer to set this up.
 _preparer = sa_create_engine("postgresql://").dialect.identifier_preparer
@@ -154,7 +152,7 @@ def get_ma_qualified_schema():
  """
  Should usually return `mathesar_types`
  """
- return _preparer.quote_schema(SCHEMA)
+ return _preparer.quote_schema(TYPES_SCHEMA)
 
 
 # TODO rename to get_qualified_mathesar_obj_name

db/types/install.py

@@ -1,12 +1,12 @@
 from db.types.custom import email, money, multicurrency, uri, json_array, json_object
-from db.types.base import SCHEMA
+from db.constants import TYPES_SCHEMA
 from db.schemas.operations.create import create_schema
 from db.types.operations.cast import install_all_casts
 import psycopg
 
 
 def create_type_schema(engine):
- create_schema(SCHEMA, engine, if_not_exists=True)
+ create_schema(TYPES_SCHEMA, engine, if_not_exists=True)
 
 
 def install_mathesar_on_database(engine):
@@ -24,4 +24,4 @@ def install_mathesar_on_database(engine):
 def uninstall_mathesar_from_database(engine):
  conn_str = str(engine.url)
  with psycopg.connect(conn_str) as conn:
- conn.execute(f"DROP SCHEMA IF EXISTS __msar, msar, {SCHEMA} CASCADE")
+ conn.execute(f"DROP SCHEMA IF EXISTS __msar, msar, {TYPES_SCHEMA} CASCADE")

docs/docs/api/rpc.md

@@ -48,6 +48,14 @@ To use an RPC function:
 
 ---
 
+::: mathesar.rpc.schemas
+ options:
+ members:
+ - list_
+ - SchemaInfo
+
+---
+
 ::: mathesar.rpc.columns
  options:
  members:

mathesar/rpc/schemas.py

@@ -0,0 +1,55 @@
+"""
+Classes and functions exposed to the RPC endpoint for managing schemas.
+"""
+from typing import Optional, TypedDict
+
+from modernrpc.core import rpc_method, REQUEST_KEY
+from modernrpc.auth.basic import http_basic_auth_login_required
+
+from db.constants import INTERNAL_SCHEMAS
+from db.schemas.operations.select import get_schemas
+from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions
+from mathesar.rpc.utils import connect
+
+
+class SchemaInfo(TypedDict):
+ """
+ Information about a schema
+
+ Attributes:
+ oid: The OID of the schema
+ name: The name of the schema
+ description: A description of the schema
+ table_count: The number of tables in the schema
+ exploration_count: The number of explorations in the schema
+ """
+ oid: int
+ name: str
+ description: Optional[str]
+ table_count: int
+ exploration_count: int
+
+
+@rpc_method(name="schemas.list")
+@http_basic_auth_login_required
+@handle_rpc_exceptions
+def list_(*, database_id: int, **kwargs) -> list[SchemaInfo]:
+ """
+ List information about schemas in a database. Exposed as `list`.
+
+ Args:
+ database_id: The Django id of the database containing the table.
+
+ Returns:
+ A list of schema details
+ """
+ user = kwargs.get(REQUEST_KEY).user
+ with connect(database_id, user) as conn:
+ schemas = get_schemas(conn)
+
+ user_defined_schemas = [s for s in schemas if s['name'] not in INTERNAL_SCHEMAS]
+
+ # TODO_FOR_BETA: join exploration count from internal DB here after we've
+ # refactored the models so that each exploration is associated with a schema
+ # (by oid) in a specific database.
+ return [{**s, "exploration_count": 0} for s in user_defined_schemas]

mathesar/tests/rpc/test_endpoints.py

@@ -10,7 +10,7 @@
 
 from mathesar.rpc import columns
 from mathesar.rpc import connections
-
+from mathesar.rpc import schemas
 
 METHODS = [
  (
@@ -32,7 +32,12 @@
  connections.add_from_scratch,
  "connections.add_from_scratch",
  [user_is_superuser]
- )
+ ),
+ (
+ schemas.list_,
+ "schemas.list",
+ [user_is_authenticated]
+ ),
 ]