Skip to content

Commit 47492d4

Browse files
guowl3guowl3
authored
Merge pull request #4323 from oceanbase/feat/zijia_mv_adapt_sensitive_column
feat(materialized view):data desensitization supports materialized views
2 parents 2ac40f6 + f10d9e9 commit 47492d4

File tree

18 files changed

+257
-10
lines changed

18 files changed

+257
-10
lines changed

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java

+11
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,17 @@ default List<String> showExternalTables(String schemaName) {
268268
*/
269269
List<DBTableColumn> listBasicExternalTableColumns(String schemaName, String externalTableName);
270270

271+
/**
272+
* Get all materialized view columns(hold only basic info) in the specified schema
273+
*/
274+
Map<String, List<DBTableColumn>> listBasicMViewColumns(String schemaName);
275+
276+
/**
277+
* Get all materialized view columns(hold only basic info) in the specified schema and materialized
278+
* view
279+
*/
280+
List<DBTableColumn> listBasicMViewColumns(String schemaName, String externalTableName);
281+
271282
/**
272283
* Get all table and view columns info (hold only basic info: schema, table and column name) in the
273284
* specified schema

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java

+3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
* @Description: []
2222
*/
2323
public final class Statements {
24+
public static final String LIST_BASIC_MATERIALIZED_VIEW_COLUMNS = "list-basic-materialized-view-columns";
25+
public static final String LIST_BASIC_SCHEMA_MATERIALIZED_VIEW_COLUMNS =
26+
"list-basic-schema-materialized-view-columns";
2427
public static final String LIST_BASIC_EXTERNAL_TABLE_COLUMNS = "list-basic-external-table-columns";
2528
public static final String LIST_BASIC_SCHEMA_EXTERNAL_TABLE_COLUMNS = "list-basic-schema-external-table-columns";
2629
public static final String LIST_BASIC_TABLE_COLUMNS = "list-basic-table-columns";

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java

+10
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,16 @@ public List<DBTableColumn> listBasicExternalTableColumns(String schemaName, Stri
516516
throw new UnsupportedOperationException("Not supported yet");
517517
}
518518

519+
@Override
520+
public Map<String, List<DBTableColumn>> listBasicMViewColumns(String schemaName) {
521+
throw new UnsupportedOperationException("Not supported yet");
522+
}
523+
524+
@Override
525+
public List<DBTableColumn> listBasicMViewColumns(String schemaName, String externalTableName) {
526+
throw new UnsupportedOperationException("Not supported yet");
527+
}
528+
519529
@Override
520530
public Map<String, List<DBTableColumn>> listBasicColumnsInfo(String schemaName) {
521531
String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO);

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java

+10
Original file line numberDiff line numberDiff line change
@@ -538,6 +538,16 @@ public List<DBTableColumn> listBasicExternalTableColumns(String schemaName, Stri
538538
throw new UnsupportedOperationException("not support yet");
539539
}
540540

541+
@Override
542+
public Map<String, List<DBTableColumn>> listBasicMViewColumns(String schemaName) {
543+
throw new UnsupportedOperationException("not support yet");
544+
}
545+
546+
@Override
547+
public List<DBTableColumn> listBasicMViewColumns(String schemaName, String externalTableName) {
548+
throw new UnsupportedOperationException("not support yet");
549+
}
550+
541551
@Override
542552
public Map<String, List<DBTableColumn>> listBasicColumnsInfo(String schemaName) {
543553
String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO);

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween432And4351SchemaAccessor.java

+11
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package com.oceanbase.tools.dbbrowser.schema.mysql;
1717

1818
import java.util.List;
19+
import java.util.Map;
1920

2021
import org.springframework.jdbc.core.JdbcOperations;
2122

@@ -24,6 +25,7 @@
2425
import com.oceanbase.tools.dbbrowser.model.DBMViewRefreshRecordParam;
2526
import com.oceanbase.tools.dbbrowser.model.DBMaterializedView;
2627
import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity;
28+
import com.oceanbase.tools.dbbrowser.model.DBTableColumn;
2729
import com.oceanbase.tools.dbbrowser.model.DBTableConstraint;
2830

2931
/**
@@ -68,4 +70,13 @@ public List<DBMViewRefreshRecord> listMViewRefreshRecords(DBMViewRefreshRecordPa
6870
throw new UnsupportedOperationException("not support yet");
6971
}
7072

73+
@Override
74+
public Map<String, List<DBTableColumn>> listBasicMViewColumns(String schemaName) {
75+
throw new UnsupportedOperationException("not support yet");
76+
}
77+
78+
public List<DBTableColumn> listBasicMViewColumns(String schemaName, String externalTableName) {
79+
throw new UnsupportedOperationException("not support yet");
80+
}
81+
7182
}

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java

+19-5
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ public class OBMySQLSchemaAccessor extends MySQLNoLessThan5700SchemaAccessor {
7878
ESCAPE_SCHEMA_SET.add("__public");
7979
}
8080

81+
public OBMySQLSchemaAccessor(JdbcOperations jdbcOperations) {
82+
super(jdbcOperations);
83+
this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBMYSQL_432x);
84+
}
85+
8186
@Override
8287
public List<DBObjectIdentity> listMViews(String schemaName) {
8388
MySQLSqlBuilder sb = new MySQLSqlBuilder();
@@ -174,11 +179,6 @@ public List<String> showDatabases() {
174179
.collect(Collectors.toList());
175180
}
176181

177-
public OBMySQLSchemaAccessor(JdbcOperations jdbcOperations) {
178-
super(jdbcOperations);
179-
this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBMYSQL_432x);
180-
}
181-
182182
@Override
183183
public DBDatabase getDatabase(String schemaName) {
184184
DBDatabase database = new DBDatabase();
@@ -556,6 +556,20 @@ public List<DBTableColumn> listBasicExternalTableColumns(String schemaName, Stri
556556
return jdbcOperations.query(sql, new Object[] {schemaName, externalTableName}, listBasicTableColumnRowMapper());
557557
}
558558

559+
560+
@Override
561+
public Map<String, List<DBTableColumn>> listBasicMViewColumns(String schemaName) {
562+
String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_MATERIALIZED_VIEW_COLUMNS);
563+
List<DBTableColumn> tableColumns = jdbcOperations.query(sql, new Object[] {schemaName, schemaName},
564+
listBasicTableColumnRowMapper());
565+
return tableColumns.stream().collect(Collectors.groupingBy(DBTableColumn::getTableName));
566+
}
567+
568+
public List<DBTableColumn> listBasicMViewColumns(String schemaName, String externalTableName) {
569+
String sql = sqlMapper.getSql(Statements.LIST_BASIC_MATERIALIZED_VIEW_COLUMNS);
570+
return jdbcOperations.query(sql, new Object[] {schemaName, externalTableName}, listBasicTableColumnRowMapper());
571+
}
572+
559573
@Override
560574
protected void correctColumnPrecisionIfNeed(List<DBTableColumn> tableColumns) {}
561575
}

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java

+13
Original file line numberDiff line numberDiff line change
@@ -1253,4 +1253,17 @@ public List<DBMViewRefreshRecord> listMViewRefreshRecords(DBMViewRefreshRecordPa
12531253
return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBMViewRefreshRecord.class));
12541254
}
12551255

1256+
@Override
1257+
public Map<String, List<DBTableColumn>> listBasicMViewColumns(String schemaName) {
1258+
String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_MATERIALIZED_VIEW_COLUMNS);
1259+
List<DBTableColumn> tableColumns = jdbcOperations.query(sql, new Object[] {schemaName, schemaName},
1260+
listBasicColumnsRowMapper());
1261+
return tableColumns.stream().collect(Collectors.groupingBy(DBTableColumn::getTableName));
1262+
}
1263+
1264+
public List<DBTableColumn> listBasicMViewColumns(String schemaName, String externalTableName) {
1265+
String sql = sqlMapper.getSql(Statements.LIST_BASIC_MATERIALIZED_VIEW_COLUMNS);
1266+
return jdbcOperations.query(sql, new Object[] {schemaName, externalTableName}, listBasicColumnsRowMapper());
1267+
}
1268+
12561269
}

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java

+10
Original file line numberDiff line numberDiff line change
@@ -688,6 +688,16 @@ public List<DBTableColumn> listBasicExternalTableColumns(String schemaName, Stri
688688
throw new UnsupportedOperationException("not support yet");
689689
}
690690

691+
@Override
692+
public Map<String, List<DBTableColumn>> listBasicMViewColumns(String schemaName) {
693+
throw new UnsupportedOperationException("not support yet");
694+
}
695+
696+
@Override
697+
public List<DBTableColumn> listBasicMViewColumns(String schemaName, String externalTableName) {
698+
throw new UnsupportedOperationException("not support yet");
699+
}
700+
691701
@Override
692702
public Map<String, List<DBTableColumn>> listBasicColumnsInfo(String schemaName) {
693703
String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO);

‎libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java

+10
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,16 @@ public List<DBTableColumn> listBasicExternalTableColumns(String schemaName, Stri
325325
throw new UnsupportedOperationException("not support yet");
326326
}
327327

328+
@Override
329+
public Map<String, List<DBTableColumn>> listBasicMViewColumns(String schemaName) {
330+
throw new UnsupportedOperationException("not support yet");
331+
}
332+
333+
@Override
334+
public List<DBTableColumn> listBasicMViewColumns(String schemaName, String externalTableName) {
335+
throw new UnsupportedOperationException("not support yet");
336+
}
337+
328338
@Override
329339
public Map<String, List<DBTableColumn>> listBasicColumnsInfo(String schemaName) {
330340
throw new UnsupportedOperationException("Not supported yet");

‎libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_3_2_x.yaml

+30
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,36 @@ sqls:
6969
ORDER BY
7070
TABLE_NAME ASC,
7171
ORDINAL_POSITION ASC
72+
list-basic-materialized-view-columns: |-
73+
SELECT
74+
TABLE_SCHEMA,
75+
TABLE_NAME,
76+
COLUMN_NAME,
77+
DATA_TYPE,
78+
COLUMN_COMMENT
79+
FROM
80+
information_schema.columns
81+
WHERE
82+
TABLE_SCHEMA = ? AND TABLE_NAME = ?
83+
ORDER BY
84+
ORDINAL_POSITION ASC
85+
list-basic-schema-materialized-view-columns: |-
86+
SELECT
87+
TABLE_SCHEMA,
88+
TABLE_NAME,
89+
COLUMN_NAME,
90+
DATA_TYPE,
91+
COLUMN_COMMENT
92+
FROM
93+
information_schema.columns
94+
WHERE
95+
TABLE_SCHEMA = ?
96+
AND TABLE_NAME IN (
97+
select MVIEW_NAME FROM OCEANBASE.DBA_MVIEWS WHERE OWNER = ?
98+
)
99+
ORDER BY
100+
TABLE_NAME ASC,
101+
ORDINAL_POSITION ASC
72102
list-basic-view-columns: |-
73103
SELECT
74104
TABLE_SCHEMA,

‎libs/db-browser/src/main/resources/schema/sql/oboracle/oboracle_4_3_2_x.yaml

+35
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,41 @@ sqls:
104104
ORDER BY
105105
TABLE_NAME ASC,
106106
COLUMN_ID ASC
107+
list-basic-materialized-view-columns: |-
108+
SELECT
109+
OWNER,
110+
TABLE_NAME,
111+
COLUMN_NAME,
112+
DATA_TYPE,
113+
COMMENTS
114+
FROM
115+
SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS
116+
WHERE
117+
OWNER = ? AND TABLE_NAME = ? AND USER_GENERATED='YES'
118+
ORDER BY
119+
COLUMN_ID ASC
120+
list-basic-schema-materialized-view-columns: |-
121+
SELECT
122+
OWNER,
123+
TABLE_NAME,
124+
COLUMN_NAME,
125+
DATA_TYPE,
126+
COMMENTS
127+
FROM
128+
SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS
129+
WHERE
130+
OWNER = ?
131+
AND TABLE_NAME IN (
132+
SELECT
133+
MVIEW_NAME
134+
FROM
135+
SYS.ALL_MVIEWS
136+
WHERE
137+
OWNER = ?
138+
) AND USER_GENERATED='YES'
139+
ORDER BY
140+
TABLE_NAME ASC,
141+
COLUMN_ID ASC
107142
list-basic-schema-columns-info: |-
108143
SELECT
109144
OWNER,

‎libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OBMySQLSchemaAccessorTest.java

+34
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.Arrays;
2222
import java.util.List;
2323
import java.util.Map;
24+
import java.util.stream.Collectors;
2425

2526
import org.junit.AfterClass;
2627
import org.junit.Assert;
@@ -217,6 +218,39 @@ public void listMViewRefreshRecords_Success() {
217218
}
218219
}
219220

221+
@Test
222+
public void listBasicMViewColumns_InSchema_Success() {
223+
if (isSupportMaterializedView) {
224+
Map<String, List<DBTableColumn>> columns = accessor.listBasicMViewColumns(getOBMySQLDataBaseName());
225+
Assert.assertTrue(columns.containsKey("test_mv_all_syntax"));
226+
Assert.assertTrue(columns.get("test_mv_all_syntax").stream()
227+
.allMatch(column -> column.getSchemaName().equals(getOBMySQLDataBaseName())));
228+
Assert.assertEquals(Arrays.asList("prim", "col2", "col3", "col4"), (columns.get("test_mv_all_syntax")
229+
.stream().map(DBTableColumn::getName).collect(Collectors.toList())));
230+
231+
Assert.assertTrue(columns.containsKey("test_mv_computation"));
232+
Assert.assertTrue(columns.get("test_mv_computation").stream()
233+
.allMatch(column -> column.getSchemaName().equals(getOBMySQLDataBaseName())));
234+
Assert.assertEquals(Arrays.asList("col1", "count(*)"), (columns.get("test_mv_computation").stream()
235+
.map(DBTableColumn::getName).collect(Collectors.toList())));
236+
}
237+
}
238+
239+
@Test
240+
public void listBasicMViewColumns_InMView_Success() {
241+
if (isSupportMaterializedView) {
242+
List<DBTableColumn> columns =
243+
accessor.listBasicMViewColumns(getOBMySQLDataBaseName(), "test_mv_all_syntax");
244+
Assert.assertEquals(4, columns.size());
245+
List<String> expect = Arrays.asList("prim", "col2", "col3", "col4");
246+
columns.forEach(column -> {
247+
Assert.assertTrue(expect.contains(column.getName()));
248+
Assert.assertEquals(column.getTableName(), "test_mv_all_syntax");
249+
Assert.assertEquals(column.getSchemaName(), getOBMySQLDataBaseName());
250+
});
251+
}
252+
}
253+
220254
@Test
221255
public void listUsers_Success() {
222256
List<DBObjectIdentity> dbUsers = accessor.listUsers();

‎libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OBOracleSchemaAccessorTest.java

+33
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,39 @@ public void listMViewRefreshRecords_Success() {
231231
}
232232
}
233233

234+
@Test
235+
public void listBasicMViewColumns_InSchema_Success() {
236+
if (isSupportMaterializedView) {
237+
Map<String, List<DBTableColumn>> columns = accessor.listBasicMViewColumns(getOBOracleSchema());
238+
Assert.assertTrue(columns.containsKey("TEST_MV_ALLSYNTAX"));
239+
Assert.assertTrue(columns.get("TEST_MV_ALLSYNTAX").stream()
240+
.allMatch(column -> column.getSchemaName().equals(getOBOracleSchema())));
241+
Assert.assertEquals(Arrays.asList("PRIM", "COL2", "COL3", "COL4"), (columns.get("TEST_MV_ALLSYNTAX")
242+
.stream().map(DBTableColumn::getName).collect(Collectors.toList())));
243+
244+
Assert.assertTrue(columns.containsKey("TEST_MV_COMPUTATION"));
245+
Assert.assertTrue(columns.get("TEST_MV_COMPUTATION").stream()
246+
.allMatch(column -> column.getSchemaName().equals(getOBOracleSchema())));
247+
Assert.assertEquals(Arrays.asList("COL1", "CNT"), (columns.get("TEST_MV_COMPUTATION").stream()
248+
.map(DBTableColumn::getName).collect(Collectors.toList())));
249+
}
250+
}
251+
252+
@Test
253+
public void listBasicMViewColumns_InMView_Success() {
254+
if (isSupportMaterializedView) {
255+
List<DBTableColumn> columns =
256+
accessor.listBasicMViewColumns(getOBOracleSchema(), "TEST_MV_ALLSYNTAX");
257+
Assert.assertEquals(4, columns.size());
258+
List<String> expect = Arrays.asList("PRIM", "COL2", "COL3", "COL4");
259+
columns.forEach(column -> {
260+
Assert.assertTrue(expect.contains(column.getName()));
261+
Assert.assertEquals(column.getTableName(), "TEST_MV_ALLSYNTAX");
262+
Assert.assertEquals(column.getSchemaName(), getOBOracleSchema());
263+
});
264+
}
265+
}
266+
234267
@Test
235268
public void listUsers_Success() {
236269
List<DBObjectIdentity> dbUsers = accessor.listUsers();

‎server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/OdcConstants.java

+2
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ public class OdcConstants {
5454
public static final String PROCEDURE_GET_LINE = "OBODC_PROCEDURE_GET_LINE";
5555

5656
public static final String VALIDATE_DDL_TABLE_POSTFIX = "___ODC___TMP";
57+
public static final String MATERIALIZED_VIEW_LOG_PREFIX = "MLOG$_";
58+
public static final String CONTAINER_TABLE_PREFIX = "__MV_CONTAINER_";
5759

5860
public static final String PL_OBJECT_STATUS_VALID = "VALID";
5961
public static final String PL_OBJECT_STATUS_INVALID = "INVALID";

‎server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -146,8 +146,15 @@ public List<DatabaseWithAllColumns> listColumns(@NotNull Long projectId, @NotEmp
146146
databaseColumn.setExternalTable2Columns(getFilteringExistColumns(database.getId(),
147147
accessor.listBasicExternalTableColumns(database.getName()), exists));
148148
}
149+
if (versionDiffConfigService.isMViewSupported(dialectType, version)) {
150+
databaseColumn.setMaterializedView2Columns(getFilteringExistColumns(database.getId(),
151+
accessor.listBasicMViewColumns(database.getName()), exists));
152+
}
149153
databaseColumn.setDataTypeUnits(versionDiffConfigService.getDatatypeList(session));
150-
if (!databaseColumn.getTable2Columns().isEmpty() || !databaseColumn.getView2Columns().isEmpty()) {
154+
if (!databaseColumn.getTable2Columns().isEmpty()
155+
|| !databaseColumn.getView2Columns().isEmpty()
156+
|| !databaseColumn.getExternalTable2Columns().isEmpty()
157+
|| !databaseColumn.getMaterializedView2Columns().isEmpty()) {
151158
databaseColumns.add(databaseColumn);
152159
}
153160
} finally {

‎server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public class DatabaseWithAllColumns {
3636
private Map<String, List<DBTableColumn>> table2Columns;
3737
private Map<String, List<DBTableColumn>> view2Columns;
3838
private Map<String, List<DBTableColumn>> externalTable2Columns;
39+
private Map<String, List<DBTableColumn>> materializedView2Columns;
3940
/**
4041
* Mapping from database type to show type, used for displaying column type icon
4142
*/

0 commit comments

Comments
 (0)