alexey.rassokhin/tet123
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

DBZ-5045 Remove deprecated TABLE_WHITELIST

Browse Source
This commit is contained in:
Vojtech Juranek 2022-05-31 14:06:34 +02:00 committed by Jiri Pechanec
parent e867bd1772
commit db08e894fc
12 changed files with 25 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
debezium-connector-mysql/src/main/java/io/debezium/connector/mysql/MySqlConnectorConfig.java
View File

@ -702,7 +702,7 @@ public static GtidNewChannelPosition parse(String value, String defaultValue) {
.withGroup(Field.createGroupEntry(Field.Group.CONNECTOR, 24))
.withWidth(Width.LONG)
.withImportance(Importance.HIGH)
.withDependents(TABLE_INCLUDE_LIST_NAME, TABLE_WHITELIST_NAME)
.withDependents(TABLE_INCLUDE_LIST_NAME)
.withDescription("The source UUIDs used to include GTID ranges when determine the starting position in the MySQL server's binlog.");
/**

6
debezium-connector-mysql/src/test/java/io/debezium/connector/mysql/MySqlConnectorIT.java
View File

@ -137,7 +137,6 @@ public void shouldFailToValidateInvalidConfiguration() {
assertNoConfigurationErrors(result, MySqlConnectorConfig.DATABASE_INCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.DATABASE_BLACKLIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.DATABASE_EXCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_WHITELIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_INCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_BLACKLIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_EXCLUDE_LIST);
@ -189,7 +188,6 @@ public void shouldValidateAcceptableConfiguration() {
assertNoConfigurationErrors(result, MySqlConnectorConfig.DATABASE_INCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.DATABASE_BLACKLIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.DATABASE_EXCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_WHITELIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_INCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_BLACKLIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_EXCLUDE_LIST);
@ -1043,12 +1041,12 @@ public void shouldHandleIncludeListTables() throws SQLException, InterruptedExce
}
@Test
public void shouldHandleWhitelistedTables() throws SQLException, InterruptedException {
public void shouldHandleIncludedTables() throws SQLException, InterruptedException {
Testing.Files.delete(DB_HISTORY_PATH);
final String tables = String.format("%s.customers, %s.orders", DATABASE.getDatabaseName(), DATABASE.getDatabaseName());
config = DATABASE.defaultConfig()
.with(MySqlConnectorConfig.TABLE_WHITELIST, tables)
.with(MySqlConnectorConfig.TABLE_INCLUDE_LIST, tables)
.with(MySqlConnectorConfig.DATABASE_WHITELIST, ".*")
.build();

4
debezium-connector-mysql/src/test/java/io/debezium/connector/mysql/SnapshotSourceIT.java
View File

@ -681,9 +681,9 @@ public void shouldSnapshotTablesInOrderSpecifiedInTableIncludeList() throws Exce
}
@Test
public void shouldSnapshotTablesInOrderSpecifiedInTablesWhitelist() throws Exception {
public void shouldSnapshotTablesInOrderSpecifiedInTablesIncludeList() throws Exception {
config = simpleConfig()
.with(MySqlConnectorConfig.TABLE_WHITELIST,
.with(MySqlConnectorConfig.TABLE_INCLUDE_LIST,
"connector_test_ro_(.*).orders,connector_test_ro_(.*).Products,connector_test_ro_(.*).products_on_hand,connector_test_ro_(.*).dbz_342_timetest")
.build();

27
debezium-connector-oracle/src/test/java/io/debezium/connector/oracle/OracleConnectorFilterIT.java
View File

@ -125,14 +125,9 @@ public void after() throws SQLException {
adminConnection.execute("DROP USER debezium2");
}
@Test
public void shouldApplyTableWhitelistConfiguration() throws Exception {
shouldApplyTableInclusionConfiguration(true);
}
@Test
public void shouldApplyTableIncludeListConfiguration() throws Exception {
shouldApplyTableInclusionConfiguration(false);
shouldApplyTableInclusionConfiguration();
}
@Test
@ -145,16 +140,10 @@ public void shouldApplyTableExcludeListConfiguration() throws Exception {
shouldApplyTableExclusionsConfiguration(false);
}
@Test
@FixFor("DBZ-3009")
public void shouldApplySchemaAndTableWhitelistConfiguration() throws Exception {
shouldApplySchemaAndTableInclusionConfiguration(true);
}
@Test
@FixFor("DBZ-3009")
public void shouldApplySchemaAndTableIncludeListConfiguration() throws Exception {
shouldApplySchemaAndTableInclusionConfiguration(false);
shouldApplySchemaAndTableInclusionConfiguration();
}
@Test
@ -297,12 +286,8 @@ public void shouldApplyColumnExcludeListConfiguration() throws Exception {
}
}
private void shouldApplyTableInclusionConfiguration(boolean useLegacyOption) throws Exception {
private void shouldApplyTableInclusionConfiguration() throws Exception {
Field option = OracleConnectorConfig.TABLE_INCLUDE_LIST;
if (useLegacyOption) {
option = OracleConnectorConfig.TABLE_WHITELIST;
}
boolean includeDdlChanges = true;
if (TestHelper.adapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
// LogMiner currently does not support DDL changes during streaming phase
@ -363,12 +348,8 @@ private void shouldApplyTableInclusionConfiguration(boolean useLegacyOption) thr
}
}
private void shouldApplySchemaAndTableInclusionConfiguration(boolean useLegacyOption) throws Exception {
private void shouldApplySchemaAndTableInclusionConfiguration() throws Exception {
Field option = OracleConnectorConfig.TABLE_INCLUDE_LIST;
if (useLegacyOption) {
option = OracleConnectorConfig.TABLE_WHITELIST;
}
boolean includeDdlChanges = true;
if (TestHelper.adapter().equals(OracleConnectorConfig.ConnectorAdapter.LOG_MINER)) {
// LogMiner currently does not support DDL changes during streaming phase

3
debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/PostgresConnectorIT.java
View File

@ -233,7 +233,6 @@ public void shouldValidateConfiguration() throws Exception {
validateConfigField(validatedConfig, PostgresConnectorConfig.SCHEMA_INCLUDE_LIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.SCHEMA_BLACKLIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.SCHEMA_EXCLUDE_LIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.TABLE_WHITELIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.TABLE_INCLUDE_LIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.TABLE_BLACKLIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.TABLE_EXCLUDE_LIST, null);
@ -1211,7 +1210,7 @@ public void shouldRemoveWhiteSpaceCharsOld() throws Exception {
.with(PostgresConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL.getValue())
.with(PostgresConnectorConfig.DROP_SLOT_ON_STOP, Boolean.TRUE)
.with(PostgresConnectorConfig.SCHEMA_WHITELIST, "s1")
.with(PostgresConnectorConfig.TABLE_WHITELIST, tableWhitelistWithWhitespace);
.with(PostgresConnectorConfig.TABLE_INCLUDE_LIST, tableWhitelistWithWhitespace);
start(PostgresConnector.class, configBuilder.build());
assertConnectorIsRunning();

2
debezium-connector-sqlserver/src/test/java/io/debezium/connector/sqlserver/SnapshotIT.java
View File

@ -328,7 +328,7 @@ public void testBlacklistColumn() throws Exception {
final Configuration config = TestHelper.defaultConfig()
.with(SqlServerConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL)
.with(SqlServerConnectorConfig.COLUMN_BLACKLIST, "dbo.blacklist_column_table_a.amount")
.with(SqlServerConnectorConfig.TABLE_WHITELIST, "dbo.blacklist_column_table_a,dbo.blacklist_column_table_b")
.with(TABLE_INCLUDE_LIST, "dbo.blacklist_column_table_a,dbo.blacklist_column_table_b")
.build();
start(SqlServerConnector.class, config);

4
debezium-connector-sqlserver/src/test/java/io/debezium/connector/sqlserver/SqlServerConnectorIT.java
View File

@ -862,13 +862,13 @@ public void verifyOffsets() throws Exception {
}
@Test
public void testWhitelistTable() throws Exception {
public void testIncludeTable() throws Exception {
final int RECORDS_PER_TABLE = 5;
final int TABLES = 1;
final int ID_START = 10;
final Configuration config = TestHelper.defaultConfig()
.with(SqlServerConnectorConfig.SNAPSHOT_MODE, SnapshotMode.SCHEMA_ONLY)
.with(SqlServerConnectorConfig.TABLE_WHITELIST, "dbo.tableb")
.with(SqlServerConnectorConfig.TABLE_INCLUDE_LIST, "dbo.tableb")
.build();
connection.execute(
"INSERT INTO tableb VALUES(1, 'b')");

28
debezium-core/src/main/java/io/debezium/relational/RelationalDatabaseConnectorConfig.java
View File

@ -58,13 +58,11 @@ public abstract class RelationalDatabaseConnectorConfig extends CommonConnectorC
protected static final String DATABASE_EXCLUDE_LIST_NAME = "database.exclude.list";
protected static final String TABLE_BLACKLIST_NAME = "table.blacklist";
protected static final String TABLE_EXCLUDE_LIST_NAME = "table.exclude.list";
protected static final String TABLE_WHITELIST_NAME = "table.whitelist";
protected static final String TABLE_INCLUDE_LIST_NAME = "table.include.list";
protected static final Pattern SERVER_NAME_PATTERN = Pattern.compile("^[a-zA-Z0-9_.\\-]+$");
public static final String TABLE_INCLUDE_LIST_ALREADY_SPECIFIED_ERROR_MSG = "\"table.include.list\" is already specified";
public static final String TABLE_WHITELIST_ALREADY_SPECIFIED_ERROR_MSG = "\"table.whitelist\" is already specified";
public static final String COLUMN_INCLUDE_LIST_ALREADY_SPECIFIED_ERROR_MSG = "\"column.include.list\" is already specified";
public static final String COLUMN_WHITELIST_ALREADY_SPECIFIED_ERROR_MSG = "\"column.whitelist\" is already specified";
public static final String SCHEMA_INCLUDE_LIST_ALREADY_SPECIFIED_ERROR_MSG = "\"schema.include.list\" is already specified";
@ -227,19 +225,6 @@ public static DecimalHandlingMode parse(String value, String defaultValue) {
.withValidation(Field::isListOfRegex)
.withDescription("The tables for which changes are to be captured");
/**
* Old, backwards-compatible "whitelist" property.
*/
@Deprecated
public static final Field TABLE_WHITELIST = Field.create(TABLE_WHITELIST_NAME)
.withDisplayName("Deprecated: Include Tables")
.withType(Type.LIST)
.withWidth(Width.LONG)
.withImportance(Importance.LOW)
.withValidation(Field::isListOfRegex)
.withInvisibleRecommender()
.withDescription("The tables for which changes are to be captured (deprecated, use \"" + TABLE_INCLUDE_LIST.name() + "\" instead)");
/**
* A comma-separated list of regular expressions that match the fully-qualified names of tables to be excluded from
* monitoring. Fully-qualified names for tables are of the form {@code <databaseName>.<tableName>} or
@ -440,7 +425,7 @@ public static DecimalHandlingMode parse(String value, String defaultValue) {
.withGroup(Field.createGroupEntry(Field.Group.FILTERS, 0))
.withWidth(Width.LONG)
.withImportance(Importance.HIGH)
.withDependents(TABLE_INCLUDE_LIST_NAME, TABLE_WHITELIST_NAME)
.withDependents(TABLE_INCLUDE_LIST_NAME)
.withValidation(Field::isListOfRegex)
.withDescription("The databases for which changes are to be captured");
@ -600,7 +585,6 @@ public static DecimalHandlingMode parse(String value, String defaultValue) {
COLUMN_INCLUDE_LIST,
COLUMN_BLACKLIST,
COLUMN_EXCLUDE_LIST,
TABLE_WHITELIST,
TABLE_INCLUDE_LIST,
TABLE_BLACKLIST,
TABLE_EXCLUDE_LIST,
@ -725,7 +709,7 @@ public String tableExcludeList() {
}
public String tableIncludeList() {
return getConfig().getFallbackStringProperty(TABLE_INCLUDE_LIST, TABLE_WHITELIST);
return getConfig().getString(TABLE_INCLUDE_LIST);
}
public ColumnNameFilter getColumnFilter() {
@ -773,18 +757,18 @@ public TableIdToStringMapper getTableIdMapper() {
}
private static int validateTableBlacklist(Configuration config, Field field, ValidationOutput problems) {
String whitelist = config.getFallbackStringPropertyWithWarning(TABLE_INCLUDE_LIST, TABLE_WHITELIST);
String includeList = config.getString(TABLE_INCLUDE_LIST);
String blacklist = config.getFallbackStringPropertyWithWarning(TABLE_EXCLUDE_LIST, TABLE_BLACKLIST);
if (whitelist != null && blacklist != null) {
problems.accept(TABLE_BLACKLIST, blacklist, TABLE_WHITELIST_ALREADY_SPECIFIED_ERROR_MSG);
if (includeList != null && blacklist != null) {
problems.accept(TABLE_BLACKLIST, blacklist, TABLE_INCLUDE_LIST_ALREADY_SPECIFIED_ERROR_MSG);
return 1;
}
return 0;
}
private static int validateTableExcludeList(Configuration config, Field field, ValidationOutput problems) {
String includeList = config.getString(TABLE_WHITELIST);
String includeList = config.getString(TABLE_INCLUDE_LIST);
String excludeList = config.getString(TABLE_BLACKLIST);
if (includeList != null && excludeList != null) {

4
debezium-core/src/main/java/io/debezium/relational/RelationalTableFilters.java
View File

@ -63,9 +63,7 @@ public RelationalTableFilters(Configuration config, TableFilter systemTablesFilt
// Define the filter using the include and exclude lists for tables and database names ...
Predicate<TableId> tablePredicate = eligibleTables
.includeTables(
config.getFallbackStringProperty(
RelationalDatabaseConnectorConfig.TABLE_INCLUDE_LIST,
RelationalDatabaseConnectorConfig.TABLE_WHITELIST),
config.getString(RelationalDatabaseConnectorConfig.TABLE_INCLUDE_LIST),
tableIdMapper)
.excludeTables(
config.getFallbackStringProperty(

2
debezium-e2e-benchmark/py/tpc-run-test.py
View File

@ -140,7 +140,7 @@ def main(argv):
table = tpcconfig['jdbc'][connectiontype].get('table')
if table == None:
table = 'TPC.TEST'
config['config']['table.whitelist'] = table
config['config']['table.include.list'] = table
lowercase = tpcconfig['jdbc'][connectiontype].get('lowercase')
if lowercase == None:

5
debezium-server/debezium-server-core/src/test/java/io/debezium/server/DebeziumServerTest.java
View File

@ -19,6 +19,7 @@
import org.junit.jupiter.api.Test;
import io.debezium.DebeziumException;
import io.debezium.relational.RelationalDatabaseConnectorConfig;
import io.debezium.server.events.ConnectorStartedEvent;
import io.debezium.util.Collect;
import io.debezium.util.Testing;
@ -51,8 +52,8 @@ void setupDependencies(@Observes ConnectorStartedEvent event) {
@Test
public void testProps() {
Properties properties = server.getProps();
Assertions.assertThat(properties.getProperty("table.whitelist")).isNotNull();
Assertions.assertThat(properties.getProperty("table.whitelist")).isEqualTo("public.table_name");
Assertions.assertThat(properties.getProperty(RelationalDatabaseConnectorConfig.TABLE_INCLUDE_LIST.name())).isNotNull();
Assertions.assertThat(properties.getProperty(RelationalDatabaseConnectorConfig.TABLE_INCLUDE_LIST.name())).isEqualTo("public.table_name");
Assertions.assertThat(properties.getProperty("offset.flush.interval.ms.test")).isNotNull();
Assertions.assertThat(properties.getProperty("offset.flush.interval.ms.test")).isEqualTo("0");

2
debezium-server/debezium-server-core/src/test/java/io/debezium/server/TestConfigSource.java
View File

@ -68,7 +68,7 @@ public TestConfigSource() {
unitTest.put("debezium.transforms.hoist.field", "line");
// DBZ-2622 For testing properties passed via smallrye/microprofile environment variables
unitTest.put("DEBEZIUM_SOURCE_TABLE_WHITELIST", "public.table_name");
unitTest.put("DEBEZIUM_SOURCE_TABLE_INCLUDE_LIST", "public.table_name");
unitTest.put("debezium_source_offset_flush_interval_ms_Test", "0");
unitTest.put("debezium.source.snapshot.select.statement.overrides.public.table_name", "SELECT * FROM table_name WHERE 1>2");
unitTest.put("debezium.source.database.allowPublicKeyRetrieval", "true");