alexey.rassokhin/tet123
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

DBZ-5045 Remove deprecated COLUMN_BLACKLIST

Browse Source
This commit is contained in:
Vojtech Juranek 2022-05-31 15:46:15 +02:00 committed by Jiri Pechanec
parent 5031594b75
commit b0b3066ed5
8 changed files with 3 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
debezium-connector-mysql/src/test/java/io/debezium/connector/mysql/MySqlConnectorIT.java
View File

@ -137,7 +137,6 @@ public void shouldFailToValidateInvalidConfiguration() {
assertNoConfigurationErrors(result, MySqlConnectorConfig.DATABASE_EXCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_INCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_EXCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.COLUMN_BLACKLIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.COLUMN_EXCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.COLUMN_INCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.CONNECTION_TIMEOUT_MS);
@ -185,7 +184,6 @@ public void shouldValidateAcceptableConfiguration() {
assertNoConfigurationErrors(result, MySqlConnectorConfig.DATABASE_EXCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_INCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.TABLE_EXCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.COLUMN_BLACKLIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.COLUMN_EXCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.COLUMN_INCLUDE_LIST);
assertNoConfigurationErrors(result, MySqlConnectorConfig.MSG_KEY_COLUMNS);

3
debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/PostgresConnectorIT.java
View File

@ -233,7 +233,6 @@ public void shouldValidateConfiguration() throws Exception {
validateConfigField(validatedConfig, PostgresConnectorConfig.SCHEMA_EXCLUDE_LIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.TABLE_INCLUDE_LIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.TABLE_EXCLUDE_LIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.COLUMN_BLACKLIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.COLUMN_EXCLUDE_LIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.COLUMN_INCLUDE_LIST, null);
validateConfigField(validatedConfig, PostgresConnectorConfig.MSG_KEY_COLUMNS, null);
@ -1110,7 +1109,7 @@ public void shouldTakeBlacklistFiltersIntoAccount() throws Exception {
.with(PostgresConnectorConfig.DROP_SLOT_ON_STOP, Boolean.TRUE)
.with(PostgresConnectorConfig.SCHEMA_EXCLUDE_LIST, "s2")
.with(PostgresConnectorConfig.TABLE_EXCLUDE_LIST, ".+b")
.with(PostgresConnectorConfig.COLUMN_BLACKLIST, ".+bb");
.with(PostgresConnectorConfig.COLUMN_EXCLUDE_LIST, ".+bb");
start(PostgresConnector.class, configBuilder.build());
assertConnectorIsRunning();

8
debezium-connector-postgres/src/test/java/io/debezium/connector/postgresql/PostgresSchemaIT.java
View File

@ -259,14 +259,6 @@ public void shouldApplyFilters() throws Exception {
assertColumnsExcluded("s1.a.aa", "s2.a.aa");
}
config = new PostgresConnectorConfig(TestHelper.defaultConfig().with(PostgresConnectorConfig.COLUMN_BLACKLIST, ".*aa")
.build());
schema = TestHelper.getSchema(config, typeRegistry);
try (PostgresConnection connection = TestHelper.createWithTypeRegistry()) {
schema.refresh(connection, false);
assertColumnsExcluded("s1.a.aa", "s2.a.aa");
}
config = new PostgresConnectorConfig(TestHelper.defaultConfig().with(PostgresConnectorConfig.COLUMN_INCLUDE_LIST, ".*bb")
.build());
schema = TestHelper.getSchema(config, typeRegistry);

59
debezium-connector-sqlserver/src/test/java/io/debezium/connector/sqlserver/SnapshotIT.java
View File

@ -314,65 +314,6 @@ public void takeSchemaOnlySnapshotAndSendHeartbeat() throws Exception {
Assertions.assertThat(record.topic()).startsWith("__debezium-heartbeat");
}
@Test
@FixFor("DBZ-1067")
public void testBlacklistColumn() throws Exception {
connection.execute(
"CREATE TABLE blacklist_column_table_a (id int, name varchar(30), amount integer primary key(id))",
"CREATE TABLE blacklist_column_table_b (id int, name varchar(30), amount integer primary key(id))");
connection.execute("INSERT INTO blacklist_column_table_a VALUES(10, 'some_name', 120)");
connection.execute("INSERT INTO blacklist_column_table_b VALUES(11, 'some_name', 447)");
TestHelper.enableTableCdc(connection, "blacklist_column_table_a");
TestHelper.enableTableCdc(connection, "blacklist_column_table_b");
final Configuration config = TestHelper.defaultConfig()
.with(SqlServerConnectorConfig.SNAPSHOT_MODE, SnapshotMode.INITIAL)
.with(SqlServerConnectorConfig.COLUMN_BLACKLIST, "dbo.blacklist_column_table_a.amount")
.with(TABLE_INCLUDE_LIST, "dbo.blacklist_column_table_a,dbo.blacklist_column_table_b")
.build();
start(SqlServerConnector.class, config);
assertConnectorIsRunning();
final SourceRecords records = consumeRecordsByTopic(2);
final List<SourceRecord> tableA = records.recordsForTopic("server1.dbo.blacklist_column_table_a");
final List<SourceRecord> tableB = records.recordsForTopic("server1.dbo.blacklist_column_table_b");
Schema expectedSchemaA = SchemaBuilder.struct()
.optional()
.name("server1.dbo.blacklist_column_table_a.Value")
.field("id", Schema.INT32_SCHEMA)
.field("name", Schema.OPTIONAL_STRING_SCHEMA)
.build();
Struct expectedValueA = new Struct(expectedSchemaA)
.put("id", 10)
.put("name", "some_name");
Schema expectedSchemaB = SchemaBuilder.struct()
.optional()
.name("server1.dbo.blacklist_column_table_b.Value")
.field("id", Schema.INT32_SCHEMA)
.field("name", Schema.OPTIONAL_STRING_SCHEMA)
.field("amount", Schema.OPTIONAL_INT32_SCHEMA)
.build();
Struct expectedValueB = new Struct(expectedSchemaB)
.put("id", 11)
.put("name", "some_name")
.put("amount", 447);
Assertions.assertThat(tableA).hasSize(1);
SourceRecordAssert.assertThat(tableA.get(0))
.valueAfterFieldIsEqualTo(expectedValueA)
.valueAfterFieldSchemaIsEqualTo(expectedSchemaA);
Assertions.assertThat(tableB).hasSize(1);
SourceRecordAssert.assertThat(tableB.get(0))
.valueAfterFieldIsEqualTo(expectedValueB)
.valueAfterFieldSchemaIsEqualTo(expectedSchemaB);
stopConnector();
}
@Test
@FixFor("DBZ-2456")
public void shouldSelectivelySnapshotTables() throws SQLException, InterruptedException {

62
debezium-connector-sqlserver/src/test/java/io/debezium/connector/sqlserver/SqlServerConnectorIT.java
View File

@ -1059,68 +1059,6 @@ public void blacklistColumnWhenCdcColumnsDoNotMatchWithOriginalSnapshot() throws
stopConnector();
}
@Test
@FixFor("DBZ-1067")
public void testBlacklistColumn() throws Exception {
connection.execute(
"CREATE TABLE blacklist_column_table_a (id int, name varchar(30), amount integer primary key(id))",
"CREATE TABLE blacklist_column_table_b (id int, name varchar(30), amount integer primary key(id))");
TestHelper.enableTableCdc(connection, "blacklist_column_table_a");
TestHelper.enableTableCdc(connection, "blacklist_column_table_b");
final Configuration config = TestHelper.defaultConfig()
.with(SqlServerConnectorConfig.SNAPSHOT_MODE, SnapshotMode.SCHEMA_ONLY)
.with(SqlServerConnectorConfig.COLUMN_BLACKLIST, "dbo.blacklist_column_table_a.amount")
.build();
start(SqlServerConnector.class, config);
assertConnectorIsRunning();
// Wait for snapshot completion
consumeRecordsByTopic(1);
connection.execute("INSERT INTO blacklist_column_table_a VALUES(10, 'some_name', 120)");
connection.execute("INSERT INTO blacklist_column_table_b VALUES(11, 'some_name', 447)");
final SourceRecords records = consumeRecordsByTopic(2);
final List<SourceRecord> tableA = records.recordsForTopic("server1.dbo.blacklist_column_table_a");
final List<SourceRecord> tableB = records.recordsForTopic("server1.dbo.blacklist_column_table_b");
Schema expectedSchemaA = SchemaBuilder.struct()
.optional()
.name("server1.dbo.blacklist_column_table_a.Value")
.field("id", Schema.INT32_SCHEMA)
.field("name", Schema.OPTIONAL_STRING_SCHEMA)
.build();
Struct expectedValueA = new Struct(expectedSchemaA)
.put("id", 10)
.put("name", "some_name");
Schema expectedSchemaB = SchemaBuilder.struct()
.optional()
.name("server1.dbo.blacklist_column_table_b.Value")
.field("id", Schema.INT32_SCHEMA)
.field("name", Schema.OPTIONAL_STRING_SCHEMA)
.field("amount", Schema.OPTIONAL_INT32_SCHEMA)
.build();
Struct expectedValueB = new Struct(expectedSchemaB)
.put("id", 11)
.put("name", "some_name")
.put("amount", 447);
Assertions.assertThat(tableA).hasSize(1);
SourceRecordAssert.assertThat(tableA.get(0))
.valueAfterFieldIsEqualTo(expectedValueA)
.valueAfterFieldSchemaIsEqualTo(expectedSchemaA);
Assertions.assertThat(tableB).hasSize(1);
SourceRecordAssert.assertThat(tableB.get(0))
.valueAfterFieldIsEqualTo(expectedValueB)
.valueAfterFieldSchemaIsEqualTo(expectedSchemaB);
stopConnector();
}
@Test
@FixFor("DBZ-1067")
public void testColumnExcludeList() throws Exception {

27
debezium-core/src/main/java/io/debezium/relational/RelationalDatabaseConnectorConfig.java
View File

@ -258,19 +258,6 @@ public static DecimalHandlingMode parse(String value, String defaultValue) {
.withValidation(Field::isListOfRegex, RelationalDatabaseConnectorConfig::validateColumnExcludeList)
.withDescription("Regular expressions matching columns to exclude from change events");
/**
* Old, backwards-compatible "blacklist" property.
*/
@Deprecated
public static final Field COLUMN_BLACKLIST = Field.create("column.blacklist")
.withDisplayName("Deprecated: Exclude Columns")
.withType(Type.LIST)
.withWidth(Width.LONG)
.withImportance(Importance.LOW)
.withValidation(Field::isListOfRegex, RelationalDatabaseConnectorConfig::validateColumnBlacklist)
.withInvisibleRecommender()
.withDescription("Regular expressions matching columns to exclude from change events (deprecated, use \"" + COLUMN_EXCLUDE_LIST.name() + "\" instead)");
/**
* A comma-separated list of regular expressions that match fully-qualified names of columns to be excluded from monitoring
* and change messages. The exact form of fully qualified names for columns might vary between connector types.
@ -500,7 +487,6 @@ public static DecimalHandlingMode parse(String value, String defaultValue) {
SNAPSHOT_LOCK_TIMEOUT_MS)
.events(
COLUMN_INCLUDE_LIST,
COLUMN_BLACKLIST,
COLUMN_EXCLUDE_LIST,
TABLE_INCLUDE_LIST,
TABLE_EXCLUDE_LIST,
@ -550,7 +536,7 @@ protected RelationalDatabaseConnectorConfig(Configuration config, String logical
this.tableFilters = null;
}
String columnExcludeList = config.getFallbackStringProperty(COLUMN_EXCLUDE_LIST, COLUMN_BLACKLIST);
String columnExcludeList = config.getString(COLUMN_EXCLUDE_LIST);
String columnIncludeList = config.getString(COLUMN_INCLUDE_LIST);
if (columnIncludeList != null) {
@ -634,17 +620,6 @@ public Boolean isFullColummnScanRequired() {
return getConfig().getBoolean(SNAPSHOT_FULL_COLUMN_SCAN_FORCE);
}
private static int validateColumnBlacklist(Configuration config, Field field, Field.ValidationOutput problems) {
String includeList = config.getString(COLUMN_INCLUDE_LIST);
String whitelist = config.getFallbackStringPropertyWithWarning(COLUMN_EXCLUDE_LIST, COLUMN_BLACKLIST);
if (whitelist != null && includeList != null) {
problems.accept(COLUMN_BLACKLIST, includeList, COLUMN_INCLUDE_LIST_ALREADY_SPECIFIED_ERROR_MSG);
return 1;
}
return 0;
}
private static int validateColumnExcludeList(Configuration config, Field field, Field.ValidationOutput problems) {
String includeList = config.getString(COLUMN_INCLUDE_LIST);
String excludeList = config.getString(COLUMN_EXCLUDE_LIST);

3
debezium-core/src/main/java/io/debezium/relational/RelationalTableFilters.java
View File

@ -5,7 +5,6 @@
*/
package io.debezium.relational;
import static io.debezium.relational.RelationalDatabaseConnectorConfig.COLUMN_BLACKLIST;
import static io.debezium.relational.RelationalDatabaseConnectorConfig.COLUMN_EXCLUDE_LIST;
import java.util.function.Predicate;
@ -74,7 +73,7 @@ public RelationalTableFilters(Configuration config, TableFilter systemTablesFilt
.excludeDatabases(config.getString(RelationalDatabaseConnectorConfig.DATABASE_EXCLUDE_LIST))
.build();
this.excludeColumns = config.getFallbackStringProperty(COLUMN_EXCLUDE_LIST, COLUMN_BLACKLIST);
this.excludeColumns = config.getString(COLUMN_EXCLUDE_LIST);
}
@Override

14
debezium-core/src/test/java/io/debezium/config/ConfigurationTest.java
View File

@ -5,7 +5,6 @@
*/
package io.debezium.config;
import static io.debezium.relational.RelationalDatabaseConnectorConfig.COLUMN_BLACKLIST;
import static io.debezium.relational.RelationalDatabaseConnectorConfig.COLUMN_EXCLUDE_LIST;
import static io.debezium.relational.RelationalDatabaseConnectorConfig.COLUMN_INCLUDE_LIST;
import static io.debezium.relational.RelationalDatabaseConnectorConfig.MSG_KEY_COLUMNS;
@ -81,19 +80,6 @@ public void shouldCreateInternalFields() {
assertThat(config.getString("internal.a")).isEqualTo("a1");
}
@Test
@FixFor("DBZ-1962")
public void shouldThrowValidationOnDuplicateOldColumnFilterConfigurationOld() {
config = Configuration.create()
.with(COLUMN_INCLUDE_LIST, ".+aa")
.with(COLUMN_BLACKLIST, ".+bb")
.build();
List<String> errorMessages = config.validate(Field.setOf(COLUMN_BLACKLIST)).get(COLUMN_BLACKLIST.name()).errorMessages();
assertThat(errorMessages).isNotEmpty();
assertThat(errorMessages.get(0)).isEqualTo(RelationalDatabaseConnectorConfig.COLUMN_INCLUDE_LIST_ALREADY_SPECIFIED_ERROR_MSG);
}
@Test
@FixFor("DBZ-1962")
public void shouldThrowValidationOnDuplicateOldColumnFilterConfiguration() {