DBZ-1831 Pulling up connector option retrieval to CommonConnectorConfig;
* Expanding test * Adding Sayed to COPYRIGHT.txt
This commit is contained in:
parent
63a462dbc0
commit
6318cf0f2b
@ -140,6 +140,7 @@ Sanjay Kr Singh
|
|||||||
Sanne Grinovero
|
Sanne Grinovero
|
||||||
Satyajit Vegesna
|
Satyajit Vegesna
|
||||||
Saulius Valatka
|
Saulius Valatka
|
||||||
|
Sayed Mohammad Hossein Torabi
|
||||||
Scofield Xu
|
Scofield Xu
|
||||||
Sean Rooney
|
Sean Rooney
|
||||||
Sherafudheen PM
|
Sherafudheen PM
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.OptionalLong;
|
import java.util.OptionalLong;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.concurrent.CountDownLatch;
|
import java.util.concurrent.CountDownLatch;
|
||||||
import java.util.concurrent.ExecutorService;
|
import java.util.concurrent.ExecutorService;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
@ -32,6 +33,7 @@
|
|||||||
import com.mongodb.client.model.Filters;
|
import com.mongodb.client.model.Filters;
|
||||||
|
|
||||||
import io.debezium.connector.mongodb.ConnectionContext.MongoPrimary;
|
import io.debezium.connector.mongodb.ConnectionContext.MongoPrimary;
|
||||||
|
import io.debezium.data.Envelope.Operation;
|
||||||
import io.debezium.pipeline.ErrorHandler;
|
import io.debezium.pipeline.ErrorHandler;
|
||||||
import io.debezium.pipeline.EventDispatcher;
|
import io.debezium.pipeline.EventDispatcher;
|
||||||
import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
|
import io.debezium.pipeline.source.spi.StreamingChangeEventSource;
|
||||||
@ -167,19 +169,7 @@ private void readOplog(MongoClient primary, MongoPrimary primaryClient, ReplicaS
|
|||||||
oplogContext.setIncompleteTxOrder(txOrder.getAsLong());
|
oplogContext.setIncompleteTxOrder(txOrder.getAsLong());
|
||||||
}
|
}
|
||||||
|
|
||||||
Bson operationFilter = null;
|
Bson operationFilter = getSkippedOperationsFilter();
|
||||||
String operations = connectionContext.config.getString(MongoDbConnectorConfig.SKIPPED_OPERATIONS);
|
|
||||||
if (operations != null) {
|
|
||||||
for (String operation : operations.trim().split(",")) {
|
|
||||||
operation = operation.trim();
|
|
||||||
|
|
||||||
if (operationFilter == null) {
|
|
||||||
operationFilter = Filters.ne("op", operation);
|
|
||||||
}
|
|
||||||
operationFilter = Filters.or(operationFilter, Filters.ne("op", operation));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (operationFilter != null) {
|
if (operationFilter != null) {
|
||||||
filter = Filters.and(filter, operationFilter);
|
filter = Filters.and(filter, operationFilter);
|
||||||
}
|
}
|
||||||
@ -225,6 +215,29 @@ private void readOplog(MongoClient primary, MongoPrimary primaryClient, ReplicaS
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Bson getSkippedOperationsFilter() {
|
||||||
|
Set<Operation> skippedOperations = taskContext.getConnectorConfig().getSkippedOps();
|
||||||
|
|
||||||
|
if (skippedOperations.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
Bson skippedOperationsFilter = null;
|
||||||
|
|
||||||
|
for (Operation operation : skippedOperations) {
|
||||||
|
Bson skippedOperationFilter = Filters.ne("op", operation.code());
|
||||||
|
|
||||||
|
if (skippedOperationsFilter == null) {
|
||||||
|
skippedOperationsFilter = skippedOperationFilter;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
skippedOperationsFilter = Filters.or(skippedOperationsFilter, skippedOperationFilter);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return skippedOperationsFilter;
|
||||||
|
}
|
||||||
|
|
||||||
private boolean handleOplogEvent(ServerAddress primaryAddress, Document event, Document masterEvent, long txOrder, ReplicaSetOplogContext oplogContext) {
|
private boolean handleOplogEvent(ServerAddress primaryAddress, Document event, Document masterEvent, long txOrder, ReplicaSetOplogContext oplogContext) {
|
||||||
String ns = event.getString("ns");
|
String ns = event.getString("ns");
|
||||||
Document object = event.get(OBJECT_FIELD, Document.class);
|
Document object = event.get(OBJECT_FIELD, Document.class);
|
||||||
|
@ -360,6 +360,7 @@ public void shouldConsumeAllEventsFromDatabase() throws InterruptedException, IO
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@FixFor("DBZ-1831")
|
||||||
public void shouldConsumeAllEventsFromDatabaseWithSkippedOperations() throws InterruptedException, IOException {
|
public void shouldConsumeAllEventsFromDatabaseWithSkippedOperations() throws InterruptedException, IOException {
|
||||||
// Use the DB configuration to define the connector's configuration ...
|
// Use the DB configuration to define the connector's configuration ...
|
||||||
config = TestHelper.getConfiguration().edit()
|
config = TestHelper.getConfiguration().edit()
|
||||||
@ -400,6 +401,9 @@ public void shouldConsumeAllEventsFromDatabaseWithSkippedOperations() throws Int
|
|||||||
Testing.debug("Document ID: " + id.get());
|
Testing.debug("Document ID: " + id.get());
|
||||||
});
|
});
|
||||||
|
|
||||||
|
SourceRecords insert = consumeRecordsByTopic(1);
|
||||||
|
assertThat(insert.recordsForTopic("mongo.dbit.arbitrary")).hasSize(1);
|
||||||
|
|
||||||
primary().execute("update", mongo -> {
|
primary().execute("update", mongo -> {
|
||||||
MongoDatabase db1 = mongo.getDatabase("dbit");
|
MongoDatabase db1 = mongo.getDatabase("dbit");
|
||||||
MongoCollection<Document> coll = db1.getCollection("arbitrary");
|
MongoCollection<Document> coll = db1.getCollection("arbitrary");
|
||||||
@ -415,11 +419,28 @@ public void shouldConsumeAllEventsFromDatabaseWithSkippedOperations() throws Int
|
|||||||
Testing.debug("Document: " + doc);
|
Testing.debug("Document: " + doc);
|
||||||
});
|
});
|
||||||
|
|
||||||
// Wait until we can consume the only 1 insert
|
primary().execute("delete", mongo -> {
|
||||||
SourceRecords insertAndUpdate = consumeRecordsByTopic(1);
|
MongoDatabase db1 = mongo.getDatabase("dbit");
|
||||||
assertThat(insertAndUpdate.recordsForTopic("mongo.dbit.arbitrary").size()).isEqualTo(1);
|
MongoCollection<Document> coll = db1.getCollection("arbitrary");
|
||||||
assertThat(insertAndUpdate.topics().size()).isEqualTo(1);
|
|
||||||
|
|
||||||
|
// Find the document ...
|
||||||
|
Document doc = coll.find().first();
|
||||||
|
Testing.debug("Document: " + doc);
|
||||||
|
Document filter = Document.parse("{\"a\": 1}");
|
||||||
|
|
||||||
|
// delete
|
||||||
|
coll.deleteOne(filter);
|
||||||
|
|
||||||
|
doc = coll.find().first();
|
||||||
|
Testing.debug("Document: " + doc);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Next should be the delete but not the skipped update
|
||||||
|
SourceRecords delete = consumeRecordsByTopic(1);
|
||||||
|
assertThat(delete.recordsForTopic("mongo.dbit.arbitrary")).hasSize(1);
|
||||||
|
SourceRecord deleteRecord = delete.allRecordsInOrder().get(0);
|
||||||
|
validate(deleteRecord);
|
||||||
|
verifyDeleteOperation(deleteRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -7,8 +7,11 @@
|
|||||||
|
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.time.temporal.ChronoUnit;
|
import java.time.temporal.ChronoUnit;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.apache.kafka.common.config.ConfigDef.Importance;
|
import org.apache.kafka.common.config.ConfigDef.Importance;
|
||||||
@ -19,6 +22,8 @@
|
|||||||
import io.debezium.config.Field.ValidationOutput;
|
import io.debezium.config.Field.ValidationOutput;
|
||||||
import io.debezium.connector.AbstractSourceInfo;
|
import io.debezium.connector.AbstractSourceInfo;
|
||||||
import io.debezium.connector.SourceInfoStructMaker;
|
import io.debezium.connector.SourceInfoStructMaker;
|
||||||
|
import io.debezium.data.Envelope;
|
||||||
|
import io.debezium.data.Envelope.Operation;
|
||||||
import io.debezium.heartbeat.Heartbeat;
|
import io.debezium.heartbeat.Heartbeat;
|
||||||
import io.debezium.relational.CustomConverterRegistry;
|
import io.debezium.relational.CustomConverterRegistry;
|
||||||
import io.debezium.relational.history.KafkaDatabaseHistory;
|
import io.debezium.relational.history.KafkaDatabaseHistory;
|
||||||
@ -376,6 +381,20 @@ public boolean getSanitizeFieldNames() {
|
|||||||
return sanitizeFieldNames;
|
return sanitizeFieldNames;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Set<Envelope.Operation> getSkippedOps() {
|
||||||
|
String operations = config.getString(SKIPPED_OPERATIONS);
|
||||||
|
|
||||||
|
if (operations != null) {
|
||||||
|
return Arrays.stream(operations.split(","))
|
||||||
|
.map(String::trim)
|
||||||
|
.map(Operation::forCode)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
return Collections.emptySet();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static int validateMaxQueueSize(Configuration config, Field field, Field.ValidationOutput problems) {
|
private static int validateMaxQueueSize(Configuration config, Field field, Field.ValidationOutput problems) {
|
||||||
int maxQueueSize = config.getInteger(field);
|
int maxQueueSize = config.getInteger(field);
|
||||||
int maxBatchSize = config.getInteger(MAX_BATCH_SIZE);
|
int maxBatchSize = config.getInteger(MAX_BATCH_SIZE);
|
||||||
@ -398,8 +417,9 @@ private static int validateSkippedOperation(Configuration config, Field field, V
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (String operation : operations.trim().split(",")) {
|
for (String operation : operations.split(",")) {
|
||||||
switch (operation.trim()) {
|
switch (operation.trim()) {
|
||||||
|
case "r":
|
||||||
case "i":
|
case "i":
|
||||||
case "u":
|
case "u":
|
||||||
case "d":
|
case "d":
|
||||||
|
Loading…
Reference in New Issue
Block a user